Skip to content

Commit 016749b

Browse files
manekinekkoruyadorno
manekinekko
authored and
ruyadorno
committed
test_runner: add initial TAP parser
Work in progress PR-URL: #43525 Refs: #43344 Reviewed-By: Franziska Hinkelmann <[email protected]> Reviewed-By: Colin Ihrig <[email protected]> Reviewed-By: Moshe Atlow <[email protected]>
1 parent f720c58 commit 016749b

19 files changed

+4418
-31
lines changed

‎doc/api/errors.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2690,6 +2690,25 @@ An unspecified or non-specific system error has occurred within the Node.js
26902690
process. The error object will have an `err.info` object property with
26912691
additional details.
26922692

2693+
<aid="ERR_TAP_LEXER_ERROR"></a>
2694+
2695+
### `ERR_TAP_LEXER_ERROR`
2696+
2697+
An error representing a failing lexer state.
2698+
2699+
<aid="ERR_TAP_PARSER_ERROR"></a>
2700+
2701+
### `ERR_TAP_PARSER_ERROR`
2702+
2703+
An error representing a failing parser state. Additional information about
2704+
the token causing the error is available via the `cause` property.
2705+
2706+
<aid="ERR_TAP_VALIDATION_ERROR"></a>
2707+
2708+
### `ERR_TAP_VALIDATION_ERROR`
2709+
2710+
This error represents a failed TAP validation.
2711+
26932712
<aid="ERR_TEST_FAILURE"></a>
26942713

26952714
### `ERR_TEST_FAILURE`

‎doc/api/test.md‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1042,8 +1042,7 @@ Emitted when [`context.diagnostic`][] is called.
10421042
### Event: `'test:fail'`
10431043

10441044
*`data`{Object}
1045-
*`duration`{number} The test duration.
1046-
*`error`{Error} The failure casing test to fail.
1045+
*`details`{Object} Additional execution metadata.
10471046
*`name`{string} The test name.
10481047
*`testNumber`{number} The ordinal number of the test.
10491048
*`todo`{string|undefined} Present if [`context.todo`][] is called
@@ -1054,7 +1053,7 @@ Emitted when a test fails.
10541053
### Event: `'test:pass'`
10551054

10561055
*`data`{Object}
1057-
*`duration`{number} The test duration.
1056+
*`details`{Object} Additional execution metadata.
10581057
*`name`{string} The test name.
10591058
*`testNumber`{number} The ordinal number of the test.
10601059
*`todo`{string|undefined} Present if [`context.todo`][] is called

‎lib/internal/errors.js‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1597,6 +1597,21 @@ E('ERR_STREAM_WRAP', 'Stream has StringDecoder set or is in objectMode', Error);
15971597
E('ERR_STREAM_WRITE_AFTER_END','write after end',Error);
15981598
E('ERR_SYNTHETIC','JavaScript Callstack',Error);
15991599
E('ERR_SYSTEM_ERROR','A system error occurred',SystemError);
1600+
E('ERR_TAP_LEXER_ERROR',function(errorMsg){
1601+
hideInternalStackFrames(this);
1602+
returnerrorMsg;
1603+
},Error);
1604+
E('ERR_TAP_PARSER_ERROR',function(errorMsg,details,tokenCausedError,source){
1605+
hideInternalStackFrames(this);
1606+
this.cause=tokenCausedError;
1607+
const{ column, line, start, end }=tokenCausedError.location;
1608+
consterrorDetails=`${details} at line ${line}, column ${column} (start ${start}, end ${end})`;
1609+
returnerrorMsg+errorDetails;
1610+
},SyntaxError);
1611+
E('ERR_TAP_VALIDATION_ERROR',function(errorMsg){
1612+
hideInternalStackFrames(this);
1613+
returnerrorMsg;
1614+
},Error);
16001615
E('ERR_TEST_FAILURE',function(error,failureType){
16011616
hideInternalStackFrames(this);
16021617
assert(typeoffailureType==='string',

‎lib/internal/test_runner/runner.js‎

Lines changed: 111 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
const{
33
ArrayFrom,
44
ArrayPrototypeFilter,
5+
ArrayPrototypeForEach,
56
ArrayPrototypeIncludes,
67
ArrayPrototypeJoin,
78
ArrayPrototypePush,
@@ -14,6 +15,7 @@ const{
1415
SafePromiseAllSettledReturnVoid,
1516
SafeMap,
1617
SafeSet,
18+
StringPrototypeRepeat,
1719
}=primordials;
1820

1921
const{ spawn }=require('child_process');
@@ -31,7 +33,10 @@ const{validateArray, validateBoolean } = require('internal/validators');
3133
const{ getInspectPort, isUsingInspector, isInspectorMessage }=require('internal/util/inspector');
3234
const{ kEmptyObject }=require('internal/util');
3335
const{ createTestTree }=require('internal/test_runner/harness');
34-
const{ kSubtestsFailed, Test }=require('internal/test_runner/test');
36+
const{ kDefaultIndent, kSubtestsFailed, Test }=require('internal/test_runner/test');
37+
const{ TapParser }=require('internal/test_runner/tap_parser');
38+
const{ TokenKind }=require('internal/test_runner/tap_lexer');
39+
3540
const{
3641
isSupportedFileType,
3742
doesPathMatchFilter,
@@ -120,11 +125,103 @@ function getRunArgs({path, inspectPort }){
120125
returnargv;
121126
}
122127

128+
classFileTestextendsTest{
129+
#buffer =[];
130+
#handleReportItem({ kind, node, nesting =0}){
131+
constindent=StringPrototypeRepeat(kDefaultIndent,nesting+1);
132+
133+
constdetails=(diagnostic)=>{
134+
return(
135+
diagnostic&&{
136+
__proto__: null,
137+
yaml:
138+
`${indent} `+
139+
ArrayPrototypeJoin(diagnostic,`\n${indent} `)+
140+
'\n',
141+
}
142+
);
143+
};
144+
145+
switch(kind){
146+
caseTokenKind.TAP_VERSION:
147+
// TODO(manekinekko): handle TAP version coming from the parser.
148+
// this.reporter.version(node.version);
149+
break;
150+
151+
caseTokenKind.TAP_PLAN:
152+
this.reporter.plan(indent,node.end-node.start+1);
153+
break;
154+
155+
caseTokenKind.TAP_SUBTEST_POINT:
156+
this.reporter.subtest(indent,node.name);
157+
break;
158+
159+
caseTokenKind.TAP_TEST_POINT:
160+
// eslint-disable-next-line no-case-declarations
161+
const{ todo, skip, pass }=node.status;
162+
// eslint-disable-next-line no-case-declarations
163+
letdirective;
164+
165+
if(skip){
166+
directive=this.reporter.getSkip(node.reason);
167+
}elseif(todo){
168+
directive=this.reporter.getTodo(node.reason);
169+
}else{
170+
directive=kEmptyObject;
171+
}
172+
173+
if(pass){
174+
this.reporter.ok(
175+
indent,
176+
node.id,
177+
node.description,
178+
details(node.diagnostics),
179+
directive
180+
);
181+
}else{
182+
this.reporter.fail(
183+
indent,
184+
node.id,
185+
node.description,
186+
details(node.diagnostics),
187+
directive
188+
);
189+
}
190+
break;
191+
192+
caseTokenKind.COMMENT:
193+
if(indent===kDefaultIndent){
194+
// Ignore file top level diagnostics
195+
break;
196+
}
197+
this.reporter.diagnostic(indent,node.comment);
198+
break;
199+
200+
caseTokenKind.UNKNOWN:
201+
this.reporter.diagnostic(indent,node.value);
202+
break;
203+
}
204+
}
205+
addToReport(ast){
206+
if(!this.isClearToSend()){
207+
ArrayPrototypePush(this.#buffer,ast);
208+
return;
209+
}
210+
this.reportSubtest();
211+
this.#handleReportItem(ast);
212+
}
213+
report(){
214+
this.reportSubtest();
215+
ArrayPrototypeForEach(this.#buffer,(ast)=>this.#handleReportItem(ast));
216+
super.report();
217+
}
218+
}
219+
123220
construnningProcesses=newSafeMap();
124221
construnningSubtests=newSafeMap();
125222

126223
functionrunTestFile(path,root,inspectPort,filesWatcher){
127-
constsubtest=root.createSubtest(Test,path,async(t)=>{
224+
constsubtest=root.createSubtest(FileTest,path,async(t)=>{
128225
constargs=getRunArgs({ path, inspectPort });
129226
conststdio=['pipe','pipe','pipe'];
130227
constenv={ ...process.env};
@@ -135,8 +232,7 @@ function runTestFile(path, root, inspectPort, filesWatcher){
135232

136233
constchild=spawn(process.execPath,args,{signal: t.signal,encoding: 'utf8', env, stdio });
137234
runningProcesses.set(path,child);
138-
// TODO(cjihrig): Implement a TAP parser to read the child's stdout
139-
// instead of just displaying it all if the child fails.
235+
140236
leterr;
141237
letstderr='';
142238

@@ -159,6 +255,17 @@ function runTestFile(path, root, inspectPort, filesWatcher){
159255
});
160256
}
161257

258+
constparser=newTapParser();
259+
child.stderr.pipe(parser).on('data',(ast)=>{
260+
if(ast.lexeme&&isInspectorMessage(ast.lexeme)){
261+
process.stderr.write(ast.lexeme+'\n');
262+
}
263+
});
264+
265+
child.stdout.pipe(parser).on('data',(ast)=>{
266+
subtest.addToReport(ast);
267+
});
268+
162269
const{0: {0: code,1: signal},1: stdout}=awaitSafePromiseAll([
163270
once(child,'exit',{signal: t.signal}),
164271
child.stdout.toArray({signal: t.signal}),

‎lib/internal/test_runner/tap_checker.js‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
'use strict';
2+
3+
const{
4+
ArrayPrototypeFilter,
5+
ArrayPrototypeFind,
6+
NumberParseInt,
7+
}=primordials;
8+
const{
9+
codes: {ERR_TAP_VALIDATION_ERROR},
10+
}=require('internal/errors');
11+
const{ TokenKind }=require('internal/test_runner/tap_lexer');
12+
13+
// TODO(@manekinekko): add more validation rules based on the TAP14 spec.
14+
// See https://testanything.org/tap-version-14-specification.html
15+
classTAPValidationStrategy{
16+
validate(ast){
17+
this.#validateVersion(ast);
18+
this.#validatePlan(ast);
19+
this.#validateTestPoints(ast);
20+
21+
returntrue;
22+
}
23+
24+
#validateVersion(ast){
25+
constentry=ArrayPrototypeFind(
26+
ast,
27+
(node)=>node.kind===TokenKind.TAP_VERSION
28+
);
29+
30+
if(!entry){
31+
thrownewERR_TAP_VALIDATION_ERROR('missing TAP version');
32+
}
33+
34+
const{ version }=entry.node;
35+
36+
// TAP14 specification is compatible with observed behavior of existing TAP13 consumers and producers
37+
if(version!=='14'&&version!=='13'){
38+
thrownewERR_TAP_VALIDATION_ERROR('TAP version should be 13 or 14');
39+
}
40+
}
41+
42+
#validatePlan(ast){
43+
constentry=ArrayPrototypeFind(
44+
ast,
45+
(node)=>node.kind===TokenKind.TAP_PLAN
46+
);
47+
48+
if(!entry){
49+
thrownewERR_TAP_VALIDATION_ERROR('missing TAP plan');
50+
}
51+
52+
constplan=entry.node;
53+
54+
if(!plan.start){
55+
thrownewERR_TAP_VALIDATION_ERROR('missing plan start');
56+
}
57+
58+
if(!plan.end){
59+
thrownewERR_TAP_VALIDATION_ERROR('missing plan end');
60+
}
61+
62+
constplanStart=NumberParseInt(plan.start,10);
63+
constplanEnd=NumberParseInt(plan.end,10);
64+
65+
if(planEnd!==0&&planStart>planEnd){
66+
thrownewERR_TAP_VALIDATION_ERROR(
67+
`plan start ${planStart} is greater than plan end ${planEnd}`
68+
);
69+
}
70+
}
71+
72+
// TODO(@manekinekko): since we are dealing with a flat AST, we need to
73+
// validate test points grouped by their "nesting" level. This is because a set of
74+
// Test points belongs to a TAP document. Each new subtest block creates a new TAP document.
75+
// https://testanything.org/tap-version-14-specification.html#subtests
76+
#validateTestPoints(ast){
77+
constbailoutEntry=ArrayPrototypeFind(
78+
ast,
79+
(node)=>node.kind===TokenKind.TAP_BAIL_OUT
80+
);
81+
constplanEntry=ArrayPrototypeFind(
82+
ast,
83+
(node)=>node.kind===TokenKind.TAP_PLAN
84+
);
85+
consttestPointEntries=ArrayPrototypeFilter(
86+
ast,
87+
(node)=>node.kind===TokenKind.TAP_TEST_POINT
88+
);
89+
90+
constplan=planEntry.node;
91+
92+
constplanStart=NumberParseInt(plan.start,10);
93+
constplanEnd=NumberParseInt(plan.end,10);
94+
95+
if(planEnd===0&&testPointEntries.length>0){
96+
thrownewERR_TAP_VALIDATION_ERROR(
97+
`found ${testPointEntries.length} Test Point${
98+
testPointEntries.length>1 ? 's' : ''
99+
} but plan is ${planStart}..0`
100+
);
101+
}
102+
103+
if(planEnd>0){
104+
if(testPointEntries.length===0){
105+
thrownewERR_TAP_VALIDATION_ERROR('missing Test Points');
106+
}
107+
108+
if(!bailoutEntry&&testPointEntries.length!==planEnd){
109+
thrownewERR_TAP_VALIDATION_ERROR(
110+
`test Points count ${testPointEntries.length} does not match plan count ${planEnd}`
111+
);
112+
}
113+
114+
for(leti=0;i<testPointEntries.length;i++){
115+
consttest=testPointEntries[i].node;
116+
consttestId=NumberParseInt(test.id,10);
117+
118+
if(testId<planStart||testId>planEnd){
119+
thrownewERR_TAP_VALIDATION_ERROR(
120+
`test ${testId} is out of plan range ${planStart}..${planEnd}`
121+
);
122+
}
123+
}
124+
}
125+
}
126+
}
127+
128+
// TAP14 and TAP13 are compatible with each other
129+
classTAP13ValidationStrategyextendsTAPValidationStrategy{}
130+
classTAP14ValidationStrategyextendsTAPValidationStrategy{}
131+
132+
classTapChecker{
133+
staticTAP13='13';
134+
staticTAP14='14';
135+
136+
constructor({ specs }){
137+
switch(specs){
138+
caseTapChecker.TAP13:
139+
this.strategy=newTAP13ValidationStrategy();
140+
break;
141+
default:
142+
this.strategy=newTAP14ValidationStrategy();
143+
}
144+
}
145+
146+
check(ast){
147+
returnthis.strategy.validate(ast);
148+
}
149+
}
150+
151+
module.exports={
152+
TapChecker,
153+
TAP14ValidationStrategy,
154+
TAP13ValidationStrategy,
155+
};

0 commit comments

Comments
(0)