Skip to content

Commit ff1fcab

Browse files
JakobJingleheimer2 peopleauthored and
aduh95
committed
test_runner: support expecting a test-case to fail
Co-Authored-By: Alejandro Espa <98526766+vespa7@users.noreply.github.com> PR-URL: #60669 Reviewed-By: Aviv Keller <me@aviv.sh> Reviewed-By: Pietro Marchini <pietro.marchini94@gmail.com> Reviewed-By: Moshe Atlow <moshe@atlow.co.il> Reviewed-By: Jordan Harband <ljharb@gmail.com>
1 parent 3f17acf commit ff1fcab

File tree

22 files changed

+806
-493
lines changed

22 files changed

+806
-493
lines changed

‎doc/api/test.md‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,57 @@ test('todo() method with message', (t) => {
224224
});
225225
```
226226

227+
## Expecting tests to fail
228+
229+
<!-- YAML
230+
added:
231+
- REPLACEME
232+
-->
233+
234+
This flips the pass/fail reporting for a specific test or suite: A flagged test/test-case must throw
235+
in order to "pass"; a test/test-case that does not throw, fails.
236+
237+
In the following, `doTheThing()` returns _currently_ `false` (`false` does not equal `true`, causing
238+
`strictEqual` to throw, so the test-case passes).
239+
240+
```js
241+
it.expectFailure('should do the thing', () => {
242+
assert.strictEqual(doTheThing(), true);
243+
});
244+
245+
it('should do the thing', { expectFailure: true }, () => {
246+
assert.strictEqual(doTheThing(), true);
247+
});
248+
```
249+
250+
`skip` and/or `todo` are mutually exclusive to `expectFailure`, and `skip` or `todo`
251+
will "win" when both are applied (`skip` wins against both, and `todo` wins
252+
against `expectFailure`).
253+
254+
These tests will be skipped (and not run):
255+
256+
```js
257+
it.expectFailure('should do the thing', { skip: true }, () => {
258+
assert.strictEqual(doTheThing(), true);
259+
});
260+
261+
it.skip('should do the thing', { expectFailure: true }, () => {
262+
assert.strictEqual(doTheThing(), true);
263+
});
264+
```
265+
266+
These tests will be marked "todo" (silencing errors):
267+
268+
```js
269+
it.expectFailure('should do the thing', { todo: true }, () => {
270+
assert.strictEqual(doTheThing(), true);
271+
});
272+
273+
it.todo('should do the thing', { expectFailure: true }, () => {
274+
assert.strictEqual(doTheThing(), true);
275+
});
276+
```
277+
227278
## `describe()` and `it()` aliases
228279

229280
Suites and tests can also be written using the `describe()` and `it()`

‎lib/internal/test_runner/harness.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -377,7 +377,7 @@ function runInParentContext(Factory) {
377377

378378
return run(name, options, fn, overrides);
379379
};
380-
ArrayPrototypeForEach(['skip', 'todo', 'only'], (keyword) => {
380+
ArrayPrototypeForEach(['expectFailure', 'skip', 'todo', 'only'], (keyword) => {
381381
test[keyword] = (name, options, fn) => {
382382
const overrides = {
383383
__proto__: null,

‎lib/internal/test_runner/reporter/tap.js‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,12 @@ async function * tapReporter(source) {
3333
for await (const { type, data } of source) {
3434
switch (type) {
3535
case 'test:fail': {
36-
yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo);
36+
yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure);
3737
const location = data.file ? `${data.file}:${data.line}:${data.column}` : null;
3838
yield reportDetails(data.nesting, data.details, location);
3939
break;
4040
} case 'test:pass':
41-
yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo);
41+
yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure);
4242
yield reportDetails(data.nesting, data.details, null);
4343
break;
4444
case 'test:plan':
@@ -65,7 +65,7 @@ async function * tapReporter(source) {
6565
}
6666
}
6767

68-
function reportTest(nesting, testNumber, status, name, skip, todo) {
68+
function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure) {
6969
let line = `${indent(nesting)}${status} ${testNumber}`;
7070

7171
if (name) {
@@ -76,6 +76,8 @@ function reportTest(nesting, testNumber, status, name, skip, todo) {
7676
line += ` # SKIP${typeof skip === 'string' && skip.length ? ` ${tapEscape(skip)}` : ''}`;
7777
} else if (todo !== undefined) {
7878
line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`;
79+
} else if (expectFailure !== undefined) {
80+
line += ' # EXPECTED FAILURE';
7981
}
8082

8183
line += '\n';

‎lib/internal/test_runner/reporter/utils.js‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,14 +71,16 @@ function formatError(error, indent) {
7171
function formatTestReport(type, data, prefix = '', indent = '', hasChildren = false, showErrorDetails = true) {
7272
let color = reporterColorMap[type] ?? colors.white;
7373
let symbol = reporterUnicodeSymbolMap[type] ?? ' ';
74-
const { skip, todo } = data;
74+
const { skip, todo, expectFailure } = data;
7575
const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : '';
7676
let title = `${data.name}${duration_ms}`;
7777

7878
if (skip !== undefined) {
7979
title += ` # ${typeof skip === 'string' && skip.length ? skip : 'SKIP'}`;
8080
} else if (todo !== undefined) {
8181
title += ` # ${typeof todo === 'string' && todo.length ? todo : 'TODO'}`;
82+
} else if (expectFailure !== undefined) {
83+
title += ` # EXPECTED FAILURE`;
8284
}
8385

8486
const error = showErrorDetails ? formatError(data.details?.error, indent) : '';

‎lib/internal/test_runner/test.js‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -496,7 +496,7 @@ class Test extends AsyncResource {
496496
super('Test');
497497

498498
let { fn, name, parent } = options;
499-
const { concurrency, entryFile, loc, only, timeout, todo, skip, signal, plan } = options;
499+
const { concurrency, entryFile, expectFailure, loc, only, timeout, todo, skip, signal, plan } = options;
500500

501501
if (typeof fn !== 'function') {
502502
fn = noop;
@@ -635,6 +635,7 @@ class Test extends AsyncResource {
635635
this.plan = null;
636636
this.expectedAssertions = plan;
637637
this.cancelled = false;
638+
this.expectFailure = expectFailure !== undefined && expectFailure !== false;
638639
this.skipped = skip !== undefined && skip !== false;
639640
this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo;
640641
this.startTime = null;
@@ -938,7 +939,12 @@ class Test extends AsyncResource {
938939
return;
939940
}
940941

941-
this.passed = false;
942+
if (this.expectFailure === true) {
943+
this.passed = true;
944+
} else {
945+
this.passed = false;
946+
}
947+
942948
this.error = err;
943949
}
944950

@@ -1335,6 +1341,8 @@ class Test extends AsyncResource {
13351341
directive = this.reporter.getSkip(this.message);
13361342
} else if (this.isTodo) {
13371343
directive = this.reporter.getTodo(this.message);
1344+
} else if (this.expectFailure) {
1345+
directive = this.reporter.getXFail(this.expectFailure); // TODO(@JakobJingleheimer): support specifying failure
13381346
}
13391347

13401348
if (this.reportedType) {
@@ -1349,6 +1357,7 @@ class Test extends AsyncResource {
13491357
if (this.passedAttempt !== undefined) {
13501358
details.passed_on_attempt = this.passedAttempt;
13511359
}
1360+
13521361
return { __proto__: null, details, directive };
13531362
}
13541363

‎lib/internal/test_runner/tests_stream.js‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,10 @@ class TestsStream extends Readable {
8787
return { __proto__: null, todo: reason ?? true };
8888
}
8989

90+
getXFail(expectation = undefined) {
91+
return { __proto__: null, expectFailure: expectation ?? true };
92+
}
93+
9094
enqueue(nesting, loc, name, type) {
9195
this[kEmitMessage]('test:enqueue', {
9296
__proto__: null,

‎test/fixtures/test-runner/output/describe_it.js‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,23 @@ const { describe, it, test } = require('node:test');
55
const util = require('util');
66

77

8-
it.todo('sync pass todo', () => {
8+
it.expectFailure('sync expect fail (method)', () => {
9+
throw new Error('should pass');
10+
});
11+
12+
it('sync expect fail (options)', { expectFailure: true }, () => {
13+
throw new Error('should pass');
14+
});
915

16+
it.expectFailure('async expect fail (method)', async () => {
17+
throw new Error('should pass');
18+
});
19+
20+
it('async expect fail (options)', { expectFailure: true }, async () => {
21+
throw new Error('should pass');
22+
});
23+
24+
it.todo('sync pass todo', () => {
1025
});
1126

1227
it('sync pass todo with message', { todo: 'this is a passing todo' }, () => {
@@ -16,13 +31,21 @@ it.todo('sync todo', () => {
1631
throw new Error('should not count as a failure');
1732
});
1833

34+
it.todo('sync todo with expect fail', { expectFailure: true }, () => {
35+
throw new Error('should not count as an expected failure');
36+
});
37+
1938
it('sync todo with message', { todo: 'this is a failing todo' }, () => {
2039
throw new Error('should not count as a failure');
2140
});
2241

2342
it.skip('sync skip pass', () => {
2443
});
2544

45+
it.skip('sync skip expect fail', { expectFailure: true }, () => {
46+
throw new Error('should not fail');
47+
});
48+
2649
it('sync skip pass with message', { skip: 'this is skipped' }, () => {
2750
});
2851

0 commit comments

Comments
 (0)