First stab at adding support for tests timeout

src/classes/TestResult.js

@@ -83,37 +83,59 @@ export default class TestResult extends BubblingEventTarget {
 	 * Run the test(s)
 	 */
 	async run () {
-		this.messages = await interceptConsole(async () => {
-			if (!this.parent) {
-				// We are running the test in isolation, so we need to run beforeAll (if it exists)
-				await this.test.beforeAll?.();
-			}
+		let test = this.test;
+
+		// By default, give the test 10 seconds to run
+		let timeout = 10000;
+		if (test.maxTime && ("expect" in test || test.throws !== undefined)) {
+			// For result-based and error-based tests, maxTime is the timeout
+			timeout = test.maxTime;
+		}
 
-			await this.test.beforeEach?.();
+		let timeoutId;
+		this.messages = await Promise.race([
+			new Promise(resolve => {
+				timeoutId = setTimeout(() => {
+					this.error = new Error(`Test timed out after ${ timeout }ms`);
+					this.timeTaken = timeout;
+					resolve([]);
+				}, timeout);
+			}),
+
+			interceptConsole(async () => {
+				if (!this.parent) {
+					// We are running the test in isolation, so we need to run beforeAll (if it exists)
+					await test.beforeAll?.();
+				}
 
-			let start = performance.now();
+				await test.beforeEach?.();
 
-			try {
-				this.actual = this.test.run ? this.test.run.apply(this.test, this.test.args) : this.test.args[0];
-				this.timeTaken = performance.now() - start;
+				let start = performance.now();
 
-				if (this.actual instanceof Promise) {
-					this.actual = await this.actual;
-					this.timeTakenAsync = performance.now() - start;
+				try {
+					this.actual = test.run ? test.run.apply(test, test.args) : test.args[0];
+					this.timeTaken = performance.now() - start;
+
+					if (this.actual instanceof Promise) {
+						this.actual = await this.actual;
+						this.timeTakenAsync = performance.now() - start;
+					}
 				}
-			}
-			catch (e) {
-				this.error = e;
-			}
-			finally {
-				await this.test.afterEach?.();
+				catch (e) {
+					this.error = e;
+				}
+				finally {
+					await test.afterEach?.();
 
-				if (!this.parent) {
+					if (!this.parent) {
 					// We are running the test in isolation, so we need to run afterAll
-					await this.test.afterAll?.();
+						await test.afterAll?.();
+					}
 				}
-			}
-		});
+			}),
+		]);
+
+		clearTimeout(timeoutId);
 
 		this.evaluate();
 	}
@@ -183,13 +205,14 @@ export default class TestResult extends BubblingEventTarget {
 	evaluate () {
 		let test = this.test;
 
+		if (test.maxTime || test.maxTimeAsync) {
+			Object.assign(this, this.evaluateTimeTaken());
+		}
+
 		if (test.throws !== undefined) {
 			Object.assign(this, this.evaluateThrown());
 		}
-		else if (test.maxTime || test.maxTimeAsync) {
-			Object.assign(this, this.evaluateTimeTaken());
-		}
-		else {
+		else if ("expect" in test) {
 			Object.assign(this, this.evaluateResult());
 		}
 
@@ -209,7 +232,7 @@ export default class TestResult extends BubblingEventTarget {
 	 */
 	evaluateThrown () {
 		let test = this.test;
-		let ret = {pass: !!this.error, details: []};
+		let ret = {pass: (this.pass ?? true) && !!this.error, details: this.details ?? []};
 
 		// We may have more picky criteria for the error
 		if (ret.pass) {
@@ -251,38 +274,42 @@ export default class TestResult extends BubblingEventTarget {
 	 */
 	evaluateResult () {
 		let test = this.test;
-		let ret = {pass: true, details: []};
 
-		if (test.map) {
-			try {
-				this.mapped = {
-					actual: Array.isArray(this.actual) ? this.actual.map(test.map) : test.map(this.actual),
-					expect: Array.isArray(test.expect) ? test.expect.map(test.map) : test.map(test.expect),
-				};
+		// If we are here and there is an error (e.g., the test timed out), we consider the test failed
+		let ret = {pass: (this.pass ?? true) && !this.error, details: this.details ?? []};
 
+		if (ret.pass) {
+			if (test.map) {
 				try {
-					ret.pass = test.check(this.mapped.actual, this.mapped.expect);
+					this.mapped = {
+						actual: Array.isArray(this.actual) ? this.actual.map(test.map) : test.map(this.actual),
+						expect: Array.isArray(test.expect) ? test.expect.map(test.map) : test.map(test.expect),
+					};
+
+					try {
+						ret.pass = test.check(this.mapped.actual, this.mapped.expect);
+					}
+					catch (e) {
+						this.error = new Error(`check() failed (working with mapped values). ${ e.message }`);
+					}
 				}
 				catch (e) {
-					this.error = new Error(`check() failed (working with mapped values). ${ e.message }`);
+					this.error = new Error(`map() failed. ${ e.message }`);
 				}
 			}
-			catch (e) {
-				this.error = new Error(`map() failed. ${ e.message }`);
-			}
-		}
-		else {
-			try {
-				ret.pass = test.check(this.actual, test.expect);
-			}
-			catch (e) {
-				this.error = new Error(`check() failed. ${ e.message }`);
+			else {
+				try {
+					ret.pass = test.check(this.actual, test.expect);
+				}
+				catch (e) {
+					this.error = new Error(`check() failed. ${ e.message }`);
+				}
 			}
-		}
 
-		// If `map()` or `check()` errors, consider the test failed
-		if (this.error) {
-			ret.pass = false;
+			// If `map()` or `check()` errors, consider the test failed
+			if (this.error) {
+				ret.pass = false;
+			}
 		}
 
 		if (!ret.pass) {

tests/failing-tests.js

@@ -1,6 +1,7 @@
 export default {
 	name: "Failing tests",
 	description: "These tests are designed to fail and should not break the test runner",
+	expect: 42,
 	tests: [
 		{
 			name: "map() fails",
@@ -15,7 +16,6 @@ export default {
 			map: arg => undefined,
 			check: (actual, expected) => actual.length < expected.length,
 			arg: 42,
-			expect: 42,
 		},
 	],
 };

tests/timeout.js

@@ -0,0 +1,25 @@
+export default {
+	name: "Tests for timeout",
+	description: "These tests are designed to fail.",
+	run: () => new Promise(resolve => setTimeout(resolve, 200, "foo")),
+	maxTime: 100,
+	tests: [
+		{
+			name: "Result-based test",
+			expect: "bar",
+		},
+		{
+			name: "Error-based test",
+			throws: error => !error.message.startsWith("Test timed out"),
+		},
+		{
+			name: "Time-based test",
+			maxTimeAsync: 100,
+		},
+		{
+			name: "Default timeout",
+			run: () => new Promise(resolve => setTimeout(resolve, 10200)),
+			skip: true, // Comment this line out to see the test fail after 10 seconds
+		},
+	],
+};