fix handling of missing WPT results (#890)

Guy Bedford · web-flow · commit 6e81d41be389 · 2024-08-08T09:08:12.000+01:00
diff --git a/tests/wpt-harness/run-wpt.mjs b/tests/wpt-harness/run-wpt.mjs
@@ -220,7 +220,7 @@ async function run() {
 
     if (config.tests.updateExpectations) {
       console.log(`Expectations updated: ${expectationsUpdated}`);
-    } else if (stats.unexpectedFail + stats.unexpectedPass != 0 || unexpectedFailure) {
+    } else if (stats.unexpectedFail + stats.unexpectedPass + stats.missing != 0 || unexpectedFailure) {
       process.exitCode = 1;
     }
   }
@@ -458,7 +458,8 @@ async function runTests(testPaths, viceroy, resultCallback, errorCallback) {
       await resultCallback(path, results, stats);
     } catch (e) {
       if (!results) {
-        e = new Error(`Parsing test results as JSON failed. Output was:\n  ${body}`);
+        e = new Error(`\nMISSING TEST RESULTS: ${path}\nParsing test results as JSON failed. Output was:\n  ${body}`);
+        totalStats.missing += Math.max(Object.keys(expectations).length, 1);
       }
       if (config.logLevel >= LogLevel.Verbose) {
         console.log(`Error running file ${path}: ${e.message}, stack:\n${e.stack}`);

Original file line number	Diff line number	Diff line change
`@@ -220,7 +220,7 @@ async function run() {`
`220`	`220`
`221`	`221`	`if (config.tests.updateExpectations) {`
`222`	`222`	console.log(`Expectations updated: ${expectationsUpdated}`);
`223`		`- } else if (stats.unexpectedFail + stats.unexpectedPass != 0 \|\| unexpectedFailure) {`
	`223`	`+ } else if (stats.unexpectedFail + stats.unexpectedPass + stats.missing != 0 \|\| unexpectedFailure) {`
`224`	`224`	`process.exitCode = 1;`
`225`	`225`	`}`
`226`	`226`	`}`
`@@ -458,7 +458,8 @@ async function runTests(testPaths, viceroy, resultCallback, errorCallback) {`
`458`	`458`	`await resultCallback(path, results, stats);`
`459`	`459`	`} catch (e) {`
`460`	`460`	`if (!results) {`
`461`		- e = new Error(`Parsing test results as JSON failed. Output was:\n ${body}`);
	`461`	+ e = new Error(`\nMISSING TEST RESULTS: ${path}\nParsing test results as JSON failed. Output was:\n ${body}`);
	`462`	`+ totalStats.missing += Math.max(Object.keys(expectations).length, 1);`
`462`	`463`	`}`
`463`	`464`	`if (config.logLevel >= LogLevel.Verbose) {`
`464`	`465`	console.log(`Error running file ${path}: ${e.message}, stack:\n${e.stack}`);