Add end2end tests (#470)

- Move unittests to run-unittests.mjs
- Move common test driver and server setup to separate helper.mjs module
- Add "SpeedometerReady" and "SpeedometerDone" events for easier testing
- Add simple end2end tests that invoke the benchmark via the html page

Author: camillobruni

.github/workflows/test.yml

@@ -57,3 +57,7 @@ jobs:
               run: |
                   echo "Running in $BROWSER"
                   npm run test:${{ matrix.browser }}
+            - name: Run end2end
+              run: |
+                  echo "Running in $BROWSER"
+                  npm run test-e2e:${{ matrix.browser }}

package-lock.json

@@ -22,10 +22,14 @@
         "pretty:check": "prettier --check ./",
         "pretty:fix": "prettier --write ./",
         "format": "npm run pretty:fix && npm run lint:fix",
-        "test:chrome": "node tests/run.mjs --browser chrome",
-        "test:firefox": "node tests/run.mjs --browser firefox",
-        "test:safari": "node tests/run.mjs --browser safari",
-        "test:edge": "node tests/run.mjs --browser edge"
+        "test:chrome": "node tests/run-unittests.mjs --browser chrome",
+        "test:firefox": "node tests/run-unittests.mjs --browser firefox",
+        "test:safari": "node tests/run-unittests.mjs --browser safari",
+        "test:edge": "node tests/run-unittests.mjs --browser edge",
+        "test-e2e:chrome": "node tests/run-end2end.mjs --browser chrome",
+        "test-e2e:firefox": "node tests/run-end2end.mjs --browser firefox",
+        "test-e2e:safari": "node tests/run-end2end.mjs --browser safari",
+        "test-e2e:edge": "node tests/run-end2end.mjs --browser edge"
     },
     "devDependencies": {
         "@babel/core": "^7.21.3",
@@ -45,7 +49,7 @@
         "http-server": "^14.1.1",
         "mocha": "^10.2.0",
         "prettier": "^2.8.3",
-        "selenium-webdriver": "^4.8.0",
+        "selenium-webdriver": "^4.27.0",
         "sinon": "^17.0.1",
         "typescript": "^5.0.4"
     }

package.json

@@ -23,6 +23,7 @@ class MainBenchmarkClient {
     constructor() {
         window.addEventListener("DOMContentLoaded", () => this.prepareUI());
         this._showSection(window.location.hash);
+        window.dispatchEvent(new Event("SpeedometerReady"));
     }
 
     start() {
@@ -150,6 +151,7 @@ class MainBenchmarkClient {
             this.showResultsDetails();
         else
             this.showResultsSummary();
+        globalThis.dispatchEvent(new Event("SpeedometerDone"));
     }
 
     handleError(error) {
@@ -159,6 +161,7 @@ class MainBenchmarkClient {
         this._metrics = Object.create(null);
         this._populateInvalidScore();
         this.showResultsSummary();
+        throw error;
     }
 
     _populateValidScore(scoreResults) {

resources/main.mjs

@@ -0,0 +1,90 @@
+import commandLineUsage from "command-line-usage";
+import commandLineArgs from "command-line-args";
+import serve from "./server.mjs";
+
+import { Builder, Capabilities, logging } from "selenium-webdriver";
+
+const optionDefinitions = [
+    { name: "browser", type: String, description: "Set the browser to test, choices are [safari, firefox, chrome]. By default the $BROWSER env variable is used." },
+    { name: "port", type: Number, defaultValue: 8010, description: "Set the test-server port, The default value is 8010." },
+    { name: "help", alias: "h", description: "Print this help text." },
+];
+
+function printHelp(message = "", exitStatus = 0) {
+    const usage = commandLineUsage([
+        {
+            header: "Run all tests",
+        },
+        {
+            header: "Options",
+            optionList: optionDefinitions,
+        },
+    ]);
+    if (message) {
+        console.error(message);
+        console.error();
+    }
+    console.log(usage);
+    process.exit(exitStatus);
+}
+
+export default async function testSetup(helpText) {
+    const options = commandLineArgs(optionDefinitions);
+
+    if ("help" in options)
+        printHelp(helpText);
+
+    const BROWSER = options?.browser;
+    if (!BROWSER)
+        printHelp("No browser specified, use $BROWSER or --browser", 1);
+
+    let capabilities;
+    switch (BROWSER) {
+        case "safari":
+            capabilities = Capabilities.safari();
+            break;
+
+        case "firefox": {
+            capabilities = Capabilities.firefox();
+            break;
+        }
+        case "chrome": {
+            capabilities = Capabilities.chrome();
+            break;
+        }
+        case "edge": {
+            capabilities = Capabilities.edge();
+            break;
+        }
+        default: {
+            printHelp(`Invalid browser "${BROWSER}", choices are: "safari", "firefox", "chrome", "edge"`);
+        }
+    }
+    const prefs = new logging.Preferences();
+    prefs.setLevel(logging.Type.BROWSER, logging.Level.ALL); // Capture all log levels
+    capabilities.setLoggingPrefs(prefs);
+
+    const PORT = options.port;
+    const server = serve(PORT);
+    let driver;
+
+    process.on("unhandledRejection", (err) => {
+        console.error(err);
+        process.exit(1);
+    });
+    process.once("uncaughtException", (err) => {
+        console.error(err);
+        process.exit(1);
+    });
+    process.on("exit", () => stop());
+
+    driver = await new Builder().withCapabilities(capabilities).build();
+    driver.manage().window().setRect({ width: 1200, height: 1000 });
+
+    function stop() {
+        server.close();
+        if (driver)
+            driver.close();
+    }
+    return { driver, PORT, stop };
+}

tests/helper.mjs

@@ -27,7 +27,7 @@
                 },
             });
 
-            await import("./benchmark-runner-tests.mjs");
+            await import("./unittests/benchmark-runner.mjs");
 
             globalThis.testResults = undefined;
             globalThis.testRunner = mocha.run();