WIP: convert to local runner

Author: wesleytodd

package-lock.json

@@ -1,3 +1,4 @@
+import { spawnSync, spawn } from 'node:child_process';
 import autocannon from 'autocannon';
 
 export function run (opts) {
@@ -30,3 +31,51 @@ export default async function main (_opts = {}) {
 
   return run(opts);
 }
+
+export class AutocannonBenchmarker {
+  constructor() {
+    const shell = (process.platform === 'win32');
+    this.name = 'autocannon';
+    this.opts = { shell };
+    this.executable = shell ? 'autocannon.cmd' : 'autocannon';
+    const result = spawnSync(this.executable, ['-h'], this.opts);
+    if (shell) {
+      this.present = (result.status === 0);
+    } else {
+      this.present = !(result.error && result.error.code === 'ENOENT');
+    }
+  }
+
+  create(options) {
+    const args = [
+      '-d', options.duration,
+      '-c', options.connections,
+      '-j',
+      '-n',
+    ];
+    for (const field in options.headers) {
+      if (this.opts.shell) {
+        args.push('-H', `'${field}=${options.headers[field]}'`);
+      } else {
+        args.push('-H', `${field}=${options.headers[field]}`);
+      }
+    }
+    const scheme = options.scheme || 'http';
+    args.push(`${scheme}://127.0.0.1:${options.port}${options.path}`);
+    const child = spawn(this.executable, args, this.opts);
+    return child;
+  }
+
+  processResults(output) {
+    let result;
+    try {
+      result = JSON.parse(output);
+    } catch {
+      return undefined;
+    }
+    if (!result || !result.requests || !result.requests.average) {
+      return undefined;
+    }
+    return result.requests.average;
+  }
+}

packages/autocannon/index.mjs

@@ -11,7 +11,7 @@ export function help (opts = {}) {
   Flags:
 
     --cwd=${opts.cwd || normalize(join(import.meta.dirname, '..', '..'))}
-    --runner=@expressjs/perf-runner-vanilla
+    --runner=@expressjs/perf-runner-local
     --repo=https://github.com/expressjs/perf-wg.git
     --repo-ref=master
     --test=@expressjs/perf-load-example
@@ -53,9 +53,9 @@ export default function main (_opts = {}) {
 
     const opts = {
       cwd,
-      repo: 'https://github.com/expressjs/perf-wg.git',
-      repoRef: 'master',
-      runner: '@expressjs/perf-runner-vanilla',
+      repo: null, // 'https://github.com/expressjs/perf-wg.git',
+      repoRef: null, // 'master',
+      runner: '@expressjs/perf-runner-local',
       test: '@expressjs/perf-load-example',
       node: 'lts_latest',
       ...conf,
@@ -109,6 +109,11 @@ export default function main (_opts = {}) {
       const vers = await nv(opts.node, {
         latestOfMajorOnly: true
       });
+      if (!vers?.[0]?.version) {
+        throw Object.assign(new Error(`Unable to resolve node version`), {
+          spec: opts.node
+        });
+      }
 
       const results = await runner({
         cwd: opts.cwd,

packages/cli/load.mjs

@@ -16,7 +16,7 @@ node metadata.mjs > ./results/metadata.json
 # If not, require a repo and ref
 REPO_DIR="./repo"
 if [ -d "$REPO_DIR" ]; then
-  echo "Using local benchmarks"
+  echo "Using local repo."
   cd "$REPO_DIR"
 else
   if [ -z "$REPO" ]; then

packages/runner-local-docker/scripts/start.sh

@@ -0,0 +1,116 @@
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { styleText } from 'node:util';
+import { AutocannonBenchmarker } from '@expressjs/perf-autocannon';
+import { Wrk2Benchmarker } from '@expressjs/perf-wrk2';
+
+import { setup, cleanup } from './setup.mjs';
+import { startServer } from './server.mjs';
+
+function runBenchmarker(instance, options) {
+  console.log(styleText(['blue'], `Running ${instance.name} load...`));
+  return new Promise((resolve, reject) => {
+    const proc = instance.create(options);
+    proc.stderr.pipe(process.stderr);
+
+    let stdout = '';
+    proc.stdout.setEncoding('utf8');
+    proc.stdout.on('data', (chunk) => stdout += chunk);
+
+    proc.once('close', (code) => {
+      if (code) {
+        let error_message = `${instance.name} failed with ${code}.`;
+        if (stdout !== '') {
+          error_message += ` Output: ${stdout}`;
+        }
+        reject(new Error(error_message), code);
+        return;
+      }
+
+      const result = instance.processResults(stdout);
+      resolve(result);
+    });
+  })
+}
+
+export async function startClient (_opts = {}, server) {
+  const opts = {
+    ..._opts
+  };
+  if (opts?.signal.aborted) {
+    return;
+  }
+
+  const autocannon = new AutocannonBenchmarker();
+  const wrk2 = new Wrk2Benchmarker();
+  const waitFor = [];
+
+  if (!wrk2.present) {
+    console.log(styleText(['bold', 'yellow'], 'Wrk2 not found. Please install it'));
+  } else {
+    waitFor.push(
+      runBenchmarker(wrk2, {
+        duration: 30,
+        connections: 100,
+        rate: 2000,
+        port: 3000,
+        path: '/'
+      })
+    );
+    console.log('Running Wrk2');
+  }
+
+  if (!autocannon.present) {
+    console.log(styleText(['bold', 'yellow'], 'Autocannon not found. Please install it with `npm i -g autocannon`'));
+  } else {
+    waitFor.push(
+      runBenchmarker(autocannon, {
+        duration: 30,
+        connections: 100,
+        port: 3000,
+        path: '/',
+      })
+    );
+    console.log('Running Autocannon');
+  }
+
+
+  return {
+    metadata: {},
+    close: async () => {
+      // TODO: need to get the client classes into order for this
+    },
+    results: () => {
+      return Promise.allSettled(waitFor);
+    }
+  };
+}
+
+export default async function runner (_opts = {}) {
+  // Start up the server, then the client
+  const opts = {
+    ..._opts
+  };
+
+  const cwd = fileURLToPath(dirname(import.meta.resolve(opts.test)));
+  await setup(cwd, opts);
+
+  const server = await startServer(cwd, opts);
+  // const client = await startClient(opts, server); 
+
+  // Wait for the client to finish, then the server
+  // const clientResults = await client.results();
+  const serverResults = await server.results();
+
+  // await client.close();
+  await server.close();
+
+  await cleanup(cwd);
+
+  return {
+    serverMetadata: server.metadata,
+    // clientMetadata: client.metadata,
+    // serverResults,
+    // clientResults
+  };
+}

packages/runner-local/index.mjs

@@ -0,0 +1,13 @@
+{
+  "name": "@expressjs/perf-runner-local",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.mjs",
+  "devDependencies": {},
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC"
+}

packages/runner-local/package.json

@@ -0,0 +1,44 @@
+import { execFile } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+
+export async function startServer (cwd, opts = {}) {
+  return new Promise((resolve, reject) => {
+    const test = fileURLToPath(import.meta.resolve(opts.test));
+    const cp = execFile(process.execPath, [ test ], { cwd });
+
+    const server = {
+      metadata: {
+        url: new URL('http://localhost:3000'),
+      },
+      status: 'starting',
+      close: () => {
+        return new Promise((closeResolve) => {
+          cp.on('exit', () => {
+            closeResolve();
+          });
+          cp.kill('SIGINT');
+        });
+      },
+      results: async () => {
+        return { };
+      }
+    };
+
+    opts?.signal.addEventListener('abort', () => {
+      server.status = 'aborted';
+      cp.kill('SIGINT');
+      reject(new Error('aborted'));
+    });
+    cp.on('error', reject);
+    cp.stdout.on('data', (d) => {
+      process.stdout.write(d);
+      if (server.status === 'starting' && d.toString('utf8').includes("startup:")) {
+        server.status = 'started';
+        resolve(server);
+      }
+    });
+    cp.stderr.on('data', (d) => {
+      process.stderr.write(d);
+    });
+  });
+} 

packages/runner-local/server.mjs

@@ -0,0 +1,90 @@
+import {
+  rename,
+  access,
+  writeFile,
+  rm,
+  realpath,
+  constants as fsConstants
+} from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import { promisify } from 'node:util';
+import { execFile } from 'node:child_process';
+
+const pExecFile = promisify(execFile);
+
+async function restore (cwd) {
+  const pkgPath = join(cwd, 'package.json');
+  const pkgBakPath = join(cwd, 'package.json.bak');
+  try {
+    await access(pkgBakPath, fsConstants.R_OK | fsConstants.W_OK);
+    console.log('package.json.bak exists, restoring original');
+    await rm(pkgPath);
+    await rename(pkgBakPath, pkgPath, fsConstants.COPYFILE_FICLONE);
+  } catch (e) {
+    if (e.code !== 'ENOENT') {
+      throw e;
+    }
+  }
+  return { pkgPath, pkgBakPath };
+}
+
+export async function setup (cwd, opts = {}) {
+  // Ensure we have the right node version
+  if (process.version !== `v${opts.node}`) {
+    // TODO: install node with nvm?
+    throw Object.assign(new Error(`Incorrect node.js version`), {
+      wanted: opts.node,
+      found: process.version.replace(/^v/, '')
+    });
+  }
+
+  // Setup the repo if asked
+  if (!opts.repo) {
+    console.log('Using local repo.');
+  } else {
+    // TODO: clone repo into tmp dir
+    throw new Error('cloning repo not yet implemented');
+  }
+
+  // Restore backup if necessary
+  const { pkgPath, pkgBakPath } = await restore(cwd);
+
+  // Read in package.json contets
+  const pkg = await import(pkgPath, {
+    with: {
+      type: 'json'
+    }
+  });
+
+  // Apply overrides
+  if (opts.overrides) {
+    await rename(pkgPath, pkgBakPath);
+    pkg.overrides = {
+      ...(pkg.overrides ?? {}),
+      ...opts.overrides
+    };
+    await writeFile(pkgPath, JSON.stringify(pkg, null, 2));
+  }
+
+  // node --run setup || npm run setup
+  try {
+    const nodePath = await realpath(process.execPath);
+    await pExecFile(nodePath, ['--run', 'setup'], { cwd });
+  } catch (e) {
+    const npmPath = await realpath(join(dirname(process.execPath), 'npm'));
+    await pExecFile(npmPath, ['run', 'setup'], { cwd });
+  }
+
+  return { cwd };
+}
+
+export async function cleanup (cwd) {
+  await restore(cwd);
+  await rm(join(cwd, 'package-lock.json'), {
+    force: true
+  });
+  await rm(join(cwd, 'node_modules'), {
+    force: true,
+    recursive: true
+  });
+}