Changed How Load Tests Run

.github/workflows/measure-framework.yml

@@ -226,7 +226,9 @@ jobs:
 
   measure-ssr-load:
     if: inputs.ssr-load-matrix != '[]'
-    runs-on: depot-ubuntu-24.04
+    runs-on: depot-ubuntu-24.04-16
+    env:
+      RUNNER_LABEL: 'Depot depot-ubuntu-24.04-16 GitHub Actions runner (16 CPUs, 64 GB RAM, 180 GB disk, 8 GB disk accelerator; server CPUs 0-11; Autocannon CPUs 12-15)'
     strategy:
       fail-fast: false
       matrix:
@@ -256,7 +258,7 @@ jobs:
         run: pnpm build
 
       - name: Run SSR load benchmark
-        run: pnpm --filter @framework-tracker/stats-generator run:ssr-load ${{ matrix.framework.package }}
+        run: pnpm --filter @framework-tracker/stats-generator run:ssr-load:containerized ${{ matrix.framework.package }}
 
       - name: Upload SSR load stats
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1

packages/docs/src/content/docs/methodology.md

@@ -13,13 +13,14 @@ starting a framework and the runtime cost of serving and hydrating a comparable
 app. Timing results are run multiple times and averaged, and generated JSON is
 published into the docs package.
 
-Benchmarks run on Depot GitHub Actions runners using
+Most benchmarks run on Depot GitHub Actions runners using
 [`depot-ubuntu-24.04`](https://depot.dev/docs/github-actions/runner-types),
 which Depot documents as an Intel runner with 2 CPUs, 8 GB RAM, 100 GB disk,
-and a 2 GB disk accelerator. Browser rendering benchmarks run directly on the
-Depot runner host and use the host Chrome installation rather than a job-level
-browser container. The generated runtime stats record the Chrome version used
-for browser rendering benchmarks.
+and a 2 GB disk accelerator. The SSR load test instead uses
+`depot-ubuntu-24.04-16`, with 16 CPUs and 64 GB RAM. Browser rendering
+benchmarks run directly on the Depot runner host and use the host Chrome
+installation rather than a job-level browser container. The generated runtime
+stats record the Chrome version used for browser rendering benchmarks.
 
 ## Dev Time
 
@@ -274,6 +275,13 @@ throughput, and load behavior for comparable production apps.
 - Load is applied with [autocannon](https://github.com/mcollina/autocannon) in
   staged connection counts: 1, 5, 10, 25, 50, 100, and 200 concurrent
   connections.
+- The framework server and Autocannon run in separate containers on the same
+  16-CPU Depot runner. The server container is pinned to CPUs 0-11 and the
+  Autocannon container to CPUs 12-15, preventing the two benchmark workloads
+  from competing for the same CPU cores. They still share the host's memory,
+  kernel, Docker runtime, and other system resources, so this does not provide
+  the full isolation of separate machines. Keeping both containers on one host
+  also avoids introducing cross-machine network latency.
 - Each stage runs for approximately 5 seconds.
 - Peak requests/sec is the highest successful stage throughput observed during
   the staged run.

packages/stats-generator/package.json

@@ -7,6 +7,8 @@
     "collect": "node src/collect-stats.ts",
     "run:ssr-request-throughput": "node src/run-ssr-request-throughput-benchmark.ts",
     "run:ssr-load": "node src/run-ssr-load-benchmark.ts",
+    "run:ssr-load:containerized": "node src/run-containerized-ssr-load.ts",
+    "serve:ssr-load": "node src/run-ssr-load-server.ts",
     "run:client-side-rendered": "node src/run-client-side-rendered-benchmark.ts",
     "run:server-side-rendered": "node src/run-server-side-rendered-benchmark.ts",
     "run:install": "node src/run-install-benchmark.ts",

packages/stats-generator/src/run-containerized-ssr-load.ts

@@ -0,0 +1,132 @@
+import { spawn } from 'node:child_process'
+import { cwd, env, exit, getgid, getuid, pid } from 'node:process'
+import { parseArgs } from './utils.ts'
+
+const RUN_SUFFIX = `${env.GITHUB_RUN_ID ?? 'local'}-${pid}`
+const NETWORK_NAME = `framework-tracker-ssr-load-${RUN_SUFFIX}`
+const SERVER_NAME = `framework-tracker-ssr-load-server-${RUN_SUFFIX}`
+const LOAD_GENERATOR_NAME = `framework-tracker-ssr-load-generator-${RUN_SUFFIX}`
+const NODE_IMAGE = 'node:24-bookworm-slim'
+
+function runDocker(
+  args: string[],
+  stdio: 'ignore' | 'inherit' = 'inherit',
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn('docker', args, { stdio })
+    child.once('error', reject)
+    child.once('exit', (code, signal) => {
+      if (code === 0) {
+        resolve()
+        return
+      }
+
+      reject(
+        new Error(
+          `docker ${args[0]} failed${signal ? ` with signal ${signal}` : ` with exit code ${code}`}`,
+        ),
+      )
+    })
+  })
+}
+
+async function ignoreDockerFailure(
+  args: string[],
+  stdio: 'ignore' | 'inherit' = 'ignore',
+): Promise<void> {
+  try {
+    await runDocker(args, stdio)
+  } catch {
+    // Cleanup must not hide the benchmark result.
+  }
+}
+
+async function main() {
+  const { packageName } = parseArgs(
+    'Usage: run-containerized-ssr-load <package-name>\nExample: run-containerized-ssr-load app-astro',
+  )
+  const workspaceDir = env.GITHUB_WORKSPACE ?? cwd()
+  const user = `${getuid?.() ?? 0}:${getgid?.() ?? 0}`
+  let cleanupPromise: Promise<void> | undefined
+
+  const cleanup = (showServerLogs = false): Promise<void> => {
+    cleanupPromise ??= (async () => {
+      if (showServerLogs) {
+        await ignoreDockerFailure(['logs', SERVER_NAME], 'inherit')
+      }
+      await ignoreDockerFailure([
+        'rm',
+        '--force',
+        LOAD_GENERATOR_NAME,
+        SERVER_NAME,
+      ])
+      await ignoreDockerFailure(['network', 'rm', NETWORK_NAME])
+    })()
+    return cleanupPromise
+  }
+
+  process.once('SIGINT', () => {
+    void cleanup().finally(() => exit(130))
+  })
+  process.once('SIGTERM', () => {
+    void cleanup().finally(() => exit(143))
+  })
+
+  let succeeded = false
+  try {
+    await runDocker(['network', 'create', NETWORK_NAME], 'ignore')
+
+    await runDocker([
+      'run',
+      '--detach',
+      '--name',
+      SERVER_NAME,
+      '--network',
+      NETWORK_NAME,
+      '--cpuset-cpus',
+      '0-11',
+      '--user',
+      user,
+      '--volume',
+      `${workspaceDir}:/workspace`,
+      '--workdir',
+      '/workspace',
+      NODE_IMAGE,
+      'node',
+      'packages/stats-generator/src/run-ssr-load-server.ts',
+      packageName,
+    ])
+
+    await runDocker([
+      'run',
+      '--name',
+      LOAD_GENERATOR_NAME,
+      '--network',
+      NETWORK_NAME,
+      '--cpuset-cpus',
+      '12-15',
+      '--user',
+      user,
+      '--volume',
+      `${workspaceDir}:/workspace`,
+      '--workdir',
+      '/workspace',
+      '--env',
+      'RUNNER_LABEL',
+      '--env',
+      `SSR_LOAD_TARGET_URL=http://${SERVER_NAME}:3003/server-side-rendered`,
+      NODE_IMAGE,
+      'node',
+      'packages/stats-generator/src/run-ssr-load-benchmark.ts',
+      packageName,
+    ])
+    succeeded = true
+  } finally {
+    await cleanup(!succeeded)
+  }
+}
+
+main().catch((error) => {
+  console.error(error)
+  process.exitCode = 1
+})

packages/stats-generator/src/run-ssr-load-server.ts

@@ -0,0 +1,38 @@
+import {
+  DEFAULT_SSR_LOAD_PORT,
+  SSR_LOAD_PATH,
+  startSSRLoadServer,
+} from './ssrLoad/index.ts'
+import { getHost, getPort } from './serve/common.ts'
+import { parseArgs } from './utils.ts'
+
+async function main() {
+  const { packageName } = parseArgs(
+    'Usage: run-ssr-load-server <package-name>\nExample: run-ssr-load-server app-astro',
+  )
+
+  const host = getHost('0.0.0.0')
+  process.env.HOST = host
+  const port = getPort(DEFAULT_SSR_LOAD_PORT)
+  process.env.PORT = String(port)
+
+  console.info(`Starting SSR load server for ${packageName}...`)
+  const stopServer = await startSSRLoadServer(packageName)
+  console.info(
+    `SSR load server is ready on http://${host}:${port}${SSR_LOAD_PATH}`,
+  )
+
+  await new Promise<void>((resolve) => {
+    const shutdown = () => {
+      stopServer()
+      resolve()
+    }
+    process.once('SIGINT', shutdown)
+    process.once('SIGTERM', shutdown)
+  })
+}
+
+main().catch((error) => {
+  console.error(error)
+  process.exitCode = 1
+})

packages/stats-generator/src/ssrLoad/index.ts

@@ -2,13 +2,12 @@ import { spawn } from 'node:child_process'
 import { join } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { packagesDir } from '../constants.ts'
-import { getHost } from '../serve/common.ts'
+import { getHost, getPort } from '../serve/common.ts'
 import { runLoadTest } from './run-load-test.ts'
 import type { SSRLoadBenchmarkResult } from './types.ts'
 
-const SSR_LOAD_HOST = getHost()
-const SSR_LOAD_PORT = 3003
-const SSR_LOAD_PATH = '/server-side-rendered'
+export const DEFAULT_SSR_LOAD_PORT = 3003
+export const SSR_LOAD_PATH = '/server-side-rendered'
 
 interface SSRLoadFrameworkConfig {
   name: string
@@ -75,11 +74,29 @@ const SSR_LOAD_FRAMEWORKS: SSRLoadFrameworkConfig[] = [
 ]
 
 export function supportsSSRLoadBenchmark(packageName: string): boolean {
-  return SSR_LOAD_FRAMEWORKS.some(
+  return getFrameworkConfig(packageName) !== undefined
+}
+
+function getFrameworkConfig(
+  packageName: string,
+): SSRLoadFrameworkConfig | undefined {
+  return SSR_LOAD_FRAMEWORKS.find(
     (framework) => framework.package === packageName,
   )
 }
 
+function requireFrameworkConfig(packageName: string): SSRLoadFrameworkConfig {
+  const config = getFrameworkConfig(packageName)
+
+  if (!config) {
+    throw new Error(
+      `Unknown SSR load package: ${packageName}. Available: ${SSR_LOAD_FRAMEWORKS.map((framework) => framework.package).join(', ')}`,
+    )
+  }
+
+  return config
+}
+
 async function waitForServer(url: string, timeoutMs = 30_000): Promise<void> {
   const deadline = Date.now() + timeoutMs
   while (Date.now() < deadline) {
@@ -97,9 +114,17 @@ async function waitForServer(url: string, timeoutMs = 30_000): Promise<void> {
   throw new Error(`Server at ${url} did not become ready within ${timeoutMs}ms`)
 }
 
+export async function startSSRLoadServer(
+  packageName: string,
+): Promise<() => void> {
+  return spawnServer(requireFrameworkConfig(packageName))
+}
+
 async function spawnServer(
   config: SSRLoadFrameworkConfig,
 ): Promise<() => void> {
+  const host = getHost()
+  const port = getPort(DEFAULT_SSR_LOAD_PORT)
   const appDir = join(packagesDir, config.package)
   const scriptPath = fileURLToPath(
     new URL(`../serve/${config.serveScript}`, import.meta.url),
@@ -108,9 +133,9 @@ async function spawnServer(
   const proc = spawn('node', [scriptPath, appDir], {
     env: {
       ...process.env,
-      HOST: SSR_LOAD_HOST,
+      HOST: host,
       NODE_ENV: 'production',
-      PORT: String(SSR_LOAD_PORT),
+      PORT: String(port),
     },
     stdio: ['ignore', 'pipe', 'pipe'],
   })
@@ -139,7 +164,7 @@ async function spawnServer(
   })
 
   await Promise.race([
-    waitForServer(`http://${SSR_LOAD_HOST}:${SSR_LOAD_PORT}${SSR_LOAD_PATH}`),
+    waitForServer(`http://${host}:${port}${SSR_LOAD_PATH}`),
     exitPromise,
   ])
 
@@ -151,19 +176,30 @@ async function spawnServer(
 export async function runSSRLoadBenchmark(
   packageName: string,
 ): Promise<SSRLoadBenchmarkResult> {
-  const config = SSR_LOAD_FRAMEWORKS.find(
-    (framework) => framework.package === packageName,
-  )
+  const config = requireFrameworkConfig(packageName)
 
-  if (!config) {
-    throw new Error(
-      `Unknown SSR load package: ${packageName}. Available: ${SSR_LOAD_FRAMEWORKS.map((framework) => framework.package).join(', ')}`,
-    )
+  const remoteUrl = process.env.SSR_LOAD_TARGET_URL
+  if (remoteUrl) {
+    const url = new URL(remoteUrl)
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+      throw new Error('SSR_LOAD_TARGET_URL must use http or https')
+    }
+
+    console.info(`Using remote server for ${config.displayName}: ${url}`)
+    await waitForServer(url.toString())
+    return {
+      name: config.name,
+      displayName: config.displayName,
+      package: config.package,
+      ssrLoadTests: await runLoadTest(url.toString()),
+    }
   }
 
-  const url = `http://${SSR_LOAD_HOST}:${SSR_LOAD_PORT}${SSR_LOAD_PATH}`
+  const host = getHost()
+  const port = getPort(DEFAULT_SSR_LOAD_PORT)
+  const url = `http://${host}:${port}${SSR_LOAD_PATH}`
   console.info(`Starting server for ${config.displayName}...`)
-  const killServer = await spawnServer(config)
+  const killServer = await startSSRLoadServer(packageName)
 
   try {
     console.info(`Running SSR load benchmark for ${config.displayName}...`)