diff --git a/.github/workflows/measure-framework.yml b/.github/workflows/measure-framework.yml index 4da9e7af..9bb284ca 100644 --- a/.github/workflows/measure-framework.yml +++ b/.github/workflows/measure-framework.yml @@ -226,7 +226,9 @@ jobs: measure-ssr-load: if: inputs.ssr-load-matrix != '[]' - runs-on: depot-ubuntu-24.04 + runs-on: depot-ubuntu-24.04-16 + env: + RUNNER_LABEL: 'Depot depot-ubuntu-24.04-16 GitHub Actions runner (16 CPUs, 64 GB RAM, 180 GB disk, 8 GB disk accelerator; server CPUs 0-11; Autocannon CPUs 12-15)' strategy: fail-fast: false matrix: @@ -256,7 +258,7 @@ jobs: run: pnpm build - name: Run SSR load benchmark - run: pnpm --filter @framework-tracker/stats-generator run:ssr-load ${{ matrix.framework.package }} + run: pnpm --filter @framework-tracker/stats-generator run:ssr-load:containerized ${{ matrix.framework.package }} - name: Upload SSR load stats uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/packages/docs/src/content/docs/methodology.md b/packages/docs/src/content/docs/methodology.md index 5cd48076..39cfdac9 100644 --- a/packages/docs/src/content/docs/methodology.md +++ b/packages/docs/src/content/docs/methodology.md @@ -13,13 +13,14 @@ starting a framework and the runtime cost of serving and hydrating a comparable app. Timing results are run multiple times and averaged, and generated JSON is published into the docs package. -Benchmarks run on Depot GitHub Actions runners using +Most benchmarks run on Depot GitHub Actions runners using [`depot-ubuntu-24.04`](https://depot.dev/docs/github-actions/runner-types), which Depot documents as an Intel runner with 2 CPUs, 8 GB RAM, 100 GB disk, -and a 2 GB disk accelerator. Browser rendering benchmarks run directly on the -Depot runner host and use the host Chrome installation rather than a job-level -browser container. The generated runtime stats record the Chrome version used -for browser rendering benchmarks. +and a 2 GB disk accelerator. The SSR load test instead uses +`depot-ubuntu-24.04-16`, with 16 CPUs and 64 GB RAM. Browser rendering +benchmarks run directly on the Depot runner host and use the host Chrome +installation rather than a job-level browser container. The generated runtime +stats record the Chrome version used for browser rendering benchmarks. ## Dev Time @@ -274,6 +275,13 @@ throughput, and load behavior for comparable production apps. - Load is applied with [autocannon](https://github.com/mcollina/autocannon) in staged connection counts: 1, 5, 10, 25, 50, 100, and 200 concurrent connections. +- The framework server and Autocannon run in separate containers on the same + 16-CPU Depot runner. The server container is pinned to CPUs 0-11 and the + Autocannon container to CPUs 12-15, preventing the two benchmark workloads + from competing for the same CPU cores. They still share the host's memory, + kernel, Docker runtime, and other system resources, so this does not provide + the full isolation of separate machines. Keeping both containers on one host + also avoids introducing cross-machine network latency. - Each stage runs for approximately 5 seconds. - Peak requests/sec is the highest successful stage throughput observed during the staged run. diff --git a/packages/stats-generator/package.json b/packages/stats-generator/package.json index fc929bb3..078aed00 100644 --- a/packages/stats-generator/package.json +++ b/packages/stats-generator/package.json @@ -7,6 +7,8 @@ "collect": "node src/collect-stats.ts", "run:ssr-request-throughput": "node src/run-ssr-request-throughput-benchmark.ts", "run:ssr-load": "node src/run-ssr-load-benchmark.ts", + "run:ssr-load:containerized": "node src/run-containerized-ssr-load.ts", + "serve:ssr-load": "node src/run-ssr-load-server.ts", "run:client-side-rendered": "node src/run-client-side-rendered-benchmark.ts", "run:server-side-rendered": "node src/run-server-side-rendered-benchmark.ts", "run:install": "node src/run-install-benchmark.ts", diff --git a/packages/stats-generator/src/run-containerized-ssr-load.ts b/packages/stats-generator/src/run-containerized-ssr-load.ts new file mode 100644 index 00000000..5e1b4049 --- /dev/null +++ b/packages/stats-generator/src/run-containerized-ssr-load.ts @@ -0,0 +1,132 @@ +import { spawn } from 'node:child_process' +import { cwd, env, exit, getgid, getuid, pid } from 'node:process' +import { parseArgs } from './utils.ts' + +const RUN_SUFFIX = `${env.GITHUB_RUN_ID ?? 'local'}-${pid}` +const NETWORK_NAME = `framework-tracker-ssr-load-${RUN_SUFFIX}` +const SERVER_NAME = `framework-tracker-ssr-load-server-${RUN_SUFFIX}` +const LOAD_GENERATOR_NAME = `framework-tracker-ssr-load-generator-${RUN_SUFFIX}` +const NODE_IMAGE = 'node:24-bookworm-slim' + +function runDocker( + args: string[], + stdio: 'ignore' | 'inherit' = 'inherit', +): Promise { + return new Promise((resolve, reject) => { + const child = spawn('docker', args, { stdio }) + child.once('error', reject) + child.once('exit', (code, signal) => { + if (code === 0) { + resolve() + return + } + + reject( + new Error( + `docker ${args[0]} failed${signal ? ` with signal ${signal}` : ` with exit code ${code}`}`, + ), + ) + }) + }) +} + +async function ignoreDockerFailure( + args: string[], + stdio: 'ignore' | 'inherit' = 'ignore', +): Promise { + try { + await runDocker(args, stdio) + } catch { + // Cleanup must not hide the benchmark result. + } +} + +async function main() { + const { packageName } = parseArgs( + 'Usage: run-containerized-ssr-load \nExample: run-containerized-ssr-load app-astro', + ) + const workspaceDir = env.GITHUB_WORKSPACE ?? cwd() + const user = `${getuid?.() ?? 0}:${getgid?.() ?? 0}` + let cleanupPromise: Promise | undefined + + const cleanup = (showServerLogs = false): Promise => { + cleanupPromise ??= (async () => { + if (showServerLogs) { + await ignoreDockerFailure(['logs', SERVER_NAME], 'inherit') + } + await ignoreDockerFailure([ + 'rm', + '--force', + LOAD_GENERATOR_NAME, + SERVER_NAME, + ]) + await ignoreDockerFailure(['network', 'rm', NETWORK_NAME]) + })() + return cleanupPromise + } + + process.once('SIGINT', () => { + void cleanup().finally(() => exit(130)) + }) + process.once('SIGTERM', () => { + void cleanup().finally(() => exit(143)) + }) + + let succeeded = false + try { + await runDocker(['network', 'create', NETWORK_NAME], 'ignore') + + await runDocker([ + 'run', + '--detach', + '--name', + SERVER_NAME, + '--network', + NETWORK_NAME, + '--cpuset-cpus', + '0-11', + '--user', + user, + '--volume', + `${workspaceDir}:/workspace`, + '--workdir', + '/workspace', + NODE_IMAGE, + 'node', + 'packages/stats-generator/src/run-ssr-load-server.ts', + packageName, + ]) + + await runDocker([ + 'run', + '--name', + LOAD_GENERATOR_NAME, + '--network', + NETWORK_NAME, + '--cpuset-cpus', + '12-15', + '--user', + user, + '--volume', + `${workspaceDir}:/workspace`, + '--workdir', + '/workspace', + '--env', + 'RUNNER_LABEL', + '--env', + `SSR_LOAD_TARGET_URL=http://${SERVER_NAME}:3003/server-side-rendered`, + NODE_IMAGE, + 'node', + 'packages/stats-generator/src/run-ssr-load-benchmark.ts', + packageName, + ]) + succeeded = true + } finally { + await cleanup(!succeeded) + } +} + +main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/packages/stats-generator/src/run-ssr-load-server.ts b/packages/stats-generator/src/run-ssr-load-server.ts new file mode 100644 index 00000000..a4b58d42 --- /dev/null +++ b/packages/stats-generator/src/run-ssr-load-server.ts @@ -0,0 +1,38 @@ +import { + DEFAULT_SSR_LOAD_PORT, + SSR_LOAD_PATH, + startSSRLoadServer, +} from './ssrLoad/index.ts' +import { getHost, getPort } from './serve/common.ts' +import { parseArgs } from './utils.ts' + +async function main() { + const { packageName } = parseArgs( + 'Usage: run-ssr-load-server \nExample: run-ssr-load-server app-astro', + ) + + const host = getHost('0.0.0.0') + process.env.HOST = host + const port = getPort(DEFAULT_SSR_LOAD_PORT) + process.env.PORT = String(port) + + console.info(`Starting SSR load server for ${packageName}...`) + const stopServer = await startSSRLoadServer(packageName) + console.info( + `SSR load server is ready on http://${host}:${port}${SSR_LOAD_PATH}`, + ) + + await new Promise((resolve) => { + const shutdown = () => { + stopServer() + resolve() + } + process.once('SIGINT', shutdown) + process.once('SIGTERM', shutdown) + }) +} + +main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/packages/stats-generator/src/ssrLoad/index.ts b/packages/stats-generator/src/ssrLoad/index.ts index da83166f..cf653b11 100644 --- a/packages/stats-generator/src/ssrLoad/index.ts +++ b/packages/stats-generator/src/ssrLoad/index.ts @@ -2,13 +2,12 @@ import { spawn } from 'node:child_process' import { join } from 'node:path' import { fileURLToPath } from 'node:url' import { packagesDir } from '../constants.ts' -import { getHost } from '../serve/common.ts' +import { getHost, getPort } from '../serve/common.ts' import { runLoadTest } from './run-load-test.ts' import type { SSRLoadBenchmarkResult } from './types.ts' -const SSR_LOAD_HOST = getHost() -const SSR_LOAD_PORT = 3003 -const SSR_LOAD_PATH = '/server-side-rendered' +export const DEFAULT_SSR_LOAD_PORT = 3003 +export const SSR_LOAD_PATH = '/server-side-rendered' interface SSRLoadFrameworkConfig { name: string @@ -75,11 +74,29 @@ const SSR_LOAD_FRAMEWORKS: SSRLoadFrameworkConfig[] = [ ] export function supportsSSRLoadBenchmark(packageName: string): boolean { - return SSR_LOAD_FRAMEWORKS.some( + return getFrameworkConfig(packageName) !== undefined +} + +function getFrameworkConfig( + packageName: string, +): SSRLoadFrameworkConfig | undefined { + return SSR_LOAD_FRAMEWORKS.find( (framework) => framework.package === packageName, ) } +function requireFrameworkConfig(packageName: string): SSRLoadFrameworkConfig { + const config = getFrameworkConfig(packageName) + + if (!config) { + throw new Error( + `Unknown SSR load package: ${packageName}. Available: ${SSR_LOAD_FRAMEWORKS.map((framework) => framework.package).join(', ')}`, + ) + } + + return config +} + async function waitForServer(url: string, timeoutMs = 30_000): Promise { const deadline = Date.now() + timeoutMs while (Date.now() < deadline) { @@ -97,9 +114,17 @@ async function waitForServer(url: string, timeoutMs = 30_000): Promise { throw new Error(`Server at ${url} did not become ready within ${timeoutMs}ms`) } +export async function startSSRLoadServer( + packageName: string, +): Promise<() => void> { + return spawnServer(requireFrameworkConfig(packageName)) +} + async function spawnServer( config: SSRLoadFrameworkConfig, ): Promise<() => void> { + const host = getHost() + const port = getPort(DEFAULT_SSR_LOAD_PORT) const appDir = join(packagesDir, config.package) const scriptPath = fileURLToPath( new URL(`../serve/${config.serveScript}`, import.meta.url), @@ -108,9 +133,9 @@ async function spawnServer( const proc = spawn('node', [scriptPath, appDir], { env: { ...process.env, - HOST: SSR_LOAD_HOST, + HOST: host, NODE_ENV: 'production', - PORT: String(SSR_LOAD_PORT), + PORT: String(port), }, stdio: ['ignore', 'pipe', 'pipe'], }) @@ -139,7 +164,7 @@ async function spawnServer( }) await Promise.race([ - waitForServer(`http://${SSR_LOAD_HOST}:${SSR_LOAD_PORT}${SSR_LOAD_PATH}`), + waitForServer(`http://${host}:${port}${SSR_LOAD_PATH}`), exitPromise, ]) @@ -151,19 +176,30 @@ async function spawnServer( export async function runSSRLoadBenchmark( packageName: string, ): Promise { - const config = SSR_LOAD_FRAMEWORKS.find( - (framework) => framework.package === packageName, - ) + const config = requireFrameworkConfig(packageName) - if (!config) { - throw new Error( - `Unknown SSR load package: ${packageName}. Available: ${SSR_LOAD_FRAMEWORKS.map((framework) => framework.package).join(', ')}`, - ) + const remoteUrl = process.env.SSR_LOAD_TARGET_URL + if (remoteUrl) { + const url = new URL(remoteUrl) + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error('SSR_LOAD_TARGET_URL must use http or https') + } + + console.info(`Using remote server for ${config.displayName}: ${url}`) + await waitForServer(url.toString()) + return { + name: config.name, + displayName: config.displayName, + package: config.package, + ssrLoadTests: await runLoadTest(url.toString()), + } } - const url = `http://${SSR_LOAD_HOST}:${SSR_LOAD_PORT}${SSR_LOAD_PATH}` + const host = getHost() + const port = getPort(DEFAULT_SSR_LOAD_PORT) + const url = `http://${host}:${port}${SSR_LOAD_PATH}` console.info(`Starting server for ${config.displayName}...`) - const killServer = await spawnServer(config) + const killServer = await startSSRLoadServer(packageName) try { console.info(`Running SSR load benchmark for ${config.displayName}...`)