diff --git a/.github/workflows/measure-framework.yml b/.github/workflows/measure-framework.yml index 4da9e7af..663a3db5 100644 --- a/.github/workflows/measure-framework.yml +++ b/.github/workflows/measure-framework.yml @@ -98,10 +98,6 @@ jobs: - name: Install workspace dependencies run: pnpm install --frozen-lockfile - - name: Install package dependencies - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm install --frozen-lockfile - - name: Run build benchmark run: | RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "build") | .runFrequency') diff --git a/packages/docs/src/content/docs/methodology.md b/packages/docs/src/content/docs/methodology.md index d5cd0ee3..b0d3cdfe 100644 --- a/packages/docs/src/content/docs/methodology.md +++ b/packages/docs/src/content/docs/methodology.md @@ -88,10 +88,12 @@ matches the framework version tracked by the starter project. temporary copy of the starter package with an empty pnpm store and cache. - Install benchmarks run 5 times by default and report average, minimum, and maximum duration. -- Cold build time removes the configured build output directory before running - `pnpm build`. -- Warm build time runs `pnpm build` again after the cold build, preserving - whatever cache or generated output the framework leaves in place. +- Each build repetition uses a fresh temporary copy of the tracked starter + files. Dependencies are installed outside the timed region with a frozen + lockfile and a dedicated store shared by the repetitions. +- Cold build time measures the first build in that fresh project. Warm build + time measures a second build in the same project, preserving whatever cache + or generated output the first build leaves in place. - Build benchmarks run 5 times by default and report average, minimum, and maximum duration. - Build output size is the total size of the configured production output diff --git a/packages/stats-generator/src/benchmark-utils.test.ts b/packages/stats-generator/src/benchmark-utils.test.ts new file mode 100644 index 00000000..b75b6a79 --- /dev/null +++ b/packages/stats-generator/src/benchmark-utils.test.ts @@ -0,0 +1,19 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { parseRunFrequency } from './benchmark-utils.ts' + +test('uses the default run frequency when one is not provided', () => { + assert.equal(parseRunFrequency(undefined), 5) + assert.equal(parseRunFrequency(undefined, 3), 3) +}) + +test('accepts a positive integer run frequency', () => { + assert.equal(parseRunFrequency('1'), 1) + assert.equal(parseRunFrequency('10'), 10) +}) + +test('rejects invalid run frequencies', () => { + for (const value of ['0', '-1', '1.5', 'abc', '2runs']) { + assert.throws(() => parseRunFrequency(value), /positive integer/) + } +}) diff --git a/packages/stats-generator/src/benchmark-utils.ts b/packages/stats-generator/src/benchmark-utils.ts new file mode 100644 index 00000000..2572a8e0 --- /dev/null +++ b/packages/stats-generator/src/benchmark-utils.ts @@ -0,0 +1,37 @@ +import { execFileSync } from 'node:child_process' + +export function parseRunFrequency( + value: string | undefined, + fallback = 5, +): number { + const runFrequency = value === undefined ? fallback : Number(value) + + if (!Number.isInteger(runFrequency) || runFrequency < 1) { + throw new Error(`Run frequency must be a positive integer: ${value}`) + } + + return runFrequency +} + +export function installDependencies( + cwd: string, + storeDir: string, + cacheDir: string, +): void { + execFileSync( + 'pnpm', + [ + 'install', + '--frozen-lockfile', + '--store-dir', + storeDir, + '--cache-dir', + cacheDir, + ], + { + cwd, + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }, + ) +} diff --git a/packages/stats-generator/src/run-build-benchmark.ts b/packages/stats-generator/src/run-build-benchmark.ts index 76489e45..86892373 100644 --- a/packages/stats-generator/src/run-build-benchmark.ts +++ b/packages/stats-generator/src/run-build-benchmark.ts @@ -1,6 +1,8 @@ -import { execSync } from 'node:child_process' -import { join } from 'node:path' -import { existsSync, rmSync } from 'node:fs' +import { execFileSync } from 'node:child_process' +import { cpSync, mkdirSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' +import { installDependencies, parseRunFrequency } from './benchmark-utils.ts' import { packagesDir } from './constants.ts' import { getDirectorySize, @@ -10,30 +12,66 @@ import { } from './utils.ts' import type { BuildStats } from './types.ts' -function measureBuildTime(cwd: string): number { +function copyTrackedProject(sourceDir: string, projectDir: string): void { + const repositoryDir = join(packagesDir, '..') + const sourcePathFromRepository = relative(repositoryDir, sourceDir) + const trackedPaths = execFileSync( + 'git', + ['ls-files', '-z', '--', sourcePathFromRepository], + { + cwd: repositoryDir, + encoding: 'utf-8', + }, + ) + .split('\0') + .filter(Boolean) + + for (const trackedPath of trackedPaths) { + const projectPath = relative(sourcePathFromRepository, trackedPath) + const destinationPath = join(projectDir, projectPath) + mkdirSync(dirname(destinationPath), { recursive: true }) + cpSync(join(repositoryDir, trackedPath), destinationPath) + } +} + +function measureBuildTime(cwd: string, buildScript: string): number { const start = performance.now() - execSync('pnpm build', { cwd, encoding: 'utf-8', stdio: 'inherit' }) + execFileSync('pnpm', [buildScript], { + cwd, + encoding: 'utf-8', + stdio: 'inherit', + }) const end = performance.now() return Math.round(end - start) } -function rmBuildOutput(buildOutputPath: string): void { - if (existsSync(buildOutputPath)) { - console.info('Build output found. Removing build output') - rmSync(buildOutputPath, { recursive: true, force: true }) - } else { - console.info('No build output found. Skipping removal of build output') +function getBuildOutputPath( + projectDir: string, + buildOutputDir: string, +): string { + const buildOutputPath = resolve(projectDir, buildOutputDir) + const relativeBuildOutputPath = relative(projectDir, buildOutputPath) + + if ( + !relativeBuildOutputPath || + relativeBuildOutputPath === '..' || + relativeBuildOutputPath.startsWith(`..${sep}`) || + isAbsolute(relativeBuildOutputPath) + ) { + throw new Error( + `Build output directory must be inside the project: ${buildOutputDir}`, + ) } + + return buildOutputPath } async function main() { const { packageName, args } = parseArgs( - 'Usage: run-build-benchmark \nExample: run-build-benchmark starter-astro', + 'Usage: run-build-benchmark [run-frequency]\nExample: run-build-benchmark starter-astro 5', ) - const fallbackFrequency = '5' - const base = 10 - const runFrequency = Number.parseInt(args[0] || fallbackFrequency, base) + const runFrequency = parseRunFrequency(args[0]) const { framework, testConfig } = await getFrameworkByPackage(packageName) @@ -42,68 +80,108 @@ async function main() { ) const packageDir = join(packagesDir, packageName) - const buildOutputPath = join(packageDir, testConfig.buildOutputDir) + const tempDir = join( + tmpdir(), + `framework-build-benchmark-${packageName}-${Date.now()}`, + ) + const storeDir = join(tempDir, 'store') + const cacheDir = join(tempDir, 'cache') const coldBuildTimesMs: number[] = [] const warmBuildTimesMs: number[] = [] + let finalProjectDir = '' + let previousRunDir = '' + + try { + for (let i = 1; i <= runFrequency; i++) { + if (previousRunDir) { + rmSync(previousRunDir, { recursive: true, force: true }) + } + + const runDir = join(tempDir, `run-${i}`) + const projectDir = join(runDir, 'project') + copyTrackedProject(packageDir, projectDir) + + console.info(`\nBuild run ${i}/${runFrequency}...`) + console.info('Installing dependencies outside the timed region...') + installDependencies(projectDir, storeDir, cacheDir) + + console.info('Cold build...') + const coldBuildTimeMs = measureBuildTime( + projectDir, + testConfig.buildScript, + ) + coldBuildTimesMs.push(coldBuildTimeMs) + console.info(` Cold build time: ${coldBuildTimeMs}ms`) + + console.info('\nWarm build...') + const warmBuildTimeMs = measureBuildTime( + projectDir, + testConfig.buildScript, + ) + warmBuildTimesMs.push(warmBuildTimeMs) + console.info(` Warm build time: ${warmBuildTimeMs}ms`) + + finalProjectDir = projectDir + previousRunDir = runDir + } + + const finalBuildOutputPath = getBuildOutputPath( + finalProjectDir, + testConfig.buildOutputDir, + ) + const excludedBuildOutputPaths = + testConfig.buildOutputDir === '.next' + ? [join(finalBuildOutputPath, 'cache')] + : [] + const buildOutputSize = getDirectorySize( + finalBuildOutputPath, + excludedBuildOutputPaths, + ) + console.info(`\nBuild output size: ${buildOutputSize} bytes`) + + const coldBuildTime = { + avgMs: + coldBuildTimesMs.reduce((total, cur) => total + cur, 0) / + coldBuildTimesMs.length, + minMs: Math.min(...coldBuildTimesMs), + maxMs: Math.max(...coldBuildTimesMs), + } + console.info(`\nAvg cold build time: ${coldBuildTime.avgMs} ms`) + console.info(`\nMin cold build time: ${coldBuildTime.minMs} ms`) + console.info(`\nMax cold build time: ${coldBuildTime.maxMs} ms`) + + const warmBuildTime = { + avgMs: + warmBuildTimesMs.reduce((total, cur) => total + cur, 0) / + warmBuildTimesMs.length, + minMs: Math.min(...warmBuildTimesMs), + maxMs: Math.max(...warmBuildTimesMs), + } + console.info(`\nAvg warm build time: ${warmBuildTime.avgMs} ms`) + console.info(`\nMin warm build time: ${warmBuildTime.minMs} ms`) + console.info(`\nMax warm build time: ${warmBuildTime.maxMs} ms`) + + const stats: BuildStats = { + coldBuildTime, + warmBuildTime, + buildOutputSize, + } + + const outputPath = join(packagesDir, packageName, 'build-stats.json') + writeJsonFile(outputPath, stats) + + const buildOutputPath = getBuildOutputPath( + packageDir, + testConfig.buildOutputDir, + ) + rmSync(buildOutputPath, { recursive: true, force: true }) + cpSync(finalBuildOutputPath, buildOutputPath, { recursive: true }) - for (let i = 1; i <= runFrequency; i++) { - console.info(`\nBuild run ${i}/${runFrequency}...`) - rmBuildOutput(buildOutputPath) - - console.info('Cold build...') - const coldBuildTimeMs = measureBuildTime(packageDir) - coldBuildTimesMs.push(coldBuildTimeMs) - console.info(` Cold build time: ${coldBuildTimeMs}ms`) - - console.info('\nWarm build...') - const warmBuildTimeMs = measureBuildTime(packageDir) - warmBuildTimesMs.push(warmBuildTimeMs) - console.info(` Warm build time: ${warmBuildTimeMs}ms`) - } - - const excludedBuildOutputPaths = - testConfig.buildOutputDir === '.next' - ? [join(buildOutputPath, 'cache')] - : [] - const buildOutputSize = getDirectorySize( - buildOutputPath, - excludedBuildOutputPaths, - ) - console.info(`\nBuild output size: ${buildOutputSize} bytes`) - - const coldBuildTime = { - avgMs: - coldBuildTimesMs.reduce((total, cur) => total + cur, 0) / - coldBuildTimesMs.length, - minMs: Math.min(...coldBuildTimesMs), - maxMs: Math.max(...coldBuildTimesMs), + console.info(`\n✓ Saved build stats to ${outputPath}`) + } finally { + rmSync(tempDir, { recursive: true, force: true }) } - console.info(`\nAvg cold build time: ${coldBuildTime.avgMs} ms`) - console.info(`\nMin cold build time: ${coldBuildTime.minMs} ms`) - console.info(`\nMax cold build time: ${coldBuildTime.maxMs} ms`) - - const warmBuildTime = { - avgMs: - warmBuildTimesMs.reduce((total, cur) => total + cur, 0) / - warmBuildTimesMs.length, - minMs: Math.min(...warmBuildTimesMs), - maxMs: Math.max(...warmBuildTimesMs), - } - console.info(`\nAvg warm build time: ${warmBuildTime.avgMs} ms`) - console.info(`\nMin warm build time: ${warmBuildTime.minMs} ms`) - console.info(`\nMax warm build time: ${warmBuildTime.maxMs} ms`) - - const stats: BuildStats = { - coldBuildTime, - warmBuildTime, - buildOutputSize, - } - - const outputPath = join(packagesDir, packageName, 'build-stats.json') - writeJsonFile(outputPath, stats) - - console.info(`\n✓ Saved build stats to ${outputPath}`) } main().catch((error) => { diff --git a/packages/stats-generator/src/run-install-benchmark.ts b/packages/stats-generator/src/run-install-benchmark.ts index d1150a3f..f7a3d975 100644 --- a/packages/stats-generator/src/run-install-benchmark.ts +++ b/packages/stats-generator/src/run-install-benchmark.ts @@ -1,7 +1,8 @@ -import { execFileSync, execSync } from 'node:child_process' +import { execSync } from 'node:child_process' import { cpSync, mkdirSync, rmSync } from 'node:fs' import { basename, join } from 'node:path' import { tmpdir } from 'node:os' +import { installDependencies, parseRunFrequency } from './benchmark-utils.ts' import { packagesDir } from './constants.ts' import { getDirectorySize, @@ -35,22 +36,7 @@ function measureInstallTime( cacheDir: string, ): number { const start = performance.now() - execFileSync( - 'pnpm', - [ - 'install', - '--frozen-lockfile', - '--store-dir', - storeDir, - '--cache-dir', - cacheDir, - ], - { - cwd, - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - }, - ) + installDependencies(cwd, storeDir, cacheDir) const end = performance.now() return Math.round(end - start) @@ -78,9 +64,7 @@ async function main() { 'Usage: run-install-benchmark [run-frequency]\nExample: run-install-benchmark starter-astro 5', ) - const fallbackFrequency = '5' - const base = 10 - const runFrequency = Number.parseInt(args[0] || fallbackFrequency, base) + const runFrequency = parseRunFrequency(args[0]) const { framework } = await getFrameworkByPackage(packageName)