From 9adcc7c2bc47b01c0ac3669335a0bc58c1ec2f48 Mon Sep 17 00:00:00 2001 From: Alexander Karan <47707063+AlexanderKaran@users.noreply.github.com> Date: Fri, 28 Aug 2026 12:00:36 +0800 Subject: [PATCH 1/2] Install Benchmarks now use fresh repo --- packages/docs/src/content/docs/methodology.md | 11 +-- .../src/run-install-benchmark.ts | 73 +++++++++++++------ 2 files changed, 56 insertions(+), 28 deletions(-) diff --git a/packages/docs/src/content/docs/methodology.md b/packages/docs/src/content/docs/methodology.md index 5cd48076..2c2e0eae 100644 --- a/packages/docs/src/content/docs/methodology.md +++ b/packages/docs/src/content/docs/methodology.md @@ -74,17 +74,18 @@ matches the framework version tracked by the starter project. ### Node Modules Size -- Install benchmarks copy the starter package to a temporary directory, remove - `node_modules`, prune the package manager store when possible, and run - `pnpm install --no-frozen-lockfile`. +- For every repetition, install benchmarks copy the starter package to a fresh + temporary directory and use dedicated, initially empty pnpm store and cache + directories. They run `pnpm install --frozen-lockfile` so every measurement + installs the committed dependency graph without reusing local package data. - `node_modules` size is measured after the regular install. This represents the starter's complete local installation, including development tools; it does not represent the framework's production deployment size. ### Build and Install Times -- Install time measures a clean `pnpm install --no-frozen-lockfile` in a - temporary copy of the starter package. +- Install time measures a clean `pnpm install --frozen-lockfile` in a fresh + temporary copy of the starter package with an empty pnpm store and cache. - Install benchmarks run 5 times by default and report average, minimum, and maximum duration. - Cold build time removes the configured build output directory before running diff --git a/packages/stats-generator/src/run-install-benchmark.ts b/packages/stats-generator/src/run-install-benchmark.ts index 3c987405..d1150a3f 100644 --- a/packages/stats-generator/src/run-install-benchmark.ts +++ b/packages/stats-generator/src/run-install-benchmark.ts @@ -1,6 +1,6 @@ -import { execSync } from 'node:child_process' -import { cpSync, rmSync, existsSync } from 'node:fs' -import { join } from 'node:path' +import { execFileSync, execSync } from 'node:child_process' +import { cpSync, mkdirSync, rmSync } from 'node:fs' +import { basename, join } from 'node:path' import { tmpdir } from 'node:os' import { packagesDir } from './constants.ts' import { @@ -19,24 +19,38 @@ function execCommand(command: string, cwd: string): string { }) } -function cleanForFreshInstall(cwd: string): void { - const nodeModulesPath = join(cwd, 'node_modules') - if (existsSync(nodeModulesPath)) { - rmSync(nodeModulesPath, { recursive: true, force: true }) - } - - try { - execCommand('pnpm store prune', cwd) - } catch { - // Ignore if prune fails - } +function copyFreshProject(sourceDir: string, runDir: string): string { + const projectDir = join(runDir, 'project') + mkdirSync(runDir, { recursive: true }) + cpSync(sourceDir, projectDir, { + recursive: true, + filter: (sourcePath) => basename(sourcePath) !== 'node_modules', + }) + return projectDir } -function measureInstallTime(cwd: string): number { - cleanForFreshInstall(cwd) - +function measureInstallTime( + cwd: string, + storeDir: string, + cacheDir: string, +): number { const start = performance.now() - execCommand('pnpm install --no-frozen-lockfile', cwd) + execFileSync( + 'pnpm', + [ + 'install', + '--frozen-lockfile', + '--store-dir', + storeDir, + '--cache-dir', + cacheDir, + ], + { + cwd, + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }, + ) const end = performance.now() return Math.round(end - start) @@ -81,17 +95,30 @@ async function main() { `framework-benchmark-${packageName}-${Date.now()}`, ) - console.info(`Copying ${packageName} to ${tempDir}...`) - cpSync(sourceDir, tempDir, { recursive: true }) + console.info(`Using isolated benchmark directory ${tempDir}...`) try { const installTimes: number[] = [] + let finalProjectDir = '' + let previousRunDir = '' for (let i = 1; i <= runFrequency; i++) { + if (previousRunDir) { + rmSync(previousRunDir, { recursive: true, force: true }) + } + + const runDir = join(tempDir, `run-${i}`) + const projectDir = copyFreshProject(sourceDir, runDir) + const storeDir = join(runDir, 'store') + const cacheDir = join(runDir, 'cache') + console.info(`\nInstall run ${i}/${runFrequency}...`) - const time = measureInstallTime(tempDir) + const time = measureInstallTime(projectDir, storeDir, cacheDir) installTimes.push(time) console.info(` Install time: ${time}ms`) + + finalProjectDir = projectDir + previousRunDir = runDir } const avgInstallTimeMs = @@ -102,12 +129,12 @@ async function main() { const maxInstallTimeMs = Math.max(...installTimes) const frameworkVersion = getFrameworkVersion( - tempDir, + finalProjectDir, framework.frameworkPackage, ) console.info(`\nFramework version: ${frameworkVersion}`) - const nodeModulesPath = join(tempDir, 'node_modules') + const nodeModulesPath = join(finalProjectDir, 'node_modules') const nodeModulesSize = getDirectorySize(nodeModulesPath) console.info(`node_modules size: ${nodeModulesSize} bytes`) From b0f61a1ea4c1d575c74e21fca1c58a8a505dd055 Mon Sep 17 00:00:00 2001 From: Alexander Karan <47707063+AlexanderKaran@users.noreply.github.com> Date: Fri, 28 Aug 2026 14:39:55 +0800 Subject: [PATCH 2/2] Moved Build Time To New Project Setup --- .github/workflows/measure-framework.yml | 4 - packages/docs/src/content/docs/methodology.md | 10 +- .../src/benchmark-utils.test.ts | 19 ++ .../stats-generator/src/benchmark-utils.ts | 37 +++ .../src/run-build-benchmark.ts | 222 ++++++++++++------ .../src/run-install-benchmark.ts | 24 +- 6 files changed, 216 insertions(+), 100 deletions(-) create mode 100644 packages/stats-generator/src/benchmark-utils.test.ts create mode 100644 packages/stats-generator/src/benchmark-utils.ts diff --git a/.github/workflows/measure-framework.yml b/.github/workflows/measure-framework.yml index 4da9e7af..663a3db5 100644 --- a/.github/workflows/measure-framework.yml +++ b/.github/workflows/measure-framework.yml @@ -98,10 +98,6 @@ jobs: - name: Install workspace dependencies run: pnpm install --frozen-lockfile - - name: Install package dependencies - working-directory: ./packages/${{ matrix.framework.package }} - run: pnpm install --frozen-lockfile - - name: Run build benchmark run: | RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "build") | .runFrequency') diff --git a/packages/docs/src/content/docs/methodology.md b/packages/docs/src/content/docs/methodology.md index 2c2e0eae..b295b41d 100644 --- a/packages/docs/src/content/docs/methodology.md +++ b/packages/docs/src/content/docs/methodology.md @@ -88,10 +88,12 @@ matches the framework version tracked by the starter project. temporary copy of the starter package with an empty pnpm store and cache. - Install benchmarks run 5 times by default and report average, minimum, and maximum duration. -- Cold build time removes the configured build output directory before running - `pnpm build`. -- Warm build time runs `pnpm build` again after the cold build, preserving - whatever cache or generated output the framework leaves in place. +- Each build repetition uses a fresh temporary copy of the tracked starter + files. Dependencies are installed outside the timed region with a frozen + lockfile and a dedicated store shared by the repetitions. +- Cold build time measures the first build in that fresh project. Warm build + time measures a second build in the same project, preserving whatever cache + or generated output the first build leaves in place. - Build benchmarks run 5 times by default and report average, minimum, and maximum duration. - Build output size is the total size of the configured production output diff --git a/packages/stats-generator/src/benchmark-utils.test.ts b/packages/stats-generator/src/benchmark-utils.test.ts new file mode 100644 index 00000000..b75b6a79 --- /dev/null +++ b/packages/stats-generator/src/benchmark-utils.test.ts @@ -0,0 +1,19 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { parseRunFrequency } from './benchmark-utils.ts' + +test('uses the default run frequency when one is not provided', () => { + assert.equal(parseRunFrequency(undefined), 5) + assert.equal(parseRunFrequency(undefined, 3), 3) +}) + +test('accepts a positive integer run frequency', () => { + assert.equal(parseRunFrequency('1'), 1) + assert.equal(parseRunFrequency('10'), 10) +}) + +test('rejects invalid run frequencies', () => { + for (const value of ['0', '-1', '1.5', 'abc', '2runs']) { + assert.throws(() => parseRunFrequency(value), /positive integer/) + } +}) diff --git a/packages/stats-generator/src/benchmark-utils.ts b/packages/stats-generator/src/benchmark-utils.ts new file mode 100644 index 00000000..2572a8e0 --- /dev/null +++ b/packages/stats-generator/src/benchmark-utils.ts @@ -0,0 +1,37 @@ +import { execFileSync } from 'node:child_process' + +export function parseRunFrequency( + value: string | undefined, + fallback = 5, +): number { + const runFrequency = value === undefined ? fallback : Number(value) + + if (!Number.isInteger(runFrequency) || runFrequency < 1) { + throw new Error(`Run frequency must be a positive integer: ${value}`) + } + + return runFrequency +} + +export function installDependencies( + cwd: string, + storeDir: string, + cacheDir: string, +): void { + execFileSync( + 'pnpm', + [ + 'install', + '--frozen-lockfile', + '--store-dir', + storeDir, + '--cache-dir', + cacheDir, + ], + { + cwd, + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }, + ) +} diff --git a/packages/stats-generator/src/run-build-benchmark.ts b/packages/stats-generator/src/run-build-benchmark.ts index 76489e45..86892373 100644 --- a/packages/stats-generator/src/run-build-benchmark.ts +++ b/packages/stats-generator/src/run-build-benchmark.ts @@ -1,6 +1,8 @@ -import { execSync } from 'node:child_process' -import { join } from 'node:path' -import { existsSync, rmSync } from 'node:fs' +import { execFileSync } from 'node:child_process' +import { cpSync, mkdirSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path' +import { installDependencies, parseRunFrequency } from './benchmark-utils.ts' import { packagesDir } from './constants.ts' import { getDirectorySize, @@ -10,30 +12,66 @@ import { } from './utils.ts' import type { BuildStats } from './types.ts' -function measureBuildTime(cwd: string): number { +function copyTrackedProject(sourceDir: string, projectDir: string): void { + const repositoryDir = join(packagesDir, '..') + const sourcePathFromRepository = relative(repositoryDir, sourceDir) + const trackedPaths = execFileSync( + 'git', + ['ls-files', '-z', '--', sourcePathFromRepository], + { + cwd: repositoryDir, + encoding: 'utf-8', + }, + ) + .split('\0') + .filter(Boolean) + + for (const trackedPath of trackedPaths) { + const projectPath = relative(sourcePathFromRepository, trackedPath) + const destinationPath = join(projectDir, projectPath) + mkdirSync(dirname(destinationPath), { recursive: true }) + cpSync(join(repositoryDir, trackedPath), destinationPath) + } +} + +function measureBuildTime(cwd: string, buildScript: string): number { const start = performance.now() - execSync('pnpm build', { cwd, encoding: 'utf-8', stdio: 'inherit' }) + execFileSync('pnpm', [buildScript], { + cwd, + encoding: 'utf-8', + stdio: 'inherit', + }) const end = performance.now() return Math.round(end - start) } -function rmBuildOutput(buildOutputPath: string): void { - if (existsSync(buildOutputPath)) { - console.info('Build output found. Removing build output') - rmSync(buildOutputPath, { recursive: true, force: true }) - } else { - console.info('No build output found. Skipping removal of build output') +function getBuildOutputPath( + projectDir: string, + buildOutputDir: string, +): string { + const buildOutputPath = resolve(projectDir, buildOutputDir) + const relativeBuildOutputPath = relative(projectDir, buildOutputPath) + + if ( + !relativeBuildOutputPath || + relativeBuildOutputPath === '..' || + relativeBuildOutputPath.startsWith(`..${sep}`) || + isAbsolute(relativeBuildOutputPath) + ) { + throw new Error( + `Build output directory must be inside the project: ${buildOutputDir}`, + ) } + + return buildOutputPath } async function main() { const { packageName, args } = parseArgs( - 'Usage: run-build-benchmark \nExample: run-build-benchmark starter-astro', + 'Usage: run-build-benchmark [run-frequency]\nExample: run-build-benchmark starter-astro 5', ) - const fallbackFrequency = '5' - const base = 10 - const runFrequency = Number.parseInt(args[0] || fallbackFrequency, base) + const runFrequency = parseRunFrequency(args[0]) const { framework, testConfig } = await getFrameworkByPackage(packageName) @@ -42,68 +80,108 @@ async function main() { ) const packageDir = join(packagesDir, packageName) - const buildOutputPath = join(packageDir, testConfig.buildOutputDir) + const tempDir = join( + tmpdir(), + `framework-build-benchmark-${packageName}-${Date.now()}`, + ) + const storeDir = join(tempDir, 'store') + const cacheDir = join(tempDir, 'cache') const coldBuildTimesMs: number[] = [] const warmBuildTimesMs: number[] = [] + let finalProjectDir = '' + let previousRunDir = '' + + try { + for (let i = 1; i <= runFrequency; i++) { + if (previousRunDir) { + rmSync(previousRunDir, { recursive: true, force: true }) + } + + const runDir = join(tempDir, `run-${i}`) + const projectDir = join(runDir, 'project') + copyTrackedProject(packageDir, projectDir) + + console.info(`\nBuild run ${i}/${runFrequency}...`) + console.info('Installing dependencies outside the timed region...') + installDependencies(projectDir, storeDir, cacheDir) + + console.info('Cold build...') + const coldBuildTimeMs = measureBuildTime( + projectDir, + testConfig.buildScript, + ) + coldBuildTimesMs.push(coldBuildTimeMs) + console.info(` Cold build time: ${coldBuildTimeMs}ms`) + + console.info('\nWarm build...') + const warmBuildTimeMs = measureBuildTime( + projectDir, + testConfig.buildScript, + ) + warmBuildTimesMs.push(warmBuildTimeMs) + console.info(` Warm build time: ${warmBuildTimeMs}ms`) + + finalProjectDir = projectDir + previousRunDir = runDir + } + + const finalBuildOutputPath = getBuildOutputPath( + finalProjectDir, + testConfig.buildOutputDir, + ) + const excludedBuildOutputPaths = + testConfig.buildOutputDir === '.next' + ? [join(finalBuildOutputPath, 'cache')] + : [] + const buildOutputSize = getDirectorySize( + finalBuildOutputPath, + excludedBuildOutputPaths, + ) + console.info(`\nBuild output size: ${buildOutputSize} bytes`) + + const coldBuildTime = { + avgMs: + coldBuildTimesMs.reduce((total, cur) => total + cur, 0) / + coldBuildTimesMs.length, + minMs: Math.min(...coldBuildTimesMs), + maxMs: Math.max(...coldBuildTimesMs), + } + console.info(`\nAvg cold build time: ${coldBuildTime.avgMs} ms`) + console.info(`\nMin cold build time: ${coldBuildTime.minMs} ms`) + console.info(`\nMax cold build time: ${coldBuildTime.maxMs} ms`) + + const warmBuildTime = { + avgMs: + warmBuildTimesMs.reduce((total, cur) => total + cur, 0) / + warmBuildTimesMs.length, + minMs: Math.min(...warmBuildTimesMs), + maxMs: Math.max(...warmBuildTimesMs), + } + console.info(`\nAvg warm build time: ${warmBuildTime.avgMs} ms`) + console.info(`\nMin warm build time: ${warmBuildTime.minMs} ms`) + console.info(`\nMax warm build time: ${warmBuildTime.maxMs} ms`) + + const stats: BuildStats = { + coldBuildTime, + warmBuildTime, + buildOutputSize, + } + + const outputPath = join(packagesDir, packageName, 'build-stats.json') + writeJsonFile(outputPath, stats) + + const buildOutputPath = getBuildOutputPath( + packageDir, + testConfig.buildOutputDir, + ) + rmSync(buildOutputPath, { recursive: true, force: true }) + cpSync(finalBuildOutputPath, buildOutputPath, { recursive: true }) - for (let i = 1; i <= runFrequency; i++) { - console.info(`\nBuild run ${i}/${runFrequency}...`) - rmBuildOutput(buildOutputPath) - - console.info('Cold build...') - const coldBuildTimeMs = measureBuildTime(packageDir) - coldBuildTimesMs.push(coldBuildTimeMs) - console.info(` Cold build time: ${coldBuildTimeMs}ms`) - - console.info('\nWarm build...') - const warmBuildTimeMs = measureBuildTime(packageDir) - warmBuildTimesMs.push(warmBuildTimeMs) - console.info(` Warm build time: ${warmBuildTimeMs}ms`) - } - - const excludedBuildOutputPaths = - testConfig.buildOutputDir === '.next' - ? [join(buildOutputPath, 'cache')] - : [] - const buildOutputSize = getDirectorySize( - buildOutputPath, - excludedBuildOutputPaths, - ) - console.info(`\nBuild output size: ${buildOutputSize} bytes`) - - const coldBuildTime = { - avgMs: - coldBuildTimesMs.reduce((total, cur) => total + cur, 0) / - coldBuildTimesMs.length, - minMs: Math.min(...coldBuildTimesMs), - maxMs: Math.max(...coldBuildTimesMs), + console.info(`\n✓ Saved build stats to ${outputPath}`) + } finally { + rmSync(tempDir, { recursive: true, force: true }) } - console.info(`\nAvg cold build time: ${coldBuildTime.avgMs} ms`) - console.info(`\nMin cold build time: ${coldBuildTime.minMs} ms`) - console.info(`\nMax cold build time: ${coldBuildTime.maxMs} ms`) - - const warmBuildTime = { - avgMs: - warmBuildTimesMs.reduce((total, cur) => total + cur, 0) / - warmBuildTimesMs.length, - minMs: Math.min(...warmBuildTimesMs), - maxMs: Math.max(...warmBuildTimesMs), - } - console.info(`\nAvg warm build time: ${warmBuildTime.avgMs} ms`) - console.info(`\nMin warm build time: ${warmBuildTime.minMs} ms`) - console.info(`\nMax warm build time: ${warmBuildTime.maxMs} ms`) - - const stats: BuildStats = { - coldBuildTime, - warmBuildTime, - buildOutputSize, - } - - const outputPath = join(packagesDir, packageName, 'build-stats.json') - writeJsonFile(outputPath, stats) - - console.info(`\n✓ Saved build stats to ${outputPath}`) } main().catch((error) => { diff --git a/packages/stats-generator/src/run-install-benchmark.ts b/packages/stats-generator/src/run-install-benchmark.ts index d1150a3f..f7a3d975 100644 --- a/packages/stats-generator/src/run-install-benchmark.ts +++ b/packages/stats-generator/src/run-install-benchmark.ts @@ -1,7 +1,8 @@ -import { execFileSync, execSync } from 'node:child_process' +import { execSync } from 'node:child_process' import { cpSync, mkdirSync, rmSync } from 'node:fs' import { basename, join } from 'node:path' import { tmpdir } from 'node:os' +import { installDependencies, parseRunFrequency } from './benchmark-utils.ts' import { packagesDir } from './constants.ts' import { getDirectorySize, @@ -35,22 +36,7 @@ function measureInstallTime( cacheDir: string, ): number { const start = performance.now() - execFileSync( - 'pnpm', - [ - 'install', - '--frozen-lockfile', - '--store-dir', - storeDir, - '--cache-dir', - cacheDir, - ], - { - cwd, - encoding: 'utf-8', - stdio: ['pipe', 'pipe', 'pipe'], - }, - ) + installDependencies(cwd, storeDir, cacheDir) const end = performance.now() return Math.round(end - start) @@ -78,9 +64,7 @@ async function main() { 'Usage: run-install-benchmark [run-frequency]\nExample: run-install-benchmark starter-astro 5', ) - const fallbackFrequency = '5' - const base = 10 - const runFrequency = Number.parseInt(args[0] || fallbackFrequency, base) + const runFrequency = parseRunFrequency(args[0]) const { framework } = await getFrameworkByPackage(packageName)