diff --git a/packages/docs/src/content/docs/methodology.md b/packages/docs/src/content/docs/methodology.md index 5cd48076..2c2e0eae 100644 --- a/packages/docs/src/content/docs/methodology.md +++ b/packages/docs/src/content/docs/methodology.md @@ -74,17 +74,18 @@ matches the framework version tracked by the starter project. ### Node Modules Size -- Install benchmarks copy the starter package to a temporary directory, remove - `node_modules`, prune the package manager store when possible, and run - `pnpm install --no-frozen-lockfile`. +- For every repetition, install benchmarks copy the starter package to a fresh + temporary directory and use dedicated, initially empty pnpm store and cache + directories. They run `pnpm install --frozen-lockfile` so every measurement + installs the committed dependency graph without reusing local package data. - `node_modules` size is measured after the regular install. This represents the starter's complete local installation, including development tools; it does not represent the framework's production deployment size. ### Build and Install Times -- Install time measures a clean `pnpm install --no-frozen-lockfile` in a - temporary copy of the starter package. +- Install time measures a clean `pnpm install --frozen-lockfile` in a fresh + temporary copy of the starter package with an empty pnpm store and cache. - Install benchmarks run 5 times by default and report average, minimum, and maximum duration. - Cold build time removes the configured build output directory before running diff --git a/packages/stats-generator/src/run-install-benchmark.ts b/packages/stats-generator/src/run-install-benchmark.ts index 3c987405..d1150a3f 100644 --- a/packages/stats-generator/src/run-install-benchmark.ts +++ b/packages/stats-generator/src/run-install-benchmark.ts @@ -1,6 +1,6 @@ -import { execSync } from 'node:child_process' -import { cpSync, rmSync, existsSync } from 'node:fs' -import { join } from 'node:path' +import { execFileSync, execSync } from 'node:child_process' +import { cpSync, mkdirSync, rmSync } from 'node:fs' +import { basename, join } from 'node:path' import { tmpdir } from 'node:os' import { packagesDir } from './constants.ts' import { @@ -19,24 +19,38 @@ function execCommand(command: string, cwd: string): string { }) } -function cleanForFreshInstall(cwd: string): void { - const nodeModulesPath = join(cwd, 'node_modules') - if (existsSync(nodeModulesPath)) { - rmSync(nodeModulesPath, { recursive: true, force: true }) - } - - try { - execCommand('pnpm store prune', cwd) - } catch { - // Ignore if prune fails - } +function copyFreshProject(sourceDir: string, runDir: string): string { + const projectDir = join(runDir, 'project') + mkdirSync(runDir, { recursive: true }) + cpSync(sourceDir, projectDir, { + recursive: true, + filter: (sourcePath) => basename(sourcePath) !== 'node_modules', + }) + return projectDir } -function measureInstallTime(cwd: string): number { - cleanForFreshInstall(cwd) - +function measureInstallTime( + cwd: string, + storeDir: string, + cacheDir: string, +): number { const start = performance.now() - execCommand('pnpm install --no-frozen-lockfile', cwd) + execFileSync( + 'pnpm', + [ + 'install', + '--frozen-lockfile', + '--store-dir', + storeDir, + '--cache-dir', + cacheDir, + ], + { + cwd, + encoding: 'utf-8', + stdio: ['pipe', 'pipe', 'pipe'], + }, + ) const end = performance.now() return Math.round(end - start) @@ -81,17 +95,30 @@ async function main() { `framework-benchmark-${packageName}-${Date.now()}`, ) - console.info(`Copying ${packageName} to ${tempDir}...`) - cpSync(sourceDir, tempDir, { recursive: true }) + console.info(`Using isolated benchmark directory ${tempDir}...`) try { const installTimes: number[] = [] + let finalProjectDir = '' + let previousRunDir = '' for (let i = 1; i <= runFrequency; i++) { + if (previousRunDir) { + rmSync(previousRunDir, { recursive: true, force: true }) + } + + const runDir = join(tempDir, `run-${i}`) + const projectDir = copyFreshProject(sourceDir, runDir) + const storeDir = join(runDir, 'store') + const cacheDir = join(runDir, 'cache') + console.info(`\nInstall run ${i}/${runFrequency}...`) - const time = measureInstallTime(tempDir) + const time = measureInstallTime(projectDir, storeDir, cacheDir) installTimes.push(time) console.info(` Install time: ${time}ms`) + + finalProjectDir = projectDir + previousRunDir = runDir } const avgInstallTimeMs = @@ -102,12 +129,12 @@ async function main() { const maxInstallTimeMs = Math.max(...installTimes) const frameworkVersion = getFrameworkVersion( - tempDir, + finalProjectDir, framework.frameworkPackage, ) console.info(`\nFramework version: ${frameworkVersion}`) - const nodeModulesPath = join(tempDir, 'node_modules') + const nodeModulesPath = join(finalProjectDir, 'node_modules') const nodeModulesSize = getDirectorySize(nodeModulesPath) console.info(`node_modules size: ${nodeModulesSize} bytes`)