Build times not use cache

.github/workflows/measure-framework.yml

@@ -98,10 +98,6 @@ jobs:
       - name: Install workspace dependencies
         run: pnpm install --frozen-lockfile
 
-      - name: Install package dependencies
-        working-directory: ./packages/${{ matrix.framework.package }}
-        run: pnpm install --frozen-lockfile
-
       - name: Run build benchmark
         run: |
           RUN_FREQUENCY=$(echo '${{ toJson(matrix.framework) }}' | jq -r '.measurements[] | select(.type == "build") | .runFrequency')

packages/docs/src/content/docs/methodology.md

@@ -88,10 +88,12 @@ matches the framework version tracked by the starter project.
   temporary copy of the starter package with an empty pnpm store and cache.
 - Install benchmarks run 5 times by default and report average, minimum, and
   maximum duration.
-- Cold build time removes the configured build output directory before running
-  `pnpm build`.
-- Warm build time runs `pnpm build` again after the cold build, preserving
-  whatever cache or generated output the framework leaves in place.
+- Each build repetition uses a fresh temporary copy of the tracked starter
+  files. Dependencies are installed outside the timed region with a frozen
+  lockfile and a dedicated store shared by the repetitions.
+- Cold build time measures the first build in that fresh project. Warm build
+  time measures a second build in the same project, preserving whatever cache
+  or generated output the first build leaves in place.
 - Build benchmarks run 5 times by default and report average, minimum, and
   maximum duration.
 - Build output size is the total size of the configured production output

packages/stats-generator/src/benchmark-utils.test.ts

@@ -0,0 +1,19 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { parseRunFrequency } from './benchmark-utils.ts'
+
+test('uses the default run frequency when one is not provided', () => {
+  assert.equal(parseRunFrequency(undefined), 5)
+  assert.equal(parseRunFrequency(undefined, 3), 3)
+})
+
+test('accepts a positive integer run frequency', () => {
+  assert.equal(parseRunFrequency('1'), 1)
+  assert.equal(parseRunFrequency('10'), 10)
+})
+
+test('rejects invalid run frequencies', () => {
+  for (const value of ['0', '-1', '1.5', 'abc', '2runs']) {
+    assert.throws(() => parseRunFrequency(value), /positive integer/)
+  }
+})

packages/stats-generator/src/benchmark-utils.ts

@@ -0,0 +1,37 @@
+import { execFileSync } from 'node:child_process'
+
+export function parseRunFrequency(
+  value: string | undefined,
+  fallback = 5,
+): number {
+  const runFrequency = value === undefined ? fallback : Number(value)
+
+  if (!Number.isInteger(runFrequency) || runFrequency < 1) {
+    throw new Error(`Run frequency must be a positive integer: ${value}`)
+  }
+
+  return runFrequency
+}
+
+export function installDependencies(
+  cwd: string,
+  storeDir: string,
+  cacheDir: string,
+): void {
+  execFileSync(
+    'pnpm',
+    [
+      'install',
+      '--frozen-lockfile',
+      '--store-dir',
+      storeDir,
+      '--cache-dir',
+      cacheDir,
+    ],
+    {
+      cwd,
+      encoding: 'utf-8',
+      stdio: ['pipe', 'pipe', 'pipe'],
+    },
+  )
+}

packages/stats-generator/src/run-build-benchmark.ts

@@ -1,6 +1,8 @@
-import { execSync } from 'node:child_process'
-import { join } from 'node:path'
-import { existsSync, rmSync } from 'node:fs'
+import { execFileSync } from 'node:child_process'
+import { cpSync, mkdirSync, rmSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'
+import { installDependencies, parseRunFrequency } from './benchmark-utils.ts'
 import { packagesDir } from './constants.ts'
 import {
   getDirectorySize,
@@ -10,30 +12,66 @@ import {
 } from './utils.ts'
 import type { BuildStats } from './types.ts'
 
-function measureBuildTime(cwd: string): number {
+function copyTrackedProject(sourceDir: string, projectDir: string): void {
+  const repositoryDir = join(packagesDir, '..')
+  const sourcePathFromRepository = relative(repositoryDir, sourceDir)
+  const trackedPaths = execFileSync(
+    'git',
+    ['ls-files', '-z', '--', sourcePathFromRepository],
+    {
+      cwd: repositoryDir,
+      encoding: 'utf-8',
+    },
+  )
+    .split('\0')
+    .filter(Boolean)
+
+  for (const trackedPath of trackedPaths) {
+    const projectPath = relative(sourcePathFromRepository, trackedPath)
+    const destinationPath = join(projectDir, projectPath)
+    mkdirSync(dirname(destinationPath), { recursive: true })
+    cpSync(join(repositoryDir, trackedPath), destinationPath)
+  }
+}
+
+function measureBuildTime(cwd: string, buildScript: string): number {
   const start = performance.now()
-  execSync('pnpm build', { cwd, encoding: 'utf-8', stdio: 'inherit' })
+  execFileSync('pnpm', [buildScript], {
+    cwd,
+    encoding: 'utf-8',
+    stdio: 'inherit',
+  })
   const end = performance.now()
   return Math.round(end - start)
 }
 
-function rmBuildOutput(buildOutputPath: string): void {
-  if (existsSync(buildOutputPath)) {
-    console.info('Build output found. Removing build output')
-    rmSync(buildOutputPath, { recursive: true, force: true })
-  } else {
-    console.info('No build output found. Skipping removal of build output')
+function getBuildOutputPath(
+  projectDir: string,
+  buildOutputDir: string,
+): string {
+  const buildOutputPath = resolve(projectDir, buildOutputDir)
+  const relativeBuildOutputPath = relative(projectDir, buildOutputPath)
+
+  if (
+    !relativeBuildOutputPath ||
+    relativeBuildOutputPath === '..' ||
+    relativeBuildOutputPath.startsWith(`..${sep}`) ||
+    isAbsolute(relativeBuildOutputPath)
+  ) {
+    throw new Error(
+      `Build output directory must be inside the project: ${buildOutputDir}`,
+    )
   }
+
+  return buildOutputPath
 }
 
 async function main() {
   const { packageName, args } = parseArgs(
-    'Usage: run-build-benchmark <package-name>\nExample: run-build-benchmark starter-astro',
+    'Usage: run-build-benchmark <package-name> [run-frequency]\nExample: run-build-benchmark starter-astro 5',
   )
 
-  const fallbackFrequency = '5'
-  const base = 10
-  const runFrequency = Number.parseInt(args[0] || fallbackFrequency, base)
+  const runFrequency = parseRunFrequency(args[0])
 
   const { framework, testConfig } = await getFrameworkByPackage(packageName)
 
@@ -42,68 +80,108 @@ async function main() {
   )
 
   const packageDir = join(packagesDir, packageName)
-  const buildOutputPath = join(packageDir, testConfig.buildOutputDir)
+  const tempDir = join(
+    tmpdir(),
+    `framework-build-benchmark-${packageName}-${Date.now()}`,
+  )
+  const storeDir = join(tempDir, 'store')
+  const cacheDir = join(tempDir, 'cache')
 
   const coldBuildTimesMs: number[] = []
   const warmBuildTimesMs: number[] = []
+  let finalProjectDir = ''
+  let previousRunDir = ''
+
+  try {
+    for (let i = 1; i <= runFrequency; i++) {
+      if (previousRunDir) {
+        rmSync(previousRunDir, { recursive: true, force: true })
+      }
+
+      const runDir = join(tempDir, `run-${i}`)
+      const projectDir = join(runDir, 'project')
+      copyTrackedProject(packageDir, projectDir)
+
+      console.info(`\nBuild run ${i}/${runFrequency}...`)
+      console.info('Installing dependencies outside the timed region...')
+      installDependencies(projectDir, storeDir, cacheDir)
+
+      console.info('Cold build...')
+      const coldBuildTimeMs = measureBuildTime(
+        projectDir,
+        testConfig.buildScript,
+      )
+      coldBuildTimesMs.push(coldBuildTimeMs)
+      console.info(`  Cold build time: ${coldBuildTimeMs}ms`)
+
+      console.info('\nWarm build...')
+      const warmBuildTimeMs = measureBuildTime(
+        projectDir,
+        testConfig.buildScript,
+      )
+      warmBuildTimesMs.push(warmBuildTimeMs)
+      console.info(`  Warm build time: ${warmBuildTimeMs}ms`)
+
+      finalProjectDir = projectDir
+      previousRunDir = runDir
+    }
+
+    const finalBuildOutputPath = getBuildOutputPath(
+      finalProjectDir,
+      testConfig.buildOutputDir,
+    )
+    const excludedBuildOutputPaths =
+      testConfig.buildOutputDir === '.next'
+        ? [join(finalBuildOutputPath, 'cache')]
+        : []
+    const buildOutputSize = getDirectorySize(
+      finalBuildOutputPath,
+      excludedBuildOutputPaths,
+    )
+    console.info(`\nBuild output size: ${buildOutputSize} bytes`)
+
+    const coldBuildTime = {
+      avgMs:
+        coldBuildTimesMs.reduce((total, cur) => total + cur, 0) /
+        coldBuildTimesMs.length,
+      minMs: Math.min(...coldBuildTimesMs),
+      maxMs: Math.max(...coldBuildTimesMs),
+    }
+    console.info(`\nAvg cold build time: ${coldBuildTime.avgMs} ms`)
+    console.info(`\nMin cold build time: ${coldBuildTime.minMs} ms`)
+    console.info(`\nMax cold build time: ${coldBuildTime.maxMs} ms`)
+
+    const warmBuildTime = {
+      avgMs:
+        warmBuildTimesMs.reduce((total, cur) => total + cur, 0) /
+        warmBuildTimesMs.length,
+      minMs: Math.min(...warmBuildTimesMs),
+      maxMs: Math.max(...warmBuildTimesMs),
+    }
+    console.info(`\nAvg warm build time: ${warmBuildTime.avgMs} ms`)
+    console.info(`\nMin warm build time: ${warmBuildTime.minMs} ms`)
+    console.info(`\nMax warm build time: ${warmBuildTime.maxMs} ms`)
+
+    const stats: BuildStats = {
+      coldBuildTime,
+      warmBuildTime,
+      buildOutputSize,
+    }
+
+    const outputPath = join(packagesDir, packageName, 'build-stats.json')
+    writeJsonFile(outputPath, stats)
+
+    const buildOutputPath = getBuildOutputPath(
+      packageDir,
+      testConfig.buildOutputDir,
+    )
+    rmSync(buildOutputPath, { recursive: true, force: true })
+    cpSync(finalBuildOutputPath, buildOutputPath, { recursive: true })
 
-  for (let i = 1; i <= runFrequency; i++) {
-    console.info(`\nBuild run ${i}/${runFrequency}...`)
-    rmBuildOutput(buildOutputPath)
-
-    console.info('Cold build...')
-    const coldBuildTimeMs = measureBuildTime(packageDir)
-    coldBuildTimesMs.push(coldBuildTimeMs)
-    console.info(`  Cold build time: ${coldBuildTimeMs}ms`)
-
-    console.info('\nWarm build...')
-    const warmBuildTimeMs = measureBuildTime(packageDir)
-    warmBuildTimesMs.push(warmBuildTimeMs)
-    console.info(`  Warm build time: ${warmBuildTimeMs}ms`)
-  }
-
-  const excludedBuildOutputPaths =
-    testConfig.buildOutputDir === '.next'
-      ? [join(buildOutputPath, 'cache')]
-      : []
-  const buildOutputSize = getDirectorySize(
-    buildOutputPath,
-    excludedBuildOutputPaths,
-  )
-  console.info(`\nBuild output size: ${buildOutputSize} bytes`)
-
-  const coldBuildTime = {
-    avgMs:
-      coldBuildTimesMs.reduce((total, cur) => total + cur, 0) /
-      coldBuildTimesMs.length,
-    minMs: Math.min(...coldBuildTimesMs),
-    maxMs: Math.max(...coldBuildTimesMs),
+    console.info(`\n✓ Saved build stats to ${outputPath}`)
+  } finally {
+    rmSync(tempDir, { recursive: true, force: true })
   }
-  console.info(`\nAvg cold build time: ${coldBuildTime.avgMs} ms`)
-  console.info(`\nMin cold build time: ${coldBuildTime.minMs} ms`)
-  console.info(`\nMax cold build time: ${coldBuildTime.maxMs} ms`)
-
-  const warmBuildTime = {
-    avgMs:
-      warmBuildTimesMs.reduce((total, cur) => total + cur, 0) /
-      warmBuildTimesMs.length,
-    minMs: Math.min(...warmBuildTimesMs),
-    maxMs: Math.max(...warmBuildTimesMs),
-  }
-  console.info(`\nAvg warm build time: ${warmBuildTime.avgMs} ms`)
-  console.info(`\nMin warm build time: ${warmBuildTime.minMs} ms`)
-  console.info(`\nMax warm build time: ${warmBuildTime.maxMs} ms`)
-
-  const stats: BuildStats = {
-    coldBuildTime,
-    warmBuildTime,
-    buildOutputSize,
-  }
-
-  const outputPath = join(packagesDir, packageName, 'build-stats.json')
-  writeJsonFile(outputPath, stats)
-
-  console.info(`\n✓ Saved build stats to ${outputPath}`)
 }
 
 main().catch((error) => {

packages/stats-generator/src/run-install-benchmark.ts

@@ -1,7 +1,8 @@
-import { execFileSync, execSync } from 'node:child_process'
+import { execSync } from 'node:child_process'
 import { cpSync, mkdirSync, rmSync } from 'node:fs'
 import { basename, join } from 'node:path'
 import { tmpdir } from 'node:os'
+import { installDependencies, parseRunFrequency } from './benchmark-utils.ts'
 import { packagesDir } from './constants.ts'
 import {
   getDirectorySize,
@@ -35,22 +36,7 @@ function measureInstallTime(
   cacheDir: string,
 ): number {
   const start = performance.now()
-  execFileSync(
-    'pnpm',
-    [
-      'install',
-      '--frozen-lockfile',
-      '--store-dir',
-      storeDir,
-      '--cache-dir',
-      cacheDir,
-    ],
-    {
-      cwd,
-      encoding: 'utf-8',
-      stdio: ['pipe', 'pipe', 'pipe'],
-    },
-  )
+  installDependencies(cwd, storeDir, cacheDir)
   const end = performance.now()
 
   return Math.round(end - start)
@@ -78,9 +64,7 @@ async function main() {
     'Usage: run-install-benchmark <package-name> [run-frequency]\nExample: run-install-benchmark starter-astro 5',
   )
 
-  const fallbackFrequency = '5'
-  const base = 10
-  const runFrequency = Number.parseInt(args[0] || fallbackFrequency, base)
+  const runFrequency = parseRunFrequency(args[0])
 
   const { framework } = await getFrameworkByPackage(packageName)