Install Benchmarks now use fresh repo

packages/docs/src/content/docs/methodology.md

@@ -74,17 +74,18 @@ matches the framework version tracked by the starter project.
 
 ### Node Modules Size
 
-- Install benchmarks copy the starter package to a temporary directory, remove
-  `node_modules`, prune the package manager store when possible, and run
-  `pnpm install --no-frozen-lockfile`.
+- For every repetition, install benchmarks copy the starter package to a fresh
+  temporary directory and use dedicated, initially empty pnpm store and cache
+  directories. They run `pnpm install --frozen-lockfile` so every measurement
+  installs the committed dependency graph without reusing local package data.
 - `node_modules` size is measured after the regular install. This represents
   the starter's complete local installation, including development tools; it
   does not represent the framework's production deployment size.
 
 ### Build and Install Times
 
-- Install time measures a clean `pnpm install --no-frozen-lockfile` in a
-  temporary copy of the starter package.
+- Install time measures a clean `pnpm install --frozen-lockfile` in a fresh
+  temporary copy of the starter package with an empty pnpm store and cache.
 - Install benchmarks run 5 times by default and report average, minimum, and
   maximum duration.
 - Cold build time removes the configured build output directory before running

packages/stats-generator/src/run-install-benchmark.ts

@@ -1,6 +1,6 @@
-import { execSync } from 'node:child_process'
-import { cpSync, rmSync, existsSync } from 'node:fs'
-import { join } from 'node:path'
+import { execFileSync, execSync } from 'node:child_process'
+import { cpSync, mkdirSync, rmSync } from 'node:fs'
+import { basename, join } from 'node:path'
 import { tmpdir } from 'node:os'
 import { packagesDir } from './constants.ts'
 import {
@@ -19,24 +19,38 @@ function execCommand(command: string, cwd: string): string {
   })
 }
 
-function cleanForFreshInstall(cwd: string): void {
-  const nodeModulesPath = join(cwd, 'node_modules')
-  if (existsSync(nodeModulesPath)) {
-    rmSync(nodeModulesPath, { recursive: true, force: true })
-  }
-
-  try {
-    execCommand('pnpm store prune', cwd)
-  } catch {
-    // Ignore if prune fails
-  }
+function copyFreshProject(sourceDir: string, runDir: string): string {
+  const projectDir = join(runDir, 'project')
+  mkdirSync(runDir, { recursive: true })
+  cpSync(sourceDir, projectDir, {
+    recursive: true,
+    filter: (sourcePath) => basename(sourcePath) !== 'node_modules',
+  })
+  return projectDir
 }
 
-function measureInstallTime(cwd: string): number {
-  cleanForFreshInstall(cwd)
-
+function measureInstallTime(
+  cwd: string,
+  storeDir: string,
+  cacheDir: string,
+): number {
   const start = performance.now()
-  execCommand('pnpm install --no-frozen-lockfile', cwd)
+  execFileSync(
+    'pnpm',
+    [
+      'install',
+      '--frozen-lockfile',
+      '--store-dir',
+      storeDir,
+      '--cache-dir',
+      cacheDir,
+    ],
+    {
+      cwd,
+      encoding: 'utf-8',
+      stdio: ['pipe', 'pipe', 'pipe'],
+    },
+  )
   const end = performance.now()
 
   return Math.round(end - start)
@@ -81,17 +95,30 @@ async function main() {
     `framework-benchmark-${packageName}-${Date.now()}`,
   )
 
-  console.info(`Copying ${packageName} to ${tempDir}...`)
-  cpSync(sourceDir, tempDir, { recursive: true })
+  console.info(`Using isolated benchmark directory ${tempDir}...`)
 
   try {
     const installTimes: number[] = []
+    let finalProjectDir = ''
+    let previousRunDir = ''
 
     for (let i = 1; i <= runFrequency; i++) {
+      if (previousRunDir) {
+        rmSync(previousRunDir, { recursive: true, force: true })
+      }
+
+      const runDir = join(tempDir, `run-${i}`)
+      const projectDir = copyFreshProject(sourceDir, runDir)
+      const storeDir = join(runDir, 'store')
+      const cacheDir = join(runDir, 'cache')
+
       console.info(`\nInstall run ${i}/${runFrequency}...`)
-      const time = measureInstallTime(tempDir)
+      const time = measureInstallTime(projectDir, storeDir, cacheDir)
       installTimes.push(time)
       console.info(`  Install time: ${time}ms`)
+
+      finalProjectDir = projectDir
+      previousRunDir = runDir
     }
 
     const avgInstallTimeMs =
@@ -102,12 +129,12 @@ async function main() {
     const maxInstallTimeMs = Math.max(...installTimes)
 
     const frameworkVersion = getFrameworkVersion(
-      tempDir,
+      finalProjectDir,
       framework.frameworkPackage,
     )
     console.info(`\nFramework version: ${frameworkVersion}`)
 
-    const nodeModulesPath = join(tempDir, 'node_modules')
+    const nodeModulesPath = join(finalProjectDir, 'node_modules')
     const nodeModulesSize = getDirectorySize(nodeModulesPath)
     console.info(`node_modules size: ${nodeModulesSize} bytes`)