Updated Tests To Keep Samples

packages/docs/src/content.config.ts

@@ -6,8 +6,23 @@ import { z } from 'astro/zod'
 
 const timeSchema = z.object({
   avgMs: z.number(),
+  standardDeviationMs: z.number().nonnegative().optional(),
   minMs: z.number(),
   maxMs: z.number(),
+  samplesMs: z.array(z.number()).nonempty().optional(),
+})
+
+const interactionTimingSchema = z.object({
+  interactionLatencyMs: z.number().nonnegative(),
+  inputDelayMs: z.number().nonnegative(),
+  processingDurationMs: z.number().nonnegative(),
+  presentationDelayMs: z.number().nonnegative(),
+})
+
+const renderedTestSampleSchema = z.object({
+  firstPaintMs: z.number().positive(),
+  fcpMs: z.number().positive(),
+  interactionTests: interactionTimingSchema,
 })
 
 const dependencyStatsSchema = z.object({
@@ -125,6 +140,13 @@ const runtimeSchema = z.object({
         })
         .optional(),
       runs: z.number(),
+      standardDeviation: interactionTimingSchema
+        .extend({
+          firstPaintMs: z.number().nonnegative(),
+          fcpMs: z.number().nonnegative(),
+        })
+        .optional(),
+      samples: z.array(renderedTestSampleSchema).nonempty().optional(),
     })
     .optional(),
   serverSideRenderedTests: z
@@ -143,6 +165,13 @@ const runtimeSchema = z.object({
         })
         .optional(),
       runs: z.number(),
+      standardDeviation: interactionTimingSchema
+        .extend({
+          firstPaintMs: z.number().nonnegative(),
+          fcpMs: z.number().nonnegative(),
+        })
+        .optional(),
+      samples: z.array(renderedTestSampleSchema).nonempty().optional(),
     })
     .optional(),
   timingMeasuredAt: z.string().optional(),

packages/stats-generator/src/clientSideRendered/run-benchmark.ts

@@ -14,6 +14,7 @@ import type {
   ClientSideRenderedBenchmarkResult,
   ClientSideRenderedRunResult,
 } from './types.ts'
+import { standardDeviation } from '../sample-statistics.ts'
 
 const CLIENT_SIDE_RENDERED_PATH = '/client-side-rendered'
 
@@ -201,6 +202,27 @@ export async function runBenchmark(
       interactions.map((value) => value.presentationDelayMs),
     ),
   }
+  const samples = results.map((_, index) => ({
+    firstPaintMs: fp[index],
+    fcpMs: fcp[index],
+    interactionTests: interactions[index],
+  }))
+  const standardDeviationStats = {
+    firstPaintMs: standardDeviation(fp),
+    fcpMs: standardDeviation(fcp),
+    interactionLatencyMs: standardDeviation(
+      interactions.map((value) => value.interactionLatencyMs),
+    ),
+    inputDelayMs: standardDeviation(
+      interactions.map((value) => value.inputDelayMs),
+    ),
+    processingDurationMs: standardDeviation(
+      interactions.map((value) => value.processingDurationMs),
+    ),
+    presentationDelayMs: standardDeviation(
+      interactions.map((value) => value.presentationDelayMs),
+    ),
+  }
   return {
     name: packageName,
     displayName,
@@ -211,6 +233,8 @@ export async function runBenchmark(
       fcpMs,
       interactionTests,
       runs: results.length,
+      standardDeviation: standardDeviationStats,
+      samples,
     },
   }
 }

packages/stats-generator/src/clientSideRendered/types.ts

@@ -1,4 +1,9 @@
-import type { InteractionTestStats, InteractionTiming } from '../types.ts'
+import type {
+  InteractionTestStats,
+  InteractionTiming,
+  RenderedTestSample,
+  RenderedTestStandardDeviation,
+} from '../types.ts'
 
 export interface ClientSideRenderedRunResult {
   firstPaintMs: number | null
@@ -16,6 +21,8 @@ export interface ClientSideRenderedBenchmarkResult {
     fcpMs: number
     interactionTests: InteractionTestStats
     runs: number
+    standardDeviation: RenderedTestStandardDeviation
+    samples: RenderedTestSample[]
   }
 }
 
@@ -30,5 +37,7 @@ export interface ClientSideRenderedStats {
     inpMs?: number
     interactionTests?: InteractionTestStats
     runs: number
+    standardDeviation?: RenderedTestStandardDeviation
+    samples?: RenderedTestSample[]
   }
 }

packages/stats-generator/src/run-build-benchmark.ts

@@ -11,6 +11,7 @@ import {
   parseArgs,
 } from './utils.ts'
 import type { BuildStats } from './types.ts'
+import { summarizeSamples } from './sample-statistics.ts'
 
 function copyTrackedProject(sourceDir: string, projectDir: string): void {
   const repositoryDir = join(packagesDir, '..')
@@ -140,25 +141,19 @@ async function main() {
     )
     console.info(`\nBuild output size: ${buildOutputSize} bytes`)
 
-    const coldBuildTime = {
-      avgMs:
-        coldBuildTimesMs.reduce((total, cur) => total + cur, 0) /
-        coldBuildTimesMs.length,
-      minMs: Math.min(...coldBuildTimesMs),
-      maxMs: Math.max(...coldBuildTimesMs),
-    }
+    const coldBuildTime = summarizeSamples(coldBuildTimesMs)
     console.info(`\nAvg cold build time: ${coldBuildTime.avgMs} ms`)
+    console.info(
+      `\nCold build standard deviation: ${coldBuildTime.standardDeviationMs} ms`,
+    )
     console.info(`\nMin cold build time: ${coldBuildTime.minMs} ms`)
     console.info(`\nMax cold build time: ${coldBuildTime.maxMs} ms`)
 
-    const warmBuildTime = {
-      avgMs:
-        warmBuildTimesMs.reduce((total, cur) => total + cur, 0) /
-        warmBuildTimesMs.length,
-      minMs: Math.min(...warmBuildTimesMs),
-      maxMs: Math.max(...warmBuildTimesMs),
-    }
+    const warmBuildTime = summarizeSamples(warmBuildTimesMs)
     console.info(`\nAvg warm build time: ${warmBuildTime.avgMs} ms`)
+    console.info(
+      `\nWarm build standard deviation: ${warmBuildTime.standardDeviationMs} ms`,
+    )
     console.info(`\nMin warm build time: ${warmBuildTime.minMs} ms`)
     console.info(`\nMax warm build time: ${warmBuildTime.maxMs} ms`)
 

packages/stats-generator/src/run-client-side-rendered-benchmark.ts

@@ -52,9 +52,11 @@ async function main() {
     `\n✓ Saved ${result.displayName} v${frameworkVersion ?? 'unknown'} (${packageName})`,
   )
   console.info(
-    `  First Paint: ${result.clientSideRenderedTests.firstPaintMs}ms`,
+    `  First Paint: ${result.clientSideRenderedTests.firstPaintMs}ms ± ${result.clientSideRenderedTests.standardDeviation.firstPaintMs}ms`,
+  )
+  console.info(
+    `  FCP:         ${result.clientSideRenderedTests.fcpMs}ms ± ${result.clientSideRenderedTests.standardDeviation.fcpMs}ms`,
   )
-  console.info(`  FCP:         ${result.clientSideRenderedTests.fcpMs}ms`)
   const interaction = result.clientSideRenderedTests.interactionTests
   console.info(`  Interaction: ${interaction.interactionLatencyMs}ms`)
   console.info(`    Input delay:  ${interaction.inputDelayMs}ms`)

packages/stats-generator/src/run-install-benchmark.ts

@@ -11,6 +11,7 @@ import {
   parseArgs,
 } from './utils.ts'
 import type { InstallStats } from './types.ts'
+import { summarizeSamples } from './sample-statistics.ts'
 
 function execCommand(command: string, cwd: string): string {
   return execSync(command, {
@@ -105,13 +106,6 @@ async function main() {
       previousRunDir = runDir
     }
 
-    const avgInstallTimeMs =
-      Math.round(
-        (installTimes.reduce((a, b) => a + b, 0) / installTimes.length) * 10,
-      ) / 10
-    const minInstallTimeMs = Math.min(...installTimes)
-    const maxInstallTimeMs = Math.max(...installTimes)
-
     const frameworkVersion = getFrameworkVersion(
       finalProjectDir,
       framework.frameworkPackage,
@@ -124,11 +118,7 @@ async function main() {
 
     const stats: InstallStats = {
       frameworkVersion,
-      installTime: {
-        avgMs: avgInstallTimeMs,
-        minMs: minInstallTimeMs,
-        maxMs: maxInstallTimeMs,
-      },
+      installTime: summarizeSamples(installTimes, 1),
       nodeModulesSize,
     }
 
@@ -137,6 +127,9 @@ async function main() {
 
     console.info(`\n✓ Saved install stats to ${outputPath}`)
     console.info(`  Average: ${stats.installTime.avgMs}ms`)
+    console.info(
+      `  Standard deviation: ${stats.installTime.standardDeviationMs}ms`,
+    )
     console.info(`  Min: ${stats.installTime.minMs}ms`)
     console.info(`  Max: ${stats.installTime.maxMs}ms`)
   } finally {

packages/stats-generator/src/run-server-side-rendered-benchmark.ts

@@ -52,9 +52,11 @@ async function main() {
     `\n✓ Saved ${result.displayName} v${frameworkVersion ?? 'unknown'} (${packageName})`,
   )
   console.info(
-    `  First Paint: ${result.serverSideRenderedTests.firstPaintMs}ms`,
+    `  First Paint: ${result.serverSideRenderedTests.firstPaintMs}ms ± ${result.serverSideRenderedTests.standardDeviation.firstPaintMs}ms`,
+  )
+  console.info(
+    `  FCP:         ${result.serverSideRenderedTests.fcpMs}ms ± ${result.serverSideRenderedTests.standardDeviation.fcpMs}ms`,
   )
-  console.info(`  FCP:         ${result.serverSideRenderedTests.fcpMs}ms`)
   const interaction = result.serverSideRenderedTests.interactionTests
   console.info(`  Interaction: ${interaction.interactionLatencyMs}ms`)
   console.info(`    Input delay:  ${interaction.inputDelayMs}ms`)

packages/stats-generator/src/sample-statistics.test.ts

@@ -0,0 +1,23 @@
+import assert from 'node:assert/strict'
+import { describe, it } from 'node:test'
+import { standardDeviation, summarizeSamples } from './sample-statistics.ts'
+
+describe('sample statistics', () => {
+  it('keeps raw samples and calculates their summary', () => {
+    assert.deepEqual(summarizeSamples([10, 12, 14, 16, 18]), {
+      avgMs: 14,
+      standardDeviationMs: 3.16,
+      minMs: 10,
+      maxMs: 18,
+      samplesMs: [10, 12, 14, 16, 18],
+    })
+  })
+
+  it('reports zero deviation for one sample', () => {
+    assert.equal(standardDeviation([12]), 0)
+  })
+
+  it('rejects an empty sample set', () => {
+    assert.throws(() => summarizeSamples([]), /empty sample set/)
+  })
+})

packages/stats-generator/src/sample-statistics.ts

@@ -0,0 +1,38 @@
+import type { TimeStat } from './types.ts'
+
+function round(value: number, decimalPlaces: number): number {
+  const factor = 10 ** decimalPlaces
+  return Math.round(value * factor) / factor
+}
+
+export function summarizeSamples(
+  samplesMs: number[],
+  decimalPlaces = 2,
+): TimeStat {
+  if (samplesMs.length === 0) {
+    throw new Error('Cannot summarize an empty sample set')
+  }
+
+  const avgMs =
+    samplesMs.reduce((sum, value) => sum + value, 0) / samplesMs.length
+  const variance =
+    samplesMs.length === 1
+      ? 0
+      : samplesMs.reduce((sum, value) => sum + (value - avgMs) ** 2, 0) /
+        (samplesMs.length - 1)
+
+  return {
+    avgMs: round(avgMs, decimalPlaces),
+    standardDeviationMs: round(Math.sqrt(variance), decimalPlaces),
+    minMs: Math.min(...samplesMs),
+    maxMs: Math.max(...samplesMs),
+    samplesMs: [...samplesMs],
+  }
+}
+
+export function standardDeviation(
+  samples: number[],
+  decimalPlaces = 2,
+): number {
+  return summarizeSamples(samples, decimalPlaces).standardDeviationMs
+}

packages/stats-generator/src/schemas.ts

@@ -1,10 +1,19 @@
 import { z } from 'zod'
 
-export const TimeStatSchema = z.object({
-  avgMs: z.number(),
-  minMs: z.number(),
-  maxMs: z.number(),
-})
+export const TimeStatSchema = z
+  .object({
+    avgMs: z.number(),
+    standardDeviationMs: z.number().nonnegative().optional(),
+    minMs: z.number(),
+    maxMs: z.number(),
+    samplesMs: z.array(z.number()).nonempty().optional(),
+  })
+  .refine(
+    (stats) =>
+      (stats.standardDeviationMs === undefined) ===
+      (stats.samplesMs === undefined),
+    { message: 'Expected standard deviation and samples together' },
+  )
 
 export const InstallStatsSchema = z.object({
   frameworkVersion: z.string().min(1),
@@ -99,6 +108,19 @@ const InteractionTestsSchema = z.object({
   presentationDelayMs: z.number().nonnegative(),
 })
 
+const InteractionTimingSchema = z.object({
+  interactionLatencyMs: z.number().nonnegative(),
+  inputDelayMs: z.number().nonnegative(),
+  processingDurationMs: z.number().nonnegative(),
+  presentationDelayMs: z.number().nonnegative(),
+})
+
+const RenderedTestSampleSchema = z.object({
+  firstPaintMs: z.number().positive(),
+  fcpMs: z.number().positive(),
+  interactionTests: InteractionTimingSchema,
+})
+
 const RenderedTestsSchema = z
   .object({
     firstPaintMs: z.number().positive(),
@@ -107,6 +129,11 @@ const RenderedTestsSchema = z
     inpMs: z.number().optional(),
     interactionTests: InteractionTestsSchema.optional(),
     runs: z.number().int().positive(),
+    standardDeviation: InteractionTimingSchema.extend({
+      firstPaintMs: z.number().nonnegative(),
+      fcpMs: z.number().nonnegative(),
+    }).optional(),
+    samples: z.array(RenderedTestSampleSchema).nonempty().optional(),
   })
   .refine(
     (stats) =>
@@ -115,6 +142,16 @@ const RenderedTestsSchema = z
       message: 'Expected historical inpMs or current interactionTests',
     },
   )
+  .refine(
+    (stats) =>
+      stats.samples === undefined || stats.samples.length === stats.runs,
+    { message: 'Expected one browser sample per run' },
+  )
+  .refine(
+    (stats) =>
+      (stats.standardDeviation === undefined) === (stats.samples === undefined),
+    { message: 'Expected standard deviation and samples together' },
+  )
 
 export const ClientSideRenderedStatsSchema = z.object({
   clientSideRenderedTests: RenderedTestsSchema,