diff --git a/packages/docs/src/content.config.ts b/packages/docs/src/content.config.ts index ad7fdd75..c9b4c5e0 100644 --- a/packages/docs/src/content.config.ts +++ b/packages/docs/src/content.config.ts @@ -6,8 +6,23 @@ import { z } from 'astro/zod' const timeSchema = z.object({ avgMs: z.number(), + standardDeviationMs: z.number().nonnegative().optional(), minMs: z.number(), maxMs: z.number(), + samplesMs: z.array(z.number()).nonempty().optional(), +}) + +const interactionTimingSchema = z.object({ + interactionLatencyMs: z.number().nonnegative(), + inputDelayMs: z.number().nonnegative(), + processingDurationMs: z.number().nonnegative(), + presentationDelayMs: z.number().nonnegative(), +}) + +const renderedTestSampleSchema = z.object({ + firstPaintMs: z.number().positive(), + fcpMs: z.number().positive(), + interactionTests: interactionTimingSchema, }) const dependencyStatsSchema = z.object({ @@ -125,6 +140,13 @@ const runtimeSchema = z.object({ }) .optional(), runs: z.number(), + standardDeviation: interactionTimingSchema + .extend({ + firstPaintMs: z.number().nonnegative(), + fcpMs: z.number().nonnegative(), + }) + .optional(), + samples: z.array(renderedTestSampleSchema).nonempty().optional(), }) .optional(), serverSideRenderedTests: z @@ -143,6 +165,13 @@ const runtimeSchema = z.object({ }) .optional(), runs: z.number(), + standardDeviation: interactionTimingSchema + .extend({ + firstPaintMs: z.number().nonnegative(), + fcpMs: z.number().nonnegative(), + }) + .optional(), + samples: z.array(renderedTestSampleSchema).nonempty().optional(), }) .optional(), timingMeasuredAt: z.string().optional(), diff --git a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts index 3df93a96..ff15dc65 100644 --- a/packages/stats-generator/src/clientSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/clientSideRendered/run-benchmark.ts @@ -14,6 +14,7 @@ import type { ClientSideRenderedBenchmarkResult, ClientSideRenderedRunResult, } from './types.ts' +import { standardDeviation } from '../sample-statistics.ts' const CLIENT_SIDE_RENDERED_PATH = '/client-side-rendered' @@ -201,6 +202,27 @@ export async function runBenchmark( interactions.map((value) => value.presentationDelayMs), ), } + const samples = results.map((_, index) => ({ + firstPaintMs: fp[index], + fcpMs: fcp[index], + interactionTests: interactions[index], + })) + const standardDeviationStats = { + firstPaintMs: standardDeviation(fp), + fcpMs: standardDeviation(fcp), + interactionLatencyMs: standardDeviation( + interactions.map((value) => value.interactionLatencyMs), + ), + inputDelayMs: standardDeviation( + interactions.map((value) => value.inputDelayMs), + ), + processingDurationMs: standardDeviation( + interactions.map((value) => value.processingDurationMs), + ), + presentationDelayMs: standardDeviation( + interactions.map((value) => value.presentationDelayMs), + ), + } return { name: packageName, displayName, @@ -211,6 +233,8 @@ export async function runBenchmark( fcpMs, interactionTests, runs: results.length, + standardDeviation: standardDeviationStats, + samples, }, } } diff --git a/packages/stats-generator/src/clientSideRendered/types.ts b/packages/stats-generator/src/clientSideRendered/types.ts index 550c97ab..fe551977 100644 --- a/packages/stats-generator/src/clientSideRendered/types.ts +++ b/packages/stats-generator/src/clientSideRendered/types.ts @@ -1,4 +1,9 @@ -import type { InteractionTestStats, InteractionTiming } from '../types.ts' +import type { + InteractionTestStats, + InteractionTiming, + RenderedTestSample, + RenderedTestStandardDeviation, +} from '../types.ts' export interface ClientSideRenderedRunResult { firstPaintMs: number | null @@ -16,6 +21,8 @@ export interface ClientSideRenderedBenchmarkResult { fcpMs: number interactionTests: InteractionTestStats runs: number + standardDeviation: RenderedTestStandardDeviation + samples: RenderedTestSample[] } } @@ -30,5 +37,7 @@ export interface ClientSideRenderedStats { inpMs?: number interactionTests?: InteractionTestStats runs: number + standardDeviation?: RenderedTestStandardDeviation + samples?: RenderedTestSample[] } } diff --git a/packages/stats-generator/src/run-build-benchmark.ts b/packages/stats-generator/src/run-build-benchmark.ts index 86892373..d874584c 100644 --- a/packages/stats-generator/src/run-build-benchmark.ts +++ b/packages/stats-generator/src/run-build-benchmark.ts @@ -11,6 +11,7 @@ import { parseArgs, } from './utils.ts' import type { BuildStats } from './types.ts' +import { summarizeSamples } from './sample-statistics.ts' function copyTrackedProject(sourceDir: string, projectDir: string): void { const repositoryDir = join(packagesDir, '..') @@ -140,25 +141,19 @@ async function main() { ) console.info(`\nBuild output size: ${buildOutputSize} bytes`) - const coldBuildTime = { - avgMs: - coldBuildTimesMs.reduce((total, cur) => total + cur, 0) / - coldBuildTimesMs.length, - minMs: Math.min(...coldBuildTimesMs), - maxMs: Math.max(...coldBuildTimesMs), - } + const coldBuildTime = summarizeSamples(coldBuildTimesMs) console.info(`\nAvg cold build time: ${coldBuildTime.avgMs} ms`) + console.info( + `\nCold build standard deviation: ${coldBuildTime.standardDeviationMs} ms`, + ) console.info(`\nMin cold build time: ${coldBuildTime.minMs} ms`) console.info(`\nMax cold build time: ${coldBuildTime.maxMs} ms`) - const warmBuildTime = { - avgMs: - warmBuildTimesMs.reduce((total, cur) => total + cur, 0) / - warmBuildTimesMs.length, - minMs: Math.min(...warmBuildTimesMs), - maxMs: Math.max(...warmBuildTimesMs), - } + const warmBuildTime = summarizeSamples(warmBuildTimesMs) console.info(`\nAvg warm build time: ${warmBuildTime.avgMs} ms`) + console.info( + `\nWarm build standard deviation: ${warmBuildTime.standardDeviationMs} ms`, + ) console.info(`\nMin warm build time: ${warmBuildTime.minMs} ms`) console.info(`\nMax warm build time: ${warmBuildTime.maxMs} ms`) diff --git a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts index d3a196a0..d4724c54 100644 --- a/packages/stats-generator/src/run-client-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-client-side-rendered-benchmark.ts @@ -52,9 +52,11 @@ async function main() { `\n✓ Saved ${result.displayName} v${frameworkVersion ?? 'unknown'} (${packageName})`, ) console.info( - ` First Paint: ${result.clientSideRenderedTests.firstPaintMs}ms`, + ` First Paint: ${result.clientSideRenderedTests.firstPaintMs}ms ± ${result.clientSideRenderedTests.standardDeviation.firstPaintMs}ms`, + ) + console.info( + ` FCP: ${result.clientSideRenderedTests.fcpMs}ms ± ${result.clientSideRenderedTests.standardDeviation.fcpMs}ms`, ) - console.info(` FCP: ${result.clientSideRenderedTests.fcpMs}ms`) const interaction = result.clientSideRenderedTests.interactionTests console.info(` Interaction: ${interaction.interactionLatencyMs}ms`) console.info(` Input delay: ${interaction.inputDelayMs}ms`) diff --git a/packages/stats-generator/src/run-install-benchmark.ts b/packages/stats-generator/src/run-install-benchmark.ts index f7a3d975..6c94016b 100644 --- a/packages/stats-generator/src/run-install-benchmark.ts +++ b/packages/stats-generator/src/run-install-benchmark.ts @@ -11,6 +11,7 @@ import { parseArgs, } from './utils.ts' import type { InstallStats } from './types.ts' +import { summarizeSamples } from './sample-statistics.ts' function execCommand(command: string, cwd: string): string { return execSync(command, { @@ -105,13 +106,6 @@ async function main() { previousRunDir = runDir } - const avgInstallTimeMs = - Math.round( - (installTimes.reduce((a, b) => a + b, 0) / installTimes.length) * 10, - ) / 10 - const minInstallTimeMs = Math.min(...installTimes) - const maxInstallTimeMs = Math.max(...installTimes) - const frameworkVersion = getFrameworkVersion( finalProjectDir, framework.frameworkPackage, @@ -124,11 +118,7 @@ async function main() { const stats: InstallStats = { frameworkVersion, - installTime: { - avgMs: avgInstallTimeMs, - minMs: minInstallTimeMs, - maxMs: maxInstallTimeMs, - }, + installTime: summarizeSamples(installTimes, 1), nodeModulesSize, } @@ -137,6 +127,9 @@ async function main() { console.info(`\n✓ Saved install stats to ${outputPath}`) console.info(` Average: ${stats.installTime.avgMs}ms`) + console.info( + ` Standard deviation: ${stats.installTime.standardDeviationMs}ms`, + ) console.info(` Min: ${stats.installTime.minMs}ms`) console.info(` Max: ${stats.installTime.maxMs}ms`) } finally { diff --git a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts index ba5c6437..bb780331 100644 --- a/packages/stats-generator/src/run-server-side-rendered-benchmark.ts +++ b/packages/stats-generator/src/run-server-side-rendered-benchmark.ts @@ -52,9 +52,11 @@ async function main() { `\n✓ Saved ${result.displayName} v${frameworkVersion ?? 'unknown'} (${packageName})`, ) console.info( - ` First Paint: ${result.serverSideRenderedTests.firstPaintMs}ms`, + ` First Paint: ${result.serverSideRenderedTests.firstPaintMs}ms ± ${result.serverSideRenderedTests.standardDeviation.firstPaintMs}ms`, + ) + console.info( + ` FCP: ${result.serverSideRenderedTests.fcpMs}ms ± ${result.serverSideRenderedTests.standardDeviation.fcpMs}ms`, ) - console.info(` FCP: ${result.serverSideRenderedTests.fcpMs}ms`) const interaction = result.serverSideRenderedTests.interactionTests console.info(` Interaction: ${interaction.interactionLatencyMs}ms`) console.info(` Input delay: ${interaction.inputDelayMs}ms`) diff --git a/packages/stats-generator/src/sample-statistics.test.ts b/packages/stats-generator/src/sample-statistics.test.ts new file mode 100644 index 00000000..d6ae9fe1 --- /dev/null +++ b/packages/stats-generator/src/sample-statistics.test.ts @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict' +import { describe, it } from 'node:test' +import { standardDeviation, summarizeSamples } from './sample-statistics.ts' + +describe('sample statistics', () => { + it('keeps raw samples and calculates their summary', () => { + assert.deepEqual(summarizeSamples([10, 12, 14, 16, 18]), { + avgMs: 14, + standardDeviationMs: 3.16, + minMs: 10, + maxMs: 18, + samplesMs: [10, 12, 14, 16, 18], + }) + }) + + it('reports zero deviation for one sample', () => { + assert.equal(standardDeviation([12]), 0) + }) + + it('rejects an empty sample set', () => { + assert.throws(() => summarizeSamples([]), /empty sample set/) + }) +}) diff --git a/packages/stats-generator/src/sample-statistics.ts b/packages/stats-generator/src/sample-statistics.ts new file mode 100644 index 00000000..4ce7a1ea --- /dev/null +++ b/packages/stats-generator/src/sample-statistics.ts @@ -0,0 +1,38 @@ +import type { TimeStat } from './types.ts' + +function round(value: number, decimalPlaces: number): number { + const factor = 10 ** decimalPlaces + return Math.round(value * factor) / factor +} + +export function summarizeSamples( + samplesMs: number[], + decimalPlaces = 2, +): TimeStat { + if (samplesMs.length === 0) { + throw new Error('Cannot summarize an empty sample set') + } + + const avgMs = + samplesMs.reduce((sum, value) => sum + value, 0) / samplesMs.length + const variance = + samplesMs.length === 1 + ? 0 + : samplesMs.reduce((sum, value) => sum + (value - avgMs) ** 2, 0) / + (samplesMs.length - 1) + + return { + avgMs: round(avgMs, decimalPlaces), + standardDeviationMs: round(Math.sqrt(variance), decimalPlaces), + minMs: Math.min(...samplesMs), + maxMs: Math.max(...samplesMs), + samplesMs: [...samplesMs], + } +} + +export function standardDeviation( + samples: number[], + decimalPlaces = 2, +): number { + return summarizeSamples(samples, decimalPlaces).standardDeviationMs +} diff --git a/packages/stats-generator/src/schemas.ts b/packages/stats-generator/src/schemas.ts index 21436ccc..07290b2f 100644 --- a/packages/stats-generator/src/schemas.ts +++ b/packages/stats-generator/src/schemas.ts @@ -1,10 +1,19 @@ import { z } from 'zod' -export const TimeStatSchema = z.object({ - avgMs: z.number(), - minMs: z.number(), - maxMs: z.number(), -}) +export const TimeStatSchema = z + .object({ + avgMs: z.number(), + standardDeviationMs: z.number().nonnegative().optional(), + minMs: z.number(), + maxMs: z.number(), + samplesMs: z.array(z.number()).nonempty().optional(), + }) + .refine( + (stats) => + (stats.standardDeviationMs === undefined) === + (stats.samplesMs === undefined), + { message: 'Expected standard deviation and samples together' }, + ) export const InstallStatsSchema = z.object({ frameworkVersion: z.string().min(1), @@ -99,6 +108,19 @@ const InteractionTestsSchema = z.object({ presentationDelayMs: z.number().nonnegative(), }) +const InteractionTimingSchema = z.object({ + interactionLatencyMs: z.number().nonnegative(), + inputDelayMs: z.number().nonnegative(), + processingDurationMs: z.number().nonnegative(), + presentationDelayMs: z.number().nonnegative(), +}) + +const RenderedTestSampleSchema = z.object({ + firstPaintMs: z.number().positive(), + fcpMs: z.number().positive(), + interactionTests: InteractionTimingSchema, +}) + const RenderedTestsSchema = z .object({ firstPaintMs: z.number().positive(), @@ -107,6 +129,11 @@ const RenderedTestsSchema = z inpMs: z.number().optional(), interactionTests: InteractionTestsSchema.optional(), runs: z.number().int().positive(), + standardDeviation: InteractionTimingSchema.extend({ + firstPaintMs: z.number().nonnegative(), + fcpMs: z.number().nonnegative(), + }).optional(), + samples: z.array(RenderedTestSampleSchema).nonempty().optional(), }) .refine( (stats) => @@ -115,6 +142,16 @@ const RenderedTestsSchema = z message: 'Expected historical inpMs or current interactionTests', }, ) + .refine( + (stats) => + stats.samples === undefined || stats.samples.length === stats.runs, + { message: 'Expected one browser sample per run' }, + ) + .refine( + (stats) => + (stats.standardDeviation === undefined) === (stats.samples === undefined), + { message: 'Expected standard deviation and samples together' }, + ) export const ClientSideRenderedStatsSchema = z.object({ clientSideRenderedTests: RenderedTestsSchema, diff --git a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts index 21107f37..94dbebcc 100644 --- a/packages/stats-generator/src/serverSideRendered/run-benchmark.ts +++ b/packages/stats-generator/src/serverSideRendered/run-benchmark.ts @@ -14,6 +14,7 @@ import type { ServerSideRenderedBenchmarkResult, ServerSideRenderedRunResult, } from './types.ts' +import { standardDeviation } from '../sample-statistics.ts' const SERVER_SIDE_RENDERED_PATH = '/server-side-rendered' @@ -203,6 +204,27 @@ export async function runBenchmark( interactions.map((value) => value.presentationDelayMs), ), } + const samples = results.map((_, index) => ({ + firstPaintMs: fp[index], + fcpMs: fcp[index], + interactionTests: interactions[index], + })) + const standardDeviationStats = { + firstPaintMs: standardDeviation(fp), + fcpMs: standardDeviation(fcp), + interactionLatencyMs: standardDeviation( + interactions.map((value) => value.interactionLatencyMs), + ), + inputDelayMs: standardDeviation( + interactions.map((value) => value.inputDelayMs), + ), + processingDurationMs: standardDeviation( + interactions.map((value) => value.processingDurationMs), + ), + presentationDelayMs: standardDeviation( + interactions.map((value) => value.presentationDelayMs), + ), + } return { name: packageName, displayName, @@ -213,6 +235,8 @@ export async function runBenchmark( fcpMs, interactionTests, runs: results.length, + standardDeviation: standardDeviationStats, + samples, }, } } diff --git a/packages/stats-generator/src/serverSideRendered/types.ts b/packages/stats-generator/src/serverSideRendered/types.ts index 2460be19..b38bf633 100644 --- a/packages/stats-generator/src/serverSideRendered/types.ts +++ b/packages/stats-generator/src/serverSideRendered/types.ts @@ -1,4 +1,9 @@ -import type { InteractionTestStats, InteractionTiming } from '../types.ts' +import type { + InteractionTestStats, + InteractionTiming, + RenderedTestSample, + RenderedTestStandardDeviation, +} from '../types.ts' export interface ServerSideRenderedRunResult { firstPaintMs: number | null @@ -16,6 +21,8 @@ export interface ServerSideRenderedBenchmarkResult { fcpMs: number interactionTests: InteractionTestStats runs: number + standardDeviation: RenderedTestStandardDeviation + samples: RenderedTestSample[] } } @@ -30,5 +37,7 @@ export interface ServerSideRenderedStats { inpMs?: number interactionTests?: InteractionTestStats runs: number + standardDeviation?: RenderedTestStandardDeviation + samples?: RenderedTestSample[] } } diff --git a/packages/stats-generator/src/types.ts b/packages/stats-generator/src/types.ts index 72f30814..489b1ebb 100644 --- a/packages/stats-generator/src/types.ts +++ b/packages/stats-generator/src/types.ts @@ -33,6 +33,17 @@ export interface InteractionTestStats extends InteractionTiming { source: 'lighthouse-inp-breakdown' } +export interface RenderedTestSample { + firstPaintMs: number + fcpMs: number + interactionTests: InteractionTiming +} + +export interface RenderedTestStandardDeviation extends InteractionTiming { + firstPaintMs: number + fcpMs: number +} + export interface FrameworkConfig { name: string displayName: string @@ -98,6 +109,8 @@ export interface CIStats { inpMs?: number interactionTests?: InteractionTestStats runs: number + standardDeviation?: RenderedTestStandardDeviation + samples?: RenderedTestSample[] } // Server-side rendered route stats (browser paint + interaction timings) serverSideRenderedTests?: { @@ -106,6 +119,8 @@ export interface CIStats { inpMs?: number interactionTests?: InteractionTestStats runs: number + standardDeviation?: RenderedTestStandardDeviation + samples?: RenderedTestSample[] } // Core-js vendored polyfill stats vendoredCoreJsSize?: number @@ -180,8 +195,10 @@ export interface NodeEnginesStats { export interface TimeStat { avgMs: number + standardDeviationMs: number minMs: number maxMs: number + samplesMs: number[] } export interface FrameworkStats extends CIStats {