From ff3b0b2292ff85977b54a5c3e11adf9030b8b0b2 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 1 Sep 2026 07:23:14 +0200 Subject: [PATCH 001/173] [code-infra] Add tachometer benchmarking Extracts the tachometer harness infrastructure from mui/base-ui-mosaic#428 (itself a port of mui/base-ui-charts#634) so a repo adopts it by adding case folders instead of copying ~1,100 lines of script. The two copies were already near-identical: packWorkspace.mts differed only in two doc comments. Ships three things: - `packWorkspace`, a reusable export that packs a ref's public workspace packages to tarballs with `pnpm pack`, cached by commit SHA. It lists packages through `getWorkspacePackages({ cwd })` in-process rather than shelling out to the CLI, so a ref's checkout no longer needs code-infra installed just to be enumerated. - A vite plugin. The harness keeps its own config and runs plain vite; the plugin contributes `root`, `appType: 'mpa'`, build entries derived from the `tachometer.json` files, and a dev-server index listing every case. It never resolves refs, so `vite build` needs no git history. - `code-infra tacho run`, which packs and builds one variant per ref, runs each case as its own tachometer invocation, and writes a combined JSON report. Three pieces the originals configured are now derived. The library under test and its version pins come from the harness's `workspace:*` dependencies rather than a hardcoded name and a hand-maintained map, which also fixes competitor libraries failing to resolve in a ref build. That in turn removes the reason TACHO_PAGES existed, so it and the page glob beside it are both gone -- the configs are now the only notion of which pages exist. The base branch is detected from origin/HEAD instead of being hardcoded to master. --- packages/code-infra/package.json | 17 +- packages/code-infra/src/cli/cmdTacho.mjs | 64 ++++ packages/code-infra/src/cli/index.mjs | 2 + .../code-infra/src/tachometer/buildPages.mjs | 171 +++++++++ .../src/tachometer/discoverCases.mjs | 210 +++++++++++ .../src/tachometer/discoverCases.test.mjs | 305 ++++++++++++++++ packages/code-infra/src/tachometer/exec.mjs | 21 ++ packages/code-infra/src/tachometer/plugin.mjs | 165 +++++++++ .../code-infra/src/tachometer/plugin.test.mjs | 151 ++++++++ packages/code-infra/src/tachometer/refs.mjs | 260 +++++++++++++ .../code-infra/src/tachometer/refs.test.mjs | 48 +++ .../src/tachometer/runTachometer.mjs | 197 ++++++++++ .../src/tachometer/summarizeCase.mjs | 85 +++++ .../src/tachometer/summarizeCase.test.mjs | 89 +++++ .../code-infra/src/utils/packWorkspace.mjs | 344 ++++++++++++++++++ .../src/utils/packWorkspace.test.mjs | 206 +++++++++++ pnpm-lock.yaml | 3 + 17 files changed, 2336 insertions(+), 2 deletions(-) create mode 100644 packages/code-infra/src/cli/cmdTacho.mjs create mode 100644 packages/code-infra/src/tachometer/buildPages.mjs create mode 100644 packages/code-infra/src/tachometer/discoverCases.mjs create mode 100644 packages/code-infra/src/tachometer/discoverCases.test.mjs create mode 100644 packages/code-infra/src/tachometer/exec.mjs create mode 100644 packages/code-infra/src/tachometer/plugin.mjs create mode 100644 packages/code-infra/src/tachometer/plugin.test.mjs create mode 100644 packages/code-infra/src/tachometer/refs.mjs create mode 100644 packages/code-infra/src/tachometer/refs.test.mjs create mode 100644 packages/code-infra/src/tachometer/runTachometer.mjs create mode 100644 packages/code-infra/src/tachometer/summarizeCase.mjs create mode 100644 packages/code-infra/src/tachometer/summarizeCase.test.mjs create mode 100644 packages/code-infra/src/utils/packWorkspace.mjs create mode 100644 packages/code-infra/src/utils/packWorkspace.test.mjs diff --git a/packages/code-infra/package.json b/packages/code-infra/package.json index 5f044ad498..70d8464ae8 100644 --- a/packages/code-infra/package.json +++ b/packages/code-infra/package.json @@ -41,6 +41,14 @@ "types": "./build/stylelint/index.d.mts", "default": "./src/stylelint/index.mjs" }, + "./packWorkspace": { + "types": "./build/utils/packWorkspace.d.mts", + "default": "./src/utils/packWorkspace.mjs" + }, + "./tachometerPlugin": { + "types": "./build/tachometer/plugin.d.mts", + "default": "./src/tachometer/plugin.mjs" + }, "./brokenLinksChecker": { "types": "./build/brokenLinksChecker/index.d.mts", "default": "./src/brokenLinksChecker/index.mjs" @@ -151,7 +159,8 @@ "@typescript/native-preview": ">=7.0.0-dev.0", "eslint": "^9.0.0 || ^10.0.0", "prettier": "^3.0.0", - "typescript": "^5.0.0 || ^6.0.0" + "typescript": "^5.0.0 || ^6.0.0", + "vite": "^7.0.0 || ^8.0.0" }, "peerDependenciesMeta": { "@typescript/native-preview": { @@ -159,6 +168,9 @@ }, "typescript": { "optional": true + }, + "vite": { + "optional": true } }, "devDependencies": { @@ -180,7 +192,8 @@ "remark": "15.0.1", "remark-mdx": "3.1.1", "serve": "14.2.6", - "typescript-eslint": "8.68.0" + "typescript-eslint": "8.68.0", + "vite": "^8.2.2" }, "files": [ "bin", diff --git a/packages/code-infra/src/cli/cmdTacho.mjs b/packages/code-infra/src/cli/cmdTacho.mjs new file mode 100644 index 0000000000..348ae3f9b8 --- /dev/null +++ b/packages/code-infra/src/cli/cmdTacho.mjs @@ -0,0 +1,64 @@ +#!/usr/bin/env node + +/** + * @typedef {Object} Args + * @property {string[]} [filters] - Only run cases whose folder name contains one of these substrings + * @property {string} [baseline] - Binds the `baseline` symbol, in the ref grammar + * @property {string} [baseBranch] - Branch PRs fork from + * @property {string} [buildCmd] - Command that builds the publishable packages + * @property {boolean} [install] - Whether to install inside a ref's checkout + * @property {string} [out] - Where to write the combined JSON report + */ + +export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ + command: 'tacho run [filters...]', + describe: + 'Benchmark a tachometer harness across builds of this workspace and write a JSON report. Run from the harness package.', + builder: (yargs) => { + return yargs + .positional('filters', { + type: 'string', + array: true, + describe: 'Only run cases whose folder name contains this substring', + }) + .option('baseline', { + type: 'string', + describe: + 'Bind the "baseline" symbol, in the ref grammar (e.g. git:abc1234). Default: on the base branch HEAD~1, otherwise the fork point from the base branch', + }) + .option('base-branch', { + type: 'string', + describe: + 'Branch PRs fork from. Default: detected from origin/HEAD, falling back to master', + }) + .option('build-cmd', { + type: 'string', + default: 'pnpm release:build', + describe: 'Command that builds the publishable workspace packages', + }) + .option('install', { + type: 'boolean', + default: true, + describe: "Install dependencies inside a ref's checkout. Use --no-install to skip", + }) + .option('out', { + type: 'string', + describe: 'Write the combined JSON report here. Default: results/report.json', + }) + .epilogue( + 'Sampling (sampleSize, autoSampleConditions, timeout) is configured per case in its own tachometer.json — tachometer rejects those as CLI flags when a config file is used.', + ); + }, + handler: async (argv) => { + const { runTachometer } = await import('../tachometer/runTachometer.mjs'); + await runTachometer({ + harnessDir: process.cwd(), + filters: argv.filters ?? [], + baseline: argv.baseline, + baseBranch: argv.baseBranch, + buildCmd: argv.buildCmd, + install: argv.install, + out: argv.out, + }); + }, +}); diff --git a/packages/code-infra/src/cli/index.mjs b/packages/code-infra/src/cli/index.mjs index 58982e6ccf..cd692b6111 100644 --- a/packages/code-infra/src/cli/index.mjs +++ b/packages/code-infra/src/cli/index.mjs @@ -16,6 +16,7 @@ import cmdPublishCanary from './cmdPublishCanary.mjs'; import cmdPublishNewPackage from './cmdPublishNewPackage.mjs'; import cmdRenameScope from './cmdRenameScope.mjs'; import cmdSetVersionOverrides from './cmdSetVersionOverrides.mjs'; +import cmdTacho from './cmdTacho.mjs'; import cmdVale from './cmdVale.mjs'; import cmdValidateBuiltTypes from './cmdValidateBuiltTypes.mjs'; @@ -50,6 +51,7 @@ await yargs(hideBin(process.argv)) .command(cmdPublishNewPackage) .command(cmdRenameScope) .command(cmdSetVersionOverrides) + .command(cmdTacho) .command(cmdVale) .command(cmdValidateBuiltTypes) .fail((msg, err, yargsInstance) => { diff --git a/packages/code-infra/src/tachometer/buildPages.mjs b/packages/code-infra/src/tachometer/buildPages.mjs new file mode 100644 index 0000000000..c332193d51 --- /dev/null +++ b/packages/code-infra/src/tachometer/buildPages.mjs @@ -0,0 +1,171 @@ +/* eslint-disable no-console */ + +import * as path from 'node:path'; +import { cp, readFile, readdir, writeFile } from 'node:fs/promises'; +import chalk from 'chalk'; +import { tarballFor } from '../utils/packWorkspace.mjs'; +import { run } from './exec.mjs'; + +/** + * Builds the benchmark pages for one ref. + * + * The working tree builds in place: the harness is a workspace member depending on the library via + * `workspace:*`, and `publishConfig.directory` makes that link resolve to the package's build + * output, so vite resolves the built library with no packing and no isolated install. + * + * Any other ref was built in a throwaway checkout that has since been removed and SHA-cached as + * tarballs, so its build is not in this workspace and cannot be reached by a workspace link. Those + * pages are therefore built in an isolated install that consumes the library exactly as a published + * release would install. + */ + +/** + * @typedef {import('./refs.mjs').ResolvedRef} ResolvedRef + * @typedef {import('../utils/packWorkspace.mjs').PackedPackage} PackedPackage + */ + +/** Matches any filename vite accepts as its config. */ +const VITE_CONFIG = /^vite\.config\.(?:[cm]?[jt]s)$/; + +/** + * Finds the harness's vite config file. + * + * @param {string} harnessDir - The harness package directory + * @returns {Promise} The config's basename + */ +async function findViteConfig(harnessDir) { + const entries = await readdir(harnessDir); + const found = entries.filter((entry) => VITE_CONFIG.test(entry)).sort(); + if (found.length === 0) { + throw new Error(`No vite config found in ${harnessDir}.`); + } + return found[0]; +} + +/** + * Runs the harness's vite build, emitting into `outDir`. + * + * The page entries come from the tachometer plugin in the harness's own config, so nothing about + * which pages exist is passed here. + * + * @param {string} cwd - Directory holding the vite config and `src/` + * @param {string} outDir - Absolute output directory + * @returns {Promise} + */ +export async function runViteBuild(cwd, outDir) { + await run('pnpm', ['exec', 'vite', 'build', '--outDir', outDir], cwd); +} + +/** + * Builds the publishable workspace packages of a checkout. + * + * @param {string} checkoutDir - Checkout to build in + * @param {string} buildCmd - The build command, e.g. `pnpm release:build` + * @param {string} label - Human-readable ref label, for logging + * @returns {Promise} + */ +export async function buildPackages(checkoutDir, buildCmd, label) { + console.log(chalk.cyan(`\nBuilding workspace packages for "${label}" (${buildCmd})…`)); + const [file, ...args] = buildCmd.split(' ').filter(Boolean); + await run(file, args, checkoutDir); +} + +/** + * Builds the benchmark pages for the working tree, in place, against the workspace's own build. + * + * Rebuild the packages and re-run to measure a source change. + * + * @param {Object} options - Build inputs + * @param {string} options.harnessDir - The harness package directory + * @param {ResolvedRef} options.ref - The ref being built (the working tree) + * @param {string} options.outDir - Absolute output directory + * @returns {Promise} + */ +export async function buildWorkspacePages(options) { + const { harnessDir, ref, outDir } = options; + console.log(chalk.cyan(`\nBuilding benchmark pages for "${ref.label}"…`)); + await runViteBuild(harnessDir, outDir); +} + +/** + * Rewrites a dependency map, pointing every `workspace:`-protocol entry at its packed tarball and + * copying everything else verbatim. + * + * This is what lets the isolated install be derived rather than configured: the harness already + * declares the library under test as `workspace:*`, so there is no library name to hardcode and no + * hand-maintained list of versions to pin. Third-party dependencies — including the competitor + * libraries a cross-library case compares against — come along unchanged, so every page builds + * against every ref. + * + * @param {Record | undefined} deps - A `dependencies` or `devDependencies` map + * @param {PackedPackage[]} packages - The packed workspace packages + * @param {string[]} [omit] - Dependency names to drop entirely + * @returns {Record} + */ +export function rewriteWorkspaceDeps(deps, packages, omit = []) { + /** @type {Record} */ + const out = {}; + for (const [name, version] of Object.entries(deps ?? {})) { + if (omit.includes(name)) { + continue; + } + out[name] = version.startsWith('workspace:') ? `file:${tarballFor(packages, name)}` : version; + } + return out; +} + +/** + * Builds the benchmark pages against a non-working-tree ref's build, in an isolated throwaway + * package. + * + * The pages install the library as a `file:` dependency on its packed `.tgz`, exactly as a + * published package would install, with every workspace-internal package pinned to its own tarball + * via overrides. Third-party dependencies are pinned to the same versions as the committed harness, + * so the only thing that varies between refs is the library build. + * + * @param {Object} options - Build inputs + * @param {string} options.harnessDir - The harness package directory, copied from + * @param {ResolvedRef} options.ref - The ref being built + * @param {PackedPackage[]} options.packages - That ref's packed workspace packages + * @param {string} options.workDir - Throwaway directory to assemble the isolated package in + * @param {string} options.outDir - Absolute output directory + * @returns {Promise} + */ +export async function buildRefPages(options) { + const { harnessDir, ref, packages, workDir, outDir } = options; + console.log(chalk.cyan(`\nBuilding benchmark pages for "${ref.label}"…`)); + + const viteConfig = await findViteConfig(harnessDir); + await cp(path.join(harnessDir, 'src'), path.join(workDir, 'src'), { recursive: true }); + await cp(path.join(harnessDir, viteConfig), path.join(workDir, viteConfig)); + + const harnessPkg = JSON.parse(await readFile(path.join(harnessDir, 'package.json'), 'utf8')); + await writeFile( + path.join(workDir, 'package.json'), + `${JSON.stringify( + { + name: `tacho-bench-${ref.id}`, + private: true, + type: harnessPkg.type, + dependencies: rewriteWorkspaceDeps(harnessPkg.dependencies, packages), + // The isolated package only builds pages; tachometer itself runs from the real harness and + // would pull chromedriver for nothing. + devDependencies: rewriteWorkspaceDeps(harnessPkg.devDependencies, packages, ['tachometer']), + }, + null, + 2, + )}\n`, + ); + + // Pin every workspace-internal package to its packed tarball, so a transitive dependency between + // them (whose packed `workspace:*` becomes a concrete, unpublished version) resolves to a local + // build rather than the registry. In pnpm 11 `overrides` live in `pnpm-workspace.yaml`, not + // `package.json`; writing one here also anchors this throwaway as its own workspace root, so pnpm + // reads the overrides and never walks up to a parent workspace (which is why `--ignore-workspace` + // — which would ignore this file — is not used). + const overrides = packages.map((pkg) => ` '${pkg.name}': "file:${pkg.tarball}"`).join('\n'); + await writeFile(path.join(workDir, 'pnpm-workspace.yaml'), `overrides:\n${overrides}\n`); + + await run('pnpm', ['install', '--prefer-offline', '--config.engine-strict=false'], workDir); + await runViteBuild(workDir, outDir); +} diff --git a/packages/code-infra/src/tachometer/discoverCases.mjs b/packages/code-infra/src/tachometer/discoverCases.mjs new file mode 100644 index 0000000000..871e89f731 --- /dev/null +++ b/packages/code-infra/src/tachometer/discoverCases.mjs @@ -0,0 +1,210 @@ +import * as path from 'node:path'; +import { access, readFile, readdir } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; +import { parseRefToken } from './refs.mjs'; + +/** + * Case discovery: the `tachometer.json` files under a harness's `src/` are the source of truth for + * which benchmark cases exist and which pages they reference. + * + * Both the runner and the vite plugin read them through here, so there is exactly one notion of + * "what pages exist". The plugin passes no `resolveRef`, which keeps discovery free of git — a + * plain `vite build` must not require a checkout with full history. + */ + +/** + * @typedef {import('./refs.mjs').ResolvedRef} ResolvedRef + */ + +/** + * @typedef {Object} Leaf + * @property {{ url?: string }} node - The config node whose `url` this is; rewritten in place once builds exist + * @property {string} page - Page path relative to `src`, e.g. `data-grid-init/index.html` + * @property {ResolvedRef | null} ref - The resolved ref, or null when discovery ran without `resolveRef` + * @property {string} suffix - Everything after the page path once `ref` was consumed: remaining query plus any fragment + */ + +/** + * @typedef {Object} BenchmarkCase + * @property {string} name - The case folder name + * @property {string} configPath - Absolute path to its `tachometer.json` + * @property {any} config - The parsed config, mutated in place as urls are rewritten + * @property {Leaf[]} leaves - Every node that selects a page + */ + +/** + * @typedef {Object} DiscoverCasesOptions + * @property {string} harnessDir - The harness package directory; cases live under its `src/` + * @property {string[]} [filters] - Only include cases whose folder name contains one of these substrings + * @property {(token?: string) => ResolvedRef} [resolveRef] - Resolves a ref token. Omit to skip resolution entirely (no git) + */ + +/** The file that marks a directory under `src/` as a benchmark case. */ +const CONFIG_NAME = 'tachometer.json'; + +/** + * True if `target` exists on disk. + * + * @param {string} target - Path to check + * @returns {Promise} + */ +async function pathExists(target) { + try { + await access(target); + return true; + } catch { + return false; + } +} + +/** + * Walks a benchmark's `expand` tree and collects its leaves — the nodes that actually select a page. + * + * `expand` is recursive and a child inherits its parent's `url` unless it overrides it, so the + * effective url is threaded down and only nodes without further `expand` are leaves. + * + * @param {{ url?: string, expand?: any[] }} node - The node to walk + * @param {string | undefined} inheritedUrl - The effective url from the parent + * @param {Array<{ node: { url?: string }, url: string }>} out - Collected leaves + * @returns {void} + */ +export function collectLeafNodes(node, inheritedUrl, out) { + const url = node.url ?? inheritedUrl; + if (Array.isArray(node.expand) && node.expand.length > 0) { + for (const child of node.expand) { + collectLeafNodes(child, url, out); + } + return; + } + if (url === undefined) { + throw new Error('A benchmark variant has no "url" to resolve.'); + } + out.push({ node, url }); +} + +/** + * Resolves one leaf url into the source page it references, the ref it selects, and any leftovers. + * + * `ref` selects the build and is consumed here; everything else belongs to the page and has to + * survive the rewrite, or a parameterised benchmark would silently run its defaults. + * + * @param {string} url - The leaf's url + * @param {string} configDir - Directory the config lives in; relative urls resolve against it + * @param {string} srcDir - The harness `src/` directory + * @param {((token?: string) => ResolvedRef) | undefined} resolveRef - Ref resolver, or undefined to skip resolution + * @returns {Promise<{ page: string, ref: ResolvedRef | null, suffix: string }>} + */ +async function parseLeafUrl(url, configDir, srcDir, resolveRef) { + const parsed = new URL(url, pathToFileURL(path.join(configDir, path.sep))); + const token = parsed.searchParams.get('ref') ?? undefined; + // Validate the grammar even when not resolving, so a typo fails the build rather than silently + // building a page the runner would later reject. + parseRefToken(token); + const ref = resolveRef ? resolveRef(token) : null; + const absolute = decodeURIComponent(parsed.pathname); + + parsed.searchParams.delete('ref'); + const search = parsed.searchParams.toString(); + const suffix = `${search ? `?${search}` : ''}${parsed.hash}`; + + const page = path.relative(srcDir, absolute); + if (page.startsWith('..') || path.isAbsolute(page)) { + throw new Error(`Benchmark url "${url}" resolves outside ${srcDir}.`); + } + if (!(await pathExists(absolute))) { + throw new Error(`Benchmark url "${url}" points at a missing page (${absolute}).`); + } + return { page, ref, suffix }; +} + +/** + * Reads every `src//tachometer.json` (optionally filtered by name), expands the sugar for + * cases that declare no variants of their own, and resolves each leaf's page and ref. + * + * A benchmark with no `expand` is the common regression case: it is expanded into `[current]` (the + * working tree) versus `[baseline]`. A benchmark that declares its own `expand` owns its variant + * axis — only the refs its leaves reference are resolved. + * + * @param {DiscoverCasesOptions} options - Where to look and how to resolve refs + * @returns {Promise} + */ +export async function discoverCases(options) { + const { harnessDir, filters = [], resolveRef } = options; + const srcDir = path.join(harnessDir, 'src'); + + const dirEntries = (await readdir(srcDir, { withFileTypes: true })).filter((entry) => + entry.isDirectory(), + ); + const hasConfig = await Promise.all( + dirEntries.map((entry) => pathExists(path.join(srcDir, entry.name, CONFIG_NAME))), + ); + const caseNames = dirEntries + .filter((_, index) => hasConfig[index]) + .map((entry) => entry.name) + .filter((name) => filters.length === 0 || filters.some((filter) => name.includes(filter))) + .sort(); + + if (caseNames.length === 0) { + throw new Error( + filters.length > 0 + ? `No benchmark case under ${srcDir} matches ${filters.map((filter) => `"${filter}"`).join(', ')}.` + : `No benchmark case with a ${CONFIG_NAME} found under ${srcDir}.`, + ); + } + + const configs = await Promise.all( + caseNames.map(async (name) => { + const configPath = path.join(srcDir, name, CONFIG_NAME); + const config = JSON.parse(await readFile(configPath, 'utf8')); + // `$schema` is for editors; tachometer rejects it in a config it is handed. + delete config.$schema; + return { name, configPath, config }; + }), + ); + + /** @type {BenchmarkCase[]} */ + const cases = []; + for (const { name, configPath, config } of configs) { + const configDir = path.dirname(configPath); + + /** @type {Array<{ node: { url?: string }, url: string }>} */ + const nodes = []; + for (const benchmark of config.benchmarks ?? []) { + if (!Array.isArray(benchmark.expand) || benchmark.expand.length === 0) { + const base = benchmark.url; + if (base === undefined) { + throw new Error(`Benchmark "${benchmark.name}" in ${configPath} has no "url".`); + } + const separator = base.includes('?') ? '&' : '?'; + benchmark.expand = [ + { name: `${benchmark.name} [current]`, url: base }, + { name: `${benchmark.name} [baseline]`, url: `${base}${separator}ref=baseline` }, + ]; + // The variants carry the url now; leaving the un-rewritten source url on the parent would + // let tachometer inherit a page that was never built. + delete benchmark.url; + } + + collectLeafNodes(benchmark, undefined, nodes); + } + + // eslint-disable-next-line no-await-in-loop + const resolved = await Promise.all( + nodes.map(({ url }) => parseLeafUrl(url, configDir, srcDir, resolveRef)), + ); + const leaves = nodes.map(({ node }, index) => ({ node, ...resolved[index] })); + + cases.push({ name, configPath, config, leaves }); + } + return cases; +} + +/** + * The distinct pages referenced by a set of cases, as paths relative to `src`, sorted. + * + * @param {BenchmarkCase[]} cases - Discovered cases + * @returns {string[]} + */ +export function pagesOf(cases) { + return [...new Set(cases.flatMap((entry) => entry.leaves.map((leaf) => leaf.page)))].sort(); +} diff --git a/packages/code-infra/src/tachometer/discoverCases.test.mjs b/packages/code-infra/src/tachometer/discoverCases.test.mjs new file mode 100644 index 0000000000..3ca1782d75 --- /dev/null +++ b/packages/code-infra/src/tachometer/discoverCases.test.mjs @@ -0,0 +1,305 @@ +import * as path from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { describe, expect, it } from 'vitest'; +import { makeTempDir } from '../utils/testUtils.mjs'; +import { discoverCases, pagesOf } from './discoverCases.mjs'; + +/** + * Builds a throwaway harness: `src//tachometer.json` plus whatever pages each case owns. + * + * @param {Record} cases - Case folders to create + * @returns {Promise} The harness directory + */ +async function makeHarness(cases) { + const harnessDir = await makeTempDir(); + await Promise.all( + Object.entries(cases).map(async ([name, { config, pages = [] }]) => { + const caseDir = path.join(harnessDir, 'src', name); + await mkdir(caseDir, { recursive: true }); + await writeFile(path.join(caseDir, 'tachometer.json'), JSON.stringify(config, null, 2)); + await Promise.all( + pages.map((page) => writeFile(path.join(caseDir, page), '\n')), + ); + }), + ); + return harnessDir; +} + +/** + * A minimal single-benchmark config. + * + * @param {string} name - Benchmark name + * @param {string} url - Page url + * @returns {any} + */ +function config(name, url) { + return { + $schema: '../../node_modules/tachometer/config.schema.json', + benchmarks: [{ name, url }], + }; +} + +describe('discoverCases', () => { + it('finds every case folder holding a tachometer.json', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + beta: { config: config('beta', './index.html'), pages: ['index.html'] }, + }); + + const cases = await discoverCases({ harnessDir }); + + expect(cases.map((entry) => entry.name)).toEqual(['alpha', 'beta']); + }); + + it('narrows to cases matching a filter', async () => { + const harnessDir = await makeHarness({ + 'grid-init': { config: config('grid-init', './index.html'), pages: ['index.html'] }, + 'grid-scroll': { config: config('grid-scroll', './index.html'), pages: ['index.html'] }, + }); + + const cases = await discoverCases({ harnessDir, filters: ['scroll'] }); + + expect(cases.map((entry) => entry.name)).toEqual(['grid-scroll']); + }); + + it('strips $schema, which tachometer rejects in a config it is handed', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.config.$schema).toBeUndefined(); + }); + + it('throws when no case matches', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + await expect(discoverCases({ harnessDir, filters: ['nope'] })).rejects.toThrow( + /No benchmark case .* matches "nope"/, + ); + }); + + describe('auto-expansion', () => { + it('expands a benchmark with no variants into current versus baseline', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.config.benchmarks[0].expand).toEqual([ + { name: 'alpha [current]', url: './index.html' }, + { name: 'alpha [baseline]', url: './index.html?ref=baseline' }, + ]); + }); + + it('removes the parent url once variants carry it', async () => { + // Leaving the un-rewritten source url on the parent would let tachometer inherit a page that + // was never built. + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.config.benchmarks[0]).not.toHaveProperty('url'); + }); + + it('joins the ref with an ampersand when the url already has a query', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html?rows=100'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.config.benchmarks[0].expand[1].url).toBe('./index.html?rows=100&ref=baseline'); + }); + + it('leaves a benchmark that declares its own variants alone', async () => { + const harnessDir = await makeHarness({ + libs: { + config: { + benchmarks: [ + { + name: 'libs', + expand: [ + { name: 'libs [ours]', url: './ours.html' }, + { name: 'libs [theirs]', url: './theirs.html' }, + ], + }, + ], + }, + pages: ['ours.html', 'theirs.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.leaves.map((leaf) => leaf.page)).toEqual(['libs/ours.html', 'libs/theirs.html']); + }); + }); + + describe('leaf urls', () => { + it('inherits a parent url through a nested expand', async () => { + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [ + { + name: 'alpha', + url: './index.html', + expand: [ + { name: 'alpha [one]', expand: [{ name: 'alpha [one/a]' }] }, + { name: 'alpha [two]', url: './other.html' }, + ], + }, + ], + }, + pages: ['index.html', 'other.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.leaves.map((leaf) => leaf.page)).toEqual([ + 'alpha/index.html', + 'alpha/other.html', + ]); + }); + + it('consumes the ref and carries every other query parameter through', async () => { + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [ + { + name: 'alpha', + expand: [{ name: 'a', url: './index.html?rows=100&ref=baseline#x' }], + }, + ], + }, + pages: ['index.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.leaves[0].suffix).toBe('?rows=100#x'); + }); + + it('resolves a case that owns no page but parameterises a sibling', async () => { + const harnessDir = await makeHarness({ + updates: { config: config('updates', './index.html'), pages: ['index.html'] }, + 'updates-throttled': { + config: { + benchmarks: [ + { + name: 'updates-throttled', + expand: [{ name: 'throttled', url: '../updates/index.html?throttle=16' }], + }, + ], + }, + }, + }); + + const cases = await discoverCases({ harnessDir }); + const throttled = cases.find((entry) => entry.name === 'updates-throttled'); + + expect(throttled?.leaves[0]).toMatchObject({ + page: path.join('updates', 'index.html'), + suffix: '?throttle=16', + }); + }); + + it('rejects a url resolving outside src', async () => { + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [{ name: 'alpha', expand: [{ name: 'a', url: '../../outside.html' }] }], + }, + }, + }); + + await expect(discoverCases({ harnessDir })).rejects.toThrow(/resolves outside/); + }); + + it('rejects a url pointing at a missing page', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './missing.html') }, + }); + + await expect(discoverCases({ harnessDir })).rejects.toThrow(/missing page/); + }); + + it('rejects an unknown ref scheme even when refs are not being resolved', async () => { + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [ + { name: 'alpha', expand: [{ name: 'a', url: './index.html?ref=nonsense' }] }, + ], + }, + pages: ['index.html'], + }, + }); + + await expect(discoverCases({ harnessDir })).rejects.toThrow(/Unknown ref "nonsense"/); + }); + }); + + describe('without a ref resolver', () => { + it('leaves refs unresolved and makes no git calls', async () => { + // The temp harness is not a git repository at all, so anything reaching for git would fail. + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.leaves).toHaveLength(2); + expect(entry.leaves.every((leaf) => leaf.ref === null)).toBe(true); + }); + + it('still yields the full page list', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + libs: { + config: { + benchmarks: [ + { + name: 'libs', + expand: [ + { name: 'libs [ours]', url: './ours.html' }, + { name: 'libs [theirs]', url: './theirs.html' }, + ], + }, + ], + }, + pages: ['ours.html', 'theirs.html'], + }, + }); + + expect(pagesOf(await discoverCases({ harnessDir }))).toEqual([ + path.join('alpha', 'index.html'), + path.join('libs', 'ours.html'), + path.join('libs', 'theirs.html'), + ]); + }); + }); +}); + +describe('pagesOf', () => { + it('deduplicates the pages a case references from several variants', async () => { + // The auto-expanded [current] and [baseline] variants point at the same page. + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + expect(pagesOf(await discoverCases({ harnessDir }))).toEqual([ + path.join('alpha', 'index.html'), + ]); + }); +}); diff --git a/packages/code-infra/src/tachometer/exec.mjs b/packages/code-infra/src/tachometer/exec.mjs new file mode 100644 index 0000000000..574c9d3123 --- /dev/null +++ b/packages/code-infra/src/tachometer/exec.mjs @@ -0,0 +1,21 @@ +/* eslint-disable no-console */ + +import chalk from 'chalk'; +import { execa } from 'execa'; + +/** + * Runs a command in `cwd` with inherited stdio, throwing on a non-zero exit. + * + * Echoes the command first: a benchmark run shells out a lot, and being able to copy a failing step + * out of the log and re-run it by hand is worth the noise. + * + * @param {string} file - Executable to run + * @param {string[]} args - Arguments + * @param {string} cwd - Working directory + * @param {NodeJS.ProcessEnv} [env] - Extends the current environment + * @returns {Promise} + */ +export async function run(file, args, cwd, env) { + console.log(chalk.dim(`$ ${file} ${args.join(' ')} (in ${cwd})`)); + await execa(file, args, { cwd, stdio: 'inherit', env }); +} diff --git a/packages/code-infra/src/tachometer/plugin.mjs b/packages/code-infra/src/tachometer/plugin.mjs new file mode 100644 index 0000000000..5b71ec2f3f --- /dev/null +++ b/packages/code-infra/src/tachometer/plugin.mjs @@ -0,0 +1,165 @@ +import * as path from 'node:path'; +import { readFile, realpath } from 'node:fs/promises'; +import { discoverCases, pagesOf } from './discoverCases.mjs'; + +/** + * Vite plugin for a tachometer benchmark harness. + * + * The harness keeps its own vite config and runs plain `vite` / `vite build` / `vite preview`; this + * plugin only contributes what is derived from the benchmark cases: + * + * - `root` is the harness's `src/`, and the app is an MPA (vite defaults to `spa`, whose fallback + * would rewrite unmatched urls toward a root `index.html` that does not exist here). + * - The build's entry points come from the `tachometer.json` files, so the configs are the single + * source of truth for which pages exist. + * - The dev server serves a generated index listing every case, including cases that own no page of + * their own and only parameterise a sibling's. + * + * It deliberately never resolves refs, so a plain `vite build` needs no git history. + */ + +/** + * @typedef {Object} TachometerPluginOptions + * @property {string} [harnessDir] - The harness package directory. Defaults to the current working directory, which is where vite is run from + */ + +/** + * Checks that the workspace packages the harness links to have actually been built. + * + * A harness depends on the library under test via `workspace:*`, and `publishConfig.directory` + * makes that link point at the package's build output — which does not exist until the library is + * built. The symlink is then dangling, and vite's failure ("failed to resolve import") does not + * say why. + * + * @param {string} harnessDir - The harness package directory + * @param {Record} deps - The harness's combined dependency map + * @returns {Promise} + */ +async function assertWorkspaceDepsBuilt(harnessDir, deps) { + const linked = Object.entries(deps).filter(([, version]) => version.startsWith('workspace:')); + await Promise.all( + linked.map(async ([name]) => { + try { + await realpath(path.join(harnessDir, 'node_modules', ...name.split('/'))); + } catch { + throw new Error( + `"${name}" is linked into ${harnessDir} but its target does not exist. ` + + `Build the workspace packages first (e.g. \`pnpm release:build\`) — the harness ` + + `resolves the library through its build output, not its source.`, + ); + } + }), + ); +} + +/** + * Renders the dev-server index: one link per case, carrying whatever query the case parameterises + * its page with. + * + * @param {import('./discoverCases.mjs').BenchmarkCase[]} cases - Discovered cases + * @returns {string} An HTML document + */ +function renderIndex(cases) { + const items = cases + .map((entry) => { + // A case's variants can point at several pages (a cross-library comparison) or at one page + // with different queries (a parameterised case), so dedupe on the pair. + const targets = [...new Set(entry.leaves.map((leaf) => `${leaf.page}${leaf.suffix}`))].sort(); + const links = targets + .map((target) => `
  • ${target}
    • `) + .join('\n '); + return `
  • \n ${entry.name}\n
      \n ${links}\n
    \n
    • `; + }) + .join('\n'); + + return ` + + + + Tachometer benchmark cases + + + +

    Tachometer benchmark cases

    +

    Pages are served from the working tree's build of the library. Rebuild it and reload to pick up a source change.

    +
      +${items} +
    + + +`; +} + +/** + * Creates the tachometer harness plugin. + * + * @param {TachometerPluginOptions} [options] - Plugin options + * @returns {import('vite').Plugin} + */ +export function tachometer(options = {}) { + const harnessDir = options.harnessDir ?? process.cwd(); + const srcDir = path.join(harnessDir, 'src'); + + return { + name: 'mui-code-infra:tachometer', + + async config(userConfig, env) { + const pkg = JSON.parse(await readFile(path.join(harnessDir, 'package.json'), 'utf8')); + await assertWorkspaceDepsBuilt(harnessDir, { + ...pkg.dependencies, + ...pkg.devDependencies, + }); + + if (env.command !== 'build') { + return { root: srcDir, appType: 'mpa' }; + } + + const cases = await discoverCases({ harnessDir }); + // Key the input map by the page path without its extension. Vite emits each page at its path + // relative to `root` regardless, so `/index.html` lands at `//index.html`; + // a distinct key per page is what lets one case folder hold several pages — a cross-library + // comparison with `mosaic.html` and `tanstack.html` side by side — without them colliding. + const input = Object.fromEntries( + pagesOf(cases).map((page) => [page.replace(/\.html$/, ''), path.join(srcDir, page)]), + ); + + return { + root: srcDir, + appType: 'mpa', + // Relative asset urls: tachometer serves the build directory and the pages live under + // `//`, so absolute "/assets/…" paths would 404. + base: './', + build: { + // The output directory is outside `root` — and, for an isolated ref build, outside the + // temporary package entirely; allow vite to clean it anyway. + emptyOutDir: true, + chunkSizeWarningLimit: 9999, + rollupOptions: { input }, + }, + }; + }, + + configureServer(server) { + server.middlewares.use(async (req, res, next) => { + const url = (req.url ?? '/').split('?')[0]; + if (url !== '/' && url !== '/index.html') { + next(); + return; + } + try { + const cases = await discoverCases({ harnessDir }); + res.setHeader('Content-Type', 'text/html; charset=utf-8'); + res.end(renderIndex(cases)); + } catch (error) { + next(error); + } + }); + }, + }; +} diff --git a/packages/code-infra/src/tachometer/plugin.test.mjs b/packages/code-infra/src/tachometer/plugin.test.mjs new file mode 100644 index 0000000000..2bb38fa0b9 --- /dev/null +++ b/packages/code-infra/src/tachometer/plugin.test.mjs @@ -0,0 +1,151 @@ +import * as path from 'node:path'; +import { mkdir, symlink, writeFile } from 'node:fs/promises'; +import { describe, expect, it } from 'vitest'; +import { makeTempDir } from '../utils/testUtils.mjs'; +import { tachometer } from './plugin.mjs'; + +/** + * Builds a throwaway harness with a package.json, so the plugin can read its dependencies. + * + * @param {Object} options - Harness contents + * @param {Record} options.cases - Case folders to create + * @param {Record} [options.dependencies] - The harness's dependencies + * @returns {Promise} The harness directory + */ +async function makeHarness({ cases, dependencies = {} }) { + const harnessDir = await makeTempDir(); + await writeFile( + path.join(harnessDir, 'package.json'), + JSON.stringify({ name: 'harness', private: true, dependencies }, null, 2), + ); + await Promise.all( + Object.entries(cases).map(async ([name, { config, pages = [] }]) => { + const caseDir = path.join(harnessDir, 'src', name); + await mkdir(caseDir, { recursive: true }); + await writeFile(path.join(caseDir, 'tachometer.json'), JSON.stringify(config, null, 2)); + await Promise.all( + pages.map((page) => writeFile(path.join(caseDir, page), '\n')), + ); + }), + ); + return harnessDir; +} + +/** + * Invokes the plugin's `config` hook. + * + * @param {string} harnessDir - The harness directory + * @param {'build' | 'serve'} command - Which vite command is running + * @returns {Promise} + */ +async function callConfig(harnessDir, command) { + const plugin = tachometer({ harnessDir }); + const hook = /** @type {any} */ (plugin.config); + return hook({}, { command, mode: command === 'build' ? 'production' : 'development' }); +} + +const oneCase = { + alpha: { + config: { benchmarks: [{ name: 'alpha', url: './index.html' }] }, + pages: ['index.html'], + }, +}; + +describe('tachometer plugin', () => { + describe('config hook', () => { + it('roots the app at src and declares it multi-page', async () => { + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'serve'); + + expect(config.root).toBe(path.join(harnessDir, 'src')); + // Vite defaults to 'spa', whose fallback rewrites unmatched urls toward a root index.html + // that does not exist in this harness. + expect(config.appType).toBe('mpa'); + }); + + it('derives no build input while serving', async () => { + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'serve'); + + expect(config.build).toBeUndefined(); + }); + + it('uses relative asset urls for a build', async () => { + // Tachometer serves the build directory and pages live under //, so absolute + // "/assets/…" paths would 404. + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'build'); + + expect(config.base).toBe('./'); + }); + + it('derives the build input from the discovered cases', async () => { + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'build'); + + expect(config.build.rollupOptions.input).toEqual({ + 'alpha/index': path.join(harnessDir, 'src', 'alpha', 'index.html'), + }); + }); + + it('keys several pages in one case folder separately', async () => { + // A distinct key per page is what stops a cross-library case's two pages colliding into one + // entry. + const harnessDir = await makeHarness({ + cases: { + libs: { + config: { + benchmarks: [ + { + name: 'libs', + expand: [ + { name: 'libs [ours]', url: './ours.html' }, + { name: 'libs [theirs]', url: './theirs.html' }, + ], + }, + ], + }, + pages: ['ours.html', 'theirs.html'], + }, + }, + }); + + const config = await callConfig(harnessDir, 'build'); + + expect(Object.keys(config.build.rollupOptions.input).sort()).toEqual([ + 'libs/ours', + 'libs/theirs', + ]); + }); + }); + + describe('workspace link check', () => { + it('passes when the harness links to nothing', async () => { + const harnessDir = await makeHarness({ cases: oneCase }); + + await expect(callConfig(harnessDir, 'serve')).resolves.toBeTruthy(); + }); + + it('explains that the library has to be built when its link dangles', async () => { + // A harness resolves the library through its build output, so before the first build the + // link points at a directory that does not exist yet. + const harnessDir = await makeHarness({ + cases: oneCase, + dependencies: { '@scope/library': 'workspace:*' }, + }); + await mkdir(path.join(harnessDir, 'node_modules', '@scope'), { recursive: true }); + await symlink( + path.join(harnessDir, 'packages', 'library', 'build'), + path.join(harnessDir, 'node_modules', '@scope', 'library'), + ); + + await expect(callConfig(harnessDir, 'serve')).rejects.toThrow( + /"@scope\/library" is linked into .* but its target does not exist/, + ); + }); + }); +}); diff --git a/packages/code-infra/src/tachometer/refs.mjs b/packages/code-infra/src/tachometer/refs.mjs new file mode 100644 index 0000000000..65b448dd1b --- /dev/null +++ b/packages/code-infra/src/tachometer/refs.mjs @@ -0,0 +1,260 @@ +import chalk from 'chalk'; +import { execaSync } from 'execa'; + +/** + * The `?ref=` grammar: which build of the workspace a benchmark page should load. + * + * A ref always denotes a commit and covers every package in the workspace. Parsing is split in two + * so that consumers who only need to know *which pages exist* never shell out to git: + * + * - {@link parseRefToken} validates the grammar and returns a descriptor. Pure. + * - {@link createRefResolver} turns a descriptor into a {@link ResolvedRef} with an immutable SHA. + */ + +/** + * @typedef {Object} RefDescriptor + * @property {'worktree'|'baseline'|'git'} kind - Which scheme the token used + * @property {string} [committish] - The revision, for `git:` refs + */ + +/** + * @typedef {Object} ResolvedRef + * @property {'worktree'|'git'} kind - Whether this is the working tree or a committed revision + * @property {string} id - Build directory name, e.g. `current` or `git-230342ee2`. Doubles as the dedupe identity + * @property {string} label - Human-readable label for logs and the report + * @property {string} [sha] - The immutable commit this ref denotes. Absent only for the working tree + * @property {string} [committish] - The committish to hand to `packRef`. Absent only for the working tree + */ + +/** + * @typedef {Object} RefResolverOptions + * @property {string} repoRoot - Repository to resolve revisions in + * @property {string} [baseBranch] - Branch PRs fork from. Defaults to detection via `origin/HEAD`, then `master` + * @property {string} [baselineOverride] - Binds the `baseline` symbol, in the ref grammar (e.g. `git:abc1234`) + */ + +/** The working tree — never cached, since it has no immutable identity. */ +export const WORKTREE_REF = /** @type {ResolvedRef} */ ({ + kind: 'worktree', + id: 'current', + label: 'working tree', +}); + +/** + * Parses a `?ref=` token into a descriptor, validating the grammar. Makes no git calls. + * + * Bare values are never auto-prefixed: an absent ref already means "working tree", so letting a + * bare value mean a git revision would give one token two meanings depending on where it appears. + * + * @param {string} [token] - The raw `ref` query value + * @returns {RefDescriptor} + */ +export function parseRefToken(token) { + if (token === undefined || token === '') { + return { kind: 'worktree' }; + } + + if (token === 'baseline') { + return { kind: 'baseline' }; + } + + const git = /^git:(.+)$/.exec(token); + if (git) { + return { kind: 'git', committish: git[1] }; + } + + if (/^github:/.test(token) || /^preview:/.test(token)) { + const scheme = token.split(':', 1)[0]; + throw new Error( + `Ref scheme "${scheme}:" is recognised but not implemented yet (in "${token}"). ` + + `Supported today: an absent ref (working tree), "baseline", and "git:".`, + ); + } + + // Only advertise the schemes that actually work here; the reserved `github:`/`preview:` schemes + // are handled above with their own message, so listing them as valid would mislead. + throw new Error( + `Unknown ref "${token}". Expected an absent ref (working tree), "baseline", or "git:"` + + `${/^[0-9a-zA-Z._/~^-]+$/.test(token) ? ` — did you mean "git:${token}"?` : '.'}`, + ); +} + +/** + * Runs a git command, throwing on a non-zero exit. + * + * @param {string[]} args - Arguments to git + * @param {string} cwd - Working directory + * @returns {string} Trimmed stdout + */ +function gitCapture(args, cwd) { + const result = execaSync('git', args, { cwd, reject: false }); + if (result.exitCode !== 0) { + throw new Error(`git ${args.join(' ')} failed: ${result.stderr.trim()}`); + } + return result.stdout.trim(); +} + +/** + * Detects the branch PRs fork from, from `origin`'s default branch, falling back to `master`. + * + * @param {string} cwd - Repository to inspect + * @returns {string} The base branch name + */ +export function detectBaseBranch(cwd) { + const result = execaSync('git', ['symbolic-ref', '--short', 'refs/remotes/origin/HEAD'], { + cwd, + reject: false, + }); + if (result.exitCode !== 0) { + return 'master'; + } + // `origin/main` → `main`. Only the last segment is the branch name. + const shortRef = result.stdout.trim(); + const slash = shortRef.indexOf('/'); + return slash === -1 ? shortRef : shortRef.slice(slash + 1); +} + +/** + * Among the available base branches — every remote's `/` plus a local `` — + * returns the one whose merge base with HEAD is the most recent commit (the closest fork point), or + * undefined if none exist. + * + * Picking by most-recent merge base prefers an up-to-date upstream over a stale fork's base branch + * without hardcoding which remote is authoritative: locally `origin` may be a months-behind fork + * while `upstream` tracks the real repo, yet in CI `origin` *is* that repo. + * + * @param {string} repoRoot - Repository to inspect + * @param {string} baseBranch - Base branch name, e.g. `master` + * @returns {{ ref: string, mergeBase: string } | undefined} + */ +export function closestBaseBranch(repoRoot, baseBranch) { + // On ties (same merge base), prefer upstream's, then origin's, then a local base branch. + const preference = [`upstream/${baseBranch}`, `origin/${baseBranch}`, baseBranch]; + /** + * @param {string} ref - Candidate ref name + * @returns {number} Sort priority, lower first + */ + const priority = (ref) => { + const index = preference.indexOf(ref); + return index === -1 ? preference.length : index; + }; + // Match only real base branches from full refnames: a remote's `/` (exactly one + // segment before it) or the local ``. Filtering short names on `/` would also catch a + // local branch literally named e.g. `wip/master`. + const remoteBase = new RegExp(`^refs/remotes/[^/]+/${baseBranch}$`); + const candidates = gitCapture( + ['for-each-ref', '--format=%(refname)', 'refs/remotes', 'refs/heads'], + repoRoot, + ) + .split('\n') + .filter((ref) => remoteBase.test(ref) || ref === `refs/heads/${baseBranch}`) + .map((ref) => ref.replace(/^refs\/(remotes|heads)\//, '')) + .sort((a, b) => priority(a) - priority(b)); + + /** @type {{ ref: string, mergeBase: string, when: number } | undefined} */ + let best; + for (const ref of candidates) { + const result = execaSync('git', ['merge-base', 'HEAD', ref], { cwd: repoRoot, reject: false }); + if (result.exitCode !== 0) { + continue; + } + const mergeBase = result.stdout.trim(); + const when = Number(gitCapture(['show', '-s', '--format=%ct', mergeBase], repoRoot)); + // Strictly greater, so on ties the higher-priority (earlier-sorted) ref keeps a clean label. + if (!best || when > best.when) { + best = { ref, mergeBase, when }; + } + } + return best && { ref: best.ref, mergeBase: best.mergeBase }; +} + +/** + * Creates a resolver that turns {@link RefDescriptor}s into {@link ResolvedRef}s, memoizing the + * `baseline` symbol so it is computed at most once per run. + * + * @param {RefResolverOptions} options - Repository and baseline policy + * @returns {{ resolve: (descriptor: RefDescriptor) => ResolvedRef, parse: (token?: string) => ResolvedRef }} + */ +export function createRefResolver(options) { + const { repoRoot, baselineOverride } = options; + const baseBranch = options.baseBranch ?? detectBaseBranch(repoRoot); + + /** @type {ResolvedRef | undefined} */ + let baselineRef; + + /** + * Resolves a git committish to a `ResolvedRef` keyed by its immutable SHA. + * + * @param {string} committish - Revision to resolve + * @param {string} [label] - Override for the human-readable label + * @returns {ResolvedRef} + */ + function gitRef(committish, label) { + const sha = gitCapture(['rev-parse', committish], repoRoot); + return { + kind: 'git', + id: `git-${sha.slice(0, 9)}`, + label: label ?? `${committish} (${sha.slice(0, 9)})`, + sha, + committish: sha, + }; + } + + /** + * Computes the `baseline` symbol: the override when given, otherwise the sensible default — on + * the base branch the previous commit, on any other branch the fork point from the closest base + * branch, which isolates what this branch changed. + * + * @returns {ResolvedRef} + */ + function computeBaselineRef() { + if (baselineOverride !== undefined) { + if (baselineOverride.trim() === 'baseline') { + throw new Error('The baseline cannot itself be "baseline" — that is the symbol it binds.'); + } + const descriptor = parseRefToken(baselineOverride); + if (descriptor.kind === 'baseline') { + throw new Error('The baseline cannot itself be "baseline" — that is the symbol it binds.'); + } + return resolve(descriptor); + } + + if (gitCapture(['rev-parse', '--abbrev-ref', 'HEAD'], repoRoot) === baseBranch) { + return gitRef('HEAD~1', 'HEAD~1'); + } + + const base = closestBaseBranch(repoRoot, baseBranch); + if (!base) { + console.warn(chalk.yellow(`No ${baseBranch} branch found; using HEAD~1 as the baseline.`)); + return gitRef('HEAD~1', 'HEAD~1'); + } + // Fork point is HEAD itself (HEAD already contained in the base branch) — nothing to diff. + if (base.mergeBase === gitCapture(['rev-parse', 'HEAD'], repoRoot)) { + return gitRef('HEAD~1', 'HEAD~1'); + } + return gitRef(base.mergeBase, `merge-base with ${base.ref}`); + } + + /** + * @param {RefDescriptor} descriptor - A parsed ref token + * @returns {ResolvedRef} + */ + function resolve(descriptor) { + switch (descriptor.kind) { + case 'worktree': + return WORKTREE_REF; + case 'baseline': + baselineRef ??= computeBaselineRef(); + return baselineRef; + case 'git': + return gitRef(/** @type {string} */ (descriptor.committish)); + default: + throw new Error(`Unhandled ref kind "${descriptor.kind}".`); + } + } + + return { + resolve, + parse: (token) => resolve(parseRefToken(token)), + }; +} diff --git a/packages/code-infra/src/tachometer/refs.test.mjs b/packages/code-infra/src/tachometer/refs.test.mjs new file mode 100644 index 0000000000..28d7825671 --- /dev/null +++ b/packages/code-infra/src/tachometer/refs.test.mjs @@ -0,0 +1,48 @@ +import { describe, expect, it } from 'vitest'; +import { parseRefToken } from './refs.mjs'; + +describe('parseRefToken', () => { + it('treats an absent ref as the working tree', () => { + expect(parseRefToken(undefined)).toEqual({ kind: 'worktree' }); + }); + + it('treats an empty ref as the working tree', () => { + expect(parseRefToken('')).toEqual({ kind: 'worktree' }); + }); + + it('recognises the baseline symbol', () => { + expect(parseRefToken('baseline')).toEqual({ kind: 'baseline' }); + }); + + it('parses a git revision', () => { + expect(parseRefToken('git:abc1234')).toEqual({ kind: 'git', committish: 'abc1234' }); + }); + + it('keeps the whole revision, including characters that look like a scheme', () => { + expect(parseRefToken('git:origin/master')).toEqual({ + kind: 'git', + committish: 'origin/master', + }); + }); + + describe('reserved schemes', () => { + it.each(['github:owner/repo#abc1234', 'preview:abc1234'])( + 'reports %s as recognised but not implemented', + (token) => { + expect(() => parseRefToken(token)).toThrow(/recognised but not implemented/); + }, + ); + }); + + describe('unknown tokens', () => { + it('suggests the git scheme for a bare revision-shaped value', () => { + expect(() => parseRefToken('HEAD~1')).toThrow(/did you mean "git:HEAD~1"\?/); + }); + + it('does not suggest the git scheme for a value that cannot be a revision', () => { + // A bare value is never auto-prefixed, since an absent ref already means "working tree". + expect(() => parseRefToken('what is this')).toThrow(/Unknown ref "what is this"/); + expect(() => parseRefToken('what is this')).not.toThrow(/did you mean/); + }); + }); +}); diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs new file mode 100644 index 0000000000..b707ce4826 --- /dev/null +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -0,0 +1,197 @@ +/* eslint-disable no-console */ + +import * as path from 'node:path'; +import * as os from 'node:os'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import chalk from 'chalk'; +import { execaSync } from 'execa'; +import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; +import { packRef } from '../utils/packWorkspace.mjs'; +import { createRefResolver } from './refs.mjs'; +import { discoverCases, pagesOf } from './discoverCases.mjs'; +import { buildPackages, buildRefPages, buildWorkspacePages } from './buildPages.mjs'; +import { summarizeCase } from './summarizeCase.mjs'; +import { run } from './exec.mjs'; + +/** + * @typedef {Object} RunTachometerOptions + * @property {string} harnessDir - The harness package directory (where the command was run) + * @property {string[]} [filters] - Only run cases whose folder name contains one of these substrings + * @property {string} [baseline] - Binds the `baseline` symbol, in the ref grammar (e.g. `git:abc1234`) + * @property {string} [baseBranch] - Branch PRs fork from. Defaults to detection via `origin/HEAD` + * @property {string} [buildCmd] - Command that builds the publishable packages. Defaults to `pnpm release:build` + * @property {boolean} [install] - Whether to install inside a ref's checkout. Defaults to true + * @property {string} [out] - Where to write the combined JSON report. Defaults to `results/report.json` + */ + +/** + * Benchmarks the harness's pages across one or more builds of the workspace with tachometer, and + * writes a combined JSON report. + * + * Cases are the source of truth: the selected `tachometer.json` files determine which pages to + * build, one variant is built per distinct ref, each url is then rewritten to that ref's built page, + * and each case runs as its own tachometer invocation so its result is a clean table for that + * scenario rather than a grid comparing unrelated scenarios against each other. + * + * @param {RunTachometerOptions} options - What to run + * @returns {Promise} + */ +export async function runTachometer(options) { + const { + harnessDir, + filters = [], + baseline, + baseBranch, + buildCmd = 'pnpm release:build', + install = true, + out, + } = options; + + const repoRoot = await findWorkspaceDir(harnessDir); + if (!repoRoot) { + throw new Error(`Could not find a pnpm workspace root above ${harnessDir}.`); + } + + const buildsDir = path.join(harnessDir, 'builds'); + // Non-working-tree refs are packed into a folder keyed by their commit SHA and cached here, so + // repeating a comparison against the same ref reuses the tarballs instead of rebuilding. + const cacheDir = path.join(harnessDir, '.cache'); + + const resolver = createRefResolver({ repoRoot, baseBranch, baselineOverride: baseline }); + const cases = await discoverCases({ harnessDir, filters, resolveRef: resolver.parse }); + + /** @type {Map} */ + const refs = new Map(); + for (const entry of cases) { + for (const leaf of entry.leaves) { + if (leaf.ref) { + refs.set(leaf.ref.id, leaf.ref); + } + } + } + + // Show the resolved commit next to each ref — a symbolic baseline (e.g. "merge-base with + // upstream/master") otherwise hides which commit it actually chose, which matters when several + // base branches exist and only the closest fork point is used. + const describeRef = (/** @type {import('./refs.mjs').ResolvedRef} */ ref) => + ref.sha ? `${ref.label} (${ref.sha.slice(0, 9)})` : ref.label; + + console.log(chalk.cyan(`Cases: ${cases.map((entry) => entry.name).join(', ')}`)); + console.log(chalk.cyan(`Pages: ${pagesOf(cases).join(', ')}`)); + console.log(chalk.cyan(`Refs: ${[...refs.values()].map(describeRef).join(', ')}`)); + + const tmpBase = await mkdtemp(path.join(os.tmpdir(), 'tacho-')); + + try { + for (const ref of refs.values()) { + const outDir = path.join(buildsDir, ref.id); + if (ref.kind === 'worktree') { + // The working tree's build is in this workspace, so build it and let the workspace link + // resolve it — no packing and no isolated install. This is also the fast dev loop. + // eslint-disable-next-line no-await-in-loop + await buildPackages(repoRoot, buildCmd, ref.label); + // eslint-disable-next-line no-await-in-loop + await buildWorkspacePages({ harnessDir, ref, outDir }); + continue; + } + + // packRef packs every workspace package at the ref into a folder cached by SHA. On a cache hit + // it skips the checkout, install, and build entirely; on a miss it builds in a throwaway + // checkout and removes it as soon as the tarballs are packed. + // eslint-disable-next-line no-await-in-loop + const packed = await packRef({ + repoRoot, + ref: /** @type {string} */ (ref.committish), + outRoot: cacheDir, + // An empty install command means "skip"; otherwise packRef's default install runs. + installCmd: install ? undefined : '', + buildCmd, + }); + + // eslint-disable-next-line no-await-in-loop + await buildRefPages({ + harnessDir, + ref, + packages: packed.packages, + workDir: path.join(tmpBase, `bench-${ref.id}`), + outDir, + }); + } + + // Rewrite each leaf url to its ref's built page, then run each case's own config. Tachometer + // resolves relative urls against the config file's directory and these configs are written to a + // temp dir, so the rewritten urls are absolute. + /** @type {Array<{ name: string, json: import('./summarizeCase.mjs').TachometerJson }>} */ + const results = []; + for (const entry of cases) { + for (const leaf of entry.leaves) { + const refId = leaf.ref ? leaf.ref.id : 'current'; + leaf.node.url = `${path.join(buildsDir, refId, leaf.page)}${leaf.suffix}`; + } + const configPath = path.join(tmpBase, `tachometer-${entry.name}.json`); + // eslint-disable-next-line no-await-in-loop + await writeFile( + configPath, + `${JSON.stringify({ ...entry.config, root: harnessDir }, null, 2)}\n`, + ); + + console.log(chalk.cyan(`\nRunning "${entry.name}"…`)); + const jsonPath = path.join(tmpBase, `result-${entry.name}.json`); + // eslint-disable-next-line no-await-in-loop + await run( + 'pnpm', + ['exec', 'tachometer', '--config', configPath, '--json-file', jsonPath], + harnessDir, + ); + // Cases run sequentially on purpose — concurrent browser benchmarks would contend for the + // same machine and skew timings — so reading each result in turn is fine. + // eslint-disable-next-line no-await-in-loop + results.push({ name: entry.name, json: JSON.parse(await readFile(jsonPath, 'utf8')) }); + } + + const outPath = out ? path.resolve(out) : path.join(harnessDir, 'results', 'report.json'); + const report = { + generatedAt: new Date().toISOString(), + head: { + ref: 'HEAD', + sha: execaSync('git', ['rev-parse', 'HEAD'], { cwd: repoRoot }).stdout.trim(), + }, + // Symbols are resolved to concrete SHAs here so a run stays interpretable after the fact. + refs: [...refs.values()].map((ref) => ({ + id: ref.id, + kind: ref.kind, + label: ref.label, + sha: ref.sha, + })), + // Summarising is best-effort per case: a case that produced no usable benchmarks must not + // cost the whole run its report, since `raw` below is the only surviving copy of every other + // case's samples once the temp dir is cleaned up. + cases: results.map(({ name, json }) => { + try { + return summarizeCase(name, json); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(chalk.yellow(`Could not summarize "${name}": ${message}`)); + return { name, error: message }; + } + }), + raw: Object.fromEntries(results.map(({ name, json }) => [name, json])), + }; + await mkdir(path.dirname(outPath), { recursive: true }); + await writeFile(outPath, `${JSON.stringify(report, null, 2)}\n`); + console.log(chalk.green(`\nWrote JSON report to ${outPath}`)); + for (const entry of report.cases) { + if (!('comparisons' in entry)) { + console.log(` ${entry.name}: no result (${entry.error})`); + continue; + } + for (const comparison of entry.comparisons) { + console.log( + ` ${entry.name}: ${entry.reference} is ${comparison.verdict} vs ${comparison.variant}`, + ); + } + } + } finally { + await rm(tmpBase, { recursive: true, force: true }); + } +} diff --git a/packages/code-infra/src/tachometer/summarizeCase.mjs b/packages/code-infra/src/tachometer/summarizeCase.mjs new file mode 100644 index 0000000000..ff8fdf8eb1 --- /dev/null +++ b/packages/code-infra/src/tachometer/summarizeCase.mjs @@ -0,0 +1,85 @@ +/** + * Reduces tachometer's raw JSON output to a flat, interpretable summary. + */ + +/** + * @typedef {Object} ConfidenceInterval + * @property {number} low - Lower bound + * @property {number} high - Upper bound + */ + +/** + * @typedef {Object} TachometerDifference + * @property {ConfidenceInterval} absolute - Absolute difference, in milliseconds + * @property {ConfidenceInterval} percentChange - Relative difference, as a percentage + */ + +/** + * @typedef {Object} TachometerBenchmark + * @property {string} name - Variant name + * @property {number} bytesSent - Bytes transferred + * @property {ConfidenceInterval} mean - Mean duration, as a 95% confidence interval + * @property {number[]} samples - Individual sample durations + * @property {Array} differences - This variant against each other, by index + */ + +/** + * @typedef {Object} TachometerJson + * @property {TachometerBenchmark[]} benchmarks - One entry per variant + */ + +/** + * @typedef {'faster'|'slower'|'unsure'} Verdict + */ + +/** + * Reduces one case's tachometer output to a flat summary: every variant's mean 95% CI (ms) and + * sample count, plus the first variant's difference against each of the others and a verdict per + * comparison. + * + * The first variant is the reference because that is the one under test — for the auto-expanded + * regression case it is `[current]`. A verdict reads the sign of the difference's confidence + * interval: entirely below 0 is `faster`, entirely above is `slower`, and a CI that still straddles + * 0 is `unsure` (the result did not resolve within the sampling budget). + * + * @param {string} name - The case name, used in error messages + * @param {TachometerJson} json - Raw tachometer output for that case + * @returns {{ name: string, reference: string, variants: Array<{ name: string, meanMs: ConfidenceInterval, samples: number }>, comparisons: Array<{ variant: string, verdict: Verdict, difference: { absoluteMs: ConfidenceInterval, percentChange: ConfidenceInterval } | null | undefined }> }} + */ +export function summarizeCase(name, json) { + const [reference, ...others] = json.benchmarks; + if (!reference) { + throw new Error(`Case "${name}" produced no benchmarks.`); + } + + const comparisons = others.map((other, index) => { + // `others` is `benchmarks` without the reference at index 0, so `other` is `benchmarks[index + 1]`. + const diff = reference.differences[index + 1]; + /** @type {Verdict} */ + let verdict = 'unsure'; + if (diff) { + if (diff.percentChange.low > 0 && diff.percentChange.high > 0) { + verdict = 'slower'; + } else if (diff.percentChange.low < 0 && diff.percentChange.high < 0) { + verdict = 'faster'; + } + } + return { + variant: other.name, + verdict, + // `reference` relative to `variant`: positive means the reference is slower. + difference: diff && { absoluteMs: diff.absolute, percentChange: diff.percentChange }, + }; + }); + + return { + name, + reference: reference.name, + variants: json.benchmarks.map((benchmark) => ({ + name: benchmark.name, + meanMs: benchmark.mean, + samples: benchmark.samples.length, + })), + comparisons, + }; +} diff --git a/packages/code-infra/src/tachometer/summarizeCase.test.mjs b/packages/code-infra/src/tachometer/summarizeCase.test.mjs new file mode 100644 index 0000000000..6332b575ec --- /dev/null +++ b/packages/code-infra/src/tachometer/summarizeCase.test.mjs @@ -0,0 +1,89 @@ +import { describe, expect, it } from 'vitest'; +import { summarizeCase } from './summarizeCase.mjs'; + +/** + * Builds a tachometer benchmark entry. + * + * @param {string} name - Variant name + * @param {Array<{ low: number, high: number } | null>} differences - Percent-change CIs against each variant, by index + * @returns {import('./summarizeCase.mjs').TachometerBenchmark} + */ +function benchmark(name, differences = []) { + return { + name, + bytesSent: 0, + mean: { low: 1, high: 2 }, + samples: [1, 2, 3], + differences: differences.map((percentChange) => + percentChange ? { absolute: { low: -1, high: 1 }, percentChange } : null, + ), + }; +} + +describe('summarizeCase', () => { + it('reports every variant with its mean and sample count', () => { + const summary = summarizeCase('example', { + benchmarks: [benchmark('a', [null, { low: -5, high: -1 }]), benchmark('b')], + }); + + expect(summary.reference).toBe('a'); + expect(summary.variants).toEqual([ + { name: 'a', meanMs: { low: 1, high: 2 }, samples: 3 }, + { name: 'b', meanMs: { low: 1, high: 2 }, samples: 3 }, + ]); + }); + + describe('verdicts', () => { + it('is faster when the whole confidence interval is below zero', () => { + const summary = summarizeCase('example', { + benchmarks: [benchmark('a', [null, { low: -5, high: -1 }]), benchmark('b')], + }); + expect(summary.comparisons[0]).toMatchObject({ variant: 'b', verdict: 'faster' }); + }); + + it('is slower when the whole confidence interval is above zero', () => { + const summary = summarizeCase('example', { + benchmarks: [benchmark('a', [null, { low: 1, high: 5 }]), benchmark('b')], + }); + expect(summary.comparisons[0]).toMatchObject({ variant: 'b', verdict: 'slower' }); + }); + + it('is unsure when the confidence interval straddles zero', () => { + const summary = summarizeCase('example', { + benchmarks: [benchmark('a', [null, { low: -3, high: 4 }]), benchmark('b')], + }); + expect(summary.comparisons[0]).toMatchObject({ variant: 'b', verdict: 'unsure' }); + }); + + it('is unsure when there is no difference to read', () => { + const summary = summarizeCase('example', { + benchmarks: [benchmark('a', [null, null]), benchmark('b')], + }); + expect(summary.comparisons[0]).toMatchObject({ variant: 'b', verdict: 'unsure' }); + expect(summary.comparisons[0].difference).toBeFalsy(); + }); + }); + + it('reads each comparison from the difference that belongs to it', () => { + // `differences` is indexed by variant, so the reference's difference against the Nth other + // variant lives at index N + 1. An off-by-one here silently mislabels every comparison. + const summary = summarizeCase('example', { + benchmarks: [ + benchmark('a', [null, { low: -9, high: -8 }, { low: 8, high: 9 }]), + benchmark('b'), + benchmark('c'), + ], + }); + + expect(summary.comparisons).toMatchObject([ + { variant: 'b', verdict: 'faster' }, + { variant: 'c', verdict: 'slower' }, + ]); + }); + + it('throws when a case produced no benchmarks', () => { + expect(() => summarizeCase('example', { benchmarks: [] })).toThrow( + 'Case "example" produced no benchmarks.', + ); + }); +}); diff --git a/packages/code-infra/src/utils/packWorkspace.mjs b/packages/code-infra/src/utils/packWorkspace.mjs new file mode 100644 index 0000000000..047bf57107 --- /dev/null +++ b/packages/code-infra/src/utils/packWorkspace.mjs @@ -0,0 +1,344 @@ +/* eslint-disable no-console */ + +import * as path from 'node:path'; +import * as os from 'node:os'; +import { access, mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import chalk from 'chalk'; +import { execa, parseCommandString } from 'execa'; +import { getWorkspacePackages } from './pnpm.mjs'; + +/** + * Packs the public workspace packages at a given git ref into a folder of tarballs. + * + * The tarballs are produced with `pnpm pack`, so each is exactly what `pnpm publish` would upload — + * honoring the package's `files`, `.npmignore`, and `publishConfig.directory`. That makes them + * suitable for installing a ref's build the way a consumer would install a published release, which + * is what benchmark harnesses and end-to-end install tests need. + */ + +/** + * @typedef {Object} PackedPackage + * @property {string} name - Package name from its package.json + * @property {string} version - Package version + * @property {string} tarball - Absolute path to the packed `.tgz` + */ + +/** + * @typedef {Object} PackedWorkspace + * @property {string} ref - The ref that was packed (verbatim input) + * @property {string} sha - The resolved commit SHA + * @property {string} dir - Absolute path to the folder holding the tarballs and `manifest.json` + * @property {PackedPackage[]} packages - One entry per packed package + */ + +/** + * @typedef {Object} RawManifestPackage + * @property {string} name - Package name + * @property {string} version - Package version + * @property {string} tarball - Tarball name, relative to the folder, so the cache is relocatable + */ + +/** + * @typedef {Object} RawManifest + * @property {string} ref - The ref that was packed + * @property {string} sha - The resolved commit SHA + * @property {string} buildCmd - The build command used, part of the cache key + * @property {RawManifestPackage[]} packages - The packed packages + */ + +/** + * @typedef {Object} PackRefOptions + * @property {string} repoRoot - The repository the ref lives in; also where the worktree is added from + * @property {string} ref - A git ref (SHA, branch, or tag) to pack + * @property {string} outRoot - Cache root. The packed folder lands at `/` and is reused on the next call + * @property {string} [installCmd] - Command run in the checkout to install before building. Defaults to a frozen-lockfile `pnpm install`; pass `''` to skip installing + * @property {string} [buildCmd] - Command run in the checkout to build before packing. Defaults to `pnpm release:build`. Part of the cache key + */ + +/** Name of the file written into each packed folder describing its contents. */ +const MANIFEST = 'manifest.json'; + +/** + * Runs a command in `cwd` with inherited stdio, throwing on a non-zero exit. + * + * @param {string} file - Executable to run + * @param {string[]} args - Arguments + * @param {string} cwd - Working directory + * @param {NodeJS.ProcessEnv} [env] - Extends the current environment + * @returns {Promise} + */ +async function run(file, args, cwd, env) { + console.log(chalk.dim(`$ ${file} ${args.join(' ')} (in ${cwd})`)); + await execa(file, args, { cwd, stdio: 'inherit', env }); +} + +/** + * Resolves a git ref to its commit SHA. + * + * @param {string} repoRoot - Repository to resolve in + * @param {string} ref - The ref to resolve + * @returns {Promise} The commit SHA + */ +async function resolveCommit(repoRoot, ref) { + const result = await execa('git', ['rev-parse', '--verify', `${ref}^{commit}`], { + cwd: repoRoot, + reject: false, + }); + if (result.exitCode !== 0) { + throw new Error(`Could not resolve git ref "${ref}": ${result.stderr.trim()}`); + } + return result.stdout.trim(); +} + +/** + * Turns a package name into a filesystem-safe tarball basename (`@scope/pkg` → `scope-pkg.tgz`). + * + * @param {string} pkgName - The package name + * @returns {string} The tarball basename + */ +export function tarballName(pkgName) { + return `${pkgName.replace(/^@/, '').replace(/\//g, '-')}.tgz`; +} + +/** + * Removes a temporary git worktree, best-effort. + * + * Removal can fail with "Directory not empty" when a background task runner (nx, via lerna) + * repopulates the tree while git is deleting it; fall back to force-removing the directory and + * pruning git's bookkeeping. Never throws — by the time this runs the packed tarballs are already + * safe, so a leftover temp checkout must not fail the run. + * + * @param {string} repoRoot - Repository the checkout belongs to + * @param {string} checkout - Checkout directory to remove + * @returns {Promise} + */ +async function removeCheckout(repoRoot, checkout) { + const removed = await execa('git', ['worktree', 'remove', '--force', checkout], { + cwd: repoRoot, + reject: false, + }); + if (removed.exitCode !== 0) { + try { + await rm(checkout, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); + } catch (error) { + console.warn( + chalk.yellow( + `Could not remove temporary checkout ${checkout}: ${error instanceof Error ? error.message : error}. ` + + `Delete it and prune git's bookkeeping to clean up.`, + ), + ); + } + // Drop git's now-dangling admin entry regardless of whether the directory delete stuck. + await execa('git', ['worktree', 'prune'], { cwd: repoRoot, reject: false }); + } +} + +/** + * True if `target` exists on disk. + * + * @param {string} target - Path to check + * @returns {Promise} + */ +async function pathExists(target) { + try { + await access(target); + return true; + } catch { + return false; + } +} + +/** + * Reads a folder's raw `manifest.json`. + * + * @param {string} dir - The packed folder + * @returns {Promise} + */ +async function readRawManifest(dir) { + return JSON.parse(await readFile(path.join(dir, MANIFEST), 'utf8')); +} + +/** + * Resolves a raw manifest's stored (relative) tarball names to absolute paths under `dir`. + * + * @param {RawManifest} raw - The parsed manifest + * @param {string} dir - The folder it describes + * @returns {PackedWorkspace} + */ +function resolveManifest(raw, dir) { + return { + ref: raw.ref, + sha: raw.sha, + dir, + packages: raw.packages.map((pkg) => ({ + name: pkg.name, + version: pkg.version, + tarball: path.join(dir, pkg.tarball), + })), + }; +} + +/** + * Returns `dir`'s manifest if it is a usable cache for the requested build — produced with the same + * `buildCmd`, with every tarball it references still present — or `null` otherwise. + * + * Returning the parsed manifest lets the caller reuse it without a second read. Guards against + * reusing a folder built with a different build script, or one whose tarballs were partially + * evicted or deleted. + * + * @param {string} dir - Candidate cache folder + * @param {string} buildCmd - The build command the caller wants + * @returns {Promise} + */ +export async function readFreshCache(dir, buildCmd) { + if (!(await pathExists(path.join(dir, MANIFEST)))) { + return null; + } + const raw = await readRawManifest(dir); + if (raw.buildCmd !== buildCmd) { + return null; + } + const tarballsPresent = await Promise.all( + raw.packages.map((pkg) => pathExists(path.join(dir, pkg.tarball))), + ); + return tarballsPresent.every(Boolean) ? raw : null; +} + +/** + * Packs the already-built public workspace packages of `checkoutDir` into `outDir`, one `.tgz` per + * package. Packages must already be built. + * + * Uses pnpm's own workspace resolution rather than scanning a directory, so the set follows + * `pnpm-workspace.yaml` and mirrors exactly what a release publishes. + * + * A public package that (transitively) depends on a *private* workspace package cannot be resolved + * by a consumer — the private package is never packed, so `pnpm install` looks for it on the + * registry and 404s. That is a real packaging bug in the ref, not something to paper over here: + * make the internal dependency public so it ships alongside the package that needs it. + * + * @param {string} checkoutDir - The (already built, already installed) checkout to pack + * @param {string} outDir - Folder to write tarballs into + * @returns {Promise} The packed packages, with absolute tarball paths + */ +export async function packBuiltPackages(checkoutDir, outDir) { + await mkdir(outDir, { recursive: true }); + const packages = await getWorkspacePackages({ cwd: checkoutDir, publicOnly: true }); + if (packages.length === 0) { + throw new Error(`No public workspace packages found in ${checkoutDir}.`); + } + return Promise.all( + packages.map(async ({ path: pkgDir, name, version }) => { + const tarball = path.join(outDir, tarballName(name)); + await run('pnpm', ['pack', '--out', tarball], pkgDir); + return { name, version, tarball }; + }), + ); +} + +/** + * Packs every public workspace package at `ref` into `/`, returning the folder and a + * manifest of the packed tarballs. + * + * The build is deterministic for a commit, so the folder is cached by SHA: a later call for the + * same commit reuses it and skips the checkout, install, and build entirely — but only when it was + * built with the same `buildCmd` and its tarballs are all present, so the cache can never hand back + * stale or mismatched output. + * + * On a cache miss the ref is checked out in a throwaway location, installed, built, and packed; the + * checkout is then removed, so the small tarballs are the only lasting artifact. The folder is + * assembled in a staging directory and atomically renamed into place, and its `manifest.json` + * stores tarball names relative to the folder — so the cache is safe to move between machines (for + * example restored from a CI cache under a different absolute path). + * + * @param {PackRefOptions} options - What to pack, and where + * @returns {Promise} + */ +export async function packRef(options) { + const { + repoRoot, + ref, + outRoot, + installCmd = 'pnpm install --frozen-lockfile --prefer-offline --config.engine-strict=false', + buildCmd = 'pnpm release:build', + } = options; + + const sha = await resolveCommit(repoRoot, ref); + const dir = path.join(outRoot, sha); + + // Cache hit: reuse the folder only if it was built the same way and its tarballs are all present. + const cached = await readFreshCache(dir, buildCmd); + if (cached) { + console.log(chalk.green(`\nReusing packed workspace for "${ref}" (${sha.slice(0, 9)}).`)); + return resolveManifest(cached, dir); + } + + await mkdir(outRoot, { recursive: true }); + // Assemble in a staging sibling on the same filesystem so the final rename into `/` + // is atomic — an interrupted run can never leave a partial folder that looks like a cache hit. + const staging = await mkdtemp(path.join(outRoot, '.staging-')); + const checkout = await mkdtemp(path.join(os.tmpdir(), 'pack-workspace-')); + try { + console.log(chalk.cyan(`\nChecking out "${ref}" (${sha.slice(0, 9)}) at ${checkout}`)); + await run('git', ['worktree', 'add', '--detach', checkout, sha], repoRoot); + if (installCmd) { + console.log(chalk.cyan(`\nInstalling dependencies for ${sha.slice(0, 9)}…`)); + const [installFile, ...installArgs] = parseCommandString(installCmd); + await run(installFile, installArgs, checkout); + } + console.log(chalk.cyan(`\nBuilding packages for ${sha.slice(0, 9)}…`)); + // Disable the nx daemon (lerna runs builds through nx): a lingering daemon keeps writing into + // the checkout and makes removal fail with "Directory not empty". execa's `env` extends the + // current environment, so only NX_DAEMON is overridden. + const [buildFile, ...buildArgs] = parseCommandString(buildCmd); + await run(buildFile, buildArgs, checkout, { NX_DAEMON: 'false' }); + + const packages = await packBuiltPackages(checkout, staging); + // Store tarballs by basename so the folder is relocatable; record buildCmd so a later run can + // tell whether the cache matches. Write the manifest last — it marks completeness. + await writeFile( + path.join(staging, MANIFEST), + `${JSON.stringify( + { + ref, + sha, + buildCmd, + packages: packages.map(({ name, version, tarball }) => ({ + name, + version, + tarball: path.basename(tarball), + })), + }, + null, + 2, + )}\n`, + ); + // Replace any pre-existing folder — a stale/partial one, or a mismatched cache we chose to + // rebuild — so the rename can't fail with ENOTEMPTY. + await rm(dir, { recursive: true, force: true }); + await rename(staging, dir); + return resolveManifest(await readRawManifest(dir), dir); + } catch (error) { + await rm(staging, { recursive: true, force: true }); + throw error; + } finally { + // Runs even if the checkout failed (leaving an empty temp dir) — removeCheckout falls back to a + // plain delete, so the temp checkout never leaks. + await removeCheckout(repoRoot, checkout); + } +} + +/** + * Looks up one packed package's tarball by name, throwing a clear error if it was not packed. + * + * @param {PackedPackage[]} packages - The packed packages + * @param {string} name - Package name to find + * @returns {string} Absolute path to the tarball + */ +export function tarballFor(packages, name) { + const found = packages.find((pkg) => pkg.name === name); + if (!found) { + const available = packages.map((pkg) => pkg.name).join(', ') || '(none)'; + throw new Error(`Package "${name}" was not packed. Packed: ${available}.`); + } + return found.tarball; +} diff --git a/packages/code-infra/src/utils/packWorkspace.test.mjs b/packages/code-infra/src/utils/packWorkspace.test.mjs new file mode 100644 index 0000000000..db81e29cbb --- /dev/null +++ b/packages/code-infra/src/utils/packWorkspace.test.mjs @@ -0,0 +1,206 @@ +import * as path from 'node:path'; +import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises'; +import { execa } from 'execa'; +import { describe, expect, it } from 'vitest'; +import { makeTempDir } from './testUtils.mjs'; +import { packRef, readFreshCache, tarballFor, tarballName } from './packWorkspace.mjs'; + +describe('tarballName', () => { + it('flattens a scoped name into a filesystem-safe basename', () => { + expect(tarballName('@base-ui/mosaic')).toBe('base-ui-mosaic.tgz'); + }); + + it('leaves an unscoped name alone', () => { + expect(tarballName('mosaic')).toBe('mosaic.tgz'); + }); +}); + +describe('tarballFor', () => { + const packages = [{ name: '@scope/one', version: '1.0.0', tarball: '/tmp/scope-one.tgz' }]; + + it('finds a packed package by name', () => { + expect(tarballFor(packages, '@scope/one')).toBe('/tmp/scope-one.tgz'); + }); + + it('lists what was packed when a package is missing', () => { + // A workspace dependency that was never packed is a packaging bug in the ref — usually a + // private package a public one depends on — so the message has to name what is available. + expect(() => tarballFor(packages, '@scope/two')).toThrow( + 'Package "@scope/two" was not packed. Packed: @scope/one.', + ); + }); + + it('says so when nothing was packed at all', () => { + expect(() => tarballFor([], '@scope/two')).toThrow(/Packed: \(none\)\./); + }); +}); + +describe('readFreshCache', () => { + /** + * Writes a packed folder with a manifest and its tarballs. + * + * @param {Object} options - Folder contents + * @param {string} options.buildCmd - The build command to record + * @param {boolean} [options.withTarball] - Whether to actually create the tarball file + * @returns {Promise} The folder + */ + async function makePackedDir({ buildCmd, withTarball = true }) { + const dir = await makeTempDir(); + await writeFile( + path.join(dir, 'manifest.json'), + JSON.stringify({ + ref: 'HEAD', + sha: 'abc', + buildCmd, + packages: [{ name: '@scope/one', version: '1.0.0', tarball: 'scope-one.tgz' }], + }), + ); + if (withTarball) { + await writeFile(path.join(dir, 'scope-one.tgz'), 'tarball'); + } + return dir; + } + + it('returns the manifest when the folder matches the requested build', async () => { + const dir = await makePackedDir({ buildCmd: 'pnpm release:build' }); + + expect(await readFreshCache(dir, 'pnpm release:build')).toMatchObject({ sha: 'abc' }); + }); + + it('misses when the folder was built with a different command', async () => { + const dir = await makePackedDir({ buildCmd: 'pnpm build' }); + + expect(await readFreshCache(dir, 'pnpm release:build')).toBeNull(); + }); + + it('misses when a referenced tarball has been evicted', async () => { + const dir = await makePackedDir({ buildCmd: 'pnpm release:build', withTarball: false }); + + expect(await readFreshCache(dir, 'pnpm release:build')).toBeNull(); + }); + + it('misses when there is no manifest', async () => { + expect(await readFreshCache(await makeTempDir(), 'pnpm release:build')).toBeNull(); + }); +}); + +/** + * Creates a git repository holding a two-package pnpm workspace: one publishable, one private. + * + * The build command is a script that appends to `marker`, so a later assertion can tell whether a + * build actually ran or the cache was reused. + * + * @returns {Promise<{ repoRoot: string, marker: string, buildCmd: string }>} + */ +async function makeFixtureRepo() { + const repoRoot = await makeTempDir(); + const marker = path.join(repoRoot, 'builds.log'); + + await writeFile(path.join(repoRoot, 'pnpm-workspace.yaml'), "packages:\n - 'packages/*'\n"); + await writeFile( + path.join(repoRoot, 'package.json'), + JSON.stringify({ name: 'fixture-root', private: true, version: '0.0.0' }, null, 2), + ); + // Appends one line per real build; a cache hit must not add one. + await writeFile( + path.join(repoRoot, 'build.mjs'), + "import { appendFileSync } from 'node:fs';\nappendFileSync(process.argv[2], 'built\\n');\n", + ); + + await Promise.all( + [ + { dir: 'public', pkg: { name: '@fixture/public', version: '1.0.0' } }, + { dir: 'private', pkg: { name: '@fixture/private', version: '1.0.0', private: true } }, + ].map(async ({ dir, pkg }) => { + const pkgDir = path.join(repoRoot, 'packages', dir); + await mkdir(pkgDir, { recursive: true }); + await writeFile(path.join(pkgDir, 'package.json'), JSON.stringify(pkg, null, 2)); + await writeFile(path.join(pkgDir, 'index.js'), 'export default 1;\n'); + }), + ); + + await execa('git', ['init', '--initial-branch=main'], { cwd: repoRoot }); + await execa('git', ['config', 'user.email', 'fixture@example.com'], { cwd: repoRoot }); + await execa('git', ['config', 'user.name', 'Fixture'], { cwd: repoRoot }); + await execa('git', ['add', '.'], { cwd: repoRoot }); + await execa('git', ['commit', '-m', 'initial'], { cwd: repoRoot }); + + return { repoRoot, marker, buildCmd: `node build.mjs ${marker}` }; +} + +describe('packRef', () => { + it( + 'packs only the publishable packages, and caches by commit', + { timeout: 120_000 }, + async () => { + const { repoRoot, marker, buildCmd } = await makeFixtureRepo(); + const outRoot = path.join(await makeTempDir(), 'cache'); + + const packed = await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); + + // The private package is never packed: a release would not publish it. + expect(packed.packages.map((pkg) => pkg.name)).toEqual(['@fixture/public']); + expect(packed.dir).toBe(path.join(outRoot, packed.sha)); + await expect(stat(packed.packages[0].tarball)).resolves.toBeTruthy(); + + // Tarball names are stored relative to the folder, so a cache restored under a different + // absolute path still resolves. + const manifest = JSON.parse(await readFile(path.join(packed.dir, 'manifest.json'), 'utf8')); + expect(manifest.packages).toEqual([ + { name: '@fixture/public', version: '1.0.0', tarball: 'fixture-public.tgz' }, + ]); + expect(manifest.buildCmd).toBe(buildCmd); + + expect(await readFile(marker, 'utf8')).toBe('built\n'); + + // The temporary checkout is cleaned up, leaving the tarballs as the only artifact. + const worktrees = await execa('git', ['worktree', 'list'], { cwd: repoRoot }); + expect(worktrees.stdout.trim().split('\n')).toHaveLength(1); + + // A second call for the same commit reuses the folder and never builds again. + const again = await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); + expect(again.sha).toBe(packed.sha); + expect(await readFile(marker, 'utf8')).toBe('built\n'); + }, + ); + + it( + 'rebuilds when the cached folder used a different build command', + { timeout: 120_000 }, + async () => { + const { repoRoot, marker, buildCmd } = await makeFixtureRepo(); + const outRoot = path.join(await makeTempDir(), 'cache'); + + await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); + await packRef({ + repoRoot, + ref: 'HEAD', + outRoot, + installCmd: '', + buildCmd: `${buildCmd} --other`, + }); + + expect(await readFile(marker, 'utf8')).toBe('built\nbuilt\n'); + }, + ); + + it('leaves no staging directory behind', { timeout: 120_000 }, async () => { + const { repoRoot, buildCmd } = await makeFixtureRepo(); + const outRoot = path.join(await makeTempDir(), 'cache'); + + const packed = await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); + + // The folder is assembled in a staging sibling and atomically renamed, so an interrupted run + // can never leave something that looks like a cache hit. + expect(await readdir(outRoot)).toEqual([packed.sha]); + }); + + it('reports a ref it cannot resolve', async () => { + const { repoRoot, buildCmd } = await makeFixtureRepo(); + const outRoot = path.join(await makeTempDir(), 'cache'); + + await expect( + packRef({ repoRoot, ref: 'no-such-ref', outRoot, installCmd: '', buildCmd }), + ).rejects.toThrow(/Could not resolve git ref "no-such-ref"/); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3d805ad746..b2fc474707 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -786,6 +786,9 @@ importers: serve: specifier: 14.2.6 version: 14.2.6(supports-color@10.2.2) + vite: + specifier: ^8.2.2 + version: 8.2.2(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.44.0)(tsx@4.23.12)(yaml@2.9.0) packages/docs-infra: dependencies: From da12b902fb565d68d5d0f15c57e0e94303ef4762 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 1 Sep 2026 09:02:01 +0200 Subject: [PATCH 002/173] [code-infra] Add a tachometer smoke-test harness Adds test/tachometer, a working harness for `code-infra tacho run`. This repository ships build tooling rather than a browser library, so there is nothing here whose render time is worth tracking; the cases run a deterministic CPU workload purely to give the pipeline something real to drive. Both pages import a workspace package and put its value on screen, so a break in either resolution path -- the workspace link for the working tree, a packed tarball for every other ref -- fails the build instead of quietly measuring nothing. Two cases, chosen to walk the parts most likely to break: `workload` is the ordinary auto-expanded regression shape, and `workload-large` owns no page of its own, parameterising its sibling's with `?size=`. Running it end to end found two bugs in the tooling: - The isolated install is its own workspace root, so it inherited none of the repository's build-script approvals and pnpm failed the install outright. It now copies the repository's allowBuilds policy across, minus strictDepBuilds -- an unapproved transitive script the pages do not need should not fail a run. - A plugin's returned config is merged *over* the inline config, so the plugin's default outDir silently overrode `vite build --outDir`, and every ref's pages landed in builds/manual. It now only fills outDir in when the caller left it unset. The harness's generated directories are listed in the root .gitignore rather than a nested one because ESLint is fed from the root file and never reads nested ones. --- .gitignore | 7 + package.json | 1 + .../code-infra/src/tachometer/buildPages.mjs | 58 +- packages/code-infra/src/tachometer/plugin.mjs | 8 + .../code-infra/src/tachometer/plugin.test.mjs | 26 +- .../src/tachometer/runTachometer.mjs | 1 + pnpm-lock.yaml | 1144 ++++++++++++++++- test/tachometer/README.md | 72 ++ test/tachometer/package.json | 20 + test/tachometer/src/_shared/measure.ts | 51 + .../src/workload-large/tachometer.json | 18 + test/tachometer/src/workload/index.html | 11 + test/tachometer/src/workload/index.ts | 42 + test/tachometer/src/workload/tachometer.json | 18 + test/tachometer/tsconfig.json | 15 + test/tachometer/vite.config.mjs | 6 + 16 files changed, 1491 insertions(+), 7 deletions(-) create mode 100644 test/tachometer/README.md create mode 100644 test/tachometer/package.json create mode 100644 test/tachometer/src/_shared/measure.ts create mode 100644 test/tachometer/src/workload-large/tachometer.json create mode 100644 test/tachometer/src/workload/index.html create mode 100644 test/tachometer/src/workload/index.ts create mode 100644 test/tachometer/src/workload/tachometer.json create mode 100644 test/tachometer/tsconfig.json create mode 100644 test/tachometer/vite.config.mjs diff --git a/.gitignore b/.gitignore index a27e62547a..2a2e6b8555 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,10 @@ next-env.d.ts # Claude Code worktrees .claude/worktrees/ .claude/cache/ + +# Tachometer harness: built pages per ref, packed workspace tarballs, and run reports. +# Listed here rather than under test/tachometer so ESLint picks them up too: it is fed from +# the root .gitignore and never reads nested ones. +/test/tachometer/builds/ +/test/tachometer/.cache/ +/test/tachometer/results/ diff --git a/package.json b/package.json index aa8b7f20d5..9d8ca3ee23 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "docs:lint": "pnpm -F \"./docs\" run lint", "docs:lib": "pnpm -F \"./packages/docs-infra\" run build", "test:benchmark": "pnpm -F ./test/performance benchmark", + "test:tacho": "pnpm -F ./test/tachometer test:tacho", "org-versions": "tsx scripts/orgVersions.mts" }, "dependencies": { diff --git a/packages/code-infra/src/tachometer/buildPages.mjs b/packages/code-infra/src/tachometer/buildPages.mjs index c332193d51..95354b6e5b 100644 --- a/packages/code-infra/src/tachometer/buildPages.mjs +++ b/packages/code-infra/src/tachometer/buildPages.mjs @@ -3,6 +3,7 @@ import * as path from 'node:path'; import { cp, readFile, readdir, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; +import { parse, stringify } from 'yaml'; import { tarballFor } from '../utils/packWorkspace.mjs'; import { run } from './exec.mjs'; @@ -87,6 +88,48 @@ export async function buildWorkspacePages(options) { await runViteBuild(harnessDir, outDir); } +/** + * The `pnpm-workspace.yaml` keys governing which dependencies may run install scripts. + * + * `strictDepBuilds` is deliberately not among them: the sandbox should mirror the repository's + * approvals, but an unapproved transitive build script the pages do not need must not fail a + * benchmark run. + */ +const BUILD_POLICY_KEYS = [ + 'allowBuilds', + 'onlyBuiltDependencies', + 'neverBuiltDependencies', + 'dangerouslyAllowAllBuilds', +]; + +/** + * Copies the repository's build-script policy, so the isolated install approves the same install + * scripts the repository does. + * + * The throwaway package is its own workspace root, so it inherits nothing. Without this, pnpm + * refuses to run install scripts it was not told to trust and fails the install outright — which + * bites on packages the build genuinely needs, such as esbuild's binary download. + * + * @param {string} repoRoot - The workspace root to copy policy from + * @returns {Promise>} + */ +async function readBuildPolicy(repoRoot) { + /** @type {Record} */ + const policy = {}; + let root; + try { + root = parse(await readFile(path.join(repoRoot, 'pnpm-workspace.yaml'), 'utf8')); + } catch { + return policy; + } + for (const key of BUILD_POLICY_KEYS) { + if (root?.[key] !== undefined) { + policy[key] = root[key]; + } + } + return policy; +} + /** * Rewrites a dependency map, pointing every `workspace:`-protocol entry at its packed tarball and * copying everything else verbatim. @@ -125,6 +168,7 @@ export function rewriteWorkspaceDeps(deps, packages, omit = []) { * * @param {Object} options - Build inputs * @param {string} options.harnessDir - The harness package directory, copied from + * @param {string} options.repoRoot - The workspace root, whose build-script policy the sandbox mirrors * @param {ResolvedRef} options.ref - The ref being built * @param {PackedPackage[]} options.packages - That ref's packed workspace packages * @param {string} options.workDir - Throwaway directory to assemble the isolated package in @@ -132,7 +176,7 @@ export function rewriteWorkspaceDeps(deps, packages, omit = []) { * @returns {Promise} */ export async function buildRefPages(options) { - const { harnessDir, ref, packages, workDir, outDir } = options; + const { harnessDir, repoRoot, ref, packages, workDir, outDir } = options; console.log(chalk.cyan(`\nBuilding benchmark pages for "${ref.label}"…`)); const viteConfig = await findViteConfig(harnessDir); @@ -162,9 +206,15 @@ export async function buildRefPages(options) { // build rather than the registry. In pnpm 11 `overrides` live in `pnpm-workspace.yaml`, not // `package.json`; writing one here also anchors this throwaway as its own workspace root, so pnpm // reads the overrides and never walks up to a parent workspace (which is why `--ignore-workspace` - // — which would ignore this file — is not used). - const overrides = packages.map((pkg) => ` '${pkg.name}': "file:${pkg.tarball}"`).join('\n'); - await writeFile(path.join(workDir, 'pnpm-workspace.yaml'), `overrides:\n${overrides}\n`); + // — which would ignore this file — is not used). Being its own root also means it inherits no + // build-script policy, hence carrying the repository's over. + await writeFile( + path.join(workDir, 'pnpm-workspace.yaml'), + stringify({ + ...(await readBuildPolicy(repoRoot)), + overrides: Object.fromEntries(packages.map((pkg) => [pkg.name, `file:${pkg.tarball}`])), + }), + ); await run('pnpm', ['install', '--prefer-offline', '--config.engine-strict=false'], workDir); await runViteBuild(workDir, outDir); diff --git a/packages/code-infra/src/tachometer/plugin.mjs b/packages/code-infra/src/tachometer/plugin.mjs index 5b71ec2f3f..d7ed688cd7 100644 --- a/packages/code-infra/src/tachometer/plugin.mjs +++ b/packages/code-infra/src/tachometer/plugin.mjs @@ -136,6 +136,14 @@ export function tachometer(options = {}) { // `//`, so absolute "/assets/…" paths would 404. base: './', build: { + // A plain `vite build` lands in `builds/manual`, next to the per-ref directories the + // runner writes. Without this it would default to `/dist` — inside `src/`, next to + // the case sources. + // + // Only fill it in when the caller said nothing: a plugin's returned config is merged + // *over* the inline config, so unconditionally setting it here would silently override + // `vite build --outDir`, which is exactly how `tacho run` directs each ref's build. + outDir: userConfig.build?.outDir ?? path.join(harnessDir, 'builds', 'manual'), // The output directory is outside `root` — and, for an isolated ref build, outside the // temporary package entirely; allow vite to clean it anyway. emptyOutDir: true, diff --git a/packages/code-infra/src/tachometer/plugin.test.mjs b/packages/code-infra/src/tachometer/plugin.test.mjs index 2bb38fa0b9..7cc12b30fb 100644 --- a/packages/code-infra/src/tachometer/plugin.test.mjs +++ b/packages/code-infra/src/tachometer/plugin.test.mjs @@ -38,10 +38,10 @@ async function makeHarness({ cases, dependencies = {} }) { * @param {'build' | 'serve'} command - Which vite command is running * @returns {Promise} */ -async function callConfig(harnessDir, command) { +async function callConfig(harnessDir, command, userConfig = {}) { const plugin = tachometer({ harnessDir }); const hook = /** @type {any} */ (plugin.config); - return hook({}, { command, mode: command === 'build' ? 'production' : 'development' }); + return hook(userConfig, { command, mode: command === 'build' ? 'production' : 'development' }); } const oneCase = { @@ -82,6 +82,28 @@ describe('tachometer plugin', () => { expect(config.base).toBe('./'); }); + it('defaults the output directory to builds/manual, outside the source root', async () => { + // Vite would otherwise emit into /dist, i.e. inside src/ next to the case sources. + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'build'); + + expect(config.build.outDir).toBe(path.join(harnessDir, 'builds', 'manual')); + }); + + it('leaves an output directory the caller chose alone', async () => { + // A plugin's returned config is merged *over* the inline config, so defaulting this + // unconditionally would silently override `vite build --outDir` — which is how `tacho run` + // directs each ref's build into its own directory. + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'build', { + build: { outDir: '/somewhere/builds/current' }, + }); + + expect(config.build.outDir).toBe('/somewhere/builds/current'); + }); + it('derives the build input from the discovered cases', async () => { const harnessDir = await makeHarness({ cases: oneCase }); diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index b707ce4826..1ff9f07023 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -111,6 +111,7 @@ export async function runTachometer(options) { // eslint-disable-next-line no-await-in-loop await buildRefPages({ harnessDir, + repoRoot, ref, packages: packed.packages, workDir: path.join(tmpBase, `bench-${ref.id}`), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b2fc474707..cc0c192ac1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1108,6 +1108,28 @@ importers: specifier: 4.1.11 version: 4.1.11(@types/node@22.20.1)(@vitest/browser-playwright@4.1.11)(@vitest/coverage-v8@4.1.11)(jsdom@28.1.0(supports-color@10.2.2))(vite@8.2.2(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.44.0)(tsx@4.23.12)(yaml@2.9.0)) + test/tachometer: + dependencies: + '@mui/internal-test-utils': + specifier: workspace:* + version: link:../../packages/test-utils/build + react: + specifier: 19.2.8 + version: 19.2.8 + devDependencies: + '@mui/internal-code-infra': + specifier: workspace:* + version: link:../../packages/code-infra + '@types/react': + specifier: 19.2.18 + version: 19.2.18 + tachometer: + specifier: ^0.7.2 + version: 0.7.2(supports-color@10.2.2) + vite: + specifier: ^8.2.2 + version: 8.2.2(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.44.0)(tsx@4.23.12)(yaml@2.9.0) + packages: '@-xun/debug@2.0.2': @@ -1841,6 +1863,9 @@ packages: '@types/react': optional: true + '@bazel/runfiles@6.5.0': + resolution: {integrity: sha512-RzahvqTkfpY2jsDxo8YItPX+/iZ6hbiikw1YhE0bA9EKBR5Og8Pa6FHn9PO9M0zaXRVsr0GFQLKbB/0rzy9SzA==} + '@bcoe/v8-coverage@1.0.2': resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} engines: {node: '>=18'} @@ -2296,6 +2321,9 @@ packages: '@floating-ui/utils@0.2.12': resolution: {integrity: sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==} + '@hapi/bourne@3.0.0': + resolution: {integrity: sha512-Waj1cwPXJDucOib4a3bAISsKJVb15MKi9IvmTI/7ssVEm6sywXGjVJDhl6/umt1pK1ZS7PacXU3A1PmFKHEZ2w==} + '@html-validate/stylish@6.0.0': resolution: {integrity: sha512-d1/lI3qIXhGVJF3+MmKEBn1dRX6U4Z+BDaOdDI3E6SXcO+OQ7hC/3mSo6BIjwQlcuV6NdDOKm6QCrzIQL1EezQ==} engines: {node: ^22.16.0 || >= 24.0.0} @@ -3635,6 +3663,10 @@ packages: '@sinclair/typebox@0.34.41': resolution: {integrity: sha512-6gS8pZzSXdyRHTIqoqSVknxolr1kzfy4/CeDnrzsVz8TTIWUbOBr6gnzOmTYJ3eXQNh4IYHIGi5aIL7sOZ2G/g==} + '@sindresorhus/is@5.6.0': + resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==} + engines: {node: '>=14.16'} + '@sindresorhus/merge-streams@4.0.0': resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} @@ -3669,6 +3701,10 @@ packages: '@swc/helpers@0.5.23': resolution: {integrity: sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==} + '@szmarczak/http-timer@5.0.1': + resolution: {integrity: sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==} + engines: {node: '>=14.16'} + '@tanstack/query-core@5.101.4': resolution: {integrity: sha512-gNwcvOJcRbLWPOLG/2OBm+zM+Yv+MKsXKEOWC57USuZDEsI71hEErQsiEGx5wX9rzWWkfwM0fVSPoiIFSsxfiw==} @@ -3804,6 +3840,9 @@ packages: '@types/etag@1.8.4': resolution: {integrity: sha512-f1z/UMth8gQ6636NBqhFmJ3zES7EuDcUnV6K1gl1osHp+85KPKX+VixYWUpqLkw1fftCagyHJjJOZjZkEi2rHw==} + '@types/execa@0.9.0': + resolution: {integrity: sha512-mgfd93RhzjYBUHHV532turHC2j4l/qxsF/PbfDmprHDEUHmNZGlDn1CEsulGK3AfsPdhkWzZQT/S/k0UGhLGsA==} + '@types/find-package-json@1.2.7': resolution: {integrity: sha512-zHPw+sj/9u0cHV8cS99fizuCSEo3K5pk6JxDu6K3DEJAEZgdvjoHnFJz/C1Vzwr75GSPlAcDZR9O30/4CuQo9Q==} @@ -3816,6 +3855,9 @@ packages: '@types/hast@3.0.5': resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==} + '@types/http-cache-semantics@4.2.0': + resolution: {integrity: sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q==} + '@types/is-empty@1.2.3': resolution: {integrity: sha512-4J1l5d79hoIvsrKh5VUKVRA1aIdsOb10Hu5j3J2VfP/msDnfTdGPmNp2E1Wg+vs97Bktzo+MZePFFXSGoykYJw==} @@ -3840,6 +3882,9 @@ packages: '@types/ms@2.1.0': resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} + '@types/node@11.15.54': + resolution: {integrity: sha512-1RWYiq+5UfozGsU6MwJyFX6BtktcT10XRjvcAQmskCtMcW3tPske88lM/nHv7BQG1w9KBXI1zPGuu5PnNCX14g==} + '@types/node@14.18.63': resolution: {integrity: sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==} @@ -3860,6 +3905,9 @@ packages: resolution: {integrity: sha512-EULJ8LApcVEPbrfND0cRQqutIOdiIgJ1Mgrhpy755r14xMohPTEpkV/k28SJvuOs9bHRFW8x+KeDAEPiGQPB9Q==} deprecated: This is a stub types definition. parse-path provides its own type definitions, so you do not need this installed. + '@types/parse5@5.0.3': + resolution: {integrity: sha512-kUNnecmtkunAoQ3CnjmMkzNU/gtxG8guhi+Fk2U/kOpIKjIMKnXGp4IJCgQJrXSgMsWYimYG4TGjz/UzbGEBTw==} + '@types/prop-types@15.7.15': resolution: {integrity: sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==} @@ -4280,6 +4328,10 @@ packages: resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} + acorn-import-phases@1.0.4: resolution: {integrity: sha512-wKmbr/DDiIXzEOiWrTTUcDm24kQ2vGfZQvM2fwg2vXqR5uW6aapr7ObPtj1th32b9u90/Pf4AItvdTh42fBmVQ==} engines: {node: '>=10.13.0'} @@ -4326,6 +4378,15 @@ packages: ansi-align@3.0.1: resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} + ansi-escape-sequences@6.2.4: + resolution: {integrity: sha512-2KJQAG1Nk4Iyu0dJENKXQJE9smEASrpu/E0F7LSnR72tQXngKGLqfRkHbkinjNct5vvAQY4BwQNt+4Tvg73nDQ==} + engines: {node: '>=12.17'} + peerDependencies: + '@75lb/nature': latest + peerDependenciesMeta: + '@75lb/nature': + optional: true + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -4334,6 +4395,10 @@ packages: resolution: {integrity: sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==} engines: {node: '>=12'} + ansi-styles@3.2.1: + resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} + engines: {node: '>=4'} + ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} @@ -4369,6 +4434,18 @@ packages: resolution: {integrity: sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==} engines: {node: '>= 0.4'} + array-back@3.1.0: + resolution: {integrity: sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==} + engines: {node: '>=6'} + + array-back@4.0.2: + resolution: {integrity: sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==} + engines: {node: '>=8'} + + array-back@6.2.3: + resolution: {integrity: sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==} + engines: {node: '>=12.17'} + array-buffer-byte-length@1.0.2: resolution: {integrity: sha512-LHE+8BuR7RYGDKvnrmcuSq3tDcKv9OFEXQt/HpbZhY7V6h0zlUXutnAD82GiFx9rdieCMjkvtcsPqBwgUl1Iiw==} engines: {node: '>= 0.4'} @@ -4571,6 +4648,9 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true + buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + buffer-from@0.1.2: resolution: {integrity: sha512-RiWIenusJsmI2KcvqQABB83tLxCByE3upSP8QU3rJDMVFGPWLvPQJt/O1Su9moRWeH7d+Q2HYb68f6+v+tw2vg==} @@ -4596,6 +4676,18 @@ packages: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} + cache-content-type@1.0.1: + resolution: {integrity: sha512-IKufZ1o4Ut42YUrZSo8+qnMTrFuKkvyoLXUywKz9GJ5BrhOFGhLdkx9sG4KAnVvbY6kEcSFjLQul+DVmBm2bgA==} + engines: {node: '>= 6.0.0'} + + cacheable-lookup@7.0.0: + resolution: {integrity: sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==} + engines: {node: '>=14.16'} + + cacheable-request@10.2.14: + resolution: {integrity: sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==} + engines: {node: '>=14.16'} + cacheable@2.5.0: resolution: {integrity: sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g==} @@ -4643,6 +4735,10 @@ packages: resolution: {integrity: sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==} engines: {node: '>=12'} + chalk@2.4.2: + resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} + engines: {node: '>=4'} + chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -4721,6 +4817,14 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + co-body@6.2.0: + resolution: {integrity: sha512-Kbpv2Yd1NdL1V/V4cwLVxraHDV6K8ayohr2rmH0J87Er8+zJjcTa6dAn9QMPC9CRgU8+aNajKbSf1TzDB1yKPA==} + engines: {node: '>=8.0.0'} + + co@4.6.0: + resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} + engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} + codsen-utils@1.8.0: resolution: {integrity: sha512-y2aaSqHOk4VBtot1oO6haBgtfc9Lf8mwQBPFePlsMdKORifG/eeRpiIJEkz6m755eErWyu5dqF/U9FffjH9uTA==} engines: {node: '>=18.20.8'} @@ -4728,10 +4832,16 @@ packages: collapse-white-space@2.1.0: resolution: {integrity: sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==} + color-convert@1.9.3: + resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} + color-name@1.1.3: + resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} + color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} @@ -4741,6 +4851,14 @@ packages: comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + command-line-args@5.2.1: + resolution: {integrity: sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==} + engines: {node: '>=4.0.0'} + + command-line-usage@6.1.3: + resolution: {integrity: sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==} + engines: {node: '>=8.0.0'} + commander@2.20.3: resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} @@ -4778,6 +4896,10 @@ packages: resolution: {integrity: sha512-kRGRZw3bLlFISDBgwTSA1TMBFN6J6GWDeubmDE3AF+3+yXL8hTWv8r5rkLbqYXY4RjPk/EzHnClI3zQf1cFmHA==} engines: {node: '>= 0.6'} + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + content-type@2.1.0: resolution: {integrity: sha512-mj7UPXE0jaqaOsukNZRUEfEi2AcL7C/vwmwcHV0O97eO1E1pxBZuyjlZrx5seTaNBg1U6+o35wpa35Qfcc+7ag==} engines: {node: '>=18'} @@ -4796,6 +4918,13 @@ packages: resolution: {integrity: sha512-JtXpxqDqJ8P0UwEHwhxLzCIXQy97vlYBZR222Sbzb1q1Erex9ASrztJ29SyhWFQjod1AeFBaPzEEC8YvtZMIYg==} engines: {node: '>=6'} + cookies@0.9.1: + resolution: {integrity: sha512-TG2hpqe4ELx54QER/S3HQ9SRVnQnGBtKUz5bLQWtYAQ+o6GpgMs6sYUvaiJjVxb+UXwhRhAEP3m7LbsIZ77Hmw==} + engines: {node: '>= 0.8'} + + copy-to@2.0.1: + resolution: {integrity: sha512-3DdaFaU/Zf1AnpLiFDeNCD4TOWe3Zl2RZaTzUvWiIk5ERzcCodOE20Vqq4fzCbNoHURFHT4/us/Lfq+S2zyY4w==} + core-js-compat@3.48.0: resolution: {integrity: sha512-OM4cAF3D6VtH/WkLtWvyNC56EZVXsZdU3iqaMG2B4WvYrlqU831pc4UtG5yp0sE9z8Y02wVN7PjW5Zf9Gt0f1Q==} @@ -4822,6 +4951,10 @@ packages: resolution: {integrity: sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==} engines: {node: '>=10.0.0'} + cross-spawn@6.0.6: + resolution: {integrity: sha512-VqCUuhcd1iB+dsv8gxPttb5iZh/D0iubSP21g36KXdEuf6I5JiioesUVjpCdHV9MZRUfVFlvwtIUyPfxo5trtw==} + engines: {node: '>=4.8'} + cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -4860,6 +4993,9 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + csv-stringify@6.8.3: + resolution: {integrity: sha512-gIeSCvq5F4VtXV3naV3VAewLhBkiZBz+PPhTOA8H3Y8h/ELa+R1ml0GZck/4/Nzo9ep2lvOluilJ6MJlbZsKMA==} + d3-array@3.2.4: resolution: {integrity: sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==} engines: {node: '>=12'} @@ -4964,6 +5100,13 @@ packages: decode-named-character-reference@1.2.0: resolution: {integrity: sha512-c6fcElNV6ShtZXmsgNgFFV5tVX2PaV4g+MOAkb8eXHvn6sryJBrZa9r0zV6+dtTyoCKxtDy5tyQ5ZwQuidtd+Q==} + decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + + deep-equal@1.0.1: + resolution: {integrity: sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==} + deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} @@ -4979,6 +5122,10 @@ packages: resolution: {integrity: sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==} engines: {node: '>=18'} + defer-to-connect@2.0.1: + resolution: {integrity: sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==} + engines: {node: '>=10'} + define-data-property@1.1.4: resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} engines: {node: '>= 0.4'} @@ -4991,10 +5138,25 @@ packages: resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} engines: {node: '>= 0.4'} + delegates@1.0.0: + resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} + + depd@1.1.2: + resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} + engines: {node: '>= 0.6'} + + depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} + destroy@1.2.0: + resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} @@ -5048,6 +5210,12 @@ packages: eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + electron-to-chromium@1.5.396: resolution: {integrity: sha512-yHiw2Y3C3H9U6TMbOfoWK/BPreiOPXRfTWPBwQBoZG6/8TB6eOPnsy5oaRYuatR7Fw2SJ4kKforgufeo7fq0EQ==} @@ -5060,6 +5228,13 @@ packages: emoji-regex@9.2.2: resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + encodeurl@1.0.2: + resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} + engines: {node: '>= 0.8'} + + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + enhanced-resolve@5.23.0: resolution: {integrity: sha512-yJN/BOOLxcOW2aQgeif9mSnaUB8KtvmMMp56oA1kx1CRfBKbhZm2pJ+NBY+3eOboHxix8lfjWpHE0Ei5U8RbSA==} engines: {node: '>=10.13.0'} @@ -5143,6 +5318,13 @@ packages: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} + escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + + escape-string-regexp@1.0.5: + resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} + engines: {node: '>=0.8.0'} + escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -5392,6 +5574,10 @@ packages: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} + execa@1.0.0: + resolution: {integrity: sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==} + engines: {node: '>=6'} + execa@10.0.1: resolution: {integrity: sha512-ge98qjkRK4IB7tL7Ju/6qmm5LHoH1eEMt5FNZrz3f4UIYhF28lggX20z3FaX1sgc67msLEn0N0BscOs29iuwyw==} engines: {node: '>=22'} @@ -5496,6 +5682,10 @@ packages: find-package-json@1.2.0: resolution: {integrity: sha512-+SOGcLGYDJHtyqHd87ysBhmaeQ95oWspDKnMXBrnQ9Eq4OkLNqejgoaD8xVWu6GPa0B6roa6KinCMEMcVeqONw==} + find-replace@3.0.0: + resolution: {integrity: sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==} + engines: {node: '>=4.0.0'} + find-root@1.1.0: resolution: {integrity: sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==} @@ -5507,6 +5697,10 @@ packages: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} + find-up@6.3.0: + resolution: {integrity: sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + flat-cache@4.0.1: resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} engines: {node: '>=16'} @@ -5528,6 +5722,10 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} + form-data-encoder@2.1.4: + resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} + engines: {node: '>= 14.17'} + format-util@1.0.5: resolution: {integrity: sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==} @@ -5538,6 +5736,14 @@ packages: fraction.js@5.3.4: resolution: {integrity: sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==} + fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + + fs-extra@10.1.0: + resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==} + engines: {node: '>=12'} + fs-readdir-recursive@1.1.0: resolution: {integrity: sha512-GNanXlVr2pf02+sPN40XN8HG+ePaNcvM0q5mZBd668Obwb0yD5GiUbZOFgwn8kGMY6I3mdyDJzieUy3PTYyTRA==} @@ -5595,6 +5801,14 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} + get-stream@4.1.0: + resolution: {integrity: sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==} + engines: {node: '>=6'} + + get-stream@5.2.0: + resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} + engines: {node: '>=8'} + get-stream@6.0.1: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} engines: {node: '>=10'} @@ -5681,6 +5895,10 @@ packages: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} + got@12.6.1: + resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==} + engines: {node: '>=14.16'} + graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} @@ -5692,6 +5910,10 @@ packages: resolution: {integrity: sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg==} engines: {node: '>= 0.4'} + has-flag@3.0.0: + resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} + engines: {node: '>=4'} + has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} @@ -5818,10 +6040,33 @@ packages: html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + http-assert@1.5.0: + resolution: {integrity: sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==} + engines: {node: '>= 0.8'} + + http-cache-semantics@4.2.0: + resolution: {integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==} + + http-errors@1.6.3: + resolution: {integrity: sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==} + engines: {node: '>= 0.6'} + + http-errors@1.8.1: + resolution: {integrity: sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==} + engines: {node: '>= 0.6'} + + http-errors@2.0.1: + resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} + engines: {node: '>= 0.8'} + http-proxy-agent@7.0.2: resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} engines: {node: '>= 14'} + http2-wrapper@2.2.1: + resolution: {integrity: sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==} + engines: {node: '>=10.19.0'} + https-proxy-agent@7.0.6: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} @@ -5838,6 +6083,10 @@ packages: resolution: {integrity: sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==} engines: {node: '>=18.18.0'} + iconv-lite@0.4.24: + resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} + engines: {node: '>=0.10.0'} + iconv-lite@0.7.3: resolution: {integrity: sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==} engines: {node: '>=0.10.0'} @@ -5874,10 +6123,17 @@ packages: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} engines: {node: '>=0.8.19'} + inflation@2.1.0: + resolution: {integrity: sha512-t54PPJHG1Pp7VQvxyVCJ9mBbjG3Hqryges9bXoOO6GExCPa+//i/d5GSuFtpx3ALLd7lgIAur6zrIlBQyJuMlQ==} + engines: {node: '>= 0.8.0'} + inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. + inherits@2.0.3: + resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==} + inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} @@ -6060,6 +6316,10 @@ packages: is-ssh@1.4.1: resolution: {integrity: sha512-JNeu1wQsHjyHgn9NcWTaXq6zWSR6hqE0++zhfZlkFBbScNkyvxCdeV8sRkSBaeLKxmbpR21brail63ACNxJ0Tg==} + is-stream@1.1.0: + resolution: {integrity: sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==} + engines: {node: '>=0.10.0'} + is-stream@2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} @@ -6226,6 +6486,19 @@ packages: engines: {node: ^18.0.0 || >=20.0.0} hasBin: true + jsonfile@6.2.1: + resolution: {integrity: sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==} + + jsonschema@1.5.0: + resolution: {integrity: sha512-K+A9hhqbn0f3pJX17Q/7H6yQfD/5OXgdrR5UE12gMXCiN9D5Xq2o5mddV2QEcX/bjla99ASsAAQUyMCCRWAEhw==} + + jsonwebtoken@9.0.3: + resolution: {integrity: sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==} + engines: {node: '>=12', npm: '>=6'} + + jstat@1.9.6: + resolution: {integrity: sha512-rPBkJbK2TnA8pzs93QcDDPlKcrtZWuuCo2dVR0TFLOJSxhqfWOVCSp8aV3/oSbn+4uY4yw1URtLpHQedtmXfug==} + jsx-ast-utils@3.3.5: resolution: {integrity: sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==} engines: {node: '>=4.0'} @@ -6233,9 +6506,19 @@ packages: jszip@3.10.1: resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==} + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + + jws@4.0.1: + resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + kebab-case@2.0.2: resolution: {integrity: sha512-NhIP7vecGtqJfNZ85ct0yRQfx//gCJHapJCBZP6/BBcBw/U218Jw7YX7rO6TI5/cCp5dJvlIjN3vbcRTqq7nWA==} + keygrip@1.1.0: + resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} + engines: {node: '>= 0.6'} + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -6254,6 +6537,36 @@ packages: resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} engines: {node: '>=6'} + koa-bodyparser@4.4.1: + resolution: {integrity: sha512-kBH3IYPMb+iAXnrxIhXnW+gXV8OTzCu8VPDqvcDHW9SQrbkHmqPQtiZwrltNmSq6/lpipHnT7k7PsjlVD7kK0w==} + engines: {node: '>=8.0.0'} + + koa-compose@4.1.0: + resolution: {integrity: sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==} + + koa-convert@2.0.0: + resolution: {integrity: sha512-asOvN6bFlSnxewce2e/DK3p4tltyfC4VM7ZwuTuepI7dEQVcvpyFuBcEARu1+Hxg8DIwytce2n7jrZtRlPrARA==} + engines: {node: '>= 10'} + + koa-mount@4.2.0: + resolution: {integrity: sha512-2iHQc7vbA9qLeVq5gKAYh3m5DOMMlMfIKjW/REPAS18Mf63daCJHHVXY9nbu7ivrnYn5PiPC4CE523Tf5qvjeQ==} + engines: {node: '>= 7.6.0'} + + koa-node-resolve@1.0.0-pre.9: + resolution: {integrity: sha512-WKgqe5TGVD6zuR3NrKnmbb/NNHIbWOCezQVqqnyQLdtLLXWgiothlUQT23S5qQGE0Z623jp6jxpMjvAqyrcZFQ==} + + koa-send@5.0.1: + resolution: {integrity: sha512-tmcyQ/wXXuxpDxyNXv5yNNkdAMdFRqwtegBXUaowiQzUKqJehttS0x2j0eOZDQAyloAth5w6wwBImnFzkUz3pQ==} + engines: {node: '>= 8'} + + koa-static@5.0.0: + resolution: {integrity: sha512-UqyYyH5YEXaJrf9S8E23GoJFQZXkBVJ9zYYMPGz919MSX1KuvAcycIuS0ci150HCoPf4XQVhQ84Qf8xRPWxFaQ==} + engines: {node: '>= 7.6.0'} + + koa@2.16.4: + resolution: {integrity: sha512-3An0GCLDSR34tsCO4H8Tef8Pp2ngtaZDAZnsWJYelqXUK5wyiHvGItgK/xcSkmHLSTn1Jcho1mRQs2ehRzvKKw==} + engines: {node: ^4.8.4 || ^6.10.1 || ^7.10.1 || >= 8.1.4} + language-subtag-registry@0.3.23: resolution: {integrity: sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ==} @@ -6364,6 +6677,13 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} + locate-path@7.2.0: + resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + lodash.camelcase@4.3.0: + resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + lodash.clonedeep@4.5.0: resolution: {integrity: sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==} @@ -6376,15 +6696,30 @@ packages: lodash.groupby@4.6.0: resolution: {integrity: sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==} + lodash.includes@4.3.0: + resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} + lodash.isboolean@3.0.3: resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} lodash.isfunction@3.0.9: resolution: {integrity: sha512-AirXNj15uRIMMPihnkInB4i3NHeb4iBtNg9WRWuK2o31S+ePwwNmDPaTL3o7dTJ+VXNZim7rFs4rxN4YU1oUJw==} + lodash.isinteger@4.0.4: + resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==} + lodash.isnil@4.0.0: resolution: {integrity: sha512-up2Mzq3545mwVnMhTDMdfoG1OurpA/s5t88JmQX809eH3C8491iu2sfKhTfhQtKY78oPNhiaHJUpT/dUDAAtng==} + lodash.isnumber@3.0.3: + resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==} + + lodash.isplainobject@4.0.6: + resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} + + lodash.isstring@4.0.1: + resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==} + lodash.isundefined@3.0.1: resolution: {integrity: sha512-MXB1is3s899/cD8jheYYE2V9qTHwKvt+npCwpD+1Sxm3Q3cECXCiYHjeHWXNwr6Q0SOBPrYUDxendrO6goVTEA==} @@ -6397,6 +6732,9 @@ packages: lodash.mergewith@4.6.2: resolution: {integrity: sha512-GK3g5RPZWTRSeLSpgP8Xhra+pnjBC56q9FZYe1d5RN3TJ35dbkGy3YqBSMbyCrlbi+CM9Z3Jk5yTL7RCsqboyQ==} + lodash.once@4.1.1: + resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==} + lodash.truncate@4.4.2: resolution: {integrity: sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==} @@ -6416,6 +6754,10 @@ packages: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true + lowercase-keys@3.0.0: + resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} @@ -6535,6 +6877,10 @@ packages: mdn-data@2.27.1: resolution: {integrity: sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==} + media-typer@0.3.0: + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} + engines: {node: '>= 0.6'} + meow@14.1.0: resolution: {integrity: sha512-EDYo6VlmtnumlcBCbh1gLJ//9jvM/ndXHfVXIFrZVr6fGcwTUyCTFNTLCKuY3ffbK8L/+3Mzqnd58RojiZqHVw==} engines: {node: '>=20'} @@ -6662,6 +7008,10 @@ packages: resolution: {integrity: sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==} engines: {node: '>= 0.6'} + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + mime-db@1.54.0: resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} engines: {node: '>= 0.6'} @@ -6670,6 +7020,10 @@ packages: resolution: {integrity: sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==} engines: {node: '>= 0.6'} + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + mime-types@3.0.2: resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} engines: {node: '>=18'} @@ -6682,6 +7036,14 @@ packages: resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==} engines: {node: '>=12'} + mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + + mimic-response@4.0.0: + resolution: {integrity: sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + min-indent@1.0.1: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} engines: {node: '>=4'} @@ -6754,6 +7116,10 @@ packages: natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + negotiator@0.6.4: resolution: {integrity: sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==} engines: {node: '>= 0.6'} @@ -6782,6 +7148,9 @@ packages: sass: optional: true + nice-try@1.0.5: + resolution: {integrity: sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==} + node-html-parser@9.0.1: resolution: {integrity: sha512-QrdiYYm1NnLRXsMXThUgVcF/syWfWIgHFmy8hylWMGFbHFtnRuXLBxxGQvIm3xaIJMUDJ0ayOmT/FGJYT3pZIw==} @@ -6805,6 +7174,10 @@ packages: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} + normalize-url@8.1.1: + resolution: {integrity: sha512-JYc0DPlpGWB40kH5g07gGTrYuMqV653k3uBKY6uITPWds3M0ov3GaWGp9lbE3Bzngx8+XkfzgvASb9vk9JDFXQ==} + engines: {node: '>=14.16'} + npm-install-checks@6.3.0: resolution: {integrity: sha512-W29RiK/xtpCGqn6f3ixfRYGk+zRyr+Ew9F2E20BfXxT5/euLdA/Nm7fO7OeTGuAmTs30cpgInyJ0cYe708YTZw==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -6821,6 +7194,10 @@ packages: resolution: {integrity: sha512-nkc+3pIIhqHVQr085X9d2JzPzLyjzQS96zbruppqC9aZRm/x8xx6xhI98gHtsfELP2bE+loHq8ZaHFHhe+NauA==} engines: {node: ^16.14.0 || >=18.0.0} + npm-run-path@2.0.2: + resolution: {integrity: sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==} + engines: {node: '>=4'} + npm-run-path@4.0.1: resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==} engines: {node: '>=8'} @@ -6879,6 +7256,10 @@ packages: resolution: {integrity: sha512-9miFgM2OFba7hB+pRgvtV84pYTBaoTHohvmIgiRt6dRIzbwEOIaNaP+dIlGs2fNFoB0SeISs0Jz5WFVRid6Xyg==} engines: {node: '>=12.20.0'} + on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + on-headers@1.1.0: resolution: {integrity: sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==} engines: {node: '>= 0.8'} @@ -6894,6 +7275,9 @@ packages: resolution: {integrity: sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==} engines: {node: '>=12'} + only@0.0.2: + resolution: {integrity: sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==} + open@11.0.1: resolution: {integrity: sha512-NzwMUB6C1D0+Kd+9iMS/H4k+Ck3cTX6Ckyfr/gAGlmvSE1LUQZnEZvWBi4PYmMwH/S5SMeTXnE+9uAz8uF+pWw==} engines: {node: '>=20'} @@ -6916,6 +7300,14 @@ packages: resolution: {integrity: sha512-qFOyK5PjiWZd+QQIh+1jhdb9LpxTF0qs7Pm8o5QHYZ0M3vKqSqzsZaEB6oWlxZ+q2sJBMI/Ktgd2N5ZwQoRHfg==} engines: {node: '>= 0.4'} + p-cancelable@3.0.0: + resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==} + engines: {node: '>=12.20'} + + p-finally@1.0.0: + resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} + engines: {node: '>=4'} + p-limit@2.3.0: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} engines: {node: '>=6'} @@ -6924,6 +7316,10 @@ packages: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} + p-limit@4.0.0: + resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + p-locate@4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -6932,6 +7328,10 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} + p-locate@6.0.0: + resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + p-retry@8.0.0: resolution: {integrity: sha512-kFVqH1HxOHp8LupNsOys7bSV09VYTRLxarH/mokO4Rqhk6wGi70E0jh4VzvVGXfEVNggHoHLAMWsQqHyU1Ey9A==} engines: {node: '>=22'} @@ -6975,16 +7375,27 @@ packages: resolution: {integrity: sha512-bCgsFI+GeGWPAvAiUv63ZorMeif3/U0zaXABGJbOWt5OH2KCaPHF6S+0ok4aqM9RuIPGyZdx9tR9l13PsW4AYQ==} engines: {node: '>=14.13.0'} + parse5@5.1.1: + resolution: {integrity: sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==} + parse5@7.3.0: resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} parse5@8.0.0: resolution: {integrity: sha512-9m4m5GSgXjL4AjumKzq1Fgfp3Z8rsvjRNbnkVwfu2ImRqE5D0LnY2QfDen18FSY9C573YU5XxSapdHZTZ2WolA==} + parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} + path-exists@5.0.0: + resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + path-is-absolute@1.0.1: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} @@ -6992,6 +7403,10 @@ packages: path-is-inside@1.0.2: resolution: {integrity: sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==} + path-key@2.0.1: + resolution: {integrity: sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==} + engines: {node: '>=4'} + path-key@3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} @@ -7047,10 +7462,17 @@ packages: resolution: {integrity: sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==} engines: {node: '>=8'} + pkg-install@1.0.0: + resolution: {integrity: sha512-UGI8bfhrDb1KN01RZ7Bq08GRQc8rmVjxQ2up0g4mUHPCYDTK1FzQ0PMmLOBCHg3yaIijZ2U3Fn9ofLa4N392Ug==} + pkg-pr-new@0.0.88: resolution: {integrity: sha512-Xc6PMJ2gher0WZP+rtjefFk26hIb7V1PTLL30bmy1Z2vsRmSvqiss7Ag1XUdyplTGYzIlFNJp4L3vMNmK44N6g==} hasBin: true + pkg-up@4.0.0: + resolution: {integrity: sha512-N4zdA4sfOe6yCv+ulPCmpnIBQ5I60xfhDr1otdBBhKte9QtEf3bhfrfkW7dTb+IQ0iEx4ZDzas0kc1o5rdWpYg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + playwright-core@1.62.1: resolution: {integrity: sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==} engines: {node: '>=20'} @@ -7167,6 +7589,10 @@ packages: process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + progress@2.0.3: + resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} + engines: {node: '>=0.4.0'} + promise-inflight@1.0.1: resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==} peerDependencies: @@ -7195,6 +7621,9 @@ packages: protocols@2.0.2: resolution: {integrity: sha512-hHVTzba3wboROl0/aWRRG9dMytgH6ow//STBZh43l/wQgmMhYhOFi0EHWAPtoCz9IAUymsyP0TSBHkhgMEGNnQ==} + pump@3.0.4: + resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} + punycode@1.4.1: resolution: {integrity: sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==} @@ -7218,6 +7647,10 @@ packages: queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + quick-lru@5.1.1: + resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} + engines: {node: '>=10'} + quotation@2.0.3: resolution: {integrity: sha512-yEc24TEgCFLXx7D4JHJJkK4JFVtatO8fziwUxY4nB/Jbea9o9CVS3gt22mA0W7rPYAGW2fWzYDSOtD94PwOyqA==} @@ -7244,6 +7677,10 @@ packages: rasterizehtml@1.4.1: resolution: {integrity: sha512-mcE6b5cKhQO0Qka9vduSQW/F/crQ7EUjrTpXDPAdcj9uNtYijBiNmUOxOrIuMMq7GO/X3CR6UckHTgh5ZVqqBA==} + raw-body@2.5.3: + resolution: {integrity: sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==} + engines: {node: '>= 0.8'} + rc@1.2.8: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true @@ -7307,6 +7744,10 @@ packages: recma-stringify@1.0.0: resolution: {integrity: sha512-cjwII1MdIIVloKvC9ErQ+OgAtwHBmcZ0Bg4ciz78FtbT8In39aAYbaA7zvxQ61xVMSPE8WxhLwLbhif4Js2C+g==} + reduce-flatten@2.0.0: + resolution: {integrity: sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w==} + engines: {node: '>=6'} + reflect.getprototypeof@1.0.10: resolution: {integrity: sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==} engines: {node: '>= 0.4'} @@ -7438,10 +7879,17 @@ packages: reselect@5.2.0: resolution: {integrity: sha512-AgZ3UOZm3YndfrJ4OYjgrT7bmCm/1iqkjvEfH/oYjzh6PD2qw4QuT3jjnXIrpdt4MTpMXclMT3lXbmRY+XRakw==} + resolve-alpn@1.2.1: + resolution: {integrity: sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==} + resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} + resolve-path@1.4.0: + resolution: {integrity: sha512-i1xevIst/Qa+nA9olDxLWnLk8YZbi8R/7JPbCMcgyWaFR6bKWaexgJgEB5oc2PKMjYdrHynyz0NY+if+H98t1w==} + engines: {node: '>= 0.8'} + resolve-pkg-maps@1.0.0: resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} @@ -7454,6 +7902,10 @@ packages: resolution: {integrity: sha512-U7WjGVG9sH8tvjW5SmGbQuui75FiyjAX72HX15DwBBwF9dNiQZRQAg9nnPhYy+TUnE0+VcrttuvNI8oSxZcocA==} hasBin: true + responselike@3.0.0: + resolution: {integrity: sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==} + engines: {node: '>=14.16'} + retry@0.12.0: resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} engines: {node: '>= 4'} @@ -7520,6 +7972,9 @@ packages: sane-domparser-error@0.2.0: resolution: {integrity: sha512-wxjDV5jty95tNv8N/4WA15UNGqqaor/xX7rnNYY961hifN3bheYoKqtXN+V/M6EUgmUAs6pMul3klwUPMEiVXA==} + sanitize-filename@1.6.4: + resolution: {integrity: sha512-9ZyI08PsvdQl2r/bBIGubpVdR3RR9sY6RDiWFPreA21C/EFlQhmgo20UZlNjZMMZNubusLhAQozkA0Od5J21Eg==} + saxes@5.0.1: resolution: {integrity: sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==} engines: {node: '>=10'} @@ -7535,6 +7990,10 @@ packages: resolution: {integrity: sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA==} engines: {node: '>= 10.13.0'} + selenium-webdriver@4.48.0: + resolution: {integrity: sha512-rKM9uXFRWcF9aThrZQDNQH2/9Et/WvMZbg3/x1rnSYWoXiwJuShYeH0IAli8Cuw+c3lEV0UWPfUz88H+fvW9Hg==} + engines: {node: '>= 22.0.0'} + semver@5.7.2: resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} hasBin: true @@ -7577,6 +8036,12 @@ packages: setimmediate@1.0.5: resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==} + setprototypeof@1.1.0: + resolution: {integrity: sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==} + + setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + sharp@0.35.3: resolution: {integrity: sha512-ej0zVHuZGHCiABXcNxeYhpRnPNPAcvbG8RMdBAhDAxLKkCRVSpK3Iyu7qbqw3JMzoj0REeM6f3tJLtVwl0023Q==} engines: {node: '>=20.9.0'} @@ -7586,10 +8051,18 @@ packages: '@types/node': optional: true + shebang-command@1.2.0: + resolution: {integrity: sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==} + engines: {node: '>=0.10.0'} + shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} + shebang-regex@1.0.0: + resolution: {integrity: sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==} + engines: {node: '>=0.10.0'} + shebang-regex@3.0.0: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} @@ -7696,6 +8169,14 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + statuses@1.5.0: + resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==} + engines: {node: '>= 0.6'} + + statuses@2.0.2: + resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} + engines: {node: '>= 0.8'} + std-env@4.1.0: resolution: {integrity: sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==} @@ -7794,6 +8275,10 @@ packages: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} + strip-eof@1.0.0: + resolution: {integrity: sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==} + engines: {node: '>=0.10.0'} + strip-final-newline@2.0.0: resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} engines: {node: '>=6'} @@ -7870,6 +8355,10 @@ packages: resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} engines: {node: '>=18'} + supports-color@5.5.0: + resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} + engines: {node: '>=4'} + supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} @@ -7904,10 +8393,24 @@ packages: resolution: {integrity: sha512-ulAk51I9UVUyJgxlv9M6lFot2WP3e7t8Kz9+IS6D4rVba1tR9kON+Ey69f+1R4Q8cd45Lod6a4IcJIxnzGc/zA==} engines: {node: '>=18'} + systeminformation@5.33.6: + resolution: {integrity: sha512-hMOQG/eRUzuopuYGGdl8ntkau0nEC7fOaRoTUg1RSr2GTQIk2VNa76DA0+ApajkGfzmcgAupgIP/vt+jtoe5EA==} + engines: {node: '>=10.0.0'} + os: [darwin, linux, win32, freebsd, openbsd, netbsd, sunos, android] + hasBin: true + + table-layout@1.0.2: + resolution: {integrity: sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==} + engines: {node: '>=8.0.0'} + table@6.9.0: resolution: {integrity: sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==} engines: {node: '>=10.0.0'} + tachometer@0.7.2: + resolution: {integrity: sha512-e9dtDyWuMFyomQxo16vzti6ymoJd3Dp2yGfKrhAiNzCkb2HiP6A0+SWtqE84hcO6tSKZF8eTFqrxM2uKbliJyQ==} + hasBin: true + tapable@2.3.3: resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} engines: {node: '>=6'} @@ -8010,6 +8513,10 @@ packages: resolution: {integrity: sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw==} engines: {node: '>=12'} + toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + totalist@3.0.1: resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} engines: {node: '>=6'} @@ -8028,6 +8535,9 @@ packages: trough@2.2.0: resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==} + truncate-utf8-bytes@1.0.2: + resolution: {integrity: sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==} + ts-api-utils@2.5.0: resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} engines: {node: '>=18.12'} @@ -8048,6 +8558,10 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsscmp@1.0.6: + resolution: {integrity: sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==} + engines: {node: '>=0.6.x'} + tsx@4.23.12: resolution: {integrity: sha512-FDf4L4sYzKtzWYhU/Xm0AQFdTjdIxNo9ElTf2mxXM6k8YMHXzYUe4yODVaXP4V9uMFbVg8c0qyBccK2OOxb45Q==} engines: {node: '>=18.0.0'} @@ -8080,6 +8594,10 @@ packages: resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} engines: {node: '>=16'} + type-is@1.6.18: + resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} + engines: {node: '>= 0.6'} + typed-array-buffer@1.0.3: resolution: {integrity: sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==} engines: {node: '>= 0.4'} @@ -8115,6 +8633,18 @@ packages: engines: {node: '>=14.17'} hasBin: true + typical@4.0.0: + resolution: {integrity: sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==} + engines: {node: '>=8'} + + typical@5.2.0: + resolution: {integrity: sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==} + engines: {node: '>=8'} + + ua-parser-js@1.0.41: + resolution: {integrity: sha512-LbBDqdIC5s8iROCUjMbW1f5dJQTEFB1+KO9ogbvlb3nm9n4YHa5p4KTvFPWvh2Hs8gZMBuiB1/8+pdfe/tDPug==} + hasBin: true + uint8-to-base64@0.2.1: resolution: {integrity: sha512-uO/84GaoDUfiAxpa8EksjVLE77A9Kc7ZTziN4zRpq4de9yLaLcZn3jx1/sVjyupsywcVX6RKWbqLe7gUNyzH+Q==} @@ -8209,6 +8739,14 @@ packages: universal-user-agent@7.0.3: resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==} + universalify@2.0.1: + resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} + engines: {node: '>= 10.0.0'} + + unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + unrs-resolver@1.11.1: resolution: {integrity: sha512-bSjt9pjaEBnNiGgc9rUiHGKv5l4/TGzDmYw3RhnkJGtLhbnnA/5qJj7x3dNDCRx/PJxu774LlH8lCOlB4hEfKg==} @@ -8233,6 +8771,9 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + utf8-byte-length@1.0.5: + resolution: {integrity: sha512-Xn0w3MtiQ6zoz2vFyUVruaCL53O/DwUvkEeOvj+uulMm0BkUGYWmBYVyElqZaSLhY6ZD0ulfU3aBra2aVT4xfA==} + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -8465,6 +9006,10 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + wordwrapjs@4.0.1: + resolution: {integrity: sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==} + engines: {node: '>=8.0.0'} + wrap-ansi@6.2.0: resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} engines: {node: '>=8'} @@ -8512,6 +9057,18 @@ packages: utf-8-validate: optional: true + ws@8.21.3: + resolution: {integrity: sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + wsl-utils@1.0.0: resolution: {integrity: sha512-Hl0ZOAs672vg+06kfujwRhoS6/jehvULrlFkuF2dRu6pHgA8U06h3xqNIqNNU1LTXPcedxByAR4GS6pwQK0mgA==} engines: {node: '>=20'} @@ -8569,10 +9126,18 @@ packages: resolution: {integrity: sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg==} engines: {node: ^20.19.0 || ^22.12.0 || >=23} + ylru@1.4.0: + resolution: {integrity: sha512-2OQsPNEmBCvXuFlIni/a+Rn+R2pHW9INm0BxXJ4hVDA8TirqMj+J/Rp9ItLatT/5pZqWwefVrTQcHpixsxnVlA==} + engines: {node: '>= 4.0.0'} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + yocto-queue@1.2.2: + resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} + engines: {node: '>=12.20'} + yoctocolors@2.1.2: resolution: {integrity: sha512-CzhO+pFNo8ajLM2d2IW/R93ipy99LWjtwblvC1RsoSUMZgyLbYFr221TnSNT7GjGdYui6P459mw9JH/g/zW2ug==} engines: {node: '>=18'} @@ -9703,6 +10268,8 @@ snapshots: optionalDependencies: '@types/react': 19.2.18 + '@bazel/runfiles@6.5.0': {} + '@bcoe/v8-coverage@1.0.2': {} '@blazediff/core@1.9.1': {} @@ -10117,6 +10684,8 @@ snapshots: '@floating-ui/utils@0.2.12': {} + '@hapi/bourne@3.0.0': {} + '@html-validate/stylish@6.0.0': {} '@humanfs/core@0.19.2': @@ -11299,6 +11868,8 @@ snapshots: '@sinclair/typebox@0.34.41': {} + '@sindresorhus/is@5.6.0': {} + '@sindresorhus/merge-streams@4.0.0': {} '@smithy/core@3.33.2': @@ -11340,6 +11911,10 @@ snapshots: dependencies: tslib: 2.8.1 + '@szmarczak/http-timer@5.0.1': + dependencies: + defer-to-connect: 2.0.1 + '@tanstack/query-core@5.101.4': {} '@tanstack/react-query@5.101.4(react@19.2.8)': @@ -11494,6 +12069,10 @@ snapshots: dependencies: '@types/node': 22.20.1 + '@types/execa@0.9.0': + dependencies: + '@types/node': 22.20.1 + '@types/find-package-json@1.2.7': dependencies: '@types/node': 22.20.1 @@ -11506,6 +12085,8 @@ snapshots: dependencies: '@types/unist': 3.0.3 + '@types/http-cache-semantics@4.2.0': {} + '@types/is-empty@1.2.3': {} '@types/jsdom@28.0.3': @@ -11531,6 +12112,8 @@ snapshots: '@types/ms@2.1.0': {} + '@types/node@11.15.54': {} + '@types/node@14.18.63': {} '@types/node@18.19.130': @@ -11551,6 +12134,8 @@ snapshots: dependencies: parse-path: 7.1.0 + '@types/parse5@5.0.3': {} + '@types/prop-types@15.7.15': {} '@types/proper-lockfile@4.1.4': @@ -12013,6 +12598,11 @@ snapshots: abbrev@2.0.0: {} + accepts@1.3.8: + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + acorn-import-phases@1.0.4(acorn@8.16.0): dependencies: acorn: 8.16.0 @@ -12056,10 +12646,18 @@ snapshots: dependencies: string-width: 4.2.3 + ansi-escape-sequences@6.2.4: + dependencies: + array-back: 6.2.3 + ansi-regex@5.0.1: {} ansi-regex@6.2.2: {} + ansi-styles@3.2.1: + dependencies: + color-convert: 1.9.3 + ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 @@ -12088,6 +12686,12 @@ snapshots: aria-query@5.3.2: {} + array-back@3.1.0: {} + + array-back@4.0.2: {} + + array-back@6.2.3: {} + array-buffer-byte-length@1.0.2: dependencies: call-bound: 1.0.4 @@ -12355,6 +12959,8 @@ snapshots: node-releases: 2.0.51 update-browserslist-db: 1.2.3(browserslist@4.28.7) + buffer-equal-constant-time@1.0.1: {} + buffer-from@0.1.2: optional: true @@ -12375,6 +12981,23 @@ snapshots: bytes@3.1.2: {} + cache-content-type@1.0.1: + dependencies: + mime-types: 2.1.18 + ylru: 1.4.0 + + cacheable-lookup@7.0.0: {} + + cacheable-request@10.2.14: + dependencies: + '@types/http-cache-semantics': 4.2.0 + get-stream: 6.0.1 + http-cache-semantics: 4.2.0 + keyv: 4.5.4 + mimic-response: 4.0.0 + normalize-url: 8.1.1 + responselike: 3.0.0 + cacheable@2.5.0: dependencies: '@cacheable/memory': 2.2.0 @@ -12420,6 +13043,12 @@ snapshots: dependencies: chalk: 4.1.2 + chalk@2.4.2: + dependencies: + ansi-styles: 3.2.1 + escape-string-regexp: 1.0.5 + supports-color: 5.5.0 + chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -12497,20 +13126,50 @@ snapshots: clsx@2.1.1: {} + co-body@6.2.0: + dependencies: + '@hapi/bourne': 3.0.0 + inflation: 2.1.0 + qs: 6.15.2 + raw-body: 2.5.3 + type-is: 1.6.18 + + co@4.6.0: {} + codsen-utils@1.8.0: {} collapse-white-space@2.1.0: {} + color-convert@1.9.3: + dependencies: + color-name: 1.1.3 + color-convert@2.0.1: dependencies: color-name: 1.1.4 + color-name@1.1.3: {} + color-name@1.1.4: {} colord@2.9.3: {} comma-separated-tokens@2.0.3: {} + command-line-args@5.2.1: + dependencies: + array-back: 3.1.0 + find-replace: 3.0.0 + lodash.camelcase: 4.3.0 + typical: 4.0.0 + + command-line-usage@6.1.3: + dependencies: + array-back: 4.0.2 + chalk: 2.4.2 + table-layout: 1.0.2 + typical: 5.2.0 + commander@2.20.3: {} commander@4.1.1: {} @@ -12548,6 +13207,8 @@ snapshots: content-disposition@0.5.2: {} + content-type@1.0.5: {} + content-type@2.1.0: {} content-type@3.0.0: {} @@ -12561,6 +13222,13 @@ snapshots: lodash.clonedeep: 4.5.0 yargs-parser: 20.2.9 + cookies@0.9.1: + dependencies: + depd: 2.0.0 + keygrip: 1.1.0 + + copy-to@2.0.1: {} + core-js-compat@3.48.0: dependencies: browserslist: 4.28.7 @@ -12592,6 +13260,14 @@ snapshots: nan: 2.23.0 optional: true + cross-spawn@6.0.6: + dependencies: + nice-try: 1.0.5 + path-key: 2.0.1 + semver: 5.7.2 + shebang-command: 1.2.0 + which: 1.3.1 + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -12633,6 +13309,8 @@ snapshots: csstype@3.2.3: {} + csv-stringify@6.8.3: {} + d3-array@3.2.4: dependencies: internmap: 2.0.3 @@ -12736,6 +13414,12 @@ snapshots: dependencies: character-entities: 2.0.2 + decompress-response@6.0.0: + dependencies: + mimic-response: 3.1.0 + + deep-equal@1.0.1: {} + deep-extend@0.6.0: {} deep-is@0.1.4: {} @@ -12747,6 +13431,8 @@ snapshots: bundle-name: 4.1.0 default-browser-id: 5.0.0 + defer-to-connect@2.0.1: {} + define-data-property@1.1.4: dependencies: es-define-property: 1.0.1 @@ -12761,8 +13447,16 @@ snapshots: has-property-descriptors: 1.0.2 object-keys: 1.1.1 + delegates@1.0.0: {} + + depd@1.1.2: {} + + depd@2.0.0: {} + dequal@2.0.3: {} + destroy@1.2.0: {} + detect-libc@2.1.2: {} devlop@1.1.0: @@ -12819,6 +13513,12 @@ snapshots: eastasianwidth@0.2.0: {} + ecdsa-sig-formatter@1.0.11: + dependencies: + safe-buffer: 5.2.1 + + ee-first@1.1.1: {} + electron-to-chromium@1.5.396: {} emoji-regex@10.6.0: {} @@ -12827,6 +13527,12 @@ snapshots: emoji-regex@9.2.2: {} + encodeurl@1.0.2: {} + + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + enhanced-resolve@5.23.0: dependencies: graceful-fs: 4.2.11 @@ -13001,6 +13707,10 @@ snapshots: escalade@3.2.0: {} + escape-html@1.0.3: {} + + escape-string-regexp@1.0.5: {} + escape-string-regexp@4.0.0: {} escape-string-regexp@5.0.0: {} @@ -13391,6 +14101,16 @@ snapshots: events@3.3.0: {} + execa@1.0.0: + dependencies: + cross-spawn: 6.0.6 + get-stream: 4.1.0 + is-stream: 1.1.0 + npm-run-path: 2.0.2 + p-finally: 1.0.0 + signal-exit: 3.0.7 + strip-eof: 1.0.0 + execa@10.0.1: dependencies: '@sindresorhus/merge-streams': 4.0.0 @@ -13530,6 +14250,10 @@ snapshots: find-package-json@1.2.0: {} + find-replace@3.0.0: + dependencies: + array-back: 3.1.0 + find-root@1.1.0: {} find-up@4.1.0: @@ -13542,6 +14266,11 @@ snapshots: locate-path: 6.0.0 path-exists: 4.0.0 + find-up@6.3.0: + dependencies: + locate-path: 7.2.0 + path-exists: 5.0.0 + flat-cache@4.0.1: dependencies: flatted: 3.4.2 @@ -13566,12 +14295,22 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 + form-data-encoder@2.1.4: {} + format-util@1.0.5: {} format@0.2.2: {} fraction.js@5.3.4: {} + fresh@0.5.2: {} + + fs-extra@10.1.0: + dependencies: + graceful-fs: 4.2.11 + jsonfile: 6.2.1 + universalify: 2.0.1 + fs-readdir-recursive@1.1.0: {} fs.realpath@1.0.0: {} @@ -13627,6 +14366,14 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.2 + get-stream@4.1.0: + dependencies: + pump: 3.0.4 + + get-stream@5.2.0: + dependencies: + pump: 3.0.4 + get-stream@6.0.1: {} get-stream@8.0.1: {} @@ -13722,6 +14469,20 @@ snapshots: gopd@1.2.0: {} + got@12.6.1: + dependencies: + '@sindresorhus/is': 5.6.0 + '@szmarczak/http-timer': 5.0.1 + cacheable-lookup: 7.0.0 + cacheable-request: 10.2.14 + decompress-response: 6.0.0 + form-data-encoder: 2.1.4 + get-stream: 6.0.1 + http2-wrapper: 2.2.1 + lowercase-keys: 3.0.0 + p-cancelable: 3.0.0 + responselike: 3.0.0 + graceful-fs@4.2.11: {} gzip-size@6.0.0: @@ -13730,6 +14491,8 @@ snapshots: has-bigints@1.1.0: {} + has-flag@3.0.0: {} + has-flag@4.0.0: {} has-flag@5.0.1: {} @@ -13917,6 +14680,36 @@ snapshots: html-void-elements@3.0.0: {} + http-assert@1.5.0: + dependencies: + deep-equal: 1.0.1 + http-errors: 1.8.1 + + http-cache-semantics@4.2.0: {} + + http-errors@1.6.3: + dependencies: + depd: 1.1.2 + inherits: 2.0.3 + setprototypeof: 1.1.0 + statuses: 1.5.0 + + http-errors@1.8.1: + dependencies: + depd: 1.1.2 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 1.5.0 + toidentifier: 1.0.1 + + http-errors@2.0.1: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.2 + toidentifier: 1.0.1 + http-proxy-agent@7.0.2(supports-color@10.2.2): dependencies: agent-base: 7.1.4 @@ -13924,6 +14717,11 @@ snapshots: transitivePeerDependencies: - supports-color + http2-wrapper@2.2.1: + dependencies: + quick-lru: 5.1.1 + resolve-alpn: 1.2.1 + https-proxy-agent@7.0.6(supports-color@10.2.2): dependencies: agent-base: 7.1.4 @@ -13937,6 +14735,10 @@ snapshots: human-signals@8.0.1: {} + iconv-lite@0.4.24: + dependencies: + safer-buffer: 2.1.2 + iconv-lite@0.7.3: dependencies: safer-buffer: 2.1.2 @@ -13962,11 +14764,15 @@ snapshots: imurmurhash@0.1.4: {} + inflation@2.1.0: {} + inflight@1.0.6: dependencies: once: 1.4.0 wrappy: 1.0.2 + inherits@2.0.3: {} + inherits@2.0.4: {} ini@1.3.8: {} @@ -14127,6 +14933,8 @@ snapshots: dependencies: protocols: 2.0.2 + is-stream@1.1.0: {} + is-stream@2.0.1: {} is-stream@3.0.0: {} @@ -14287,6 +15095,29 @@ snapshots: dependencies: '@dmsnell/diff-match-patch': 1.1.0 + jsonfile@6.2.1: + dependencies: + universalify: 2.0.1 + optionalDependencies: + graceful-fs: 4.2.11 + + jsonschema@1.5.0: {} + + jsonwebtoken@9.0.3: + dependencies: + jws: 4.0.1 + lodash.includes: 4.3.0 + lodash.isboolean: 3.0.3 + lodash.isinteger: 4.0.4 + lodash.isnumber: 3.0.3 + lodash.isplainobject: 4.0.6 + lodash.isstring: 4.0.1 + lodash.once: 4.1.1 + ms: 2.1.3 + semver: 7.8.5 + + jstat@1.9.6: {} + jsx-ast-utils@3.3.5: dependencies: array-includes: 3.1.9 @@ -14301,8 +15132,23 @@ snapshots: readable-stream: 2.3.8 setimmediate: 1.0.5 + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + + jws@4.0.1: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.2.1 + kebab-case@2.0.2: {} + keygrip@1.1.0: + dependencies: + tsscmp: 1.0.6 + keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -14317,6 +15163,82 @@ snapshots: kleur@4.1.5: {} + koa-bodyparser@4.4.1: + dependencies: + co-body: 6.2.0 + copy-to: 2.0.1 + type-is: 1.6.18 + + koa-compose@4.1.0: {} + + koa-convert@2.0.0: + dependencies: + co: 4.6.0 + koa-compose: 4.1.0 + + koa-mount@4.2.0(supports-color@10.2.2): + dependencies: + debug: 4.4.3(supports-color@10.2.2) + koa-compose: 4.1.0 + transitivePeerDependencies: + - supports-color + + koa-node-resolve@1.0.0-pre.9(supports-color@10.2.2): + dependencies: + '@babel/generator': 7.29.7 + '@babel/parser': 7.29.7 + '@babel/traverse': 7.29.7(supports-color@10.2.2) + '@types/babel__generator': 7.27.0 + '@types/parse5': 5.0.3 + get-stream: 5.2.0 + parse5: 5.1.1 + resolve: 1.22.12 + transitivePeerDependencies: + - supports-color + + koa-send@5.0.1(supports-color@10.2.2): + dependencies: + debug: 4.4.3(supports-color@10.2.2) + http-errors: 1.8.1 + resolve-path: 1.4.0 + transitivePeerDependencies: + - supports-color + + koa-static@5.0.0(supports-color@10.2.2): + dependencies: + debug: 3.2.7(supports-color@10.2.2) + koa-send: 5.0.1(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + koa@2.16.4(supports-color@10.2.2): + dependencies: + accepts: 1.3.8 + cache-content-type: 1.0.1 + content-disposition: 0.5.2 + content-type: 1.0.5 + cookies: 0.9.1 + debug: 4.4.3(supports-color@10.2.2) + delegates: 1.0.0 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 1.0.2 + escape-html: 1.0.3 + fresh: 0.5.2 + http-assert: 1.5.0 + http-errors: 1.8.1 + is-generator-function: 1.1.2 + koa-compose: 4.1.0 + koa-convert: 2.0.0 + on-finished: 2.4.1 + only: 0.0.2 + parseurl: 1.3.3 + statuses: 1.5.0 + type-is: 1.6.18 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + language-subtag-registry@0.3.23: {} language-tags@1.0.9: @@ -14402,6 +15324,12 @@ snapshots: dependencies: p-locate: 5.0.0 + locate-path@7.2.0: + dependencies: + p-locate: 6.0.0 + + lodash.camelcase@4.3.0: {} + lodash.clonedeep@4.5.0: {} lodash.debounce@4.0.8: {} @@ -14410,12 +15338,22 @@ snapshots: lodash.groupby@4.6.0: {} + lodash.includes@4.3.0: {} + lodash.isboolean@3.0.3: {} lodash.isfunction@3.0.9: {} + lodash.isinteger@4.0.4: {} + lodash.isnil@4.0.0: {} + lodash.isnumber@3.0.3: {} + + lodash.isplainobject@4.0.6: {} + + lodash.isstring@4.0.1: {} + lodash.isundefined@3.0.1: {} lodash.memoize@4.1.2: {} @@ -14424,6 +15362,8 @@ snapshots: lodash.mergewith@4.6.2: {} + lodash.once@4.1.1: {} + lodash.truncate@4.4.2: {} lodash.uniq@4.5.0: {} @@ -14438,6 +15378,8 @@ snapshots: dependencies: js-tokens: 4.0.0 + lowercase-keys@3.0.0: {} + lru-cache@10.4.3: {} lru-cache@11.2.7: {} @@ -14676,6 +15618,8 @@ snapshots: mdn-data@2.27.1: {} + media-typer@0.3.0: {} + meow@14.1.0: {} merge-stream@2.0.0: {} @@ -14960,12 +15904,18 @@ snapshots: mime-db@1.33.0: {} + mime-db@1.52.0: {} + mime-db@1.54.0: {} mime-types@2.1.18: dependencies: mime-db: 1.33.0 + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + mime-types@3.0.2: dependencies: mime-db: 1.54.0 @@ -14974,6 +15924,10 @@ snapshots: mimic-fn@4.0.0: {} + mimic-response@3.1.0: {} + + mimic-response@4.0.0: {} + min-indent@1.0.1: {} minimatch@10.2.6: @@ -15038,6 +15992,8 @@ snapshots: natural-compare@1.4.0: {} + negotiator@0.6.3: {} + negotiator@0.6.4: {} neo-async@2.6.2: {} @@ -15069,6 +16025,8 @@ snapshots: - '@types/node' - babel-plugin-macros + nice-try@1.0.5: {} + node-html-parser@9.0.1: dependencies: css-select: 5.2.2 @@ -15090,6 +16048,8 @@ snapshots: normalize-path@3.0.0: {} + normalize-url@8.1.1: {} + npm-install-checks@6.3.0: dependencies: semver: 7.8.5 @@ -15110,6 +16070,10 @@ snapshots: npm-package-arg: 11.0.3 semver: 7.8.5 + npm-run-path@2.0.2: + dependencies: + path-key: 2.0.1 + npm-run-path@4.0.1: dependencies: path-key: 3.1.1 @@ -15176,6 +16140,10 @@ snapshots: obug@2.1.3: {} + on-finished@2.4.1: + dependencies: + ee-first: 1.1.1 + on-headers@1.1.0: {} once@1.4.0: @@ -15190,6 +16158,8 @@ snapshots: dependencies: mimic-fn: 4.0.0 + only@0.0.2: {} + open@11.0.1: dependencies: default-browser: 5.5.0 @@ -15222,6 +16192,10 @@ snapshots: object-keys: 1.1.1 safe-push-apply: 1.0.0 + p-cancelable@3.0.0: {} + + p-finally@1.0.0: {} + p-limit@2.3.0: dependencies: p-try: 2.2.0 @@ -15230,6 +16204,10 @@ snapshots: dependencies: yocto-queue: 0.1.0 + p-limit@4.0.0: + dependencies: + yocto-queue: 1.2.2 + p-locate@4.1.0: dependencies: p-limit: 2.3.0 @@ -15238,6 +16216,10 @@ snapshots: dependencies: p-limit: 3.1.0 + p-locate@6.0.0: + dependencies: + p-limit: 4.0.0 + p-retry@8.0.0: dependencies: is-network-error: 1.3.2 @@ -15290,6 +16272,8 @@ snapshots: '@types/parse-path': 7.1.0 parse-path: 7.1.0 + parse5@5.1.1: {} + parse5@7.3.0: dependencies: entities: 6.0.1 @@ -15298,12 +16282,18 @@ snapshots: dependencies: entities: 6.0.1 + parseurl@1.3.3: {} + path-exists@4.0.0: {} + path-exists@5.0.0: {} + path-is-absolute@1.0.1: {} path-is-inside@1.0.2: {} + path-key@2.0.1: {} + path-key@3.1.1: {} path-key@4.0.0: {} @@ -15341,8 +16331,18 @@ snapshots: dependencies: find-up: 4.1.0 + pkg-install@1.0.0: + dependencies: + '@types/execa': 0.9.0 + '@types/node': 11.15.54 + execa: 1.0.0 + pkg-pr-new@0.0.88: {} + pkg-up@4.0.0: + dependencies: + find-up: 6.3.0 + playwright-core@1.62.1: {} playwright@1.62.1: @@ -15445,6 +16445,8 @@ snapshots: process-nextick-args@2.0.1: {} + progress@2.0.3: {} + promise-inflight@1.0.1(bluebird@3.4.7): optionalDependencies: bluebird: 3.4.7 @@ -15475,6 +16477,11 @@ snapshots: protocols@2.0.2: {} + pump@3.0.4: + dependencies: + end-of-stream: 1.4.5 + once: 1.4.0 + punycode@1.4.1: optional: true @@ -15493,10 +16500,11 @@ snapshots: qs@6.15.2: dependencies: side-channel: 1.1.0 - optional: true queue-microtask@1.2.3: {} + quick-lru@5.1.1: {} + quotation@2.0.3: {} range-parser@1.2.0: {} @@ -15529,6 +16537,13 @@ snapshots: xmlserializer: 0.6.1 optional: true + raw-body@2.5.3: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.1 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + rc@1.2.8: dependencies: deep-extend: 0.6.0 @@ -15623,6 +16638,8 @@ snapshots: unified: 11.0.5 vfile: 6.0.3 + reduce-flatten@2.0.0: {} + reflect.getprototypeof@1.0.10: dependencies: call-bind: 1.0.8 @@ -15916,8 +16933,15 @@ snapshots: reselect@5.2.0: {} + resolve-alpn@1.2.1: {} + resolve-from@4.0.0: {} + resolve-path@1.4.0: + dependencies: + http-errors: 1.6.3 + path-is-absolute: 1.0.1 + resolve-pkg-maps@1.0.0: {} resolve@1.22.12: @@ -15933,6 +16957,10 @@ snapshots: path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 + responselike@3.0.0: + dependencies: + lowercase-keys: 3.0.0 + retry@0.12.0: {} reusify@1.1.0: {} @@ -16040,6 +17068,10 @@ snapshots: sane-domparser-error@0.2.0: optional: true + sanitize-filename@1.6.4: + dependencies: + truncate-utf8-bytes: 1.0.2 + saxes@5.0.1: dependencies: xmlchars: 2.2.0 @@ -16057,6 +17089,16 @@ snapshots: ajv-formats: 2.1.1(ajv@8.18.0) ajv-keywords: 5.1.0(ajv@8.18.0) + selenium-webdriver@4.48.0: + dependencies: + '@bazel/runfiles': 6.5.0 + jszip: 3.10.1 + tmp: 0.2.7 + ws: 8.21.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + semver@5.7.2: {} semver@6.3.1: {} @@ -16117,6 +17159,10 @@ snapshots: setimmediate@1.0.5: {} + setprototypeof@1.1.0: {} + + setprototypeof@1.2.0: {} + sharp@0.35.3(@types/node@22.20.1): dependencies: '@img/colour': 1.1.0 @@ -16150,10 +17196,16 @@ snapshots: '@img/sharp-win32-x64': 0.35.3 '@types/node': 22.20.1 + shebang-command@1.2.0: + dependencies: + shebang-regex: 1.0.0 + shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 + shebang-regex@1.0.0: {} + shebang-regex@3.0.0: {} side-channel-list@1.0.0: @@ -16259,6 +17311,10 @@ snapshots: stackback@0.0.2: {} + statuses@1.5.0: {} + + statuses@2.0.2: {} + std-env@4.1.0: {} stop-iteration-iterator@1.1.0: @@ -16401,6 +17457,8 @@ snapshots: strip-bom@3.0.0: {} + strip-eof@1.0.0: {} + strip-final-newline@2.0.0: {} strip-final-newline@3.0.0: {} @@ -16500,6 +17558,10 @@ snapshots: supports-color@10.2.2: {} + supports-color@5.5.0: + dependencies: + has-flag: 3.0.0 + supports-color@7.2.0: dependencies: has-flag: 4.0.0 @@ -16527,6 +17589,15 @@ snapshots: system-architecture@0.1.0: {} + systeminformation@5.33.6: {} + + table-layout@1.0.2: + dependencies: + array-back: 4.0.2 + deep-extend: 0.6.0 + typical: 5.2.0 + wordwrapjs: 4.0.1 + table@6.9.0: dependencies: ajv: 8.18.0 @@ -16535,6 +17606,41 @@ snapshots: string-width: 4.2.3 strip-ansi: 6.0.1 + tachometer@0.7.2(supports-color@10.2.2): + dependencies: + ansi-escape-sequences: 6.2.4 + command-line-args: 5.2.1 + command-line-usage: 6.1.3 + csv-stringify: 6.8.3 + fs-extra: 10.1.0 + get-stream: 6.0.1 + got: 12.6.1 + jsonschema: 1.5.0 + jsonwebtoken: 9.0.3 + jstat: 1.9.6 + koa: 2.16.4(supports-color@10.2.2) + koa-bodyparser: 4.4.1 + koa-mount: 4.2.0(supports-color@10.2.2) + koa-node-resolve: 1.0.0-pre.9(supports-color@10.2.2) + koa-send: 5.0.1(supports-color@10.2.2) + koa-static: 5.0.0(supports-color@10.2.2) + pkg-install: 1.0.0 + pkg-up: 4.0.0 + progress: 2.0.3 + sanitize-filename: 1.6.4 + selenium-webdriver: 4.48.0 + semver: 7.8.5 + source-map-support: 0.5.21 + strip-ansi: 7.2.0 + systeminformation: 5.33.6 + table: 6.9.0 + ua-parser-js: 1.0.41 + transitivePeerDependencies: + - '@75lb/nature' + - bufferutil + - supports-color + - utf-8-validate + tapable@2.3.3: {} terser-webpack-plugin@5.6.1(esbuild@0.28.1)(lightningcss@1.33.0)(postcss@8.5.26)(webpack@5.107.2(esbuild@0.28.1)(lightningcss@1.33.0)(postcss@8.5.26)): @@ -16600,6 +17706,8 @@ snapshots: toad-cache@3.7.0: {} + toidentifier@1.0.1: {} + totalist@3.0.1: {} tough-cookie@6.0.0: @@ -16614,6 +17722,10 @@ snapshots: trough@2.2.0: {} + truncate-utf8-bytes@1.0.2: + dependencies: + utf8-byte-length: 1.0.5 + ts-api-utils@2.5.0(typescript@6.0.3): dependencies: typescript: 6.0.3 @@ -16635,6 +17747,8 @@ snapshots: tslib@2.8.1: {} + tsscmp@1.0.6: {} + tsx@4.23.12: dependencies: esbuild: 0.28.1 @@ -16657,6 +17771,11 @@ snapshots: type-fest@4.41.0: {} + type-is@1.6.18: + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + typed-array-buffer@1.0.3: dependencies: call-bound: 1.0.4 @@ -16710,6 +17829,12 @@ snapshots: typescript@6.0.3: {} + typical@4.0.0: {} + + typical@5.2.0: {} + + ua-parser-js@1.0.41: {} + uint8-to-base64@0.2.1: {} unbox-primitive@1.1.0: @@ -16843,6 +17968,10 @@ snapshots: universal-user-agent@7.0.3: {} + universalify@2.0.1: {} + + unpipe@1.0.0: {} + unrs-resolver@1.11.1: dependencies: napi-postinstall: 0.3.4 @@ -16892,6 +18021,8 @@ snapshots: dependencies: react: 19.2.8 + utf8-byte-length@1.0.5: {} + util-deprecate@1.0.2: {} validate-npm-package-license@3.0.4: @@ -17155,6 +18286,11 @@ snapshots: word-wrap@1.2.5: {} + wordwrapjs@4.0.1: + dependencies: + reduce-flatten: 2.0.0 + typical: 5.2.0 + wrap-ansi@6.2.0: dependencies: ansi-styles: 4.3.0 @@ -17189,6 +18325,8 @@ snapshots: ws@8.21.0: {} + ws@8.21.3: {} + wsl-utils@1.0.0: dependencies: is-wsl: 3.1.1 @@ -17248,8 +18386,12 @@ snapshots: y18n: 5.0.8 yargs-parser: 22.0.0 + ylru@1.4.0: {} + yocto-queue@0.1.0: {} + yocto-queue@1.2.2: {} + yoctocolors@2.1.2: {} zod-validation-error@3.5.4(zod@3.25.76): diff --git a/test/tachometer/README.md b/test/tachometer/README.md new file mode 100644 index 0000000000..75e01dac76 --- /dev/null +++ b/test/tachometer/README.md @@ -0,0 +1,72 @@ +# Tachometer smoke test + +A working harness for `code-infra tacho run`, used to exercise that tooling end to end. + +This is **not** a meaningful benchmark, and it is not trying to be. This repository ships build and +test tooling, not a browser library, so there is nothing here whose render time is worth tracking. +The cases run a deterministic CPU workload purely so the pipeline has something real to drive: +discovering cases, resolving refs, packing a ref's workspace to tarballs, building each variant, and +running Chrome under tachometer. + +For the real thing, see the consumers this tooling was extracted from — `base-ui-mosaic` and +`base-ui-charts` benchmark actual data grids and charts. + +## What it covers + +The two cases are chosen to walk the parts of the pipeline most likely to break: + +| Case | Covers | +| :--------------- | :------------------------------------------------------------------------------- | +| `workload` | The ordinary regression shape: one page, auto-expanded into current vs baseline | +| `workload-large` | A case owning **no page of its own**, parameterising its sibling's with `?size=` | + +`workload-large` is the interesting one. It exercises that a case's query parameters survive the url +rewrite, that `?ref=` is appended with `&` when a url already has a query, and that two cases +referencing one page build it only once. + +Both pages import `@mui/internal-test-utils` and put its value on screen. That is the load-bearing +part: the working tree resolves it through the workspace link to the package's build output, while +every other ref resolves it from a packed tarball in an isolated install. If either path breaks the +page fails to build, rather than quietly measuring nothing. + +## Running + +```bash +pnpm release:build # the harness resolves workspace packages through their build output +pnpm test:tacho # from the repo root +``` + +Or from here, with the usual filters and flags: + +```bash +pnpm -F ./test/tachometer test:tacho large +pnpm -F ./test/tachometer test:tacho --baseline git:HEAD +``` + +The report lands in `results/report.json`. Cases run sequentially, and each auto-samples until its +difference resolves or the (deliberately short) timeout is hit. Because both sides run identical +code, the expected verdict is `unsure` — that is the "no change" outcome, not a failure. + +### Prerequisites + +A real Chrome. Tachometer drives it over WebDriver and installs a matching `chromedriver` on demand; +the driver's major version must match Chrome's, or you will see `SessionNotCreatedError: This +version of ChromeDriver only supports Chrome version N`. Fix it by updating Chrome, or set +`browser.binary` in a case's `tachometer.json` to an existing Chromium. + +### A note on the baseline + +The default baseline is the fork point from the base branch, so a ref older than the introduction of +`code-infra tacho` will fail to build its pages: this harness's vite config imports a plugin that +did not exist at that commit. Pass `--baseline git:HEAD` to compare the working tree against the +current commit while that is still true. + +## Working on a case by hand + +```bash +pnpm -F ./test/tachometer dev +``` + +The dev server lists every discovered case at `/`, including `workload-large`, which owns no page +and would otherwise be invisible. `pnpm -F ./test/tachometer build:pages` and `preview` produce and +serve the production bundle, which is what actually gets measured. diff --git a/test/tachometer/package.json b/test/tachometer/package.json new file mode 100644 index 0000000000..5f0afec9d0 --- /dev/null +++ b/test/tachometer/package.json @@ -0,0 +1,20 @@ +{ + "private": true, + "description": "Tachometer smoke-test harness for the code-infra tacho tooling", + "scripts": { + "dev": "vite", + "build:pages": "vite build", + "preview": "vite preview", + "test:tacho": "code-infra tacho run" + }, + "dependencies": { + "@mui/internal-test-utils": "workspace:*", + "react": "19.2.8" + }, + "devDependencies": { + "@mui/internal-code-infra": "workspace:*", + "@types/react": "19.2.18", + "tachometer": "^0.7.2", + "vite": "^8.2.2" + } +} diff --git a/test/tachometer/src/_shared/measure.ts b/test/tachometer/src/_shared/measure.ts new file mode 100644 index 0000000000..00eefa0ffa --- /dev/null +++ b/test/tachometer/src/_shared/measure.ts @@ -0,0 +1,51 @@ +/** + * Shared measurement helper. + * + * Tachometer's `performance` measurement mode polls the page for a `performance.measure()` span of + * a given name, so a page's only obligation is to produce one. + */ + +/** The measure name each case's `tachometer.json` reads. */ +const MEASURE = 'mount'; + +/** + * Runs `work`, writes its result into the page, and closes the `mount` measure once the browser has + * painted it. + * + * The paint is inside the measured window on purpose: a workload whose result is never displayed + * can be optimized away, and the number would then track nothing. + */ +export function measureMount(work: () => string): void { + performance.mark(`${MEASURE}-start`); + const text = work(); + + const root = document.getElementById('root'); + if (!root) { + throw new Error('The page is missing its #root element.'); + } + root.textContent = text; + + // A double rAF closes the measure after the DOM write *and* the paint it triggers; a single one + // would still be inside the frame that performs the paint. + requestAnimationFrame(() => { + requestAnimationFrame(() => { + performance.mark(`${MEASURE}-end`); + performance.measure(MEASURE, `${MEASURE}-start`, `${MEASURE}-end`); + }); + }); +} + +/** + * Reads a positive integer from the page's query string, so one page can back several cases. + */ +export function numberParam(name: string, fallback: number): number { + const raw = new URLSearchParams(window.location.search).get(name); + if (raw === null) { + return fallback; + } + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`Expected "${name}" to be a positive integer, got "${raw}".`); + } + return value; +} diff --git a/test/tachometer/src/workload-large/tachometer.json b/test/tachometer/src/workload-large/tachometer.json new file mode 100644 index 0000000000..ae102d0e4d --- /dev/null +++ b/test/tachometer/src/workload-large/tachometer.json @@ -0,0 +1,18 @@ +{ + "$schema": "../../node_modules/tachometer/config.schema.json", + "sampleSize": 10, + "timeout": 1, + "autoSampleConditions": ["10%"], + "benchmarks": [ + { + "name": "workload-large", + "url": "../workload/index.html?size=400000", + "measurement": { "mode": "performance", "entryName": "mount" }, + "browser": { + "name": "chrome", + "headless": true, + "windowSize": { "width": 1024, "height": 768 } + } + } + ] +} diff --git a/test/tachometer/src/workload/index.html b/test/tachometer/src/workload/index.html new file mode 100644 index 0000000000..556d47228b --- /dev/null +++ b/test/tachometer/src/workload/index.html @@ -0,0 +1,11 @@ + + + + + workload + + +
    + + + diff --git a/test/tachometer/src/workload/index.ts b/test/tachometer/src/workload/index.ts new file mode 100644 index 0000000000..23977b00f5 --- /dev/null +++ b/test/tachometer/src/workload/index.ts @@ -0,0 +1,42 @@ +import { reactMajor } from '@mui/internal-test-utils/env'; +import { measureMount, numberParam } from '../_shared/measure'; + +/** + * A deterministic CPU workload, sized by `?size=`. + * + * The point of this page is not the number it produces — this repository ships build tooling, not a + * browser library, so there is nothing here whose render time is worth tracking. It exists to give + * `code-infra tacho run` something real to drive end to end. + * + * It does import a workspace package (`@mui/internal-test-utils`) and put its value on screen. That + * is the part that matters: the working tree resolves it through the workspace link to the + * package's build output, while every other ref resolves it from a packed tarball in an isolated + * install. If either path breaks, this page fails to build rather than silently measuring nothing. + */ + +/** A small linear congruential generator, so every sample does identical work. */ +function createRandom(seed: number): () => number { + let state = seed; + return () => { + state = (state * 1664525 + 1013904223) % 0x100000000; + return state / 0x100000000; + }; +} + +measureMount(() => { + const size = numberParam('size', 50_000); + const random = createRandom(42); + + const values = new Float64Array(size); + for (let index = 0; index < size; index += 1) { + values[index] = random(); + } + values.sort(); + + let checksum = 0; + for (let index = 0; index < size; index += 1) { + checksum += values[index] * index; + } + + return `size=${size} checksum=${checksum.toFixed(3)} react=${reactMajor}`; +}); diff --git a/test/tachometer/src/workload/tachometer.json b/test/tachometer/src/workload/tachometer.json new file mode 100644 index 0000000000..49d6cecbf3 --- /dev/null +++ b/test/tachometer/src/workload/tachometer.json @@ -0,0 +1,18 @@ +{ + "$schema": "../../node_modules/tachometer/config.schema.json", + "sampleSize": 10, + "timeout": 1, + "autoSampleConditions": ["10%"], + "benchmarks": [ + { + "name": "workload", + "url": "./index.html", + "measurement": { "mode": "performance", "entryName": "mount" }, + "browser": { + "name": "chrome", + "headless": true, + "windowSize": { "width": 1024, "height": 768 } + } + } + ] +} diff --git a/test/tachometer/tsconfig.json b/test/tachometer/tsconfig.json new file mode 100644 index 0000000000..cc3ed604af --- /dev/null +++ b/test/tachometer/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "types": ["vite/client"], + "strict": true, + "noEmit": true, + "isolatedModules": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true + }, + "include": ["src", "vite.config.mjs"] +} diff --git a/test/tachometer/vite.config.mjs b/test/tachometer/vite.config.mjs new file mode 100644 index 0000000000..cbee30cf55 --- /dev/null +++ b/test/tachometer/vite.config.mjs @@ -0,0 +1,6 @@ +import { defineConfig } from 'vite'; +import { tachometer } from '@mui/internal-code-infra/tachometerPlugin'; + +// The plugin contributes everything derived from the benchmark cases: the `src/` root, the +// multi-page app type, the build's entry points, and the dev-server case index. +export default defineConfig({ plugins: [tachometer()] }); From 8a6a438883da2a5e67a9d395014bd50d0c761c61 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 1 Sep 2026 14:48:08 +0200 Subject: [PATCH 003/173] Dedupe the lockfile after adding the tachometer dependencies --- pnpm-lock.yaml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cc0c192ac1..85e729289b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9045,18 +9045,6 @@ packages: utf-8-validate: optional: true - ws@8.21.0: - resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - ws@8.21.3: resolution: {integrity: sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==} engines: {node: '>=10.0.0'} @@ -12431,7 +12419,7 @@ snapshots: sirv: 3.0.2 tinyrainbow: 3.1.0 vitest: 4.1.11(@types/node@22.20.1)(@vitest/browser-playwright@4.1.11)(@vitest/coverage-v8@4.1.11)(jsdom@28.1.0(supports-color@10.2.2))(vite@8.2.2(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.44.0)(tsx@4.23.12)(yaml@2.9.0)) - ws: 8.21.0 + ws: 8.21.3 transitivePeerDependencies: - bufferutil - msw @@ -12983,7 +12971,7 @@ snapshots: cache-content-type@1.0.1: dependencies: - mime-types: 2.1.18 + mime-types: 2.1.35 ylru: 1.4.0 cacheable-lookup@7.0.0: {} @@ -18323,8 +18311,6 @@ snapshots: ws@7.5.11: {} - ws@8.21.0: {} - ws@8.21.3: {} wsl-utils@1.0.0: From 99b9d72bd260cde3b55d19403886069c0f708c63 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:00:48 +0200 Subject: [PATCH 004/173] [code-infra] Bring tacho run up to base-ui-mosaic's current design The extraction in this PR was taken from mui/base-ui-mosaic#428 at e62e7e7, which turned out to be stale: the branch had ten further commits that were never pushed, changing precisely the parts that were extracted. Those have since been pushed. This ports them, so the shared tooling encodes the current design rather than a superseded one. Every ref now goes through an isolated install, the working tree included. Building one side through a workspace link and the other from a tarball compared two different resolution paths -- source exports against an installed package's -- and that difference landed in the measurement rather than in the library. packWorkingTree names each tarball by a hash of its bytes, which is what lets the installs persist: an unchanged build keeps its path, so pnpm has nothing to do. The cache splits into .cache/packed (what CI should cache) and .cache/installs. Verdicts now require the millisecond interval and the percent interval to exclude zero on the same side. They are computed separately from the same samples, so near zero one can exclude it while the other does not, and a verdict resting on one of them flips depending which is read. Results are paired by (variant, measurement) rather than by position. Tachometer flattens variants x measurements into one list with a difference matrix over the whole thing, so the previous differences[index + 1] silently compared unrelated measurements as soon as a case declared more than one. Benchmarks run on Playwright's pinned Chrome for Testing instead of whatever Chrome the machine auto-updated to, and the chromedriver major is checked against it before anything is built -- a mismatch now fails in seconds naming both versions instead of after minutes of packing. The isolated install uses --ignore-scripts rather than copying the repository's allowBuilds policy: nothing installed there needs its scripts, since the pages are built from the packages exactly as packed. --- package.json | 1 + packages/code-infra/package.json | 5 + .../code-infra/src/tachometer/browser.mjs | 121 ++++++ .../src/tachometer/browser.test.mjs | 27 ++ .../code-infra/src/tachometer/buildPages.mjs | 158 +++---- .../src/tachometer/discoverCases.mjs | 70 +++- .../src/tachometer/discoverCases.test.mjs | 93 ++++- .../src/tachometer/runTachometer.mjs | 110 ++--- .../src/tachometer/summarizeCase.mjs | 167 ++++++-- .../src/tachometer/summarizeCase.test.mjs | 221 +++++++--- .../code-infra/src/utils/packWorkspace.mjs | 50 +++ .../src/utils/packWorkspace.test.mjs | 37 +- pnpm-lock.yaml | 385 ++++++++++++++++++ pnpm-workspace.yaml | 1 + test/tachometer/README.md | 22 +- test/tachometer/package.json | 1 + 16 files changed, 1209 insertions(+), 260 deletions(-) create mode 100644 packages/code-infra/src/tachometer/browser.mjs create mode 100644 packages/code-infra/src/tachometer/browser.test.mjs diff --git a/package.json b/package.json index 9d8ca3ee23..2337d566c8 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "@vitest/browser-playwright": "4.1.11", "@vitest/coverage-v8": "4.1.11", "baseline-browser-mapping": "2.11.16", + "chromedriver": "^151.0.5", "eslint-plugin-n": "18.3.0", "pkg-pr-new": "0.0.88", "playwright": "1.62.1", diff --git a/packages/code-infra/package.json b/packages/code-infra/package.json index 70d8464ae8..db2b17bd32 100644 --- a/packages/code-infra/package.json +++ b/packages/code-infra/package.json @@ -156,6 +156,7 @@ }, "peerDependencies": { "@next/eslint-plugin-next": "*", + "@playwright/test": ">=1.50.0", "@typescript/native-preview": ">=7.0.0-dev.0", "eslint": "^9.0.0 || ^10.0.0", "prettier": "^3.0.0", @@ -171,10 +172,14 @@ }, "vite": { "optional": true + }, + "@playwright/test": { + "optional": true } }, "devDependencies": { "@octokit/types": "17.0.0", + "@playwright/test": "1.62.1", "@types/babel__core": "7.20.5", "@types/babel__preset-env": "7.10.0", "@types/env-ci": "3.1.4", diff --git a/packages/code-infra/src/tachometer/browser.mjs b/packages/code-infra/src/tachometer/browser.mjs new file mode 100644 index 0000000000..738ee0eba8 --- /dev/null +++ b/packages/code-infra/src/tachometer/browser.mjs @@ -0,0 +1,121 @@ +import * as path from 'node:path'; +import { createRequire } from 'node:module'; +import { access } from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; +import chalk from 'chalk'; + +/** + * Which browser the benchmarks run on, and whether the driver can actually drive it. + * + * Both checks happen before anything is built: a driver that cannot open the browser fails the run + * either way, and finding out now costs seconds instead of minutes of packing and installing. + */ + +/** + * Resolves the browser every case runs on: Playwright's pinned Chrome for Testing, rather than + * whatever Chrome the machine has auto-updated to. + * + * The path is machine-specific, so callers inject it into each config instead of committing it. + * + * @param {string} harnessDir - The harness package directory, which `@playwright/test` resolves from + * @returns {Promise} Absolute path to the browser binary + */ +export async function resolveBrowserBinary(harnessDir) { + const require = createRequire(path.join(harnessDir, 'package.json')); + // `@playwright/test` is CommonJS, so `require` hands back its exports directly; a dynamic + // `import()` would bury them under `default`. + /** @type {typeof import('@playwright/test')} */ + let playwright; + try { + playwright = require('@playwright/test'); + } catch { + throw new Error( + `Could not resolve "@playwright/test" from ${harnessDir}. Add it as a devDependency — ` + + `the benchmarks run on its pinned Chrome for Testing so results do not move when the ` + + `machine's own Chrome updates.`, + ); + } + + const binary = playwright.chromium.executablePath(); + if (!binary || !(await pathExists(binary))) { + throw new Error( + `Playwright's Chromium is not installed at "${binary}". Run \`pnpm exec playwright install chromium\`.`, + ); + } + return binary; +} + +/** + * True if `target` exists on disk. + * + * @param {string} target - Path to check + * @returns {Promise} + */ +async function pathExists(target) { + try { + await access(target); + return true; + } catch { + return false; + } +} + +/** + * Extracts the major version out of output like `Google Chrome for Testing 151.0.7922.34`. + * + * @param {string} versionOutput - A version string or `--version` output + * @returns {number | undefined} + */ +export function majorVersionOf(versionOutput) { + const match = /(\d+)\.\d+\.\d+/.exec(versionOutput); + return match ? Number(match[1]) : undefined; +} + +/** + * Throws when the chromedriver tachometer will use cannot drive `binary`. + * + * A chromedriver only drives its own Chrome major. The driver is resolved the way tachometer + * resolves it — from tachometer's own package root — so this checks the pair that will actually be + * used rather than whatever else is installed. + * + * @param {string} harnessDir - The harness package directory, which tachometer resolves from + * @param {string} binary - The browser binary that will be driven + * @returns {void} + */ +export function assertDriverMatchesBrowser(harnessDir, binary) { + const browserMajor = majorVersionOf( + execFileSync(binary, ['--version'], { encoding: 'utf8' }).trim(), + ); + + /** @type {string | undefined} */ + let driverVersion; + try { + const fromHarness = createRequire(path.join(harnessDir, 'package.json')); + const fromTachometer = createRequire(fromHarness.resolve('tachometer/package.json')); + driverVersion = fromTachometer('chromedriver/package.json').version; + } catch { + console.warn( + chalk.yellow( + 'Could not resolve chromedriver from tachometer; it will install the latest major itself, ' + + 'which fails against any browser not on that major yet. Pin a "chromedriver" ' + + 'devDependency to keep the two aligned.', + ), + ); + return; + } + + const driverMajor = majorVersionOf(driverVersion ?? ''); + if (browserMajor === undefined || driverMajor === undefined) { + console.warn( + chalk.yellow('Could not read the browser or chromedriver version; skipping the check.'), + ); + return; + } + if (browserMajor !== driverMajor) { + throw new Error( + `chromedriver ${driverVersion} cannot drive Chrome for Testing ${browserMajor}. ` + + `Update the "chromedriver" devDependency to ${browserMajor}.x, or align ` + + `"@playwright/test" with it — bump the two together.`, + ); + } +} diff --git a/packages/code-infra/src/tachometer/browser.test.mjs b/packages/code-infra/src/tachometer/browser.test.mjs new file mode 100644 index 0000000000..fa355f12a1 --- /dev/null +++ b/packages/code-infra/src/tachometer/browser.test.mjs @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest'; +import { majorVersionOf } from './browser.mjs'; + +describe('majorVersionOf', () => { + it('reads the major from Chrome for Testing output', () => { + expect(majorVersionOf('Google Chrome for Testing 151.0.7922.34')).toBe(151); + }); + + it('reads the major from plain Chrome output', () => { + expect(majorVersionOf('Google Chrome 152.0.6367.60')).toBe(152); + }); + + it('reads the major from a bare package version', () => { + // chromedriver's package.json version, which is what the driver side is read from. + expect(majorVersionOf('151.0.5')).toBe(151); + }); + + it('returns undefined when there is no version to read', () => { + expect(majorVersionOf('')).toBeUndefined(); + expect(majorVersionOf('no version here')).toBeUndefined(); + }); + + it('ignores a leading number that is not part of a version', () => { + // Requires three dot-separated components, so a stray count cannot be mistaken for a version. + expect(majorVersionOf('2 browsers found: Chrome 151.0.7922.34')).toBe(151); + }); +}); diff --git a/packages/code-infra/src/tachometer/buildPages.mjs b/packages/code-infra/src/tachometer/buildPages.mjs index 95354b6e5b..98541bcad8 100644 --- a/packages/code-infra/src/tachometer/buildPages.mjs +++ b/packages/code-infra/src/tachometer/buildPages.mjs @@ -1,23 +1,19 @@ /* eslint-disable no-console */ import * as path from 'node:path'; -import { cp, readFile, readdir, writeFile } from 'node:fs/promises'; +import { cp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; -import { parse, stringify } from 'yaml'; +import { stringify } from 'yaml'; import { tarballFor } from '../utils/packWorkspace.mjs'; import { run } from './exec.mjs'; /** - * Builds the benchmark pages for one ref. + * Builds the benchmark pages for one ref, in an isolated install. * - * The working tree builds in place: the harness is a workspace member depending on the library via - * `workspace:*`, and `publishConfig.directory` makes that link resolve to the package's build - * output, so vite resolves the built library with no packing and no isolated install. - * - * Any other ref was built in a throwaway checkout that has since been removed and SHA-cached as - * tarballs, so its build is not in this workspace and cannot be reached by a workspace link. Those - * pages are therefore built in an isolated install that consumes the library exactly as a published - * release would install. + * Every ref goes through this, the working tree included, so both sides of a comparison resolve the + * library the way a consumer installs it: from a tarball, through its own `exports` map and its own + * dependency ranges. Building one side through a workspace link instead would compare two different + * resolution paths, and that difference lands in the measurement rather than in the library. */ /** @@ -57,79 +53,6 @@ export async function runViteBuild(cwd, outDir) { await run('pnpm', ['exec', 'vite', 'build', '--outDir', outDir], cwd); } -/** - * Builds the publishable workspace packages of a checkout. - * - * @param {string} checkoutDir - Checkout to build in - * @param {string} buildCmd - The build command, e.g. `pnpm release:build` - * @param {string} label - Human-readable ref label, for logging - * @returns {Promise} - */ -export async function buildPackages(checkoutDir, buildCmd, label) { - console.log(chalk.cyan(`\nBuilding workspace packages for "${label}" (${buildCmd})…`)); - const [file, ...args] = buildCmd.split(' ').filter(Boolean); - await run(file, args, checkoutDir); -} - -/** - * Builds the benchmark pages for the working tree, in place, against the workspace's own build. - * - * Rebuild the packages and re-run to measure a source change. - * - * @param {Object} options - Build inputs - * @param {string} options.harnessDir - The harness package directory - * @param {ResolvedRef} options.ref - The ref being built (the working tree) - * @param {string} options.outDir - Absolute output directory - * @returns {Promise} - */ -export async function buildWorkspacePages(options) { - const { harnessDir, ref, outDir } = options; - console.log(chalk.cyan(`\nBuilding benchmark pages for "${ref.label}"…`)); - await runViteBuild(harnessDir, outDir); -} - -/** - * The `pnpm-workspace.yaml` keys governing which dependencies may run install scripts. - * - * `strictDepBuilds` is deliberately not among them: the sandbox should mirror the repository's - * approvals, but an unapproved transitive build script the pages do not need must not fail a - * benchmark run. - */ -const BUILD_POLICY_KEYS = [ - 'allowBuilds', - 'onlyBuiltDependencies', - 'neverBuiltDependencies', - 'dangerouslyAllowAllBuilds', -]; - -/** - * Copies the repository's build-script policy, so the isolated install approves the same install - * scripts the repository does. - * - * The throwaway package is its own workspace root, so it inherits nothing. Without this, pnpm - * refuses to run install scripts it was not told to trust and fails the install outright — which - * bites on packages the build genuinely needs, such as esbuild's binary download. - * - * @param {string} repoRoot - The workspace root to copy policy from - * @returns {Promise>} - */ -async function readBuildPolicy(repoRoot) { - /** @type {Record} */ - const policy = {}; - let root; - try { - root = parse(await readFile(path.join(repoRoot, 'pnpm-workspace.yaml'), 'utf8')); - } catch { - return policy; - } - for (const key of BUILD_POLICY_KEYS) { - if (root?.[key] !== undefined) { - policy[key] = root[key]; - } - } - return policy; -} - /** * Rewrites a dependency map, pointing every `workspace:`-protocol entry at its packed tarball and * copying everything else verbatim. @@ -158,30 +81,52 @@ export function rewriteWorkspaceDeps(deps, packages, omit = []) { } /** - * Builds the benchmark pages against a non-working-tree ref's build, in an isolated throwaway - * package. + * Replaces the install directory's `.env*` files with the harness's. + * + * Vite reads `VITE_*` variables from `.env*` files sitting next to the config, and a harness may + * need one (a licence key, say). Stale files are cleared first, so removing one from the harness + * removes it here too. + * + * @param {string} harnessDir - The harness package directory + * @param {string} workDir - The isolated install directory + * @returns {Promise} + */ +async function syncEnvFiles(harnessDir, workDir) { + const existing = (await readdir(workDir)).filter((name) => name.startsWith('.env')); + await Promise.all(existing.map((name) => rm(path.join(workDir, name), { force: true }))); + + const incoming = (await readdir(harnessDir)).filter((name) => name.startsWith('.env')); + await Promise.all( + incoming.map((name) => cp(path.join(harnessDir, name), path.join(workDir, name))), + ); +} + +/** + * Builds the benchmark pages against one ref's packed build, in an isolated install. * - * The pages install the library as a `file:` dependency on its packed `.tgz`, exactly as a - * published package would install, with every workspace-internal package pinned to its own tarball - * via overrides. Third-party dependencies are pinned to the same versions as the committed harness, - * so the only thing that varies between refs is the library build. + * `workDir` persists between runs and only its inputs are refreshed. Combined with tarball paths + * that change only when their content does — a ref's embeds its commit SHA, the working tree's a + * hash of its bytes — an unchanged run is a no-op install and a changed one swaps a single package. * * @param {Object} options - Build inputs * @param {string} options.harnessDir - The harness package directory, copied from - * @param {string} options.repoRoot - The workspace root, whose build-script policy the sandbox mirrors * @param {ResolvedRef} options.ref - The ref being built * @param {PackedPackage[]} options.packages - That ref's packed workspace packages - * @param {string} options.workDir - Throwaway directory to assemble the isolated package in - * @param {string} options.outDir - Absolute output directory + * @param {string} options.workDir - Persistent directory holding this ref's isolated install + * @param {string} options.outDir - Absolute output directory for the built pages * @returns {Promise} */ export async function buildRefPages(options) { - const { harnessDir, repoRoot, ref, packages, workDir, outDir } = options; + const { harnessDir, ref, packages, workDir, outDir } = options; console.log(chalk.cyan(`\nBuilding benchmark pages for "${ref.label}"…`)); const viteConfig = await findViteConfig(harnessDir); + await mkdir(workDir, { recursive: true }); + // Sources are replaced rather than overlaid, so a page deleted from the harness cannot linger. + await rm(path.join(workDir, 'src'), { recursive: true, force: true }); await cp(path.join(harnessDir, 'src'), path.join(workDir, 'src'), { recursive: true }); await cp(path.join(harnessDir, viteConfig), path.join(workDir, viteConfig)); + await syncEnvFiles(harnessDir, workDir); const harnessPkg = JSON.parse(await readFile(path.join(harnessDir, 'package.json'), 'utf8')); await writeFile( @@ -192,8 +137,8 @@ export async function buildRefPages(options) { private: true, type: harnessPkg.type, dependencies: rewriteWorkspaceDeps(harnessPkg.dependencies, packages), - // The isolated package only builds pages; tachometer itself runs from the real harness and - // would pull chromedriver for nothing. + // No page imports tachometer; installing it here would only slow every ref down, and it + // pulls a chromedriver besides. devDependencies: rewriteWorkspaceDeps(harnessPkg.devDependencies, packages, ['tachometer']), }, null, @@ -201,21 +146,24 @@ export async function buildRefPages(options) { )}\n`, ); - // Pin every workspace-internal package to its packed tarball, so a transitive dependency between - // them (whose packed `workspace:*` becomes a concrete, unpublished version) resolves to a local - // build rather than the registry. In pnpm 11 `overrides` live in `pnpm-workspace.yaml`, not - // `package.json`; writing one here also anchors this throwaway as its own workspace root, so pnpm - // reads the overrides and never walks up to a parent workspace (which is why `--ignore-workspace` - // — which would ignore this file — is not used). Being its own root also means it inherits no - // build-script policy, hence carrying the repository's over. + // Marks this folder as its own workspace root, so pnpm does not walk up into the monorepo. The + // overrides pin every packed workspace package, so a transitive dependency between them (whose + // packed `workspace:*` became a concrete, unpublished version) resolves to a local build rather + // than 404ing on the registry. await writeFile( path.join(workDir, 'pnpm-workspace.yaml'), stringify({ - ...(await readBuildPolicy(repoRoot)), overrides: Object.fromEntries(packages.map((pkg) => [pkg.name, `file:${pkg.tarball}`])), }), ); - await run('pnpm', ['install', '--prefer-offline', '--config.engine-strict=false'], workDir); + // `--ignore-scripts`: this workspace has no build-script approvals, and pnpm fails an install + // over unapproved ones rather than warning. Nothing installed here needs its scripts either — the + // pages are built from the packages exactly as packed. + await run( + 'pnpm', + ['install', '--prefer-offline', '--ignore-scripts', '--config.engine-strict=false'], + workDir, + ); await runViteBuild(workDir, outDir); } diff --git a/packages/code-infra/src/tachometer/discoverCases.mjs b/packages/code-infra/src/tachometer/discoverCases.mjs index 871e89f731..bb82549242 100644 --- a/packages/code-infra/src/tachometer/discoverCases.mjs +++ b/packages/code-infra/src/tachometer/discoverCases.mjs @@ -24,12 +24,20 @@ import { parseRefToken } from './refs.mjs'; * @property {string} suffix - Everything after the page path once `ref` was consumed: remaining query plus any fragment */ +/** + * @typedef {Object} CaseVariant + * @property {string} name - The variant's name, as tachometer will report it + * @property {string | null} refId - The ref this variant loads, or null when refs were not resolved + */ + /** * @typedef {Object} BenchmarkCase * @property {string} name - The case folder name * @property {string} configPath - Absolute path to its `tachometer.json` * @property {any} config - The parsed config, mutated in place as urls are rewritten * @property {Leaf[]} leaves - Every node that selects a page + * @property {CaseVariant[]} variants - Declared variants in order; the first is every comparison's reference + * @property {string[]} measurements - Measurement names, as tachometer will name them in its output */ /** @@ -57,15 +65,42 @@ async function pathExists(target) { } } +/** + * The name tachometer gives a measurement: an explicit `name`, else the expression, else the entry + * name. + * + * With more than one measurement on a page, tachometer appends ` []` to each benchmark's + * name, and results have to be paired back by that name — pairing by position would compare + * unrelated measurements. + * + * @param {any} measurement - A tachometer `measurement` entry + * @returns {string} + */ +export function measurementNameOf(measurement) { + if (typeof measurement === 'string') { + return measurement; + } + if (measurement.name) { + return measurement.name; + } + if (measurement.mode === 'expression') { + return measurement.expression; + } + if (measurement.mode === 'callback') { + return 'callback'; + } + return measurement.entryName === 'first-contentful-paint' ? 'fcp' : measurement.entryName; +} + /** * Walks a benchmark's `expand` tree and collects its leaves — the nodes that actually select a page. * * `expand` is recursive and a child inherits its parent's `url` unless it overrides it, so the * effective url is threaded down and only nodes without further `expand` are leaves. * - * @param {{ url?: string, expand?: any[] }} node - The node to walk + * @param {{ url?: string, name?: string, expand?: any[] }} node - The node to walk * @param {string | undefined} inheritedUrl - The effective url from the parent - * @param {Array<{ node: { url?: string }, url: string }>} out - Collected leaves + * @param {Array<{ node: { url?: string, name?: string }, url: string }>} out - Collected leaves * @returns {void} */ export function collectLeafNodes(node, inheritedUrl, out) { @@ -167,9 +202,16 @@ export async function discoverCases(options) { for (const { name, configPath, config } of configs) { const configDir = path.dirname(configPath); - /** @type {Array<{ node: { url?: string }, url: string }>} */ + /** @type {Array<{ node: { url?: string, name?: string }, url: string, benchmarkName: string }>} */ const nodes = []; + /** @type {Set} */ + const measurements = new Set(); for (const benchmark of config.benchmarks ?? []) { + // `measurement` may be a single entry or a list; a page with several is exactly the case + // whose results have to be paired by measurement name rather than by position. + for (const measurement of [benchmark.measurement ?? 'callback'].flat()) { + measurements.add(measurementNameOf(measurement)); + } if (!Array.isArray(benchmark.expand) || benchmark.expand.length === 0) { const base = benchmark.url; if (base === undefined) { @@ -185,7 +227,14 @@ export async function discoverCases(options) { delete benchmark.url; } - collectLeafNodes(benchmark, undefined, nodes); + /** @type {Array<{ node: { url?: string, name?: string }, url: string }>} */ + const benchmarkNodes = []; + collectLeafNodes(benchmark, undefined, benchmarkNodes); + // A leaf's own name is what tachometer reports; only an unexpanded benchmark falls back to + // the benchmark's. + for (const entry of benchmarkNodes) { + nodes.push({ ...entry, benchmarkName: benchmark.name }); + } } // eslint-disable-next-line no-await-in-loop @@ -193,8 +242,19 @@ export async function discoverCases(options) { nodes.map(({ url }) => parseLeafUrl(url, configDir, srcDir, resolveRef)), ); const leaves = nodes.map(({ node }, index) => ({ node, ...resolved[index] })); + const variants = nodes.map(({ node, benchmarkName }, index) => ({ + name: node.name ?? benchmarkName, + refId: resolved[index].ref?.id ?? null, + })); - cases.push({ name, configPath, config, leaves }); + cases.push({ + name, + configPath, + config, + leaves, + variants, + measurements: [...measurements], + }); } return cases; } diff --git a/packages/code-infra/src/tachometer/discoverCases.test.mjs b/packages/code-infra/src/tachometer/discoverCases.test.mjs index 3ca1782d75..9517736906 100644 --- a/packages/code-infra/src/tachometer/discoverCases.test.mjs +++ b/packages/code-infra/src/tachometer/discoverCases.test.mjs @@ -2,7 +2,7 @@ import * as path from 'node:path'; import { mkdir, writeFile } from 'node:fs/promises'; import { describe, expect, it } from 'vitest'; import { makeTempDir } from '../utils/testUtils.mjs'; -import { discoverCases, pagesOf } from './discoverCases.mjs'; +import { discoverCases, measurementNameOf, pagesOf } from './discoverCases.mjs'; /** * Builds a throwaway harness: `src//tachometer.json` plus whatever pages each case owns. @@ -303,3 +303,94 @@ describe('pagesOf', () => { ]); }); }); + +describe('measurementNameOf', () => { + it('uses an explicit name', () => { + expect(measurementNameOf({ mode: 'performance', entryName: 'mount', name: 'paint' })).toBe( + 'paint', + ); + }); + + it('uses the expression for an expression measurement', () => { + expect(measurementNameOf({ mode: 'expression', expression: 'window.total' })).toBe( + 'window.total', + ); + }); + + it('names a callback measurement', () => { + expect(measurementNameOf({ mode: 'callback' })).toBe('callback'); + expect(measurementNameOf('callback')).toBe('callback'); + }); + + it('falls back to the entry name', () => { + expect(measurementNameOf({ mode: 'performance', entryName: 'mount' })).toBe('mount'); + }); + + it('abbreviates first-contentful-paint the way tachometer does', () => { + expect(measurementNameOf({ mode: 'performance', entryName: 'first-contentful-paint' })).toBe( + 'fcp', + ); + }); +}); + +describe('case variants and measurements', () => { + it('lists the auto-expanded variants in order, the reference first', async () => { + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [ + { + name: 'alpha', + url: './index.html', + measurement: { mode: 'performance', entryName: 'mount' }, + }, + ], + }, + pages: ['index.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.variants.map((variant) => variant.name)).toEqual([ + 'alpha [current]', + 'alpha [baseline]', + ]); + expect(entry.measurements).toEqual(['mount']); + }); + + it('collects every measurement a benchmark declares', async () => { + // A page with several measurements is exactly the case whose results must be paired by name. + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [ + { + name: 'alpha', + url: './index.html', + measurement: [ + { mode: 'performance', entryName: 'mount' }, + { mode: 'performance', entryName: 'scroll' }, + ], + }, + ], + }, + pages: ['index.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.measurements).toEqual(['mount', 'scroll']); + }); + + it('defaults to the callback measurement when none is declared', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.measurements).toEqual(['callback']); + }); +}); diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index 1ff9f07023..bdef5b3110 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -6,10 +6,11 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; import { execaSync } from 'execa'; import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; -import { packRef } from '../utils/packWorkspace.mjs'; +import { packRef, packWorkingTree } from '../utils/packWorkspace.mjs'; import { createRefResolver } from './refs.mjs'; import { discoverCases, pagesOf } from './discoverCases.mjs'; -import { buildPackages, buildRefPages, buildWorkspacePages } from './buildPages.mjs'; +import { buildRefPages } from './buildPages.mjs'; +import { assertDriverMatchesBrowser, resolveBrowserBinary } from './browser.mjs'; import { summarizeCase } from './summarizeCase.mjs'; import { run } from './exec.mjs'; @@ -53,9 +54,12 @@ export async function runTachometer(options) { } const buildsDir = path.join(harnessDir, 'builds'); - // Non-working-tree refs are packed into a folder keyed by their commit SHA and cached here, so - // repeating a comparison against the same ref reuses the tarballs instead of rebuilding. + // Everything reusable between runs lives under `.cache`. `packed` holds tarballs — a ref's keyed + // by commit SHA, the working tree's by content hash — and is what CI should cache; `installs` + // holds each ref's isolated install, cheap to recreate. Both are gitignored and safe to delete. const cacheDir = path.join(harnessDir, '.cache'); + const packedDir = path.join(cacheDir, 'packed'); + const installsDir = path.join(cacheDir, 'installs'); const resolver = createRefResolver({ repoRoot, baseBranch, baselineOverride: baseline }); const cases = await discoverCases({ harnessDir, filters, resolveRef: resolver.parse }); @@ -76,59 +80,69 @@ export async function runTachometer(options) { const describeRef = (/** @type {import('./refs.mjs').ResolvedRef} */ ref) => ref.sha ? `${ref.label} (${ref.sha.slice(0, 9)})` : ref.label; - console.log(chalk.cyan(`Cases: ${cases.map((entry) => entry.name).join(', ')}`)); - console.log(chalk.cyan(`Pages: ${pagesOf(cases).join(', ')}`)); - console.log(chalk.cyan(`Refs: ${[...refs.values()].map(describeRef).join(', ')}`)); + // Before any build: a driver that cannot open the browser fails the run either way, and finding + // out now costs seconds instead of minutes of packing and installing. + const browserBinary = await resolveBrowserBinary(harnessDir); + assertDriverMatchesBrowser(harnessDir, browserBinary); + + console.log(chalk.cyan(`Cases: ${cases.map((entry) => entry.name).join(', ')}`)); + console.log(chalk.cyan(`Pages: ${pagesOf(cases).join(', ')}`)); + console.log(chalk.cyan(`Refs: ${[...refs.values()].map(describeRef).join(', ')}`)); + console.log(chalk.cyan(`Browser: ${browserBinary}`)); const tmpBase = await mkdtemp(path.join(os.tmpdir(), 'tacho-')); try { + // Build one variant per distinct ref, every one through its own isolated install so both sides + // of a comparison resolve the library identically. for (const ref of refs.values()) { - const outDir = path.join(buildsDir, ref.id); - if (ref.kind === 'worktree') { - // The working tree's build is in this workspace, so build it and let the workspace link - // resolve it — no packing and no isolated install. This is also the fast dev loop. - // eslint-disable-next-line no-await-in-loop - await buildPackages(repoRoot, buildCmd, ref.label); - // eslint-disable-next-line no-await-in-loop - await buildWorkspacePages({ harnessDir, ref, outDir }); - continue; - } - - // packRef packs every workspace package at the ref into a folder cached by SHA. On a cache hit - // it skips the checkout, install, and build entirely; on a miss it builds in a throwaway - // checkout and removes it as soon as the tarballs are packed. - // eslint-disable-next-line no-await-in-loop - const packed = await packRef({ - repoRoot, - ref: /** @type {string} */ (ref.committish), - outRoot: cacheDir, - // An empty install command means "skip"; otherwise packRef's default install runs. - installCmd: install ? undefined : '', - buildCmd, - }); + const packages = + ref.kind === 'worktree' + ? // eslint-disable-next-line no-await-in-loop + await packWorkingTree({ + repoRoot, + outRoot: path.join(packedDir, 'current'), + buildCmd, + }) + : // packRef caches a ref's tarballs by SHA; a hit skips the checkout, install, and build. + // eslint-disable-next-line no-await-in-loop + await packRef({ + repoRoot, + ref: /** @type {string} */ (ref.committish), + outRoot: packedDir, + // An empty install command means "skip"; otherwise packRef's default install runs. + installCmd: install ? undefined : '', + buildCmd, + }).then((packed) => packed.packages); // eslint-disable-next-line no-await-in-loop await buildRefPages({ harnessDir, - repoRoot, ref, - packages: packed.packages, - workDir: path.join(tmpBase, `bench-${ref.id}`), - outDir, + packages, + workDir: path.join(installsDir, ref.id), + outDir: path.join(buildsDir, ref.id), }); } // Rewrite each leaf url to its ref's built page, then run each case's own config. Tachometer // resolves relative urls against the config file's directory and these configs are written to a // temp dir, so the rewritten urls are absolute. - /** @type {Array<{ name: string, json: import('./summarizeCase.mjs').TachometerJson }>} */ + /** @type {Array<{ entry: import('./discoverCases.mjs').BenchmarkCase, json: any }>} */ const results = []; for (const entry of cases) { for (const leaf of entry.leaves) { const refId = leaf.ref ? leaf.ref.id : 'current'; leaf.node.url = `${path.join(buildsDir, refId, leaf.page)}${leaf.suffix}`; } + // `browser` is inherited down the `expand` tree, so setting it per benchmark covers every + // variant. An explicitly configured binary wins. + for (const benchmark of entry.config.benchmarks ?? []) { + benchmark.browser = { + ...benchmark.browser, + binary: benchmark.browser?.binary ?? browserBinary, + }; + } const configPath = path.join(tmpBase, `tachometer-${entry.name}.json`); // eslint-disable-next-line no-await-in-loop await writeFile( @@ -147,7 +161,7 @@ export async function runTachometer(options) { // Cases run sequentially on purpose — concurrent browser benchmarks would contend for the // same machine and skew timings — so reading each result in turn is fine. // eslint-disable-next-line no-await-in-loop - results.push({ name: entry.name, json: JSON.parse(await readFile(jsonPath, 'utf8')) }); + results.push({ entry, json: JSON.parse(await readFile(jsonPath, 'utf8')) }); } const outPath = out ? path.resolve(out) : path.join(harnessDir, 'results', 'report.json'); @@ -157,6 +171,7 @@ export async function runTachometer(options) { ref: 'HEAD', sha: execaSync('git', ['rev-parse', 'HEAD'], { cwd: repoRoot }).stdout.trim(), }, + browser: browserBinary, // Symbols are resolved to concrete SHAs here so a run stays interpretable after the fact. refs: [...refs.values()].map((ref) => ({ id: ref.id, @@ -167,29 +182,32 @@ export async function runTachometer(options) { // Summarising is best-effort per case: a case that produced no usable benchmarks must not // cost the whole run its report, since `raw` below is the only surviving copy of every other // case's samples once the temp dir is cleaned up. - cases: results.map(({ name, json }) => { + cases: results.map(({ entry, json }) => { try { - return summarizeCase(name, json); + return summarizeCase(entry, json); } catch (error) { const message = error instanceof Error ? error.message : String(error); - console.error(chalk.yellow(`Could not summarize "${name}": ${message}`)); - return { name, error: message }; + console.error(chalk.yellow(`Could not summarize "${entry.name}": ${message}`)); + return { name: entry.name, error: message }; } }), - raw: Object.fromEntries(results.map(({ name, json }) => [name, json])), + raw: Object.fromEntries(results.map(({ entry, json }) => [entry.name, json])), }; await mkdir(path.dirname(outPath), { recursive: true }); await writeFile(outPath, `${JSON.stringify(report, null, 2)}\n`); console.log(chalk.green(`\nWrote JSON report to ${outPath}`)); for (const entry of report.cases) { - if (!('comparisons' in entry)) { + if (!('measurements' in entry)) { console.log(` ${entry.name}: no result (${entry.error})`); continue; } - for (const comparison of entry.comparisons) { - console.log( - ` ${entry.name}: ${entry.reference} is ${comparison.verdict} vs ${comparison.variant}`, - ); + for (const measurement of entry.measurements) { + for (const comparison of measurement.comparisons) { + console.log( + ` ${entry.name} [${measurement.name}]: ${entry.reference} is ` + + `${comparison.verdict} vs ${comparison.variant}`, + ); + } } } } finally { diff --git a/packages/code-infra/src/tachometer/summarizeCase.mjs b/packages/code-infra/src/tachometer/summarizeCase.mjs index ff8fdf8eb1..18a3ea5e03 100644 --- a/packages/code-infra/src/tachometer/summarizeCase.mjs +++ b/packages/code-infra/src/tachometer/summarizeCase.mjs @@ -1,7 +1,13 @@ +import chalk from 'chalk'; + /** * Reduces tachometer's raw JSON output to a flat, interpretable summary. */ +/** + * @typedef {import('./discoverCases.mjs').BenchmarkCase} BenchmarkCase + */ + /** * @typedef {Object} ConfidenceInterval * @property {number} low - Lower bound @@ -16,16 +22,17 @@ /** * @typedef {Object} TachometerBenchmark - * @property {string} name - Variant name + * @property {string} name - Variant name, suffixed with ` []` when a page has several * @property {number} bytesSent - Bytes transferred * @property {ConfidenceInterval} mean - Mean duration, as a 95% confidence interval * @property {number[]} samples - Individual sample durations - * @property {Array} differences - This variant against each other, by index + * @property {{ name?: string }} [measurement] - The measurement this entry reports + * @property {Array} differences - This entry against every other, by flat index */ /** * @typedef {Object} TachometerJson - * @property {TachometerBenchmark[]} benchmarks - One entry per variant + * @property {TachometerBenchmark[]} benchmarks - One entry per variant × measurement */ /** @@ -33,53 +40,131 @@ */ /** - * Reduces one case's tachometer output to a flat summary: every variant's mean 95% CI (ms) and - * sample count, plus the first variant's difference against each of the others and a verdict per - * comparison. + * Judges a difference. * - * The first variant is the reference because that is the one under test — for the auto-expanded - * regression case it is `[current]`. A verdict reads the sign of the difference's confidence - * interval: entirely below 0 is `faster`, entirely above is `slower`, and a CI that still straddles - * 0 is `unsure` (the result did not resolve within the sampling budget). + * A difference only counts as `faster`/`slower` when the millisecond interval and the percent + * interval both exclude zero on the same side. They come from the same samples but are computed + * separately, so near zero one can exclude it while the other does not — and a verdict that rests + * on only one of them flips depending on which is read. * - * @param {string} name - The case name, used in error messages + * @param {TachometerDifference} difference - A difference from tachometer's matrix + * @returns {Verdict} + */ +export function verdictOf(difference) { + if (difference.absolute.low > 0 && difference.percentChange.low > 0) { + return 'slower'; + } + if (difference.absolute.high < 0 && difference.percentChange.high < 0) { + return 'faster'; + } + return 'unsure'; +} + +/** + * Indexes tachometer's flat benchmark list by measurement and then by variant, keeping each entry's + * position so the difference matrix can be read back. + * + * @param {TachometerJson} json - Raw tachometer output + * @returns {Map>} + */ +function indexByMeasurement(json) { + /** @type {Map>} */ + const byMeasurement = new Map(); + json.benchmarks.forEach((benchmark, index) => { + const measurement = benchmark.measurement?.name ?? ''; + const suffix = ` [${measurement}]`; + const variant = benchmark.name.endsWith(suffix) + ? benchmark.name.slice(0, -suffix.length) + : benchmark.name; + const forMeasurement = byMeasurement.get(measurement) ?? new Map(); + forMeasurement.set(variant, { benchmark, index }); + byMeasurement.set(measurement, forMeasurement); + }); + return byMeasurement; +} + +/** + * Reduces one case's tachometer output to a per-measurement summary. + * + * Tachometer flattens variants × measurements into one list, naming entries + * ` []` when a page declares more than one, with a difference matrix over the + * whole flat list. Results are paired back by (variant, measurement) name — pairing by position + * would compare unrelated measurements as soon as a case has more than one. + * + * The first declared variant is the reference, because that is the one under test: for the + * auto-expanded regression case it is `[current]`. + * + * @param {BenchmarkCase} entry - The discovered case, which declares the variants and measurements * @param {TachometerJson} json - Raw tachometer output for that case - * @returns {{ name: string, reference: string, variants: Array<{ name: string, meanMs: ConfidenceInterval, samples: number }>, comparisons: Array<{ variant: string, verdict: Verdict, difference: { absoluteMs: ConfidenceInterval, percentChange: ConfidenceInterval } | null | undefined }> }} + * @returns {{ name: string, reference: string, measurements: Array<{ name: string, variants: Array<{ variant: string, refId: string | null, meanMs: ConfidenceInterval, samples: number, bytesSent: number }>, comparisons: Array<{ variant: string, verdict: Verdict, absoluteMs: ConfidenceInterval, percentChange: ConfidenceInterval, versusReference?: { verdict: Verdict, absoluteMs: ConfidenceInterval, percentChange: ConfidenceInterval } }> }> }} */ -export function summarizeCase(name, json) { - const [reference, ...others] = json.benchmarks; +export function summarizeCase(entry, json) { + const [reference] = entry.variants; if (!reference) { - throw new Error(`Case "${name}" produced no benchmarks.`); + throw new Error(`Case "${entry.name}" declares no variants.`); } - const comparisons = others.map((other, index) => { - // `others` is `benchmarks` without the reference at index 0, so `other` is `benchmarks[index + 1]`. - const diff = reference.differences[index + 1]; - /** @type {Verdict} */ - let verdict = 'unsure'; - if (diff) { - if (diff.percentChange.low > 0 && diff.percentChange.high > 0) { - verdict = 'slower'; - } else if (diff.percentChange.low < 0 && diff.percentChange.high < 0) { - verdict = 'faster'; + const byMeasurement = indexByMeasurement(json); + + const measurements = entry.measurements.map((measurement) => { + /** + * @param {string} variant - Variant name to look up + * @returns {{ benchmark: TachometerBenchmark, index: number } | undefined} + */ + const lookup = (variant) => byMeasurement.get(measurement)?.get(variant); + + const referenceResult = lookup(reference.name); + if (!referenceResult) { + throw new Error( + `Case "${entry.name}": no "${measurement}" result for the reference "${reference.name}".`, + ); + } + + const variants = []; + const comparisons = []; + for (const variant of entry.variants) { + const found = lookup(variant.name); + if (!found) { + console.warn( + chalk.yellow( + ` ${entry.name}: no "${measurement}" result for "${variant.name}"; skipping.`, + ), + ); + continue; + } + variants.push({ + variant: variant.name, + refId: variant.refId, + meanMs: found.benchmark.mean, + samples: found.benchmark.samples.length, + bytesSent: found.benchmark.bytesSent, + }); + if (variant.name === reference.name) { + continue; + } + const difference = referenceResult.benchmark.differences[found.index]; + if (!difference) { + continue; } + // The same pair read from the variant's own matrix row. Each direction has its own + // denominator, so it is not the negation of `difference`. + const reverse = found.benchmark.differences[referenceResult.index]; + comparisons.push({ + variant: variant.name, + verdict: verdictOf(difference), + absoluteMs: difference.absolute, + percentChange: difference.percentChange, + versusReference: reverse + ? { + verdict: verdictOf(reverse), + absoluteMs: reverse.absolute, + percentChange: reverse.percentChange, + } + : undefined, + }); } - return { - variant: other.name, - verdict, - // `reference` relative to `variant`: positive means the reference is slower. - difference: diff && { absoluteMs: diff.absolute, percentChange: diff.percentChange }, - }; + return { name: measurement, variants, comparisons }; }); - return { - name, - reference: reference.name, - variants: json.benchmarks.map((benchmark) => ({ - name: benchmark.name, - meanMs: benchmark.mean, - samples: benchmark.samples.length, - })), - comparisons, - }; + return { name: entry.name, reference: reference.name, measurements }; } diff --git a/packages/code-infra/src/tachometer/summarizeCase.test.mjs b/packages/code-infra/src/tachometer/summarizeCase.test.mjs index 6332b575ec..cedbb32a7f 100644 --- a/packages/code-infra/src/tachometer/summarizeCase.test.mjs +++ b/packages/code-infra/src/tachometer/summarizeCase.test.mjs @@ -1,89 +1,202 @@ -import { describe, expect, it } from 'vitest'; -import { summarizeCase } from './summarizeCase.mjs'; +import { describe, expect, it, vi } from 'vitest'; +import { summarizeCase, verdictOf } from './summarizeCase.mjs'; /** - * Builds a tachometer benchmark entry. + * A tachometer benchmark entry. * - * @param {string} name - Variant name - * @param {Array<{ low: number, high: number } | null>} differences - Percent-change CIs against each variant, by index - * @returns {import('./summarizeCase.mjs').TachometerBenchmark} + * @param {Object} options - Entry fields + * @param {string} options.name - Entry name, as tachometer reports it + * @param {string} [options.measurement] - Measurement name + * @param {Array<{ ms: [number, number], pct: [number, number] } | null>} [options.differences] - Differences against every entry, by flat index + * @returns {any} */ -function benchmark(name, differences = []) { +function benchmark({ name, measurement, differences = [] }) { return { name, - bytesSent: 0, + bytesSent: 100, mean: { low: 1, high: 2 }, samples: [1, 2, 3], - differences: differences.map((percentChange) => - percentChange ? { absolute: { low: -1, high: 1 }, percentChange } : null, + measurement: measurement === undefined ? undefined : { name: measurement }, + differences: differences.map((difference) => + difference + ? { + absolute: { low: difference.ms[0], high: difference.ms[1] }, + percentChange: { low: difference.pct[0], high: difference.pct[1] }, + } + : null, ), }; } +/** + * A discovered case, as `discoverCases` would return it. + * + * @param {string[]} variants - Variant names in order; the first is the reference + * @param {string[]} measurements - Measurement names + * @returns {any} + */ +function discovered(variants, measurements) { + return { + name: 'example', + configPath: '/tmp/tachometer.json', + config: {}, + leaves: [], + variants: variants.map((name) => ({ name, refId: 'current' })), + measurements, + }; +} + +describe('verdictOf', () => { + it('is slower when both intervals sit above zero', () => { + expect(verdictOf({ absolute: { low: 1, high: 3 }, percentChange: { low: 2, high: 5 } })).toBe( + 'slower', + ); + }); + + it('is faster when both intervals sit below zero', () => { + expect( + verdictOf({ absolute: { low: -3, high: -1 }, percentChange: { low: -5, high: -2 } }), + ).toBe('faster'); + }); + + it('is unsure when the intervals disagree', () => { + // The two are computed separately from the same samples, so near zero one can exclude it while + // the other does not. A verdict resting on only one of them would flip depending which is read. + expect(verdictOf({ absolute: { low: 1, high: 3 }, percentChange: { low: -1, high: 5 } })).toBe( + 'unsure', + ); + }); + + it('is unsure when both intervals straddle zero', () => { + expect(verdictOf({ absolute: { low: -1, high: 3 }, percentChange: { low: -2, high: 5 } })).toBe( + 'unsure', + ); + }); +}); + describe('summarizeCase', () => { - it('reports every variant with its mean and sample count', () => { - const summary = summarizeCase('example', { - benchmarks: [benchmark('a', [null, { low: -5, high: -1 }]), benchmark('b')], + it('reports every variant of a single-measurement case', () => { + const entry = discovered(['a', 'b'], ['mount']); + const summary = summarizeCase(entry, { + benchmarks: [ + benchmark({ + name: 'a', + measurement: 'mount', + differences: [null, { ms: [-3, -1], pct: [-5, -2] }], + }), + benchmark({ + name: 'b', + measurement: 'mount', + differences: [{ ms: [1, 3], pct: [2, 5] }, null], + }), + ], }); expect(summary.reference).toBe('a'); - expect(summary.variants).toEqual([ - { name: 'a', meanMs: { low: 1, high: 2 }, samples: 3 }, - { name: 'b', meanMs: { low: 1, high: 2 }, samples: 3 }, + expect(summary.measurements).toHaveLength(1); + expect(summary.measurements[0].variants.map((variant) => variant.variant)).toEqual(['a', 'b']); + expect(summary.measurements[0].comparisons).toMatchObject([ + { variant: 'b', verdict: 'faster' }, ]); }); - describe('verdicts', () => { - it('is faster when the whole confidence interval is below zero', () => { - const summary = summarizeCase('example', { - benchmarks: [benchmark('a', [null, { low: -5, high: -1 }]), benchmark('b')], - }); - expect(summary.comparisons[0]).toMatchObject({ variant: 'b', verdict: 'faster' }); + it('pairs results by measurement rather than by position', () => { + // Tachometer flattens variants × measurements into one list and names entries + // ` []`. Reading the difference matrix positionally would compare `mount` + // against `scroll`. Here the flat order deliberately interleaves the two measurements. + const entry = discovered(['a', 'b'], ['mount', 'scroll']); + const summary = summarizeCase(entry, { + benchmarks: [ + // index 0 + benchmark({ + name: 'a [mount]', + measurement: 'mount', + differences: [null, null, { ms: [-3, -1], pct: [-5, -2] }, null], + }), + // index 1 + benchmark({ + name: 'a [scroll]', + measurement: 'scroll', + differences: [null, null, null, { ms: [1, 3], pct: [2, 5] }], + }), + // index 2 + benchmark({ name: 'b [mount]', measurement: 'mount', differences: [] }), + // index 3 + benchmark({ name: 'b [scroll]', measurement: 'scroll', differences: [] }), + ], }); - it('is slower when the whole confidence interval is above zero', () => { - const summary = summarizeCase('example', { - benchmarks: [benchmark('a', [null, { low: 1, high: 5 }]), benchmark('b')], - }); - expect(summary.comparisons[0]).toMatchObject({ variant: 'b', verdict: 'slower' }); - }); + const [mount, scroll] = summary.measurements; + expect(mount.name).toBe('mount'); + expect(mount.comparisons).toMatchObject([{ variant: 'b', verdict: 'faster' }]); + expect(scroll.name).toBe('scroll'); + expect(scroll.comparisons).toMatchObject([{ variant: 'b', verdict: 'slower' }]); + }); - it('is unsure when the confidence interval straddles zero', () => { - const summary = summarizeCase('example', { - benchmarks: [benchmark('a', [null, { low: -3, high: 4 }]), benchmark('b')], - }); - expect(summary.comparisons[0]).toMatchObject({ variant: 'b', verdict: 'unsure' }); + it('strips the measurement suffix when pairing variants back', () => { + const entry = discovered(['a', 'b'], ['mount']); + const summary = summarizeCase(entry, { + benchmarks: [ + benchmark({ + name: 'a [mount]', + measurement: 'mount', + differences: [null, { ms: [-3, -1], pct: [-5, -2] }], + }), + benchmark({ name: 'b [mount]', measurement: 'mount', differences: [] }), + ], }); - it('is unsure when there is no difference to read', () => { - const summary = summarizeCase('example', { - benchmarks: [benchmark('a', [null, null]), benchmark('b')], - }); - expect(summary.comparisons[0]).toMatchObject({ variant: 'b', verdict: 'unsure' }); - expect(summary.comparisons[0].difference).toBeFalsy(); - }); + expect(summary.measurements[0].variants.map((variant) => variant.variant)).toEqual(['a', 'b']); }); - it('reads each comparison from the difference that belongs to it', () => { - // `differences` is indexed by variant, so the reference's difference against the Nth other - // variant lives at index N + 1. An off-by-one here silently mislabels every comparison. - const summary = summarizeCase('example', { + it('records the reverse comparison, which is not the negation of the forward one', () => { + // Each direction has its own denominator. + const entry = discovered(['a', 'b'], ['mount']); + const summary = summarizeCase(entry, { benchmarks: [ - benchmark('a', [null, { low: -9, high: -8 }, { low: 8, high: 9 }]), - benchmark('b'), - benchmark('c'), + benchmark({ + name: 'a', + measurement: 'mount', + differences: [null, { ms: [-3, -1], pct: [-5, -2] }], + }), + benchmark({ + name: 'b', + measurement: 'mount', + differences: [{ ms: [1, 3], pct: [3, 9] }, null], + }), ], }); - expect(summary.comparisons).toMatchObject([ - { variant: 'b', verdict: 'faster' }, - { variant: 'c', verdict: 'slower' }, - ]); + expect(summary.measurements[0].comparisons[0]).toMatchObject({ + verdict: 'faster', + versusReference: { verdict: 'slower', percentChange: { low: 3, high: 9 } }, + }); + }); + + it('skips a variant with no result for a measurement instead of failing the case', () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const entry = discovered(['a', 'b'], ['mount']); + + const summary = summarizeCase(entry, { + benchmarks: [benchmark({ name: 'a', measurement: 'mount', differences: [null] })], + }); + + expect(summary.measurements[0].variants.map((variant) => variant.variant)).toEqual(['a']); + expect(warn).toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('throws when the reference itself has no result for a measurement', () => { + const entry = discovered(['a'], ['mount']); + + expect(() => summarizeCase(entry, { benchmarks: [] })).toThrow( + 'Case "example": no "mount" result for the reference "a".', + ); }); - it('throws when a case produced no benchmarks', () => { - expect(() => summarizeCase('example', { benchmarks: [] })).toThrow( - 'Case "example" produced no benchmarks.', + it('throws when the case declares no variants', () => { + expect(() => summarizeCase(discovered([], ['mount']), { benchmarks: [] })).toThrow( + 'Case "example" declares no variants.', ); }); }); diff --git a/packages/code-infra/src/utils/packWorkspace.mjs b/packages/code-infra/src/utils/packWorkspace.mjs index 047bf57107..65cfffce22 100644 --- a/packages/code-infra/src/utils/packWorkspace.mjs +++ b/packages/code-infra/src/utils/packWorkspace.mjs @@ -2,6 +2,7 @@ import * as path from 'node:path'; import * as os from 'node:os'; +import { createHash } from 'node:crypto'; import { access, mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; import { execa, parseCommandString } from 'execa'; @@ -327,6 +328,55 @@ export async function packRef(options) { } } +/** + * Builds and packs the working tree, naming each tarball by a hash of its bytes. + * + * A tarball path then never points at different content, which is what lets a consumer's installs + * persist across runs: an unchanged build keeps its filename, so `pnpm install` has nothing to do, + * while a changed one swaps a single package. + * + * The tree is mutable, so this always builds; a good `buildCmd` is cached (nx, turbo) and an + * unchanged tree rebuilds in seconds. + * + * @param {Object} options - What to pack, and where + * @param {string} options.repoRoot - The workspace to build and pack + * @param {string} options.outRoot - Directory to hold the hashed tarballs. Replaced on each call + * @param {string} [options.buildCmd] - Command that builds the publishable packages. Defaults to `pnpm release:build` + * @returns {Promise} + */ +export async function packWorkingTree(options) { + const { repoRoot, outRoot, buildCmd = 'pnpm release:build' } = options; + + console.log(chalk.cyan(`\nBuilding workspace packages for "working tree" (${buildCmd})…`)); + // Disable the nx daemon: it keeps writing into the workspace after the build returns. + const [file, ...args] = parseCommandString(buildCmd); + await run(file, args, repoRoot, { NX_DAEMON: 'false' }); + + console.log(chalk.cyan('\nPacking the working tree…')); + // Stage next to the destination so the renames cannot cross filesystems. + const packedRoot = path.dirname(outRoot); + await mkdir(packedRoot, { recursive: true }); + const staging = await mkdtemp(path.join(packedRoot, '.pack-')); + try { + const packed = await packBuiltPackages(repoRoot, staging); + await rm(outRoot, { recursive: true, force: true }); + await mkdir(outRoot, { recursive: true }); + return await Promise.all( + packed.map(async (pkg) => { + const hash = createHash('sha256') + .update(await readFile(pkg.tarball)) + .digest('hex') + .slice(0, 12); + const tarball = path.join(outRoot, `${path.basename(pkg.tarball, '.tgz')}-${hash}.tgz`); + await rename(pkg.tarball, tarball); + return { ...pkg, tarball }; + }), + ); + } finally { + await rm(staging, { recursive: true, force: true }); + } +} + /** * Looks up one packed package's tarball by name, throwing a clear error if it was not packed. * diff --git a/packages/code-infra/src/utils/packWorkspace.test.mjs b/packages/code-infra/src/utils/packWorkspace.test.mjs index db81e29cbb..73e0020efb 100644 --- a/packages/code-infra/src/utils/packWorkspace.test.mjs +++ b/packages/code-infra/src/utils/packWorkspace.test.mjs @@ -3,7 +3,13 @@ import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises'; import { execa } from 'execa'; import { describe, expect, it } from 'vitest'; import { makeTempDir } from './testUtils.mjs'; -import { packRef, readFreshCache, tarballFor, tarballName } from './packWorkspace.mjs'; +import { + packRef, + packWorkingTree, + readFreshCache, + tarballFor, + tarballName, +} from './packWorkspace.mjs'; describe('tarballName', () => { it('flattens a scoped name into a filesystem-safe basename', () => { @@ -204,3 +210,32 @@ describe('packRef', () => { ).rejects.toThrow(/Could not resolve git ref "no-such-ref"/); }); }); + +describe('packWorkingTree', () => { + it('names each tarball by a hash of its content', { timeout: 120_000 }, async () => { + const { repoRoot, buildCmd } = await makeFixtureRepo(); + const outRoot = path.join(await makeTempDir(), 'packed', 'current'); + + const first = await packWorkingTree({ repoRoot, outRoot, buildCmd }); + const unchanged = await packWorkingTree({ repoRoot, outRoot, buildCmd }); + + // An unchanged build keeps its filename, which is what lets a consumer's isolated install + // persist: the dependency path does not move, so pnpm has nothing to do. + expect(first.map((pkg) => pkg.tarball)).toEqual(unchanged.map((pkg) => pkg.tarball)); + expect(path.basename(first[0].tarball)).toMatch(/^fixture-public-[0-9a-f]{12}\.tgz$/); + await expect(stat(first[0].tarball)).resolves.toBeTruthy(); + }); + + it('renames when the packed content changes', { timeout: 120_000 }, async () => { + const { repoRoot, buildCmd } = await makeFixtureRepo(); + const outRoot = path.join(await makeTempDir(), 'packed', 'current'); + + const before = await packWorkingTree({ repoRoot, outRoot, buildCmd }); + await writeFile(path.join(repoRoot, 'packages', 'public', 'index.js'), 'export default 2;\n'); + const after = await packWorkingTree({ repoRoot, outRoot, buildCmd }); + + expect(after[0].tarball).not.toBe(before[0].tarball); + // The directory is replaced, so a stale tarball cannot accumulate or be resolved by mistake. + expect(await readdir(outRoot)).toEqual([path.basename(after[0].tarball)]); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 85e729289b..545aa389d7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -84,6 +84,9 @@ importers: baseline-browser-mapping: specifier: 2.11.16 version: 2.11.16 + chromedriver: + specifier: ^151.0.5 + version: 151.0.5(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) eslint-plugin-n: specifier: 18.3.0 version: 18.3.0(eslint@10.9.1(jiti@2.7.0)(supports-color@10.2.2))(ts-declaration-location@1.0.7(typescript@6.0.3))(typescript@6.0.3) @@ -735,6 +738,9 @@ importers: '@octokit/types': specifier: 17.0.0 version: 17.0.0 + '@playwright/test': + specifier: 1.62.1 + version: 1.62.1 '@types/babel__core': specifier: 7.20.5 version: 7.20.5 @@ -1120,6 +1126,9 @@ importers: '@mui/internal-code-infra': specifier: workspace:* version: link:../../packages/code-infra + '@playwright/test': + specifier: 1.62.1 + version: 1.62.1 '@types/react': specifier: 19.2.18 version: 19.2.18 @@ -3713,6 +3722,9 @@ packages: peerDependencies: react: ^18 || ^19 + '@testim/chrome-version@1.1.4': + resolution: {integrity: sha512-kIhULpw9TrGYnHp/8VfdcneIcxKnLixmADtukQRtJUmsVlMg0niMkwV0xZmi8hqa57xqilIHjWFA0GKvEjVU5g==} + '@testing-library/dom@10.4.1': resolution: {integrity: sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==} engines: {node: '>=18'} @@ -4352,10 +4364,22 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + adm-zip@0.6.0: + resolution: {integrity: sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg==} + engines: {node: '>=14.0'} + + agent-base@6.0.2: + resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} + engines: {node: '>= 6.0.0'} + agent-base@7.1.4: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} + agent-base@9.0.0: + resolution: {integrity: sha512-TQf59BsZnytt8GdJKLPfUZ54g/iaUL2OWDSFCCvMOhsHduDQxO8xC4PNeyIkVcA5KwL2phPSv0douC0fgWzmnA==} + engines: {node: '>= 20'} + ajv-formats@2.1.1: resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} peerDependencies: @@ -4495,6 +4519,10 @@ packages: ast-types-flow@0.0.8: resolution: {integrity: sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==} + ast-types@0.13.4: + resolution: {integrity: sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==} + engines: {node: '>=4'} + ast-v8-to-istanbul@1.0.0: resolution: {integrity: sha512-1fSfIwuDICFA4LKkCzRPO7F0hzFf0B7+Xqrl27ynQaa+Rh0e1Es0v6kWHPott3lU10AyAr7oKHa65OppjLn3Rg==} @@ -4510,6 +4538,9 @@ packages: resolution: {integrity: sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA==} engines: {node: '>= 0.4'} + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + autoprefixer@10.5.4: resolution: {integrity: sha512-MaU0U/za7N3r6brxD4YB/l4NSrFzLPlANv6wEuQVaIPlD3L4W9rFcQPbL/EilY9BHhHvhfcz3gInDLrEtWT4EA==} engines: {node: ^10 || ^12 || >=14} @@ -4529,6 +4560,9 @@ packages: resolution: {integrity: sha512-ilYanEU8vxxBexpJd8cWM4ElSQq4QctCLKih0TSfjIfCQTeyH/6zVrmIJfLPrKTKJRbiG+cfnZbQIjAlJmF1jQ==} engines: {node: '>=4'} + axios@1.20.0: + resolution: {integrity: sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==} + axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} @@ -4600,6 +4634,10 @@ packages: engines: {node: '>=6.0.0'} hasBin: true + basic-ftp@5.3.1: + resolution: {integrity: sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw==} + engines: {node: '>=10.0.0'} + bcrypt-pbkdf@1.0.2: resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} @@ -4775,6 +4813,11 @@ packages: resolution: {integrity: sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ==} engines: {node: '>=6.0'} + chromedriver@151.0.5: + resolution: {integrity: sha512-oPI/VtTeLePbkTOI5o4yvEOlcdpmqEdqQNLZbyu5ha6kpDQYfDRuliisDiph29hBK/UzHSmOyZFKX1mE3efYOQ==} + engines: {node: '>=22'} + hasBin: true + ci-info@4.4.0: resolution: {integrity: sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==} engines: {node: '>=8'} @@ -4848,6 +4891,10 @@ packages: colord@2.9.3: resolution: {integrity: sha512-jeC1axXpnb0/2nn/Y1LPuLdgXBLH7aDcHu4KEKfqw3CUhX7ZpfBSlPKyqXE6btIgEzfWtrX3/tyBCaCvXvMkOw==} + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} @@ -4877,6 +4924,9 @@ packages: commondir@1.0.1: resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} + compare-versions@6.1.1: + resolution: {integrity: sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg==} + compressible@2.0.18: resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} engines: {node: '>= 0.6'} @@ -5043,6 +5093,10 @@ packages: damerau-levenshtein@1.0.8: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} + data-uri-to-buffer@8.0.0: + resolution: {integrity: sha512-6UHfyCux51b8PTGDgveqtz1tvphBku5DrMKKJbFAZAJOI2zsjDpDoYE1+QGj7FOMS4BdTFNJsJiR3zEB0xH0yQ==} + engines: {node: '>= 20'} + data-urls@7.0.0: resolution: {integrity: sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==} engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0} @@ -5081,6 +5135,15 @@ packages: supports-color: optional: true + debug@4.3.1: + resolution: {integrity: sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -5138,6 +5201,16 @@ packages: resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} engines: {node: '>= 0.4'} + degenerator@7.0.1: + resolution: {integrity: sha512-ABErK0IefDSyHjlPH7WUEenIAX2rPPnrDcDM+TS3z3+zu9TfyKKi07BQM+8rmxpdE2y1v5fjjdoAS/x4D2U60w==} + engines: {node: '>= 20'} + peerDependencies: + quickjs-wasi: ^2.2.0 + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + delegates@1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} @@ -5333,6 +5406,11 @@ packages: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} + escodegen@2.1.0: + resolution: {integrity: sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==} + engines: {node: '>=6.0'} + hasBin: true + eslint-compat-utils@0.5.1: resolution: {integrity: sha512-3z3vFexKIEnjHE3zCMRo6fn/e44U7T1khUjg+Hp0ZQMCigh28rALD0nPFBcGZuiLC5rLZa2ubQHDRln09JfU2Q==} engines: {node: '>=12'} @@ -5525,6 +5603,11 @@ packages: resolution: {integrity: sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} + esprima@4.0.1: + resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} + engines: {node: '>=4'} + hasBin: true + esquery@1.7.0: resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} engines: {node: '>=0.10'} @@ -5714,6 +5797,15 @@ packages: flatted@3.4.2: resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==} + follow-redirects@1.16.0: + resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + for-each@0.3.5: resolution: {integrity: sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==} engines: {node: '>= 0.4'} @@ -5726,6 +5818,10 @@ packages: resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} engines: {node: '>= 14.17'} + form-data@4.0.6: + resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} + engines: {node: '>= 6'} + format-util@1.0.5: resolution: {integrity: sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==} @@ -5828,6 +5924,10 @@ packages: get-tsconfig@4.13.0: resolution: {integrity: sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==} + get-uri@8.0.1: + resolution: {integrity: sha512-/5N/P4Lrh0p/mDwlDRi7Y1+P2o/OyzZI3l6Iz1Ov6XXwwm1y3RlZLuo3gVgML99djrEDtV980bBxSuOeHLk8ww==} + engines: {node: '>= 20'} + git-up@8.1.1: resolution: {integrity: sha512-FDenSF3fVqBYSaJoYy1KSc2wosx0gCvKP+c+PRBht7cAaiCeQlBtfBDX9vgnNOHmdePlSFITVcn4pFfcgNvx3g==} @@ -6063,14 +6163,26 @@ packages: resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} engines: {node: '>= 14'} + http-proxy-agent@9.1.0: + resolution: {integrity: sha512-2NxoveTT58mjYT4n3RPTEfCZGLMbidoO8XEieXfpSYxu+PQJ1qpx4ypwH6N+uF9twBPIvRRgvkvW5HUTYWENig==} + engines: {node: '>= 20'} + http2-wrapper@2.2.1: resolution: {integrity: sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==} engines: {node: '>=10.19.0'} + https-proxy-agent@5.0.1: + resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} + engines: {node: '>= 6'} + https-proxy-agent@7.0.6: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} + https-proxy-agent@9.1.0: + resolution: {integrity: sha512-ag87y7cJJ9/3+GxFr8Oy4O5faDsGRGnBGsJj/YjOSsSx/5eadKLYTMPlzuR6obgoCDDm0abAAZitXXQkMOPSpA==} + engines: {node: '>= 20'} + human-signals@2.1.0: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} engines: {node: '>=10.17.0'} @@ -6158,6 +6270,14 @@ packages: resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==} engines: {node: '>=12'} + ip-address@10.7.0: + resolution: {integrity: sha512-BGFsyJd5mpXp3rK6jIdADLNgpJUK1jnjzvYF8lK+VyDab9JAmqN0YOKDdP17HlgKb2+ehPgDc8EtnRLbGCAMhA==} + engines: {node: '>= 12'} + + ip-regex@2.1.0: + resolution: {integrity: sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw==} + engines: {node: '>=4'} + is-alphabetical@2.0.1: resolution: {integrity: sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==} @@ -6348,6 +6468,9 @@ packages: resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==} engines: {node: '>=18'} + is-url@1.2.4: + resolution: {integrity: sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==} + is-wayland@0.1.0: resolution: {integrity: sha512-QkbMsWkIfkrzOPxenwye0h56iAXirZYHG9eHVPb22fO9y+wPbaX/CHacOWBa/I++4ohTcByimhM1/nyCsH8KNA==} engines: {node: '>=20'} @@ -6372,6 +6495,10 @@ packages: resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==} engines: {node: '>=16'} + is2@2.0.1: + resolution: {integrity: sha512-+WaJvnaA7aJySz2q/8sLjMb2Mw14KTplHmSwcSpZ/fWJPkUmqw3YTzSWbPJ7OAwRvdYTWF2Wg+yYJ1AdP5Z8CA==} + engines: {node: '>=v0.10.0'} + is64bit@2.0.0: resolution: {integrity: sha512-jv+8jaWCl0g2lSBkNSVXdzfBA0npK1HGC2KtWM9FumFRoGS94g3NbCCLVnCYHLjp4GrW2KZeeSTMo5ddtznmGw==} engines: {node: '>=18'} @@ -6768,6 +6895,10 @@ packages: lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} + lru-cache@7.18.3: + resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==} + engines: {node: '>=12'} + lru.min@1.1.4: resolution: {integrity: sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA==} engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} @@ -7077,6 +7208,9 @@ packages: ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + ms@2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -7127,6 +7261,10 @@ packages: neo-async@2.6.2: resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} + netmask@2.1.1: + resolution: {integrity: sha512-eonl3sLUha+S1GzTPxychyhnUzKyeQkZ7jLjKrBagJgPla13F+uQ71HgpFefyHgqrjEbCPkDArxYsjY8/+gLKA==} + engines: {node: '>= 0.4.0'} + next@16.3.1: resolution: {integrity: sha512-hsAp0i7Rh+/dhe7DGIeN2YlpLM1DP4MNxti9EtDMtqcO612X81MvvEj388/oTce9U1EcEIOWDlGq0zRwrBKvuA==} engines: {node: '>=20.9.0'} @@ -7340,6 +7478,16 @@ packages: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} + pac-proxy-agent@9.1.0: + resolution: {integrity: sha512-1aU+1mpj3DrQPfo3gh+3Gap3G5x+axnMx1P/y0ZF2ch7kb2meyOCAH8K2k9d27ROsTE7TnAerzxqF9aon2jqnA==} + engines: {node: '>= 20'} + + pac-resolver@9.0.1: + resolution: {integrity: sha512-lJbS008tmkj08VhoM8Hzuv/VE5tK9MS0OIQ/7+s0lIF+BYhiQWFYzkSpML7lXs9iBu2jfmzBTLzhe9n6BX+dYw==} + engines: {node: '>= 20'} + peerDependencies: + quickjs-wasi: ^2.2.0 + package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} @@ -7621,6 +7769,23 @@ packages: protocols@2.0.2: resolution: {integrity: sha512-hHVTzba3wboROl0/aWRRG9dMytgH6ow//STBZh43l/wQgmMhYhOFi0EHWAPtoCz9IAUymsyP0TSBHkhgMEGNnQ==} + proxy-agent-negotiate@1.1.0: + resolution: {integrity: sha512-N8IBcM3UgCVzz2L2Lqv8DVntDnnC8/hiV4nEDUPkqq72TPUgYWjQc+bdZlBPZK9LzPAvOY//gAt0S0DApoOXWQ==} + engines: {node: '>= 20'} + peerDependencies: + kerberos: ^2.0.0 + peerDependenciesMeta: + kerberos: + optional: true + + proxy-agent@8.0.2: + resolution: {integrity: sha512-idLLRewuemWd7GH/BDJzGiB0dWGfT2SQs3jy6NtZtGWU9uPTTSdeC1/cdbqLwgzhfv027daGFuXX426e2Eg20A==} + engines: {node: '>= 20'} + + proxy-from-env@2.1.0: + resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} + engines: {node: '>=10'} + pump@3.0.4: resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} @@ -7651,6 +7816,9 @@ packages: resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} engines: {node: '>=10'} + quickjs-wasi@2.2.0: + resolution: {integrity: sha512-zQxXmQMrEoD3S+jQdYsloq4qAuaxKFHZj6hHqOYGwB2iQZH+q9e/lf5zQPXCKOk0WJuAjzRFbO4KwHIp2D05Iw==} + quotation@2.0.3: resolution: {integrity: sha512-yEc24TEgCFLXx7D4JHJJkK4JFVtatO8fziwUxY4nB/Jbea9o9CVS3gt22mA0W7rPYAGW2fWzYDSOtD94PwOyqA==} @@ -8116,6 +8284,18 @@ packages: resolution: {integrity: sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==} engines: {node: '>=10'} + smart-buffer@4.2.0: + resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==} + engines: {node: '>= 6.0.0', npm: '>= 3.0.0'} + + socks-proxy-agent@10.1.0: + resolution: {integrity: sha512-WlMj/67cEJ6MDI1OcsnjuYKDNDoyPCCYZ249kuuXPiMDw9F8PXkVaQ7YWu3siTydfQ/4BEZcvGzu+aYvz7dDCQ==} + engines: {node: '>= 20'} + + socks@2.8.9: + resolution: {integrity: sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==} + engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -8415,6 +8595,9 @@ packages: resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} engines: {node: '>=6'} + tcp-port-used@1.0.3: + resolution: {integrity: sha512-4CEQ3qRJYo+mtEbJ+OoQu3dF4TDkwaO3RDVC4UzP5cpAOIUWwuwPjD7sdxDFFqsMUjsXVVYBMlg/boAaloThMA==} + terser-webpack-plugin@5.6.1: resolution: {integrity: sha512-201R5j+sJpK8nFWwKVyNfZot8FaJbLZDq5evriVzbV1wDtSXDjRUDRfJzHpAaxFDMEhsZL1QkeqM61wgsS3KaQ==} engines: {node: '>= 10.13.0'} @@ -11910,6 +12093,8 @@ snapshots: '@tanstack/query-core': 5.101.4 react: 19.2.8 + '@testim/chrome-version@1.1.4': {} + '@testing-library/dom@10.4.1': dependencies: '@babel/code-frame': 7.29.7 @@ -12605,8 +12790,18 @@ snapshots: acorn@8.16.0: {} + adm-zip@0.6.0: {} + + agent-base@6.0.2(supports-color@10.2.2): + dependencies: + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + agent-base@7.1.4: {} + agent-base@9.0.0: {} + ajv-formats@2.1.1(ajv@8.18.0): optionalDependencies: ajv: 8.18.0 @@ -12761,6 +12956,10 @@ snapshots: ast-types-flow@0.0.8: {} + ast-types@0.13.4: + dependencies: + tslib: 2.8.1 + ast-v8-to-istanbul@1.0.0: dependencies: '@jridgewell/trace-mapping': 0.3.31 @@ -12773,6 +12972,8 @@ snapshots: async-function@1.0.0: {} + asynckit@0.4.0: {} + autoprefixer@10.5.4(postcss@8.5.26): dependencies: browserslist: 4.28.7 @@ -12790,6 +12991,16 @@ snapshots: axe-core@4.11.0: {} + axios@1.20.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2): + dependencies: + follow-redirects: 1.16.0(debug@4.4.3(supports-color@10.2.2)) + form-data: 4.0.6 + https-proxy-agent: 5.0.1(supports-color@10.2.2) + proxy-from-env: 2.1.0 + transitivePeerDependencies: + - debug + - supports-color + axobject-query@4.1.0: {} babel-plugin-macros@3.1.0: @@ -12889,6 +13100,8 @@ snapshots: baseline-browser-mapping@2.11.16: {} + basic-ftp@5.3.1: {} + bcrypt-pbkdf@1.0.2: dependencies: tweetnacl: 0.14.5 @@ -13071,6 +13284,20 @@ snapshots: chrome-trace-event@1.0.4: {} + chromedriver@151.0.5(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2): + dependencies: + '@testim/chrome-version': 1.1.4 + adm-zip: 0.6.0 + axios: 1.20.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) + compare-versions: 6.1.1 + proxy-agent: 8.0.2(supports-color@10.2.2) + proxy-from-env: 2.1.0 + tcp-port-used: 1.0.3(supports-color@10.2.2) + transitivePeerDependencies: + - debug + - kerberos + - supports-color + ci-info@4.4.0: {} cli-boxes@3.0.0: {} @@ -13142,6 +13369,10 @@ snapshots: colord@2.9.3: {} + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + comma-separated-tokens@2.0.3: {} command-line-args@5.2.1: @@ -13168,6 +13399,8 @@ snapshots: commondir@1.0.1: {} + compare-versions@6.1.1: {} + compressible@2.0.18: dependencies: mime-db: 1.54.0 @@ -13341,6 +13574,8 @@ snapshots: damerau-levenshtein@1.0.8: {} + data-uri-to-buffer@8.0.0: {} + data-urls@7.0.0: dependencies: whatwg-mimetype: 5.0.0 @@ -13382,6 +13617,12 @@ snapshots: optionalDependencies: supports-color: 10.2.2 + debug@4.3.1(supports-color@10.2.2): + dependencies: + ms: 2.1.2 + optionalDependencies: + supports-color: 10.2.2 + debug@4.4.3(supports-color@10.2.2): dependencies: ms: 2.1.3 @@ -13435,6 +13676,15 @@ snapshots: has-property-descriptors: 1.0.2 object-keys: 1.1.1 + degenerator@7.0.1(quickjs-wasi@2.2.0): + dependencies: + ast-types: 0.13.4 + escodegen: 2.1.0 + esprima: 4.0.1 + quickjs-wasi: 2.2.0 + + delayed-stream@1.0.0: {} + delegates@1.0.0: {} depd@1.1.2: {} @@ -13703,6 +13953,14 @@ snapshots: escape-string-regexp@5.0.0: {} + escodegen@2.1.0: + dependencies: + esprima: 4.0.1 + estraverse: 5.3.0 + esutils: 2.0.3 + optionalDependencies: + source-map: 0.6.1 + eslint-compat-utils@0.5.1(eslint@10.9.1(jiti@2.7.0)(supports-color@10.2.2)): dependencies: eslint: 10.9.1(jiti@2.7.0)(supports-color@10.2.2) @@ -14038,6 +14296,8 @@ snapshots: acorn-jsx: 5.3.2(acorn@8.16.0) eslint-visitor-keys: 5.0.1 + esprima@4.0.1: {} + esquery@1.7.0: dependencies: estraverse: 5.3.0 @@ -14274,6 +14534,10 @@ snapshots: flatted@3.4.2: {} + follow-redirects@1.16.0(debug@4.4.3(supports-color@10.2.2)): + optionalDependencies: + debug: 4.4.3(supports-color@10.2.2) + for-each@0.3.5: dependencies: is-callable: 1.2.7 @@ -14285,6 +14549,14 @@ snapshots: form-data-encoder@2.1.4: {} + form-data@4.0.6: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.4 + mime-types: 2.1.35 + format-util@1.0.5: {} format@0.2.2: {} @@ -14381,6 +14653,14 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 + get-uri@8.0.1(supports-color@10.2.2): + dependencies: + basic-ftp: 5.3.1 + data-uri-to-buffer: 8.0.0 + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + git-up@8.1.1: dependencies: is-ssh: 1.4.1 @@ -14705,11 +14985,27 @@ snapshots: transitivePeerDependencies: - supports-color + http-proxy-agent@9.1.0(supports-color@10.2.2): + dependencies: + agent-base: 9.0.0 + debug: 4.4.3(supports-color@10.2.2) + proxy-agent-negotiate: 1.1.0 + transitivePeerDependencies: + - kerberos + - supports-color + http2-wrapper@2.2.1: dependencies: quick-lru: 5.1.1 resolve-alpn: 1.2.1 + https-proxy-agent@5.0.1(supports-color@10.2.2): + dependencies: + agent-base: 6.0.2(supports-color@10.2.2) + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + https-proxy-agent@7.0.6(supports-color@10.2.2): dependencies: agent-base: 7.1.4 @@ -14717,6 +15013,15 @@ snapshots: transitivePeerDependencies: - supports-color + https-proxy-agent@9.1.0(supports-color@10.2.2): + dependencies: + agent-base: 9.0.0 + debug: 4.4.3(supports-color@10.2.2) + proxy-agent-negotiate: 1.1.0 + transitivePeerDependencies: + - kerberos + - supports-color + human-signals@2.1.0: {} human-signals@5.0.0: {} @@ -14783,6 +15088,10 @@ snapshots: internmap@2.0.3: {} + ip-address@10.7.0: {} + + ip-regex@2.1.0: {} + is-alphabetical@2.0.1: {} is-alphanumerical@2.0.1: @@ -14946,6 +15255,8 @@ snapshots: is-unicode-supported@2.1.0: {} + is-url@1.2.4: {} + is-wayland@0.1.0: {} is-weakmap@2.0.2: {} @@ -14967,6 +15278,12 @@ snapshots: dependencies: is-inside-container: 1.0.0 + is2@2.0.1: + dependencies: + deep-is: 0.1.4 + ip-regex: 2.1.0 + is-url: 1.2.4 + is64bit@2.0.0: dependencies: system-architecture: 0.1.0 @@ -15376,6 +15693,8 @@ snapshots: dependencies: yallist: 3.1.1 + lru-cache@7.18.3: {} + lru.min@1.1.4: {} lucide-react@1.33.0(react@19.2.8): @@ -15940,6 +16259,8 @@ snapshots: ms@2.0.0: {} + ms@2.1.2: {} + ms@2.1.3: {} multipipe@1.0.2: @@ -15986,6 +16307,8 @@ snapshots: neo-async@2.6.2: {} + netmask@2.1.1: {} + next@16.3.1(@babel/core@7.29.7(supports-color@10.2.2))(@playwright/test@1.62.1)(@types/node@22.20.1)(babel-plugin-macros@3.1.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8): dependencies: '@next/env': 16.3.1 @@ -16214,6 +16537,26 @@ snapshots: p-try@2.2.0: {} + pac-proxy-agent@9.1.0(supports-color@10.2.2): + dependencies: + agent-base: 9.0.0 + debug: 4.4.3(supports-color@10.2.2) + get-uri: 8.0.1(supports-color@10.2.2) + http-proxy-agent: 9.1.0(supports-color@10.2.2) + https-proxy-agent: 9.1.0(supports-color@10.2.2) + pac-resolver: 9.0.1(quickjs-wasi@2.2.0) + quickjs-wasi: 2.2.0 + socks-proxy-agent: 10.1.0(supports-color@10.2.2) + transitivePeerDependencies: + - kerberos + - supports-color + + pac-resolver@9.0.1(quickjs-wasi@2.2.0): + dependencies: + degenerator: 7.0.1(quickjs-wasi@2.2.0) + netmask: 2.1.1 + quickjs-wasi: 2.2.0 + package-json-from-dist@1.0.1: {} pako@1.0.11: {} @@ -16465,6 +16808,24 @@ snapshots: protocols@2.0.2: {} + proxy-agent-negotiate@1.1.0: {} + + proxy-agent@8.0.2(supports-color@10.2.2): + dependencies: + agent-base: 9.0.0 + debug: 4.4.3(supports-color@10.2.2) + http-proxy-agent: 9.1.0(supports-color@10.2.2) + https-proxy-agent: 9.1.0(supports-color@10.2.2) + lru-cache: 7.18.3 + pac-proxy-agent: 9.1.0(supports-color@10.2.2) + proxy-from-env: 2.1.0 + socks-proxy-agent: 10.1.0(supports-color@10.2.2) + transitivePeerDependencies: + - kerberos + - supports-color + + proxy-from-env@2.1.0: {} + pump@3.0.4: dependencies: end-of-stream: 1.4.5 @@ -16493,6 +16854,8 @@ snapshots: quick-lru@5.1.1: {} + quickjs-wasi@2.2.0: {} + quotation@2.0.3: {} range-parser@1.2.0: {} @@ -17254,6 +17617,21 @@ snapshots: astral-regex: 2.0.0 is-fullwidth-code-point: 3.0.0 + smart-buffer@4.2.0: {} + + socks-proxy-agent@10.1.0(supports-color@10.2.2): + dependencies: + agent-base: 9.0.0 + debug: 4.4.3(supports-color@10.2.2) + socks: 2.8.9 + transitivePeerDependencies: + - supports-color + + socks@2.8.9: + dependencies: + ip-address: 10.7.0 + smart-buffer: 4.2.0 + source-map-js@1.2.1: {} source-map-support@0.5.21: @@ -17631,6 +18009,13 @@ snapshots: tapable@2.3.3: {} + tcp-port-used@1.0.3(supports-color@10.2.2): + dependencies: + debug: 4.3.1(supports-color@10.2.2) + is2: 2.0.1 + transitivePeerDependencies: + - supports-color + terser-webpack-plugin@5.6.1(esbuild@0.28.1)(lightningcss@1.33.0)(postcss@8.5.26)(webpack@5.107.2(esbuild@0.28.1)(lightningcss@1.33.0)(postcss@8.5.26)): dependencies: '@jridgewell/trace-mapping': 0.3.31 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index dce77aded3..c943e1469c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,6 +6,7 @@ packages: engineStrict: true strictDepBuilds: true allowBuilds: + chromedriver: true esbuild: true nx: true sharp: true diff --git a/test/tachometer/README.md b/test/tachometer/README.md index 75e01dac76..22d3fb3c40 100644 --- a/test/tachometer/README.md +++ b/test/tachometer/README.md @@ -25,9 +25,9 @@ rewrite, that `?ref=` is appended with `&` when a url already has a query, and t referencing one page build it only once. Both pages import `@mui/internal-test-utils` and put its value on screen. That is the load-bearing -part: the working tree resolves it through the workspace link to the package's build output, while -every other ref resolves it from a packed tarball in an isolated install. If either path breaks the -page fails to build, rather than quietly measuring nothing. +part: every ref — the working tree included — resolves it from a packed tarball in an isolated +install, so both sides of a comparison consume the library the way a consumer does. If that path +breaks the page fails to build, rather than quietly measuring nothing. ## Running @@ -49,10 +49,18 @@ code, the expected verdict is `unsure` — that is the "no change" outcome, not ### Prerequisites -A real Chrome. Tachometer drives it over WebDriver and installs a matching `chromedriver` on demand; -the driver's major version must match Chrome's, or you will see `SessionNotCreatedError: This -version of ChromeDriver only supports Chrome version N`. Fix it by updating Chrome, or set -`browser.binary` in a case's `tachometer.json` to an existing Chromium. +Playwright's Chromium, installed with `pnpm exec playwright install chromium`. Runs use Playwright's +pinned Chrome for Testing rather than whatever Chrome the machine has auto-updated to, so a browser +update cannot move the numbers on its own. + +A chromedriver only drives its own Chrome major, so the root `chromedriver` devDependency has to +stay aligned with `@playwright/test` — bump the two together. `tacho run` checks the pair before it +builds anything and fails with both versions named, rather than letting you find out minutes later +through `SessionNotCreatedError`. + +One wrinkle: tachometer installs a chromedriver into its own package directory on demand, and that +copy wins over the root pin. If the check reports a driver you did not install, delete +`node_modules/.pnpm/tachometer@*/node_modules/tachometer/node_modules/chromedriver` and re-run. ### A note on the baseline diff --git a/test/tachometer/package.json b/test/tachometer/package.json index 5f0afec9d0..2c865cf133 100644 --- a/test/tachometer/package.json +++ b/test/tachometer/package.json @@ -13,6 +13,7 @@ }, "devDependencies": { "@mui/internal-code-infra": "workspace:*", + "@playwright/test": "1.62.1", "@types/react": "19.2.18", "tachometer": "^0.7.2", "vite": "^8.2.2" From 9d6e638f215e55928a7b02e94796fd8ae3d1badd Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:30:22 +0200 Subject: [PATCH 005/173] [code-infra] Match the report contract mosaic's table renderer expects Two things the integration into base-ui-mosaic#428 needs, found by reading its benchmarkTable.mts and root scripts rather than by running into them later. The renderer's TachometerReport requires `version`, `reportType` and `head.branch`; the writer emitted none of them, so the table would have failed to read a report the runner considered complete. Splitting the build command matters for speed, not output. Mosaic's `build` and `release:build` run the same per-package build, differing only in that the latter skips the nx cache. A ref is built in a throwaway checkout with a cold cache and wants the thorough one, but the working tree is rebuilt on every run with a warm cache, so forcing it through --skip-nx-cache would rebuild everything each time for nothing. --working-tree-build-cmd defaults to --build-cmd, so repositories where the two are the same are unaffected. --- packages/code-infra/src/cli/cmdTacho.mjs | 11 +++++++++-- .../code-infra/src/tachometer/runTachometer.mjs | 16 ++++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/packages/code-infra/src/cli/cmdTacho.mjs b/packages/code-infra/src/cli/cmdTacho.mjs index 348ae3f9b8..5a5515df9c 100644 --- a/packages/code-infra/src/cli/cmdTacho.mjs +++ b/packages/code-infra/src/cli/cmdTacho.mjs @@ -5,7 +5,8 @@ * @property {string[]} [filters] - Only run cases whose folder name contains one of these substrings * @property {string} [baseline] - Binds the `baseline` symbol, in the ref grammar * @property {string} [baseBranch] - Branch PRs fork from - * @property {string} [buildCmd] - Command that builds the publishable packages + * @property {string} [buildCmd] - Command that builds the publishable packages of a checked-out ref + * @property {string} [workingTreeBuildCmd] - Command that builds the working tree * @property {boolean} [install] - Whether to install inside a ref's checkout * @property {string} [out] - Where to write the combined JSON report */ @@ -34,7 +35,12 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ .option('build-cmd', { type: 'string', default: 'pnpm release:build', - describe: 'Command that builds the publishable workspace packages', + describe: "Command that builds the publishable workspace packages of a ref's checkout", + }) + .option('working-tree-build-cmd', { + type: 'string', + describe: + 'Command that builds the working tree. Defaults to --build-cmd; point it at a cached build when the two differ only in caching', }) .option('install', { type: 'boolean', @@ -57,6 +63,7 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ baseline: argv.baseline, baseBranch: argv.baseBranch, buildCmd: argv.buildCmd, + workingTreeBuildCmd: argv.workingTreeBuildCmd, install: argv.install, out: argv.out, }); diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index bdef5b3110..70edc0965b 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -20,7 +20,8 @@ import { run } from './exec.mjs'; * @property {string[]} [filters] - Only run cases whose folder name contains one of these substrings * @property {string} [baseline] - Binds the `baseline` symbol, in the ref grammar (e.g. `git:abc1234`) * @property {string} [baseBranch] - Branch PRs fork from. Defaults to detection via `origin/HEAD` - * @property {string} [buildCmd] - Command that builds the publishable packages. Defaults to `pnpm release:build` + * @property {string} [buildCmd] - Command that builds the publishable packages of a checked-out ref. Defaults to `pnpm release:build` + * @property {string} [workingTreeBuildCmd] - Command that builds the working tree. Defaults to `buildCmd` * @property {boolean} [install] - Whether to install inside a ref's checkout. Defaults to true * @property {string} [out] - Where to write the combined JSON report. Defaults to `results/report.json` */ @@ -44,6 +45,10 @@ export async function runTachometer(options) { baseline, baseBranch, buildCmd = 'pnpm release:build', + // A ref is built in a throwaway checkout with a cold task cache, so it wants the thorough + // command. The working tree is rebuilt on every run and usually has a warm one, so a repository + // can point this at the cached build instead without changing what is produced. + workingTreeBuildCmd = buildCmd, install = true, out, } = options; @@ -102,7 +107,7 @@ export async function runTachometer(options) { await packWorkingTree({ repoRoot, outRoot: path.join(packedDir, 'current'), - buildCmd, + buildCmd: workingTreeBuildCmd, }) : // packRef caches a ref's tarballs by SHA; a hit skips the checkout, install, and build. // eslint-disable-next-line no-await-in-loop @@ -166,10 +171,17 @@ export async function runTachometer(options) { const outPath = out ? path.resolve(out) : path.join(harnessDir, 'results', 'report.json'); const report = { + // Consumers render reports from several benchmark axes; the pair identifies which one this + // is and how to read it. + version: 1, + reportType: 'tachometer', generatedAt: new Date().toISOString(), head: { ref: 'HEAD', sha: execaSync('git', ['rev-parse', 'HEAD'], { cwd: repoRoot }).stdout.trim(), + branch: execaSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { + cwd: repoRoot, + }).stdout.trim(), }, browser: browserBinary, // Symbols are resolved to concrete SHAs here so a run stays interpretable after the fact. From b1a08081339cc7a7da31d46ebccbd3e3af6a9044 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:00:06 +0200 Subject: [PATCH 006/173] [code-infra] Inherit the repository's overrides in the isolated install The throwaway package is its own workspace root, so it inherited none of the repository's `overrides`. That is a correctness gap, not just a convenience: a repository that pins a transitive dependency through an override would resolve a different graph in the sandbox than in its real install, quietly changing what is being benchmarked. Found by integrating into base-ui-mosaic#428, where it surfaced as a hard failure rather than a silent one: that repository sets blockExoticSubdeps, and the overrides keeping its dependencies within that policy were missing from the sandbox, so the install refused to proceed. The repository's overrides go in first and the packed workspace packages pin over them, since those are the whole point of the install. --- .../code-infra/src/tachometer/buildPages.mjs | 38 ++++++++++++++++--- .../src/tachometer/runTachometer.mjs | 1 + pnpm-lock.yaml | 18 +-------- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/packages/code-infra/src/tachometer/buildPages.mjs b/packages/code-infra/src/tachometer/buildPages.mjs index 98541bcad8..427b7f5f56 100644 --- a/packages/code-infra/src/tachometer/buildPages.mjs +++ b/packages/code-infra/src/tachometer/buildPages.mjs @@ -3,7 +3,7 @@ import * as path from 'node:path'; import { cp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; -import { stringify } from 'yaml'; +import { parse, stringify } from 'yaml'; import { tarballFor } from '../utils/packWorkspace.mjs'; import { run } from './exec.mjs'; @@ -80,6 +80,27 @@ export function rewriteWorkspaceDeps(deps, packages, omit = []) { return out; } +/** + * Reads the repository's `overrides`, so the isolated install resolves the same dependency graph + * the repository does. + * + * The throwaway package is its own workspace root and inherits nothing. Without this, a repository + * that pins a transitive dependency through an override would get a *different* graph in the + * sandbox than in its real install — quietly changing what is being benchmarked — and any policy + * the override exists to satisfy (a blocked resolution, a security pin) would fail there too. + * + * @param {string} repoRoot - The workspace root to copy overrides from + * @returns {Promise>} + */ +async function readOverrides(repoRoot) { + try { + const root = parse(await readFile(path.join(repoRoot, 'pnpm-workspace.yaml'), 'utf8')); + return root?.overrides ?? {}; + } catch { + return {}; + } +} + /** * Replaces the install directory's `.env*` files with the harness's. * @@ -110,6 +131,7 @@ async function syncEnvFiles(harnessDir, workDir) { * * @param {Object} options - Build inputs * @param {string} options.harnessDir - The harness package directory, copied from + * @param {string} options.repoRoot - The workspace root, whose overrides the sandbox inherits * @param {ResolvedRef} options.ref - The ref being built * @param {PackedPackage[]} options.packages - That ref's packed workspace packages * @param {string} options.workDir - Persistent directory holding this ref's isolated install @@ -117,7 +139,7 @@ async function syncEnvFiles(harnessDir, workDir) { * @returns {Promise} */ export async function buildRefPages(options) { - const { harnessDir, ref, packages, workDir, outDir } = options; + const { harnessDir, repoRoot, ref, packages, workDir, outDir } = options; console.log(chalk.cyan(`\nBuilding benchmark pages for "${ref.label}"…`)); const viteConfig = await findViteConfig(harnessDir); @@ -147,13 +169,17 @@ export async function buildRefPages(options) { ); // Marks this folder as its own workspace root, so pnpm does not walk up into the monorepo. The - // overrides pin every packed workspace package, so a transitive dependency between them (whose - // packed `workspace:*` became a concrete, unpublished version) resolves to a local build rather - // than 404ing on the registry. + // overrides carry the repository's own, then pin every packed workspace package on top — so a + // transitive dependency between them (whose packed `workspace:*` became a concrete, unpublished + // version) resolves to a local build rather than 404ing on the registry. The packed pins win, + // since they are the whole point of this install. await writeFile( path.join(workDir, 'pnpm-workspace.yaml'), stringify({ - overrides: Object.fromEntries(packages.map((pkg) => [pkg.name, `file:${pkg.tarball}`])), + overrides: { + ...(await readOverrides(repoRoot)), + ...Object.fromEntries(packages.map((pkg) => [pkg.name, `file:${pkg.tarball}`])), + }, }), ); diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index 70edc0965b..e0f29d5dfd 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -123,6 +123,7 @@ export async function runTachometer(options) { // eslint-disable-next-line no-await-in-loop await buildRefPages({ harnessDir, + repoRoot, ref, packages, workDir: path.join(installsDir, ref.id), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fc3407c936..7ccf73a746 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9313,18 +9313,6 @@ packages: utf-8-validate: optional: true - ws@8.21.0: - resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - ws@8.21.3: resolution: {integrity: sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==} engines: {node: '>=10.0.0'} @@ -12734,7 +12722,7 @@ snapshots: sirv: 3.0.2 tinyrainbow: 3.1.0 vitest: 4.1.11(@types/node@22.20.1)(@vitest/browser-playwright@4.1.11)(@vitest/coverage-v8@4.1.11)(jsdom@28.1.0(supports-color@10.2.2))(vite@8.2.2(@types/node@22.20.1)(esbuild@0.28.1)(jiti@2.7.0)(terser@5.44.0)(tsx@4.23.12)(yaml@2.9.0)) - ws: 8.21.0 + ws: 8.21.3 transitivePeerDependencies: - bufferutil - msw @@ -13314,7 +13302,7 @@ snapshots: cache-content-type@1.0.1: dependencies: - mime-types: 2.1.18 + mime-types: 2.1.35 ylru: 1.4.0 cacheable-lookup@7.0.0: {} @@ -18849,8 +18837,6 @@ snapshots: ws@7.5.11: {} - ws@8.21.0: {} - ws@8.21.3: {} wsl-utils@1.0.0: From 516af6e1f259294a894d560ba5a8e521ce9d5d2c Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:25:15 +0200 Subject: [PATCH 007/173] [code-infra] Render the tachometer table from tacho run The renderer lived in the consuming repository while the runner defined the report it renders, so `tacho run` could only print one-line verdicts and every consumer needed a wrapper script to get a table. Worse, the two halves could drift: a change to summarizeCase here would silently break a renderer over there. Moving it puts the report's shape and its presentation in one place. `tacho run` now renders from the object it already has, with no write-then-read hop, and a consumer's package.json script is just `code-infra tacho run`. It is also exported as `./tachometerReport` so a repository that multiplexes report types through its own CLI can still call it directly. Ported from base-ui-mosaic's scripts/benchmarkTable.mts, colours swapped to chalk. What stays there is the Vitest suite's own rendering, which measures something else entirely. --- packages/code-infra/package.json | 4 + .../src/tachometer/renderReport.mjs | 453 ++++++++++++++++++ .../src/tachometer/runTachometer.mjs | 19 +- 3 files changed, 461 insertions(+), 15 deletions(-) create mode 100644 packages/code-infra/src/tachometer/renderReport.mjs diff --git a/packages/code-infra/package.json b/packages/code-infra/package.json index c12669472d..a55c4824c8 100644 --- a/packages/code-infra/package.json +++ b/packages/code-infra/package.json @@ -49,6 +49,10 @@ "types": "./build/tachometer/plugin.d.mts", "default": "./src/tachometer/plugin.mjs" }, + "./tachometerReport": { + "types": "./build/tachometer/renderReport.d.mts", + "default": "./src/tachometer/renderReport.mjs" + }, "./brokenLinksChecker": { "types": "./build/brokenLinksChecker/index.d.mts", "default": "./src/brokenLinksChecker/index.mjs" diff --git a/packages/code-infra/src/tachometer/renderReport.mjs b/packages/code-infra/src/tachometer/renderReport.mjs new file mode 100644 index 0000000000..c07fd9ba48 --- /dev/null +++ b/packages/code-infra/src/tachometer/renderReport.mjs @@ -0,0 +1,453 @@ +/* eslint-disable no-console -- a reporter: printing the report is its output. */ + +import chalk from 'chalk'; + +/** + * Renders the report `tacho run` produces. + * + * Lives beside the runner rather than in the consuming repository because the runner defines the + * report's shape: keeping the producer and its renderer apart is how a change to one silently + * breaks the other. + */ + +/** + * @typedef {import('./summarizeCase.mjs').ConfidenceInterval} ConfidenceInterval + * @typedef {import('./summarizeCase.mjs').Verdict} Verdict + */ + +/** + * @typedef {Object} VariantResult + * @property {string} variant - Variant name + * @property {string | null} refId - The ref this variant loaded + * @property {ConfidenceInterval} meanMs - Mean duration, as a 95% confidence interval + * @property {number} samples - Sample count + * @property {number} bytesSent - Bytes transferred + */ + +/** + * @typedef {Object} Comparison + * @property {string} variant - The variant compared against the reference + * @property {Verdict} verdict - Whether the reference is faster, slower, or unresolved + * @property {ConfidenceInterval} absoluteMs - Difference in milliseconds + * @property {ConfidenceInterval} percentChange - Difference as a percentage + * @property {{ verdict: Verdict, absoluteMs: ConfidenceInterval, percentChange: ConfidenceInterval }} [versusReference] - The same pair the other way round + */ + +/** + * @typedef {Object} MeasurementResult + * @property {string} name - Measurement name + * @property {VariantResult[]} variants - One entry per variant + * @property {Comparison[]} comparisons - The reference against every other variant + */ + +/** + * @typedef {Object} CaseResult + * @property {string} name - Case name + * @property {string} [reference] - The variant every comparison is expressed against. Absent when the case failed to summarize + * @property {MeasurementResult[]} [measurements] - One entry per measurement + * @property {string} [error] - Why the case produced no summary + */ + +/** + * @typedef {Object} TachometerReport + * @property {number} version - Report format version + * @property {'tachometer'} reportType - Which benchmark axis produced this + * @property {string} generatedAt - ISO timestamp + * @property {{ ref: string, sha: string, branch?: string }} head - The commit measured + * @property {Array<{ id: string, kind: string, label: string, sha?: string }>} refs - The builds compared + * @property {CaseResult[]} cases - One entry per case + */ + +/** Milliseconds, to the precision tachometer's own table uses. */ +const MS_FORMAT = /** @type {Intl.NumberFormatOptions} */ ({ + style: 'unit', + unit: 'millisecond', + maximumFractionDigits: 2, + // Pad to the declared precision so a column lines up on the decimal point (`29.0 ms` under + // `30.9 ms`, not `29 ms`). + minimumFractionDigits: 2, +}); + +const msFormatter = new Intl.NumberFormat('en-US', MS_FORMAT); + +/** + * Matches the escape sequences chalk emits, so column widths measure visible characters. + * + * Built from a char code rather than written as a literal: an escape character inside a regular + * expression source is invisible in a diff and trips `no-control-regex`. + */ +const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g'); + +/** + * @param {string} text - Possibly coloured text + * @returns {string} The same text with colour escapes removed + */ +function stripAnsi(text) { + return text.replace(ANSI, ''); +} + +/** + * Prints a table, padding each column to its widest visible cell. + * + * @param {string[]} headers - Column headers + * @param {string[][]} rows - Row cells, in column order + * @param {boolean[]} alignLeft - Whether each column is left-aligned + * @returns {void} + */ +function printTable(headers, rows, alignLeft) { + const widths = headers.map((header, index) => + Math.max(stripAnsi(header).length, ...rows.map((row) => stripAnsi(row[index] ?? '').length)), + ); + /** + * @param {string} text - Cell contents + * @param {number} width - Target width + * @param {boolean} left - Whether to left-align + * @returns {string} + */ + const pad = (text, width, left) => { + const fill = ' '.repeat(Math.max(0, width - stripAnsi(text).length)); + return left ? text + fill : fill + text; + }; + /** + * @param {string[]} cells - One row + * @returns {string} + */ + const line = (cells) => + cells + .map((cell, index) => pad(cell ?? '', widths[index], alignLeft[index])) + .join(' ') + .trimEnd(); + + console.log(line(headers)); + console.log(widths.map((width) => '─'.repeat(width)).join(' ')); + for (const row of rows) { + console.log(line(row)); + } +} + +/** + * `low – high`, with the unit written once. + * + * @param {ConfidenceInterval} interval - The interval to format + * @returns {string} + */ +function formatInterval(interval) { + return `${msFormatter.format(interval.low).replace(/\s*ms$/, '')} – ${msFormatter.format(interval.high)}`; +} + +/** + * @param {ConfidenceInterval} interval - The interval to format + * @returns {string} Both bounds as signed percentages + */ +function formatPercentInterval(interval) { + /** + * @param {number} value - A percentage + * @returns {string} + */ + const signed = (value) => `${value >= 0 ? '+' : ''}${value.toFixed(1)}%`; + return `${signed(interval.low)} – ${signed(interval.high)}`; +} + +/** + * @param {Verdict} verdict - The verdict to colour by + * @param {string} text - Text to colour + * @returns {string} + */ +function colorVerdict(verdict, text) { + if (verdict === 'faster') { + return chalk.green(text); + } + if (verdict === 'slower') { + return chalk.red(text); + } + return chalk.dim(text); +} + +/** + * Kibibytes, matching how tachometer's own table reports `bytesSent`. + * + * @param {number} bytes - Byte count + * @returns {string} + */ +function formatBytes(bytes) { + return `${(bytes / 1024).toFixed(1)} KiB`; +} + +/** + * Tachometer names each variant ` []`, and the case is already its own column, so + * the prefix is dropped for display. Dropping it is also what makes columns line up across cases: + * every auto-expanded case then contributes the same `[current]`/`[baseline]` pair rather than a + * column of its own. + * + * @param {string} caseName - The case's name + * @param {string} variant - The variant's full name + * @returns {string} + */ +function shortNameOf(caseName, variant) { + return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; +} + +/** + * The distinct variants a case reports, in order of first appearance. + * + * @param {CaseResult & { measurements: MeasurementResult[] }} entry - A summarized case + * @returns {string[]} + */ +function variantsOf(entry) { + /** @type {string[]} */ + const variants = []; + for (const measurement of entry.measurements) { + for (const variant of measurement.variants) { + const short = shortNameOf(entry.name, variant.variant); + if (!variants.includes(short)) { + variants.push(short); + } + } + } + return variants; +} + +/** + * Renders one case as a table of variants, for a comparison with more than two of them. + * + * Rows are variants, not cases: with several libraries, a column per variant plus a Δ column per + * pair would run off the screen. Each measurement then shows the variant's interval next to its + * difference *relative to the reference* — the direction a row about that library reads in — taken + * from the variant's own row of tachometer's matrix. + * + * @param {CaseResult & { measurements: MeasurementResult[] }} entry - A summarized case + * @param {string[]} variants - The variant set, reference first + * @returns {void} + */ +function printVariantTable(entry, variants) { + const [reference] = variants; + const headers = [ + entry.name, + ...entry.measurements.flatMap((measurement) => [measurement.name, `vs ${reference}`]), + 'transferred', + 'Samples', + ]; + const alignLeft = headers.map((_, index) => index === 0); + + const rows = variants.map((variant) => { + /** @type {number | undefined} */ + let bytesSent; + /** @type {Set} */ + const sampleCounts = new Set(); + const cells = entry.measurements.flatMap((measurement) => { + const found = measurement.variants.find( + (candidate) => shortNameOf(entry.name, candidate.variant) === variant, + ); + const comparison = measurement.comparisons.find( + (candidate) => shortNameOf(entry.name, candidate.variant) === variant, + ); + if (found) { + bytesSent = found.bytesSent; + sampleCounts.add(found.samples); + } + let delta = chalk.dim('—'); + if (variant === reference) { + delta = chalk.dim('reference'); + } else if (comparison?.versusReference) { + const { verdict, percentChange } = comparison.versusReference; + delta = colorVerdict(verdict, `${verdict} ${formatPercentInterval(percentChange)}`); + } + return [found ? formatInterval(found.meanMs) : chalk.dim('—'), delta]; + }); + return [ + variant, + ...cells, + bytesSent === undefined ? chalk.dim('—') : formatBytes(bytesSent), + [...sampleCounts].join('/'), + ]; + }); + + printTable(headers, rows, alignLeft); + console.log( + chalk.dim( + `vs ${reference} is tachometer's confidence interval on the difference, that variant ` + + `relative to ${reference} — negative is faster than ${reference}.`, + ), + ); +} + +/** + * Renders a group of cases sharing a variant set, one row per case and measurement. + * + * @param {Array} cases - Cases in the group + * @param {string[]} variants - The shared variant set, reference first + * @returns {void} + */ +function printCaseTable(cases, variants) { + const [reference, ...others] = variants; + const headers = [ + 'Case', + 'Measurement', + ...variants, + ...others.map((variant) => `Δ vs ${variant}`), + 'Samples', + ]; + const alignLeft = [true, true, ...variants.map(() => false), ...others.map(() => false), false]; + + /** @type {string[][]} */ + const rows = []; + for (const [caseIndex, entry] of cases.entries()) { + if (caseIndex > 0) { + rows.push(headers.map(() => '')); + } + let firstRow = true; + for (const measurement of entry.measurements) { + const byVariant = new Map( + measurement.variants.map((variant) => [shortNameOf(entry.name, variant.variant), variant]), + ); + const byComparison = new Map( + measurement.comparisons.map((comparison) => [ + shortNameOf(entry.name, comparison.variant), + comparison, + ]), + ); + // Auto-sampling stops per case, not per variant, so the counts are normally equal; when they + // are not, saying so beats printing one number that describes neither side. + const sampleCounts = [...new Set(measurement.variants.map((variant) => variant.samples))]; + + rows.push([ + firstRow ? entry.name : '', + measurement.name, + ...variants.map((variant) => { + const found = byVariant.get(variant); + return found ? formatInterval(found.meanMs) : chalk.dim('—'); + }), + ...others.map((variant) => { + const comparison = byComparison.get(variant); + if (!comparison) { + return chalk.dim('—'); + } + return colorVerdict( + comparison.verdict, + `${comparison.verdict} ${formatPercentInterval(comparison.percentChange)}`, + ); + }), + sampleCounts.join('/'), + ]); + firstRow = false; + } + } + + printTable(headers, rows, alignLeft); + console.log( + chalk.dim( + `Δ is tachometer's confidence interval on the difference, ${reference ?? 'the reference'} ` + + `relative to the other variant${others.length > 1 ? 's' : ''} — negative is faster.`, + ), + ); +} + +/** + * Prints a tachometer report: one table per variant set, with a column per variant holding its 95% + * confidence interval and a Δ column per non-reference variant. + * + * The Δ column is **tachometer's own confidence interval on the difference**, not a delta of the + * two means. That is the whole reason this suite exists next to a profiling one: variants are + * sampled round-robin in a single session and auto-sampled until that interval clears the case's + * `autoSampleConditions`, so the interval — not a point estimate plus a post-hoc test — is the + * result. `unsure` means it never resolved within the sampling budget, which for two builds that + * really are equivalent is the expected outcome rather than a failure. + * + * Signs are the reference relative to the variant, so a faster reference reads negative; the + * verdict word carries the direction so the sign never has to be read twice. + * + * @param {TachometerReport} report - The report to render + * @returns {void} + */ +export function renderTachometerReport(report) { + const failed = report.cases.filter((entry) => !entry.measurements); + const usable = /** @type {Array} */ ( + report.cases.filter( + (entry) => entry.measurements !== undefined && entry.measurements.length > 0, + ) + ); + + if (usable.length === 0) { + console.log(chalk.dim('No tachometer results to show.')); + for (const entry of failed) { + console.log(chalk.red(` ${entry.name}: ${entry.error ?? 'no result'}`)); + } + return; + } + + // One table per variant set, in order of first appearance: a regression case carries + // `[current]`/`[baseline]`, a cross-library case `[ours]`/`[theirs]`, and each set has its own + // reference (the first variant). A single table over the union of the columns would put a blank + // cell in every row of every case, and could name only one reference for all of them. + /** @type {Map }>} */ + const groups = new Map(); + for (const entry of usable) { + const variants = variantsOf(entry); + const key = JSON.stringify(variants); + const group = groups.get(key) ?? { variants, cases: [] }; + group.cases.push(entry); + groups.set(key, group); + } + + // Cases whose table carries the transfer size as a column, so the note below need not repeat it. + /** @type {Set} */ + const withBytesColumn = new Set(); + + for (const [groupIndex, group] of [...groups.values()].entries()) { + if (groupIndex > 0) { + console.log(''); + } + if (group.variants.length > 2) { + for (const [caseIndex, entry] of group.cases.entries()) { + if (caseIndex > 0) { + console.log(''); + } + printVariantTable(entry, group.variants); + withBytesColumn.add(entry.name); + } + continue; + } + printCaseTable(group.cases, group.variants); + } + + const notes = [ + 'Each cell is a 95% confidence interval for the mean, in milliseconds; one sample = one page load.', + '"unsure" means the interval still straddles zero: the difference did not resolve within the case\'s sampling budget.', + ]; + + // Bundle weight is a property of the variant's page, not of a measurement, so it belongs in a + // note rather than repeated down every row. It comes free with the run and is the number that + // explains a cold-start difference that mount time alone does not. + for (const entry of usable) { + if (withBytesColumn.has(entry.name)) { + continue; + } + /** @type {Map} */ + const bytes = new Map(); + for (const measurement of entry.measurements) { + for (const variant of measurement.variants) { + bytes.set(shortNameOf(entry.name, variant.variant), variant.bytesSent); + } + } + const perVariant = [...bytes].map(([variant, value]) => `${variant} ${formatBytes(value)}`); + if (perVariant.length > 0) { + notes.push(`${entry.name} transferred: ${perVariant.join(' · ')}`); + } + } + + const refLabels = report.refs.map( + (ref) => `${ref.id} = ${ref.label}${ref.sha ? ` (${ref.sha.slice(0, 9)})` : ''}`, + ); + if (refLabels.length > 0) { + notes.push(`Builds: ${refLabels.join(' · ')}`); + } + notes.push( + `head: ${report.head.sha.slice(0, 9)} (${report.head.branch || '?'}) · ` + + `measured on the production bundle, installed from a packed tarball.`, + ); + console.log(chalk.dim(`\n${notes.join('\n')}`)); + + for (const entry of failed) { + console.log(chalk.red(`\n${entry.name}: ${entry.error ?? 'no result'}`)); + } +} diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index e0f29d5dfd..2004fd2649 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -12,6 +12,7 @@ import { discoverCases, pagesOf } from './discoverCases.mjs'; import { buildRefPages } from './buildPages.mjs'; import { assertDriverMatchesBrowser, resolveBrowserBinary } from './browser.mjs'; import { summarizeCase } from './summarizeCase.mjs'; +import { renderTachometerReport } from './renderReport.mjs'; import { run } from './exec.mjs'; /** @@ -175,7 +176,7 @@ export async function runTachometer(options) { // Consumers render reports from several benchmark axes; the pair identifies which one this // is and how to read it. version: 1, - reportType: 'tachometer', + reportType: /** @type {const} */ ('tachometer'), generatedAt: new Date().toISOString(), head: { ref: 'HEAD', @@ -208,21 +209,9 @@ export async function runTachometer(options) { }; await mkdir(path.dirname(outPath), { recursive: true }); await writeFile(outPath, `${JSON.stringify(report, null, 2)}\n`); + console.log(''); + renderTachometerReport(report); console.log(chalk.green(`\nWrote JSON report to ${outPath}`)); - for (const entry of report.cases) { - if (!('measurements' in entry)) { - console.log(` ${entry.name}: no result (${entry.error})`); - continue; - } - for (const measurement of entry.measurements) { - for (const comparison of measurement.comparisons) { - console.log( - ` ${entry.name} [${measurement.name}]: ${entry.reference} is ` + - `${comparison.verdict} vs ${comparison.variant}`, - ); - } - } - } } finally { await rm(tmpBase, { recursive: true, force: true }); } From 220f80917512ef6d269247ef93ce48f589955e55 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:37:14 +0200 Subject: [PATCH 008/173] [code-infra] Report tachometer results on the pull request `tacho run` produced a table in the CI log and nothing else, so a regression was only found by someone reading the job output. It now uploads its report to the CI report store and asks the dashboard to refresh the PR comment, joining bundle size, deploy previews and the Vitest benchmarks in the same comment. Regressions are the point, so they are stated above the collapsed block and mark the section heading. GitHub renders
    closed, so anything inside it is invisible until someone chooses to look -- a warning buried there would be no warning at all. Everything that did not regress collapses to one line, and `unsure` is never called a regression: it is the expected result for two equivalent builds, and treating it as a warning would make the marker meaningless within a week. Only comparisons across refs count as regressions. A cross-library case runs every variant from one build, so "slower than ag-grid" is true, worth knowing, and unrelated to what the pull request changed. Unlike the benchmark section this fetches one artifact and does no comparison: a tachometer report already contains both sides, sampled round-robin in a single session, so there is no base report to look up and no chance of pairing against a mismatched one. The dashboard mirrors the report types rather than importing them, matching what it does for @mui/internal-benchmark -- it renders artifacts written months ago by whatever version of the tooling a repository had. That is the opposite of the call made for the terminal renderer, which runs in-process at the same version. The details page is deliberately plain: every number, as tables, no charts. A confidence interval is two numbers and a verdict, and drawing it as a bar invites reading the picture as significance when the interval is what carries that. The upload transport is a small copy of the benchmark package's rather than a shared import: code-infra's dependency tree is large, and having the benchmark package depend on it would push all of that onto its consumers. --- .../[owner]/[repo]/page.tsx | 16 ++ .../api/ci-reports/sync-pr-comment/route.ts | 72 +++--- .../app/api/ci-reports/upload/route.ts | 15 +- apps/code-infra-dashboard/src/constants.ts | 1 + .../src/lib/ciReports/tachometerReport.ts | 41 ++++ .../tachometer/buildMarkdownReport.test.ts | 194 ++++++++++++++++ .../src/lib/tachometer/buildMarkdownReport.ts | 207 +++++++++++++++++ .../src/lib/tachometer/types.ts | 73 ++++++ .../src/utils/fetchCiReport.ts | 3 + .../src/views/TachometerDetails.tsx | 211 ++++++++++++++++++ packages/code-infra/package.json | 3 +- packages/code-infra/src/cli/cmdTacho.mjs | 8 + .../code-infra/src/tachometer/ciReport.mjs | 187 ++++++++++++++++ .../src/tachometer/ciReport.test.mjs | 140 ++++++++++++ .../src/tachometer/runTachometer.mjs | 42 ++++ pnpm-lock.yaml | 3 + 16 files changed, 1182 insertions(+), 34 deletions(-) create mode 100644 apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx create mode 100644 apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts create mode 100644 apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts create mode 100644 apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts create mode 100644 apps/code-infra-dashboard/src/lib/tachometer/types.ts create mode 100644 apps/code-infra-dashboard/src/views/TachometerDetails.tsx create mode 100644 packages/code-infra/src/tachometer/ciReport.mjs create mode 100644 packages/code-infra/src/tachometer/ciReport.test.mjs diff --git a/apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx b/apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx new file mode 100644 index 0000000000..6d1a42140e --- /dev/null +++ b/apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx @@ -0,0 +1,16 @@ +import * as React from 'react'; +import type { Metadata } from 'next'; +import TachometerDetails from '@/views/TachometerDetails'; + +export async function generateMetadata({ + params, +}: { + params: Promise<{ owner: string; repo: string }>; +}): Promise { + const { owner, repo } = await params; + return { title: `Tachometer Details - ${owner}/${repo}` }; +} + +export default function TachometerDetailsPage() { + return ; +} diff --git a/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts b/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts index 4542ce2e75..0cedc5ddec 100644 --- a/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts +++ b/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts @@ -10,6 +10,10 @@ import { } from '@/lib/ciReports/bundleSizeReport'; import { generateBenchmarkReport, BENCHMARK_SECTION_TITLE } from '@/lib/ciReports/benchmarkReport'; import { generateDeployPreviewReport } from '@/lib/ciReports/deployPreviewReport'; +import { + generateTachometerReport, + TACHOMETER_SECTION_TITLE, +} from '@/lib/ciReports/tachometerReport'; import type { ReportResult } from '@/lib/ciReports/types'; import { fetchParentCommits } from '@/utils/fetchCiReportWithFallback'; import { getOctokit } from '@/lib/github'; @@ -118,36 +122,50 @@ export async function POST(request: NextRequest) { // Generate all configured report sections in parallel. // Each generator is wrapped in .catch() so a failure in one doesn't block the others. - const [bundleSizeReport, benchmarkReportResult, deployPreviewReport] = await Promise.all([ - prCommentConfig?.bundleSize - ? generateBundleSizeReport(reportOptions).catch((error): ReportResult => { - console.error('Failed to generate bundle size report:', error); - return { - content: `## ${BUNDLE_SIZE_SECTION_TITLE}\n\n:warning: Failed to generate bundle size report.`, - }; - }) - : null, - prCommentConfig?.benchmark - ? generateBenchmarkReport(reportOptions).catch((error): ReportResult => { - console.error('Failed to generate benchmark report:', error); - return { - content: `## ${BENCHMARK_SECTION_TITLE}\n\n:warning: Failed to generate benchmark report.`, - }; - }) - : null, - prCommentConfig?.netlifyDocs - ? generateDeployPreviewReport(reportOptions).catch((error): ReportResult => { - console.error('Failed to generate deploy preview report:', error); - return { - content: `## Deploy preview\n\n:warning: Failed to generate deploy preview.`, - }; - }) - : null, - ]); + const [bundleSizeReport, benchmarkReportResult, tachometerReportResult, deployPreviewReport] = + await Promise.all([ + prCommentConfig?.bundleSize + ? generateBundleSizeReport(reportOptions).catch((error): ReportResult => { + console.error('Failed to generate bundle size report:', error); + return { + content: `## ${BUNDLE_SIZE_SECTION_TITLE}\n\n:warning: Failed to generate bundle size report.`, + }; + }) + : null, + prCommentConfig?.benchmark + ? generateBenchmarkReport(reportOptions).catch((error): ReportResult => { + console.error('Failed to generate benchmark report:', error); + return { + content: `## ${BENCHMARK_SECTION_TITLE}\n\n:warning: Failed to generate benchmark report.`, + }; + }) + : null, + prCommentConfig?.tachometer + ? generateTachometerReport(reportOptions).catch((error): ReportResult => { + console.error('Failed to generate tachometer report:', error); + return { + content: `## ${TACHOMETER_SECTION_TITLE}\n\n:warning: Failed to generate tachometer report.`, + }; + }) + : null, + prCommentConfig?.netlifyDocs + ? generateDeployPreviewReport(reportOptions).catch((error): ReportResult => { + console.error('Failed to generate deploy preview report:', error); + return { + content: `## Deploy preview\n\n:warning: Failed to generate deploy preview.`, + }; + }) + : null, + ]); const trailer = `
    \n\nCheck out the [code infra dashboard](${DASHBOARD_ORIGIN}/repository/${prRepo}/prs/${pr.number}) for more information about this PR.`; - const commentBody = [deployPreviewReport, bundleSizeReport, benchmarkReportResult] + const commentBody = [ + deployPreviewReport, + bundleSizeReport, + benchmarkReportResult, + tachometerReportResult, + ] .filter((report): report is ReportResult => report !== null) .map((report) => report.content) .concat(trailer) diff --git a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts index fb0e6eb338..6b1fa79906 100644 --- a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts +++ b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts @@ -5,7 +5,12 @@ import { uploadReport } from '@/lib/ciReports/s3'; import { verifyOidcToken } from '@/lib/ciReports/oidcAuth'; import { findAssociatedPr } from '@/lib/ciReports/findAssociatedPr'; -const VALID_REPORT_TYPES = new Set(['size-snapshot', 'benchmark']); +const VALID_REPORT_TYPES = new Set(['size-snapshot', 'benchmark', 'tachometer']); + +// Report types stored as the full upload envelope (version, timestamp, commitSha, repo, +// branch, prNumber, reportType, report). Older types keep their historic +// "just the inner report" storage. +const ENVELOPE_REPORT_TYPES = new Set(['benchmark', 'tachometer']); const uploadSchema = z.object({ version: z.number(), @@ -55,11 +60,9 @@ export async function POST(request: NextRequest) { const { commitSha, repo, reportType, branch, report } = parsed.data; - // For benchmark uploads, store the full wrapper (version, timestamp, commitSha, - // repo, branch, prNumber, reportType, report, base). Other report types keep - // their historic "just the inner report" storage. - const storedBody = - reportType === 'benchmark' ? JSON.stringify(parsed.data) : JSON.stringify(report); + const storedBody = ENVELOPE_REPORT_TYPES.has(reportType) + ? JSON.stringify(parsed.data) + : JSON.stringify(report); if (!VALID_REPORT_TYPES.has(reportType)) { return NextResponse.json( diff --git a/apps/code-infra-dashboard/src/constants.ts b/apps/code-infra-dashboard/src/constants.ts index 1bd59b2047..7418489909 100644 --- a/apps/code-infra-dashboard/src/constants.ts +++ b/apps/code-infra-dashboard/src/constants.ts @@ -25,6 +25,7 @@ export interface Repository { prComment?: { bundleSize?: boolean; benchmark?: boolean; + tachometer?: boolean; netlifyDocs?: | boolean | { diff --git a/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts b/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts new file mode 100644 index 0000000000..587039389f --- /dev/null +++ b/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts @@ -0,0 +1,41 @@ +import { fetchCiReport } from '@/utils/fetchCiReport'; +import { + buildTachometerMarkdownReport, + TACHOMETER_SECTION_TITLE, +} from '@/lib/tachometer/buildMarkdownReport'; +import { DASHBOARD_ORIGIN } from '@/constants'; + +import type { TachometerUpload } from '@/lib/tachometer/types'; +import type { ReportOptions, ReportResult } from './types'; + +export { TACHOMETER_SECTION_TITLE }; + +/** + * Generates the tachometer section of the pull request comment. + * + * Unlike the benchmark section, this fetches exactly one artifact and does no comparison. A + * tachometer report already contains both sides: `[current]` and `[baseline]` are sampled + * round-robin in a single browser session, and the report carries tachometer's own confidence + * interval on the difference. There is no base report to look up, and so no chance of pairing + * against a mismatched one. + * + * Returns null when the head commit has no report, which is how a repository that did not run the + * job produces no section at all. + */ +export async function generateTachometerReport( + options: ReportOptions, +): Promise { + const { repo, commitSha } = options; + + const upload: TachometerUpload | null = await fetchCiReport(repo, commitSha, 'tachometer.json'); + if (!upload?.report) { + return null; + } + + const detailsUrl = new URL(`${DASHBOARD_ORIGIN}/tachometer-details/${repo}`); + detailsUrl.searchParams.set('sha', commitSha); + + return { + content: buildTachometerMarkdownReport(upload.report, { detailsUrl: detailsUrl.href }), + }; +} diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts new file mode 100644 index 0000000000..18ad2e24a7 --- /dev/null +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts @@ -0,0 +1,194 @@ +import { describe, expect, it } from 'vitest'; +import { buildTachometerMarkdownReport, findRegressions } from './buildMarkdownReport'; +import type { + TachometerCaseResult, + TachometerMeasurementResult, + TachometerReport, + Verdict, +} from './types'; + +/** + * A variant on a given build. + */ +function variant(name: string, refId: string | null) { + return { + variant: name, + refId, + meanMs: { low: 20, high: 22 }, + samples: 120, + bytesSent: 2969, + }; +} + +/** + * A comparison of the reference against `name`. + */ +function comparison(name: string, verdict: Verdict) { + const sign = verdict === 'slower' ? 1 : -1; + return { + variant: name, + verdict, + absoluteMs: { low: sign * 6.1, high: sign * 15.2 }, + percentChange: { low: sign * 2.7, high: sign * 6.8 }, + }; +} + +/** + * A case comparing `[current]` against `[baseline]` — two different builds. + */ +function regressionCase(name: string, verdicts: Record): TachometerCaseResult { + const measurements: TachometerMeasurementResult[] = Object.entries(verdicts).map( + ([measurement, verdict]) => ({ + name: measurement, + variants: [ + variant(`${name} [current]`, 'current'), + variant(`${name} [baseline]`, 'git-abc123456'), + ], + comparisons: [comparison(`${name} [baseline]`, verdict)], + }), + ); + return { name, reference: `${name} [current]`, measurements }; +} + +function report(cases: TachometerCaseResult[]): TachometerReport { + return { + version: 1, + reportType: 'tachometer', + generatedAt: '2026-09-02T00:00:00.000Z', + head: { ref: 'HEAD', sha: 'a'.repeat(40), branch: 'feature' }, + refs: [ + { id: 'current', kind: 'worktree', label: 'working tree' }, + { id: 'git-abc123456', kind: 'git', label: 'merge-base', sha: 'b'.repeat(40) }, + ], + cases, + }; +} + +/** Everything a reader sees without expanding anything. */ +function visiblePart(markdown: string): string { + return markdown.split('
    ')[0]; +} + +describe('findRegressions', () => { + it('finds a slower verdict across two builds', () => { + const found = findRegressions(report([regressionCase('workload', { mount: 'slower' })])); + + expect(found).toMatchObject([{ caseName: 'workload', measurement: 'mount' }]); + }); + + it.each(['unsure', 'faster'])('does not count a %s verdict', (verdict) => { + expect(findRegressions(report([regressionCase('workload', { mount: verdict })]))).toEqual([]); + }); + + it('ignores a slower verdict between variants on the same build', () => { + // A cross-library case runs every variant from one build, so "slower than ag-grid" says nothing + // about what this pull request changed. Flagging it would warn on every comment forever. + const libs: TachometerCaseResult = { + name: 'libs-mount', + reference: 'libs-mount [mosaic]', + measurements: [ + { + name: 'mount', + variants: [ + variant('libs-mount [mosaic]', 'current'), + variant('libs-mount [ag-grid]', 'current'), + ], + comparisons: [comparison('libs-mount [ag-grid]', 'slower')], + }, + ], + }; + + expect(findRegressions(report([libs]))).toEqual([]); + }); + + it('reports only the measurement that regressed', () => { + const found = findRegressions( + report([regressionCase('workload', { mount: 'slower', scroll: 'unsure' })]), + ); + + expect(found).toHaveLength(1); + expect(found[0].measurement).toBe('mount'); + }); +}); + +describe('buildTachometerMarkdownReport', () => { + it('states a regression before the collapsed block, where it can be seen', () => { + // GitHub renders
    closed, so a regression inside it is invisible. + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'slower' })]), + ); + + expect(visiblePart(markdown)).toContain('workload'); + expect(visiblePart(markdown)).toContain('mount'); + expect(visiblePart(markdown)).toContain('+2.7% – +6.8%'); + }); + + it('marks the heading when something regressed', () => { + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'slower' })]), + ); + + expect(markdown.split('\n')[0]).toBe('## Tachometer ⚠️'); + }); + + it('leaves the heading unmarked when nothing regressed', () => { + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'unsure' })]), + ); + + expect(markdown.split('\n')[0]).toBe('## Tachometer'); + expect(visiblePart(markdown)).not.toContain('⚠️'); + }); + + it('summarizes a clean run in one line', () => { + const markdown = buildTachometerMarkdownReport( + report([ + regressionCase('workload', { mount: 'unsure' }), + regressionCase('scroll', { mount: 'unsure' }), + ]), + ); + + expect(visiblePart(markdown)).toContain('2 cases measured · 2 unchanged'); + }); + + it('counts a faster case separately', () => { + const markdown = buildTachometerMarkdownReport( + report([ + regressionCase('workload', { mount: 'faster' }), + regressionCase('scroll', { mount: 'unsure' }), + ]), + ); + + expect(visiblePart(markdown)).toContain('1 faster'); + }); + + it('reports a case that produced no result', () => { + const markdown = buildTachometerMarkdownReport( + report([{ name: 'broken', error: 'produced no benchmarks' }]), + ); + + expect(visiblePart(markdown)).toContain('broken'); + expect(visiblePart(markdown)).toContain('produced no benchmarks'); + }); + + it('puts every variant and measurement in the collapsed table', () => { + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'unsure', scroll: 'unsure' })]), + ); + + const details = markdown.split('
    ')[1]; + expect(details).toContain('mount'); + expect(details).toContain('scroll'); + expect(details).toContain('[current]'); + expect(details).toContain('[baseline]'); + }); + + it('links to the full run when given a url', () => { + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'unsure' })]), + { detailsUrl: 'https://example.test/run' }, + ); + + expect(markdown).toContain('[See the full run](https://example.test/run)'); + }); +}); diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts new file mode 100644 index 0000000000..937c6f5b70 --- /dev/null +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts @@ -0,0 +1,207 @@ +import type { + ConfidenceInterval, + TachometerCaseResult, + TachometerMeasurementResult, + TachometerReport, +} from './types'; + +export const TACHOMETER_SECTION_TITLE = 'Tachometer'; + +interface Regression { + caseName: string; + measurement: string; + variant: string; + percentChange: ConfidenceInterval; + absoluteMs: ConfidenceInterval; +} + +interface BuildOptions { + /** Where the full results can be inspected. Omitted in tests. */ + detailsUrl?: string; +} + +/** `+2.7% – +6.8%`, both bounds signed so the direction reads without the verdict. */ +function formatPercent(interval: ConfidenceInterval): string { + const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(1)}%`; + return `${signed(interval.low)} – ${signed(interval.high)}`; +} + +/** `+6.1 ms – +15.2 ms`. */ +function formatMs(interval: ConfidenceInterval): string { + const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(2)} ms`; + return `${signed(interval.low)} – ${signed(interval.high)}`; +} + +/** `20.50 – 22.67 ms`, with the unit written once. */ +function formatMean(interval: ConfidenceInterval): string { + return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; +} + +function formatBytes(bytes: number): string { + return `${(bytes / 1024).toFixed(1)} KiB`; +} + +/** + * Tachometer names each variant ` []`; the case is already its own heading here. + */ +function shortNameOf(caseName: string, variant: string): string { + return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; +} + +type SummarizedCase = TachometerCaseResult & { measurements: TachometerMeasurementResult[] }; + +function isSummarized(entry: TachometerCaseResult): entry is SummarizedCase { + return entry.measurements !== undefined && entry.measurements.length > 0; +} + +/** + * Collects the comparisons that mean "this pull request made something slower". + * + * `comparisons` express the reference relative to each other variant, so a `slower` verdict means + * the reference — `[current]` for an auto-expanded case — is the slow side. + * + * Only comparisons **across refs** count. A cross-library case runs every variant from the same + * build, so a `slower` verdict there says this library is slower than a competitor: true, worth + * knowing, and completely unrelated to what the pull request changed. Flagging those would put a + * warning on every comment until nobody read it. + */ +export function findRegressions(report: TachometerReport): Regression[] { + const regressions: Regression[] = []; + + for (const entry of report.cases) { + if (!isSummarized(entry)) { + continue; + } + for (const measurement of entry.measurements) { + const referenceRefId = measurement.variants.find( + (variant) => variant.variant === entry.reference, + )?.refId; + + for (const comparison of measurement.comparisons) { + if (comparison.verdict !== 'slower') { + continue; + } + const comparedRefId = measurement.variants.find( + (variant) => variant.variant === comparison.variant, + )?.refId; + if (comparedRefId === referenceRefId) { + continue; + } + regressions.push({ + caseName: entry.name, + measurement: measurement.name, + variant: shortNameOf(entry.name, comparison.variant), + percentChange: comparison.percentChange, + absoluteMs: comparison.absoluteMs, + }); + } + } + } + + return regressions; +} + +/** One flat table per case: every measurement, every variant, every number the report holds. */ +function renderCaseTable(entry: SummarizedCase): string { + const rows: string[] = []; + + for (const measurement of entry.measurements) { + const byVariant = new Map( + measurement.comparisons.map((comparison) => [comparison.variant, comparison]), + ); + for (const variant of measurement.variants) { + const comparison = byVariant.get(variant.variant); + const isReference = variant.variant === entry.reference; + const versus = isReference + ? '_reference_' + : (comparison && `${comparison.verdict} \`${formatPercent(comparison.percentChange)}\``) || + '—'; + rows.push( + `| ${measurement.name} | ${shortNameOf(entry.name, variant.variant)} | ${formatMean( + variant.meanMs, + )} | ${versus} | ${variant.samples} | ${formatBytes(variant.bytesSent)} |`, + ); + } + } + + return [ + `**${entry.name}**`, + '', + '| Measurement | Variant | Mean (95% CI) | vs reference | Samples | Transferred |', + '| :--- | :--- | ---: | :--- | ---: | ---: |', + ...rows, + ].join('\n'); +} + +/** + * Renders the tachometer section of the pull request comment. + * + * Regressions are stated above the collapsed block, and mark the heading, because a reader has to + * see them without expanding anything — GitHub renders `
    ` closed, so anything inside it is + * invisible until someone chooses to look. + */ +export function buildTachometerMarkdownReport( + report: TachometerReport, + options: BuildOptions = {}, +): string { + const regressions = findRegressions(report); + const summarized = report.cases.filter(isSummarized); + const failed = report.cases.filter((entry) => !isSummarized(entry)); + + const heading = `## ${TACHOMETER_SECTION_TITLE}${regressions.length > 0 ? ' ⚠️' : ''}`; + const lines = [heading, '']; + + if (regressions.length > 0) { + for (const regression of regressions) { + lines.push( + `⚠️ **${regression.caseName}** · ${regression.measurement} · slower than ` + + `${regression.variant} \`${formatPercent(regression.percentChange)}\` ` + + `(\`${formatMs(regression.absoluteMs)}\`)`, + ); + } + lines.push(''); + } + + // Everything that did not regress collapses to a single line, so the regressions are what the eye + // lands on. `unsure` is the expected result for two equivalent builds, not a warning. + const faster = summarized.filter((entry) => + entry.measurements.some((measurement) => + measurement.comparisons.some((comparison) => comparison.verdict === 'faster'), + ), + ).length; + const unchanged = summarized.length - regressions.length - faster; + const summary = [ + `${summarized.length} case${summarized.length === 1 ? '' : 's'} measured`, + unchanged > 0 ? `${unchanged} unchanged` : null, + faster > 0 ? `${faster} faster` : null, + regressions.length > 0 ? `${regressions.length} slower` : null, + ] + .filter(Boolean) + .join(' · '); + lines.push(summary, ''); + + for (const entry of failed) { + lines.push(`❌ **${entry.name}**: ${entry.error ?? 'produced no result'}`); + } + if (failed.length > 0) { + lines.push(''); + } + + if (summarized.length > 0) { + lines.push('
    ', 'Full results', ''); + lines.push(summarized.map(renderCaseTable).join('\n\n')); + lines.push( + '', + '_Each cell is a 95% confidence interval for the mean; one sample is one page load._', + '_"unsure" means the interval still straddles zero — the expected result for two equivalent builds._', + '', + '
    ', + ); + } + + if (options.detailsUrl) { + lines.push('', `[See the full run](${options.detailsUrl})`); + } + + return lines.join('\n'); +} diff --git a/apps/code-infra-dashboard/src/lib/tachometer/types.ts b/apps/code-infra-dashboard/src/lib/tachometer/types.ts new file mode 100644 index 0000000000..29e68f68e7 --- /dev/null +++ b/apps/code-infra-dashboard/src/lib/tachometer/types.ts @@ -0,0 +1,73 @@ +/** + * Types mirroring @mui/internal-code-infra/tachometerReport. + * + * Mirrored rather than imported on purpose: these describe artifacts already sitting in S3, written + * by whatever version of the tooling a repository had at the time. Importing the current definitions + * would make the dashboard assume every stored report matches today's shape. + */ + +export interface ConfidenceInterval { + low: number; + high: number; +} + +export type Verdict = 'faster' | 'slower' | 'unsure'; + +export interface TachometerVariantResult { + variant: string; + refId: string | null; + meanMs: ConfidenceInterval; + samples: number; + bytesSent: number; +} + +export interface TachometerComparison { + variant: string; + verdict: Verdict; + /** The reference relative to `variant`: positive means the reference is slower. */ + absoluteMs: ConfidenceInterval; + percentChange: ConfidenceInterval; + /** The same pair the other way round. Each direction has its own denominator. */ + versusReference?: { + verdict: Verdict; + absoluteMs: ConfidenceInterval; + percentChange: ConfidenceInterval; + }; +} + +export interface TachometerMeasurementResult { + name: string; + variants: TachometerVariantResult[]; + comparisons: TachometerComparison[]; +} + +export interface TachometerCaseResult { + name: string; + /** The variant every comparison is expressed against. Absent when the case failed to summarize. */ + reference?: string; + measurements?: TachometerMeasurementResult[]; + error?: string; +} + +export interface TachometerReport { + version: number; + reportType: 'tachometer'; + generatedAt: string; + head: { ref: string; sha: string; branch?: string }; + browser?: string; + refs: Array<{ id: string; kind: string; label: string; sha?: string }>; + cases: TachometerCaseResult[]; + raw?: Record; +} + +/** The stored upload envelope, as the upload route writes it. */ +export interface TachometerUpload { + version: number; + timestamp: number; + commitSha: string; + repo: string; + reportType: 'tachometer'; + prNumber?: number; + branch: string; + report: TachometerReport; +} diff --git a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts index 2b2e7cfabd..d3837fae7d 100644 --- a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts +++ b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts @@ -1,10 +1,13 @@ import type { SizeSnapshotWithMetadata } from '@/lib/bundleSize/types'; import type { BenchmarkReport, BenchmarkUpload } from '@/lib/benchmark/types'; +import type { TachometerUpload } from '@/lib/tachometer/types'; import { migrateBenchmarkReport } from '@/lib/benchmark/migrateBenchmarkReport'; export interface CiReportTypes { 'benchmark.json': BenchmarkUpload; 'size-snapshot.json': SizeSnapshotWithMetadata; + // Always stored as the full envelope, so there is no legacy flat shape to normalize. + 'tachometer.json': TachometerUpload; } export type CiReportName = keyof CiReportTypes; diff --git a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx new file mode 100644 index 0000000000..93085a83ad --- /dev/null +++ b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx @@ -0,0 +1,211 @@ +'use client'; + +import * as React from 'react'; +import { useParams, useSearchParams } from 'next/navigation'; +import { useQuery } from '@tanstack/react-query'; +import Alert from '@mui/material/Alert'; +import Box from '@mui/material/Box'; +import CircularProgress from '@mui/material/CircularProgress'; +import Paper from '@mui/material/Paper'; +import Table from '@mui/material/Table'; +import TableBody from '@mui/material/TableBody'; +import TableCell from '@mui/material/TableCell'; +import TableHead from '@mui/material/TableHead'; +import TableRow from '@mui/material/TableRow'; +import Typography from '@mui/material/Typography'; +import { fetchCiReport } from '@/utils/fetchCiReport'; +import type { + ConfidenceInterval, + TachometerCaseResult, + TachometerMeasurementResult, + TachometerUpload, + Verdict, +} from '@/lib/tachometer/types'; +import Heading from '../components/Heading'; +import ReportHeader from '../components/ReportHeader'; +import ErrorDisplay from '../components/ErrorDisplay'; + +/** + * Deliberately plain: every number the report holds, as tables. + * + * No charts or bars. A confidence interval is two numbers and a verdict, and drawing it as a length + * invites reading the picture as significance when the interval is what actually carries that. + */ + +function formatMean(interval: ConfidenceInterval): string { + return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; +} + +function formatPercent(interval: ConfidenceInterval): string { + const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(1)}%`; + return `${signed(interval.low)} – ${signed(interval.high)}`; +} + +function formatBytes(bytes: number): string { + return `${(bytes / 1024).toFixed(1)} KiB`; +} + +/** Only a regression is coloured; `unsure` is the expected result and stays neutral. */ +function verdictColor(verdict: Verdict): 'error' | 'success' | 'text.secondary' { + if (verdict === 'slower') { + return 'error'; + } + if (verdict === 'faster') { + return 'success'; + } + return 'text.secondary'; +} + +function shortNameOf(caseName: string, variant: string): string { + return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; +} + +interface CaseTableProps { + entry: TachometerCaseResult & { measurements: TachometerMeasurementResult[] }; +} + +function CaseTable({ entry }: CaseTableProps) { + return ( + + {entry.name} + + + + Measurement + Variant + Mean (95% CI) + vs reference + Samples + Transferred + + + + {entry.measurements.flatMap((measurement) => { + const byVariant = new Map( + measurement.comparisons.map((comparison) => [comparison.variant, comparison]), + ); + return measurement.variants.map((variant) => { + const comparison = byVariant.get(variant.variant); + const isReference = variant.variant === entry.reference; + return ( + + {measurement.name} + {shortNameOf(entry.name, variant.variant)} + {formatMean(variant.meanMs)} + + {isReference ? ( + + reference + + ) : ( + comparison && ( + + {comparison.verdict} {formatPercent(comparison.percentChange)} + + ) + )} + + {variant.samples} + {formatBytes(variant.bytesSent)} + + ); + }); + })} + +
    +     + ); +} + +export default function TachometerDetails() { + const params = useParams<{ owner: string; repo: string }>(); + const searchParams = useSearchParams(); + + if (!params.owner || !params.repo) { + throw new Error('Missing required path parameters'); + } + + const repo = `${params.owner}/${params.repo}`; + const sha = searchParams.get('sha'); + const prNumber = searchParams.get('prNumber'); + + const { + data: upload, + isLoading, + error, + } = useQuery({ + queryKey: ['tachometer-report', repo, sha], + queryFn: () => fetchCiReport(repo, sha!, 'tachometer.json'), + retry: 1, + enabled: Boolean(sha), + }); + + if (!sha) { + return ( + + Tachometer Details + + Missing required "sha" query parameter. + + + ); + } + + const report = upload?.report; + const summarized = (report?.cases ?? []).filter( + (entry): entry is TachometerCaseResult & { measurements: TachometerMeasurementResult[] } => + entry.measurements !== undefined && entry.measurements.length > 0, + ); + const failed = (report?.cases ?? []).filter((entry) => !entry.measurements?.length); + + // A tachometer report carries its own comparison, so the baseline is one of its own refs rather + // than a separately fetched report. + const baselineRef = report?.refs.find((ref) => ref.kind !== 'worktree'); + + return ( + + Tachometer Details + + + + + {isLoading && ( + + + Loading tachometer report… + + )} + + {error && } + + {!isLoading && !error && !report && ( + No tachometer report found for this commit. + )} + + {failed.map((entry) => ( + + {entry.name}: {entry.error ?? 'produced no result'} + + ))} + + {summarized.map((entry) => ( + + ))} + + {report && ( + + Each cell is a 95% confidence interval for the mean; one sample is one page load. + "unsure" means the interval still straddles zero — the expected result for two + equivalent builds. Measured on the production bundle, installed from a packed tarball. + + )} + + + ); +} diff --git a/packages/code-infra/package.json b/packages/code-infra/package.json index a55c4824c8..2c4b07c1c3 100644 --- a/packages/code-infra/package.json +++ b/packages/code-infra/package.json @@ -156,7 +156,8 @@ "unified-lint-rule": "^3.0.1", "unist-util-visit": "^5.1.0", "yaml": "^2.9.0", - "yargs": "^18.1.0" + "yargs": "^18.1.0", + "zod": "^4.4.3" }, "peerDependencies": { "@next/eslint-plugin-next": "*", diff --git a/packages/code-infra/src/cli/cmdTacho.mjs b/packages/code-infra/src/cli/cmdTacho.mjs index 5a5515df9c..3d4e22d714 100644 --- a/packages/code-infra/src/cli/cmdTacho.mjs +++ b/packages/code-infra/src/cli/cmdTacho.mjs @@ -9,6 +9,7 @@ * @property {string} [workingTreeBuildCmd] - Command that builds the working tree * @property {boolean} [install] - Whether to install inside a ref's checkout * @property {string} [out] - Where to write the combined JSON report + * @property {boolean} [upload] - Upload the report and refresh the pull request comment */ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ @@ -47,6 +48,12 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ default: true, describe: "Install dependencies inside a ref's checkout. Use --no-install to skip", }) + .option('upload', { + type: 'boolean', + default: process.env.TACHO_UPLOAD === 'true', + describe: + 'Upload the report and refresh the PR comment. Defaults to TACHO_UPLOAD=true, so CI can switch it on by environment. Requires CIRCLE_OIDC_TOKEN_V2', + }) .option('out', { type: 'string', describe: 'Write the combined JSON report here. Default: results/report.json', @@ -66,6 +73,7 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ workingTreeBuildCmd: argv.workingTreeBuildCmd, install: argv.install, out: argv.out, + upload: argv.upload, }); }, }); diff --git a/packages/code-infra/src/tachometer/ciReport.mjs b/packages/code-infra/src/tachometer/ciReport.mjs new file mode 100644 index 0000000000..b7315e7dd9 --- /dev/null +++ b/packages/code-infra/src/tachometer/ciReport.mjs @@ -0,0 +1,187 @@ +/* eslint-disable no-console -- progress belongs in the CI log. */ + +import { execa } from 'execa'; +import envCi from 'env-ci'; +import { z } from 'zod/v4'; + +/** + * Uploading a tachometer report to the CI report store, and asking the dashboard to refresh the + * pull request comment. + * + * The transport is deliberately a small copy of `@mui/internal-benchmark`'s rather than a shared + * import: code-infra carries a large dependency tree, and having the benchmark package depend on it + * would push all of that onto every consumer. What is duplicated — a bearer-token POST to one + * endpoint — is small and does not move. + */ + +const DEFAULT_API_URL = 'https://frontend-public.mui.com'; + +const confidenceIntervalSchema = z.object({ + low: z.number(), + high: z.number(), +}); + +const differenceSchema = z.object({ + verdict: z.enum(['faster', 'slower', 'unsure']), + absoluteMs: confidenceIntervalSchema, + percentChange: confidenceIntervalSchema, +}); + +const variantResultSchema = z.object({ + variant: z.string(), + refId: z.string().nullable(), + meanMs: confidenceIntervalSchema, + samples: z.number(), + bytesSent: z.number(), +}); + +const comparisonSchema = differenceSchema.extend({ + variant: z.string(), + versusReference: differenceSchema.optional(), +}); + +const measurementResultSchema = z.object({ + name: z.string(), + variants: z.array(variantResultSchema), + comparisons: z.array(comparisonSchema), +}); + +// A case that failed to summarize carries an `error` and no measurements. It is kept in the report +// rather than dropped, so the comment can say a case produced nothing instead of quietly omitting it. +const caseResultSchema = z.object({ + name: z.string(), + reference: z.string().optional(), + measurements: z.array(measurementResultSchema).optional(), + error: z.string().optional(), +}); + +const tachometerReportSchema = z.object({ + version: z.number(), + reportType: z.literal('tachometer'), + generatedAt: z.string(), + head: z.object({ ref: z.string(), sha: z.string(), branch: z.string().optional() }), + browser: z.string().optional(), + refs: z.array( + z.object({ + id: z.string(), + kind: z.string(), + label: z.string(), + sha: z.string().optional(), + }), + ), + cases: z.array(caseResultSchema), + // Every sample tachometer produced. Not read by the comment, but kept so a richer view can be + // built later without changing what CI uploads. + raw: z.record(z.string(), z.unknown()), +}); + +/** + * The upload envelope, matching what `@mui/internal-benchmark` sends and what the dashboard's + * upload route expects. + */ +export const tachometerUploadSchema = z.object({ + version: z.literal(1), + timestamp: z.number(), + commitSha: z.string().regex(/^[0-9a-f]{40}$/, 'Must be a 40-character hex string'), + repo: z.string().includes('/', 'Must be in owner/repo format'), + reportType: z.literal('tachometer'), + prNumber: z.number().int().positive().optional(), + branch: z.string(), + report: tachometerReportSchema, +}); + +/** + * @typedef {z.infer} TachometerUpload + */ + +/** + * Resolves the commit, branch, repository and pull request this run belongs to. + * + * The commit and branch come from git rather than the CI provider: a provider reports the branch it + * was triggered for, which on a pull request build is not necessarily the checkout's own. + * + * @returns {Promise<{ timestamp: number, repo: string, prNumber: number | undefined, branch: string, commitSha: string }>} + */ +export async function getCiMetadata() { + const ciInfo = /** @type {any} */ (envCi()); + const [commit, branch] = await Promise.all([ + execa('git', ['rev-parse', 'HEAD']), + execa('git', ['branch', '--show-current']), + ]); + + return { + timestamp: Date.now(), + repo: ciInfo.slug ?? '', + prNumber: ciInfo.pr ? Number(ciInfo.pr) : undefined, + branch: branch.stdout.trim() || (ciInfo.isPr ? ciInfo.prBranch : ciInfo.branch) || '', + commitSha: commit.stdout.trim(), + }; +} + +/** + * Uploads a report to the CI report store. + * + * Validates before sending because the server stores `report` without inspecting it — a malformed + * report would be accepted here and only fail much later, when the dashboard tried to render it. + * + * @param {TachometerUpload} upload - The envelope to send + * @returns {Promise} + */ +export async function uploadCiReport(upload) { + tachometerUploadSchema.parse(upload); + + const apiUrl = process.env.CI_REPORT_API_URL ?? DEFAULT_API_URL; + const url = new URL('/api/ci-reports/upload', apiUrl); + + const oidcToken = process.env.CIRCLE_OIDC_TOKEN_V2; + if (!oidcToken) { + throw new Error('CIRCLE_OIDC_TOKEN_V2 environment variable is required for uploads'); + } + + console.log(`Uploading tachometer report to ${url.href}`); + + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${oidcToken}` }, + body: JSON.stringify(upload), + }); + + const responseText = await response.text(); + if (!response.ok) { + throw new Error(`Upload failed (${response.status}): ${responseText}`); + } + + const result = JSON.parse(responseText); + console.log(`Tachometer report uploaded. S3 key: ${result.key}`); +} + +/** + * Asks the dashboard to regenerate the pull request comment for `repo`. + * + * @param {string} repo - The repository, as `owner/name` + * @returns {Promise<{ success: boolean, skipped?: boolean }>} + */ +export async function syncPrComment(repo) { + const oidcToken = process.env.CIRCLE_OIDC_TOKEN_V2; + if (!oidcToken) { + throw new Error('CIRCLE_OIDC_TOKEN_V2 environment variable is required for PR comment sync'); + } + + const apiUrl = process.env.CI_REPORT_API_URL ?? DEFAULT_API_URL; + const url = new URL('/api/ci-reports/sync-pr-comment', apiUrl); + + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${oidcToken}` }, + body: JSON.stringify({ repo }), + }); + + if (!response.ok) { + const responseText = await response.text(); + throw new Error( + `Sync PR comment API returned ${response.status} ${response.statusText}: ${responseText}`, + ); + } + + return response.json(); +} diff --git a/packages/code-infra/src/tachometer/ciReport.test.mjs b/packages/code-infra/src/tachometer/ciReport.test.mjs new file mode 100644 index 0000000000..f210f05ce2 --- /dev/null +++ b/packages/code-infra/src/tachometer/ciReport.test.mjs @@ -0,0 +1,140 @@ +import { describe, expect, it } from 'vitest'; +import { tachometerUploadSchema } from './ciReport.mjs'; + +/** + * A report of the shape `runTachometer` writes, trimmed to one case. + * + * @param {Partial} [overrides] - Fields to replace + * @returns {any} + */ +function report(overrides = {}) { + return { + version: 1, + reportType: 'tachometer', + generatedAt: '2026-09-02T00:00:00.000Z', + head: { ref: 'HEAD', sha: 'a'.repeat(40), branch: 'feature' }, + browser: '/path/to/Chrome for Testing', + refs: [ + { id: 'current', kind: 'worktree', label: 'working tree' }, + { id: 'git-abc123456', kind: 'git', label: 'merge-base', sha: 'b'.repeat(40) }, + ], + cases: [ + { + name: 'workload', + reference: 'workload [current]', + measurements: [ + { + name: 'mount', + variants: [ + { + variant: 'workload [current]', + refId: 'current', + meanMs: { low: 20.5, high: 22.7 }, + samples: 120, + bytesSent: 2969, + }, + ], + comparisons: [ + { + variant: 'workload [baseline]', + verdict: 'unsure', + absoluteMs: { low: -1.2, high: 2.1 }, + percentChange: { low: -5.2, high: 9.3 }, + versusReference: { + verdict: 'unsure', + absoluteMs: { low: -2.1, high: 1.2 }, + percentChange: { low: -9.3, high: 5.2 }, + }, + }, + ], + }, + ], + }, + ], + raw: { workload: { benchmarks: [] } }, + ...overrides, + }; +} + +/** + * @param {Partial} [overrides] - Envelope fields to replace + * @returns {any} + */ +function envelope(overrides = {}) { + return { + version: 1, + timestamp: 1_770_000_000_000, + commitSha: 'a'.repeat(40), + repo: 'mui/mui-public', + reportType: 'tachometer', + prNumber: 1829, + branch: 'feature', + report: report(), + ...overrides, + }; +} + +describe('tachometerUploadSchema', () => { + it('accepts a report of the shape the runner writes', () => { + expect(tachometerUploadSchema.safeParse(envelope()).success).toBe(true); + }); + + it('carries the envelope metadata through', () => { + const parsed = tachometerUploadSchema.parse(envelope()); + + expect(parsed).toMatchObject({ + repo: 'mui/mui-public', + prNumber: 1829, + branch: 'feature', + reportType: 'tachometer', + }); + }); + + it('accepts a run with no pull request', () => { + // A push to the base branch uploads too; it just has no PR to comment on. + const { prNumber, ...withoutPr } = envelope(); + + expect(tachometerUploadSchema.safeParse(withoutPr).success).toBe(true); + }); + + it('accepts a case that failed to summarize', () => { + // Kept in the report rather than dropped, so the comment can say the case produced nothing. + const withFailure = envelope({ + report: report({ cases: [{ name: 'broken', error: 'produced no benchmarks' }] }), + }); + + expect(tachometerUploadSchema.safeParse(withFailure).success).toBe(true); + }); + + describe('rejects', () => { + it('a report from another benchmark axis', () => { + // The dashboard routes on reportType, so the wrong one would render as the wrong thing. + const wrongType = envelope({ reportType: 'benchmark' }); + + expect(tachometerUploadSchema.safeParse(wrongType).success).toBe(false); + }); + + it('a commit that is not a full sha', () => { + // The artifact key is built from it, and a short sha would not match what the dashboard + // looks up for the head commit. + expect(tachometerUploadSchema.safeParse(envelope({ commitSha: 'abc1234' })).success).toBe( + false, + ); + }); + + it('a repo that is not owner/name', () => { + expect(tachometerUploadSchema.safeParse(envelope({ repo: 'mui-public' })).success).toBe( + false, + ); + }); + + it('a verdict outside the known set', () => { + const badVerdict = report(); + badVerdict.cases[0].measurements[0].comparisons[0].verdict = 'regressed'; + + expect(tachometerUploadSchema.safeParse(envelope({ report: badVerdict })).success).toBe( + false, + ); + }); + }); +}); diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index 2004fd2649..f20b5eecb4 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -13,6 +13,7 @@ import { buildRefPages } from './buildPages.mjs'; import { assertDriverMatchesBrowser, resolveBrowserBinary } from './browser.mjs'; import { summarizeCase } from './summarizeCase.mjs'; import { renderTachometerReport } from './renderReport.mjs'; +import { getCiMetadata, syncPrComment, uploadCiReport } from './ciReport.mjs'; import { run } from './exec.mjs'; /** @@ -25,6 +26,7 @@ import { run } from './exec.mjs'; * @property {string} [workingTreeBuildCmd] - Command that builds the working tree. Defaults to `buildCmd` * @property {boolean} [install] - Whether to install inside a ref's checkout. Defaults to true * @property {string} [out] - Where to write the combined JSON report. Defaults to `results/report.json` + * @property {boolean} [upload] - Upload the report and refresh the pull request comment */ /** @@ -52,6 +54,7 @@ export async function runTachometer(options) { workingTreeBuildCmd = buildCmd, install = true, out, + upload = false, } = options; const repoRoot = await findWorkspaceDir(harnessDir); @@ -212,7 +215,46 @@ export async function runTachometer(options) { console.log(''); renderTachometerReport(report); console.log(chalk.green(`\nWrote JSON report to ${outPath}`)); + + if (upload) { + await publishReport(report); + } } finally { await rm(tmpBase, { recursive: true, force: true }); } } + +/** + * Uploads a report and refreshes the pull request comment. + * + * A failure to refresh the comment does not fail the run: the numbers are already measured, written + * and uploaded, and losing them to a comment API hiccup would waste the whole benchmark. + * + * @param {any} report - The report just written + * @returns {Promise} + */ +async function publishReport(report) { + const metadata = await getCiMetadata(); + if (!metadata.repo) { + console.warn( + chalk.yellow('Skipping upload: no repository detected, which usually means this is not CI.'), + ); + return; + } + + await uploadCiReport({ version: 1, reportType: 'tachometer', ...metadata, report }); + + try { + console.log('Syncing PR comment via the dashboard API…'); + const result = await syncPrComment(metadata.repo); + console.log( + result.skipped ? 'No open PR found for this branch, skipping.' : 'PR comment synced.', + ); + } catch (error) { + console.error( + chalk.yellow( + `Failed to sync the PR comment: ${error instanceof Error ? error.message : error}`, + ), + ); + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7ccf73a746..b05dbc83aa 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -731,6 +731,9 @@ importers: yargs: specifier: ^18.1.0 version: 18.1.0 + zod: + specifier: ^4.4.3 + version: 4.4.3 devDependencies: '@octokit/types': specifier: 17.0.0 From 6cef12ba4ac313ce77972b069b7791bb552ffda6 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 2 Sep 2026 10:37:48 +0200 Subject: [PATCH 009/173] [dashboard] Enable the tachometer section for base-ui-mosaic It is the repository that runs the job. Enabling it before the first upload is safe: with no artifact for the head commit the generator returns null and the comment simply has no tachometer section. --- apps/code-infra-dashboard/src/constants.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/code-infra-dashboard/src/constants.ts b/apps/code-infra-dashboard/src/constants.ts index 7418489909..f78c4e7fe1 100644 --- a/apps/code-infra-dashboard/src/constants.ts +++ b/apps/code-infra-dashboard/src/constants.ts @@ -167,6 +167,7 @@ export const repositories = new Map( prComment: { bundleSize: true, benchmark: true, + tachometer: true, }, }, { From 15e4c33786ea3ef1bd77097744c9915987dea7d0 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:05:07 +0200 Subject: [PATCH 010/173] [code-infra] Run Chrome without its sandbox as root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Containerised CI runs as root, and Chrome will not enter its sandbox there: it exits during launch, leaving the driver to report no more than "session not created: Chrome instance exited". Adding the flag only for root keeps the sandbox on for local runs, and a case's own arguments survive the merge — the flags that hold a measurement still would otherwise be dropped. --- .../code-infra/src/tachometer/browser.mjs | 33 +++++++++++ .../src/tachometer/browser.test.mjs | 57 ++++++++++++++++++- .../src/tachometer/runTachometer.mjs | 19 ++++--- 3 files changed, 101 insertions(+), 8 deletions(-) diff --git a/packages/code-infra/src/tachometer/browser.mjs b/packages/code-infra/src/tachometer/browser.mjs index 738ee0eba8..550bab1ebe 100644 --- a/packages/code-infra/src/tachometer/browser.mjs +++ b/packages/code-infra/src/tachometer/browser.mjs @@ -119,3 +119,36 @@ export function assertDriverMatchesBrowser(harnessDir, binary) { ); } } + +/** + * A benchmark's `browser` config. Only the two fields a run touches are named; everything else + * tachometer accepts there (`name`, `headless`, `windowSize`, …) passes through untouched. + * + * @typedef {{ binary?: string, addArguments?: string[] } & Record} BrowserConfig + */ + +/** Chrome cannot enter its sandbox as root, and refusing to start is all it does about it. */ +const NO_SANDBOX = '--no-sandbox'; + +/** + * Merges what the run decides into a case's own `browser` config. + * + * Neither of these belongs in a committed `tachometer.json`. The binary is machine-specific. And + * whether the sandbox has to come off depends on the user the run happens under: as root Chrome + * exits during launch without explaining itself, leaving the driver to report nothing more than + * `session not created: Chrome instance exited`. Containerised CI runs as root, so the flag goes + * on there and nowhere else — a local run keeps the sandbox. + * + * @param {BrowserConfig | undefined} browser - The case's own browser config, if it has one + * @param {string} binary - Binary to drive, unless the case pins its own + * @param {boolean} asRoot - Whether the run is happening as root + * @returns {BrowserConfig} + */ +export function withBrowserDefaults(browser, binary, asRoot) { + const merged = { ...browser, binary: browser?.binary ?? binary }; + const addArguments = merged.addArguments ?? []; + if (asRoot && !addArguments.includes(NO_SANDBOX)) { + merged.addArguments = [...addArguments, NO_SANDBOX]; + } + return merged; +} diff --git a/packages/code-infra/src/tachometer/browser.test.mjs b/packages/code-infra/src/tachometer/browser.test.mjs index fa355f12a1..0b5aed23cf 100644 --- a/packages/code-infra/src/tachometer/browser.test.mjs +++ b/packages/code-infra/src/tachometer/browser.test.mjs @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { majorVersionOf } from './browser.mjs'; +import { majorVersionOf, withBrowserDefaults } from './browser.mjs'; describe('majorVersionOf', () => { it('reads the major from Chrome for Testing output', () => { @@ -25,3 +25,58 @@ describe('majorVersionOf', () => { expect(majorVersionOf('2 browsers found: Chrome 151.0.7922.34')).toBe(151); }); }); + +describe('withBrowserDefaults', () => { + const BINARY = '/path/to/Chrome for Testing'; + + it('fills in the binary for a case that does not pin one', () => { + expect(withBrowserDefaults({ name: 'chrome', headless: true }, BINARY, false)).toEqual({ + name: 'chrome', + headless: true, + binary: BINARY, + }); + }); + + it('leaves a case that pins its own binary alone', () => { + const pinned = withBrowserDefaults({ binary: '/opt/chrome' }, BINARY, false); + + expect(pinned.binary).toBe('/opt/chrome'); + }); + + it('keeps the sandbox when the run is not root', () => { + // The sandbox is a real protection for the pages a benchmark loads; only root cannot have it. + expect(withBrowserDefaults({ name: 'chrome' }, BINARY, false)).not.toHaveProperty( + 'addArguments', + ); + }); + + it('disables the sandbox when the run is root', () => { + expect(withBrowserDefaults({ name: 'chrome' }, BINARY, true).addArguments).toEqual([ + '--no-sandbox', + ]); + }); + + it("keeps a case's own arguments when it disables the sandbox", () => { + // Cases pass flags that hold their measurements still; dropping them would move the numbers. + const merged = withBrowserDefaults( + { addArguments: ['--disable-renderer-backgrounding'] }, + BINARY, + true, + ); + + expect(merged.addArguments).toEqual(['--disable-renderer-backgrounding', '--no-sandbox']); + }); + + it('does not repeat an argument a case already passes', () => { + const merged = withBrowserDefaults({ addArguments: ['--no-sandbox'] }, BINARY, true); + + expect(merged.addArguments).toEqual(['--no-sandbox']); + }); + + it('handles a case with no browser config at all', () => { + expect(withBrowserDefaults(undefined, BINARY, true)).toEqual({ + binary: BINARY, + addArguments: ['--no-sandbox'], + }); + }); +}); diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index f20b5eecb4..3061b4f236 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -10,7 +10,11 @@ import { packRef, packWorkingTree } from '../utils/packWorkspace.mjs'; import { createRefResolver } from './refs.mjs'; import { discoverCases, pagesOf } from './discoverCases.mjs'; import { buildRefPages } from './buildPages.mjs'; -import { assertDriverMatchesBrowser, resolveBrowserBinary } from './browser.mjs'; +import { + assertDriverMatchesBrowser, + resolveBrowserBinary, + withBrowserDefaults, +} from './browser.mjs'; import { summarizeCase } from './summarizeCase.mjs'; import { renderTachometerReport } from './renderReport.mjs'; import { getCiMetadata, syncPrComment, uploadCiReport } from './ciReport.mjs'; @@ -93,11 +97,15 @@ export async function runTachometer(options) { // out now costs seconds instead of minutes of packing and installing. const browserBinary = await resolveBrowserBinary(harnessDir); assertDriverMatchesBrowser(harnessDir, browserBinary); + // `getuid` is POSIX-only; on Windows nobody is root. + const asRoot = process.getuid?.() === 0; console.log(chalk.cyan(`Cases: ${cases.map((entry) => entry.name).join(', ')}`)); console.log(chalk.cyan(`Pages: ${pagesOf(cases).join(', ')}`)); console.log(chalk.cyan(`Refs: ${[...refs.values()].map(describeRef).join(', ')}`)); - console.log(chalk.cyan(`Browser: ${browserBinary}`)); + console.log( + chalk.cyan(`Browser: ${browserBinary}${asRoot ? ' (as root, so without its sandbox)' : ''}`), + ); const tmpBase = await mkdtemp(path.join(os.tmpdir(), 'tacho-')); @@ -146,12 +154,9 @@ export async function runTachometer(options) { leaf.node.url = `${path.join(buildsDir, refId, leaf.page)}${leaf.suffix}`; } // `browser` is inherited down the `expand` tree, so setting it per benchmark covers every - // variant. An explicitly configured binary wins. + // variant. for (const benchmark of entry.config.benchmarks ?? []) { - benchmark.browser = { - ...benchmark.browser, - binary: benchmark.browser?.binary ?? browserBinary, - }; + benchmark.browser = withBrowserDefaults(benchmark.browser, browserBinary, asRoot); } const configPath = path.join(tmpBase, `tachometer-${entry.name}.json`); // eslint-disable-next-line no-await-in-loop From 42a5b43a7b394ff3d0f9dd223d284c737ec674e9 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 2 Sep 2026 11:35:00 +0200 Subject: [PATCH 011/173] [code-infra] Run the tachometer smoke test in CI The harness only ever ran by hand, so nothing caught a break in the pipeline it exercises. Measuring the working tree against HEAD keeps both sides identical: the verdict is always "unsure", and what the job actually proves is that packing, installing, building, sampling and reporting still work end to end. Until this lands, production rejects the report type, so the upload points at this pull request's own dashboard preview. --- .circleci/config.yml | 27 ++++++++++++++++++++++ apps/code-infra-dashboard/src/constants.ts | 1 + 2 files changed, 28 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index c99262046f..9d17d8703d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -104,6 +104,31 @@ jobs: BENCHMARK_UPLOAD: 'true' command: pnpm -F ./test/performance benchmark + test_tachometer: + <<: *default-job + executor: + name: code-infra/mui-node-browser + playwright-img-version: 'v1.62.1-noble' + steps: + - checkout + - code-infra/install-deps + - run: + name: Run the tachometer smoke test + # Packing a ref installs and builds it in a quiet worktree; sampling then prints per case. + no_output_timeout: 30m + environment: + TACHO_UPLOAD: 'true' + # TEMPORARY: production does not know the tachometer report type until this pull + # request lands, so it would reject the upload. Points at this pull request's own + # dashboard preview until then — remove the line before merging. + CI_REPORT_API_URL: https://code-infra-dashboard-pr-1829.onrender.com + # This repository ships tooling rather than a browser library, so no render time here is + # worth tracking against the base branch. The working tree is measured against HEAD — + # identical code on both sides, so the verdict is always "unsure" — because the point is + # to exercise the pipeline: packing a ref, installing it in isolation, building the pages, + # sampling under Chrome, and reporting. + command: pnpm test:tacho --baseline git:HEAD + test_browser: <<: *default-job executor: @@ -145,5 +170,7 @@ workflows: <<: *default-context - test_benchmark: <<: *default-context + - test_tachometer: + <<: *default-context - test_browser: <<: *default-context diff --git a/apps/code-infra-dashboard/src/constants.ts b/apps/code-infra-dashboard/src/constants.ts index f78c4e7fe1..08e8bf2f43 100644 --- a/apps/code-infra-dashboard/src/constants.ts +++ b/apps/code-infra-dashboard/src/constants.ts @@ -195,6 +195,7 @@ export const repositories = new Map( prComment: { bundleSize: true, benchmark: true, + tachometer: true, netlifyDocs: { siteId: 'mui-internal', formatDocPath: (filePath) => { From 6a6408c6aa055e00921425c795c77e2dcd80f8b4 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:23:37 +0200 Subject: [PATCH 012/173] [dashboard] Read each tachometer row in the variant's own direction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A comparison holds the reference relative to the variant, which is the direction a regression is stated in. Both tables put that under a "vs reference" heading on the variant's row, where it says the opposite of what it means: the real report rendered ag-grid, at 324 ms against a 192 ms reference, as "faster -42.8%". The summary line beside it counted regressed measurements against a total in cases, and counted a case as "faster" for beating a competing library — neither of which the sentence claims. Both now read the cross-ref comparisons a regression is taken from, and count cases. --- .../tachometer/buildMarkdownReport.test.ts | 101 +++++++++++++++--- .../src/lib/tachometer/buildMarkdownReport.ts | 62 +++++++---- .../src/views/TachometerDetails.tsx | 10 +- 3 files changed, 129 insertions(+), 44 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts index 18ad2e24a7..2035a49909 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts @@ -20,8 +20,18 @@ function variant(name: string, refId: string | null) { }; } +/** The verdict the other way round; an unresolved difference is unresolved from both sides. */ +function mirrorOf(verdict: Verdict): Verdict { + if (verdict === 'slower') { + return 'faster'; + } + return verdict === 'faster' ? 'slower' : verdict; +} + /** - * A comparison of the reference against `name`. + * A comparison of the reference against `name`, carrying both directions the way a real report + * does. The two are not negations of each other — each has its own denominator — so the numbers + * here differ, which is what lets a test tell which direction was rendered. */ function comparison(name: string, verdict: Verdict) { const sign = verdict === 'slower' ? 1 : -1; @@ -30,6 +40,11 @@ function comparison(name: string, verdict: Verdict) { verdict, absoluteMs: { low: sign * 6.1, high: sign * 15.2 }, percentChange: { low: sign * 2.7, high: sign * 6.8 }, + versusReference: { + verdict: mirrorOf(verdict), + absoluteMs: { low: sign * -6.1, high: sign * -15.2 }, + percentChange: { low: sign * -2.6, high: sign * -6.4 }, + }, }; } @@ -50,6 +65,23 @@ function regressionCase(name: string, verdicts: Record): Tachom return { name, reference: `${name} [current]`, measurements }; } +/** + * A case comparing libraries with each other, so every variant comes from the same build. + */ +function libraryCase(name: string, verdict: Verdict): TachometerCaseResult { + return { + name, + reference: `${name} [mosaic]`, + measurements: [ + { + name: 'mount', + variants: [variant(`${name} [mosaic]`, 'current'), variant(`${name} [ag-grid]`, 'current')], + comparisons: [comparison(`${name} [ag-grid]`, verdict)], + }, + ], + }; +} + function report(cases: TachometerCaseResult[]): TachometerReport { return { version: 1, @@ -69,6 +101,14 @@ function visiblePart(markdown: string): string { return markdown.split('
    ')[0]; } +/** The collapsed table's row for `variant`, rather than any prose that names it too. */ +function tableRowFor(markdown: string, variantName: string): string | undefined { + return markdown + .split('
    ')[1] + ?.split('\n') + .find((line) => line.startsWith('|') && line.includes(variantName)); +} + describe('findRegressions', () => { it('finds a slower verdict across two builds', () => { const found = findRegressions(report([regressionCase('workload', { mount: 'slower' })])); @@ -83,22 +123,7 @@ describe('findRegressions', () => { it('ignores a slower verdict between variants on the same build', () => { // A cross-library case runs every variant from one build, so "slower than ag-grid" says nothing // about what this pull request changed. Flagging it would warn on every comment forever. - const libs: TachometerCaseResult = { - name: 'libs-mount', - reference: 'libs-mount [mosaic]', - measurements: [ - { - name: 'mount', - variants: [ - variant('libs-mount [mosaic]', 'current'), - variant('libs-mount [ag-grid]', 'current'), - ], - comparisons: [comparison('libs-mount [ag-grid]', 'slower')], - }, - ], - }; - - expect(findRegressions(report([libs]))).toEqual([]); + expect(findRegressions(report([libraryCase('libs-mount', 'slower')]))).toEqual([]); }); it('reports only the measurement that regressed', () => { @@ -151,6 +176,27 @@ describe('buildTachometerMarkdownReport', () => { expect(visiblePart(markdown)).toContain('2 cases measured · 2 unchanged'); }); + it('counts a case that regressed in two measurements once', () => { + // The total beside it counts cases, so counting measurements here stopped the line adding up. + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'slower', 'cold-start': 'slower' })]), + ); + + expect(visiblePart(markdown)).toContain('1 case measured · 1 slower'); + }); + + it('does not count beating another library as having got faster', () => { + const markdown = buildTachometerMarkdownReport( + report([ + libraryCase('libs-mount', 'faster'), + regressionCase('workload', { mount: 'unsure' }), + ]), + ); + + expect(visiblePart(markdown)).toContain('2 cases measured · 2 unchanged'); + expect(visiblePart(markdown)).not.toContain('faster'); + }); + it('counts a faster case separately', () => { const markdown = buildTachometerMarkdownReport( report([ @@ -183,6 +229,27 @@ describe('buildTachometerMarkdownReport', () => { expect(details).toContain('[baseline]'); }); + it('states each row as the variant against the reference, not the reference against it', () => { + // The row sits under a "vs reference" heading next to the variant's own mean, so it has to + // read as a statement about that variant. Rendering the direction a regression is stated in + // makes a variant that is slower than the reference look faster than it. + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'slower' })]), + ); + + expect(tableRowFor(markdown, '[baseline]')).toContain('faster `-2.6% – -6.4%`'); + }); + + it('leaves the comparison out when the report has only one direction', () => { + // Older reports, written before both directions were captured. + const oneDirection = report([regressionCase('workload', { mount: 'slower' })]); + delete oneDirection.cases[0].measurements![0].comparisons[0].versusReference; + + const markdown = buildTachometerMarkdownReport(oneDirection); + + expect(tableRowFor(markdown, '[baseline]')).toContain('—'); + }); + it('links to the full run when given a url', () => { const markdown = buildTachometerMarkdownReport( report([regressionCase('workload', { mount: 'unsure' })]), diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts index 937c6f5b70..5451818cb9 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts @@ -65,6 +65,27 @@ function isSummarized(entry: TachometerCaseResult): entry is SummarizedCase { * knowing, and completely unrelated to what the pull request changed. Flagging those would put a * warning on every comment until nobody read it. */ +function comparisonsAcrossRefs(entry: SummarizedCase) { + const found = []; + + for (const measurement of entry.measurements) { + const referenceRefId = measurement.variants.find( + (variant) => variant.variant === entry.reference, + )?.refId; + + for (const comparison of measurement.comparisons) { + const comparedRefId = measurement.variants.find( + (variant) => variant.variant === comparison.variant, + )?.refId; + if (comparedRefId !== referenceRefId) { + found.push({ measurement, comparison }); + } + } + } + + return found; +} + export function findRegressions(report: TachometerReport): Regression[] { const regressions: Regression[] = []; @@ -72,21 +93,8 @@ export function findRegressions(report: TachometerReport): Regression[] { if (!isSummarized(entry)) { continue; } - for (const measurement of entry.measurements) { - const referenceRefId = measurement.variants.find( - (variant) => variant.variant === entry.reference, - )?.refId; - - for (const comparison of measurement.comparisons) { - if (comparison.verdict !== 'slower') { - continue; - } - const comparedRefId = measurement.variants.find( - (variant) => variant.variant === comparison.variant, - )?.refId; - if (comparedRefId === referenceRefId) { - continue; - } + for (const { measurement, comparison } of comparisonsAcrossRefs(entry)) { + if (comparison.verdict === 'slower') { regressions.push({ caseName: entry.name, measurement: measurement.name, @@ -112,9 +120,14 @@ function renderCaseTable(entry: SummarizedCase): string { for (const variant of measurement.variants) { const comparison = byVariant.get(variant.variant); const isReference = variant.variant === entry.reference; + // Each row reads as "this variant, compared to the reference", so it needs the direction + // whose subject is the variant. `comparison` itself holds the opposite one — the reference + // relative to the variant — which is what a regression is stated in, above the fold. + const againstReference = comparison?.versusReference; const versus = isReference ? '_reference_' - : (comparison && `${comparison.verdict} \`${formatPercent(comparison.percentChange)}\``) || + : (againstReference && + `${againstReference.verdict} \`${formatPercent(againstReference.percentChange)}\``) || '—'; rows.push( `| ${measurement.name} | ${shortNameOf(entry.name, variant.variant)} | ${formatMean( @@ -163,18 +176,21 @@ export function buildTachometerMarkdownReport( } // Everything that did not regress collapses to a single line, so the regressions are what the eye - // lands on. `unsure` is the expected result for two equivalent builds, not a warning. - const faster = summarized.filter((entry) => - entry.measurements.some((measurement) => - measurement.comparisons.some((comparison) => comparison.verdict === 'faster'), - ), + // lands on. `unsure` is the expected result for two equivalent builds, not a warning. Counted in + // cases, like the total beside them, and over the same cross-ref comparisons a regression is read + // from — a case that beats a competing library has not got faster. + const regressedCases = new Set(regressions.map((regression) => regression.caseName)); + const faster = summarized.filter( + (entry) => + !regressedCases.has(entry.name) && + comparisonsAcrossRefs(entry).some(({ comparison }) => comparison.verdict === 'faster'), ).length; - const unchanged = summarized.length - regressions.length - faster; + const unchanged = summarized.length - regressedCases.size - faster; const summary = [ `${summarized.length} case${summarized.length === 1 ? '' : 's'} measured`, unchanged > 0 ? `${unchanged} unchanged` : null, faster > 0 ? `${faster} faster` : null, - regressions.length > 0 ? `${regressions.length} slower` : null, + regressedCases.size > 0 ? `${regressedCases.size} slower` : null, ] .filter(Boolean) .join(' · '); diff --git a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx index 93085a83ad..50eea230a8 100644 --- a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx +++ b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx @@ -85,7 +85,9 @@ function CaseTable({ entry }: CaseTableProps) { measurement.comparisons.map((comparison) => [comparison.variant, comparison]), ); return measurement.variants.map((variant) => { - const comparison = byVariant.get(variant.variant); + // The row is about this variant, so it needs the direction whose subject is the + // variant. The comparison itself holds the other one, the reference relative to it. + const againstReference = byVariant.get(variant.variant)?.versusReference; const isReference = variant.variant === entry.reference; return ( @@ -98,9 +100,9 @@ function CaseTable({ entry }: CaseTableProps) { reference ) : ( - comparison && ( - - {comparison.verdict} {formatPercent(comparison.percentChange)} + againstReference && ( + + {againstReference.verdict} {formatPercent(againstReference.percentChange)} ) )} From ad39187b35e2f3a31edab150b19a95a93d1c08f6 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 2 Sep 2026 12:54:03 +0200 Subject: [PATCH 013/173] [dashboard] Structure the tachometer page like the tables CI prints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The page had one flat table per case, a row per measurement and variant, which is not how the run reads in a terminal. It now groups cases by their variant set the same way: two variants become one table across the group, a row per case and measurement with a column per variant and a Δ against each; more than two become a table per case, a row per variant, since a column per variant plus a Δ per pair would run off the page. Transfer size and the build labels move into the notes underneath, where the terminal keeps them. The grouping is a pure module of its own, so what decides a table's shape is testable without rendering anything. --- .../src/lib/tachometer/groupCases.test.ts | 112 +++++++ .../src/lib/tachometer/groupCases.ts | 71 +++++ .../src/views/TachometerDetails.tsx | 299 ++++++++++++++---- 3 files changed, 423 insertions(+), 59 deletions(-) create mode 100644 apps/code-infra-dashboard/src/lib/tachometer/groupCases.test.ts create mode 100644 apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts diff --git a/apps/code-infra-dashboard/src/lib/tachometer/groupCases.test.ts b/apps/code-infra-dashboard/src/lib/tachometer/groupCases.test.ts new file mode 100644 index 0000000000..8dcbfe7642 --- /dev/null +++ b/apps/code-infra-dashboard/src/lib/tachometer/groupCases.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from 'vitest'; +import { bytesPerVariant, groupCasesByVariantSet, shortNameOf, variantsOf } from './groupCases'; +import type { SummarizedCase } from './groupCases'; + +function variant(name: string, bytesSent = 2969) { + return { + variant: name, + refId: 'current', + meanMs: { low: 20, high: 22 }, + samples: 120, + bytesSent, + }; +} + +function caseWith(name: string, variantNames: string[], measurements = ['mount']): SummarizedCase { + return { + name, + reference: `${name} ${variantNames[0]}`, + measurements: measurements.map((measurement) => ({ + name: measurement, + variants: variantNames.map((short) => variant(`${name} ${short}`)), + comparisons: [], + })), + }; +} + +describe('shortNameOf', () => { + it('drops the case prefix tachometer puts on a variant', () => { + expect(shortNameOf('libs-mount', 'libs-mount [ag-grid]')).toBe('[ag-grid]'); + }); + + it('leaves a name that does not carry the prefix alone', () => { + expect(shortNameOf('scroll', 'row-updates [current]')).toBe('row-updates [current]'); + }); +}); + +describe('variantsOf', () => { + it('lists the variants in the order they first appear', () => { + expect(variantsOf(caseWith('libs-mount', ['[mosaic]', '[ag-grid]', '[mui-x]']))).toEqual([ + '[mosaic]', + '[ag-grid]', + '[mui-x]', + ]); + }); + + it('counts a variant once across measurements', () => { + const entry = caseWith('workload', ['[current]', '[baseline]'], ['mount', 'cold-start']); + + expect(variantsOf(entry)).toEqual(['[current]', '[baseline]']); + }); +}); + +describe('groupCasesByVariantSet', () => { + it('puts cases sharing a variant set in one group', () => { + // Auto-expanded cases all carry [current]/[baseline], so they belong in one table. + const groups = groupCasesByVariantSet([ + caseWith('workload', ['[current]', '[baseline]']), + caseWith('scroll', ['[current]', '[baseline]']), + ]); + + expect(groups).toHaveLength(1); + expect(groups[0].cases.map((entry) => entry.name)).toEqual(['workload', 'scroll']); + }); + + it('gives a case with a different variant set its own group', () => { + // A table over the union would leave a blank cell in every row, and could name only one + // reference for both. + const groups = groupCasesByVariantSet([ + caseWith('workload', ['[current]', '[baseline]']), + caseWith('libs-mount', ['[mosaic]', '[ag-grid]']), + ]); + + expect(groups.map((group) => group.variants)).toEqual([ + ['[current]', '[baseline]'], + ['[mosaic]', '[ag-grid]'], + ]); + }); + + it('separates variant sets that differ only in order', () => { + // The first variant is the reference, so the same names in another order is another table. + const groups = groupCasesByVariantSet([ + caseWith('a', ['[current]', '[baseline]']), + caseWith('b', ['[baseline]', '[current]']), + ]); + + expect(groups).toHaveLength(2); + }); + + it('keeps the groups in order of first appearance', () => { + const groups = groupCasesByVariantSet([ + caseWith('libs-mount', ['[mosaic]', '[ag-grid]']), + caseWith('workload', ['[current]', '[baseline]']), + caseWith('libs-scroll', ['[mosaic]', '[ag-grid]']), + ]); + + expect(groups.map((group) => group.cases.map((entry) => entry.name))).toEqual([ + ['libs-mount', 'libs-scroll'], + ['workload'], + ]); + }); +}); + +describe('bytesPerVariant', () => { + it('reports one figure per variant, not per measurement', () => { + const entry = caseWith('workload', ['[current]', '[baseline]'], ['mount', 'cold-start']); + + expect(bytesPerVariant(entry)).toEqual([ + ['[current]', 2969], + ['[baseline]', 2969], + ]); + }); +}); diff --git a/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts b/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts new file mode 100644 index 0000000000..9829732ced --- /dev/null +++ b/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts @@ -0,0 +1,71 @@ +import type { TachometerCaseResult, TachometerMeasurementResult } from './types'; + +export type SummarizedCase = TachometerCaseResult & { + measurements: TachometerMeasurementResult[]; +}; + +export interface VariantGroup { + /** The variant set the group's cases share, reference first. */ + variants: string[]; + cases: SummarizedCase[]; +} + +/** + * Tachometer names each variant ` []`, and the case is named by the table itself, so + * the prefix is dropped for display. Dropping it is also what lets cases share a table: every + * auto-expanded case then contributes the same `[current]`/`[baseline]` pair rather than a pair of + * its own. + */ +export function shortNameOf(caseName: string, variant: string): string { + return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; +} + +/** The distinct variants a case reports, in order of first appearance. */ +export function variantsOf(entry: SummarizedCase): string[] { + const variants: string[] = []; + + for (const measurement of entry.measurements) { + for (const variant of measurement.variants) { + const short = shortNameOf(entry.name, variant.variant); + if (!variants.includes(short)) { + variants.push(short); + } + } + } + + return variants; +} + +/** + * Splits cases into the groups that each get their own table, the way the terminal report does. + * + * A regression case carries `[current]`/`[baseline]`, a cross-library case `[ours]`/`[theirs]`, and + * each set has its own reference — the first variant. One table over the union of the columns would + * leave a blank cell in every row of every case, and could name only one reference for all of them. + */ +export function groupCasesByVariantSet(cases: SummarizedCase[]): VariantGroup[] { + const groups = new Map(); + + for (const entry of cases) { + const variants = variantsOf(entry); + const key = JSON.stringify(variants); + const group = groups.get(key) ?? { variants, cases: [] }; + group.cases.push(entry); + groups.set(key, group); + } + + return [...groups.values()]; +} + +/** What each of a case's variants transferred, for the cases whose table has no such column. */ +export function bytesPerVariant(entry: SummarizedCase): Array<[string, number]> { + const bytes = new Map(); + + for (const measurement of entry.measurements) { + for (const variant of measurement.variants) { + bytes.set(shortNameOf(entry.name, variant.variant), variant.bytesSent); + } + } + + return [...bytes]; +} diff --git a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx index 50eea230a8..ee896f744b 100644 --- a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx +++ b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx @@ -10,26 +10,29 @@ import Paper from '@mui/material/Paper'; import Table from '@mui/material/Table'; import TableBody from '@mui/material/TableBody'; import TableCell from '@mui/material/TableCell'; +import TableContainer from '@mui/material/TableContainer'; import TableHead from '@mui/material/TableHead'; import TableRow from '@mui/material/TableRow'; import Typography from '@mui/material/Typography'; import { fetchCiReport } from '@/utils/fetchCiReport'; -import type { - ConfidenceInterval, - TachometerCaseResult, - TachometerMeasurementResult, - TachometerUpload, - Verdict, -} from '@/lib/tachometer/types'; +import type { ConfidenceInterval, TachometerUpload, Verdict } from '@/lib/tachometer/types'; +import { + bytesPerVariant, + groupCasesByVariantSet, + shortNameOf, +} from '@/lib/tachometer/groupCases'; +import type { SummarizedCase } from '@/lib/tachometer/groupCases'; import Heading from '../components/Heading'; import ReportHeader from '../components/ReportHeader'; import ErrorDisplay from '../components/ErrorDisplay'; /** - * Deliberately plain: every number the report holds, as tables. + * The tables the run prints in CI, as HTML: cases grouped by their variant set, a group of two + * variants read down its measurements, a group of more read across its variants. * - * No charts or bars. A confidence interval is two numbers and a verdict, and drawing it as a length - * invites reading the picture as significance when the interval is what actually carries that. + * Deliberately plain — no charts or bars. A confidence interval is two numbers and a verdict, and + * drawing it as a length invites reading the picture as significance when the interval is what + * actually carries that. */ function formatMean(interval: ConfidenceInterval): string { @@ -45,7 +48,7 @@ function formatBytes(bytes: number): string { return `${(bytes / 1024).toFixed(1)} KiB`; } -/** Only a regression is coloured; `unsure` is the expected result and stays neutral. */ +/** Only a resolved difference is coloured; `unsure` is the expected result and stays neutral. */ function verdictColor(verdict: Verdict): 'error' | 'success' | 'text.secondary' { if (verdict === 'slower') { return 'error'; @@ -56,66 +59,203 @@ function verdictColor(verdict: Verdict): 'error' | 'success' | 'text.secondary' return 'text.secondary'; } -function shortNameOf(caseName: string, variant: string): string { - return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; +/** Auto-sampling stops per case, so the counts are normally equal; when they are not, say both. */ +function formatSamples(samples: number[]): string { + return [...new Set(samples)].join('/'); } -interface CaseTableProps { - entry: TachometerCaseResult & { measurements: TachometerMeasurementResult[] }; +function VerdictText({ + verdict, + percentChange, +}: { + verdict: Verdict; + percentChange: ConfidenceInterval; +}) { + return ( + + {verdict} {formatPercent(percentChange)} + + ); } -function CaseTable({ entry }: CaseTableProps) { +function Muted({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ); +} + +/** + * One case as a table of variants, for a comparison with more than two of them. + * + * Rows are variants, not measurements: with several libraries, a column per variant plus a Δ column + * per pair would run off the page. Each measurement then shows the variant's interval next to its + * difference relative to the reference — the direction a row about that library reads in. + */ +function VariantTable({ entry, variants }: { entry: SummarizedCase; variants: string[] }) { + const [reference] = variants; + return ( - - {entry.name} + - Measurement - Variant - Mean (95% CI) - vs reference - Samples + {entry.name} + {entry.measurements.map((measurement) => ( + + {measurement.name} + vs {reference} + + ))} Transferred + Samples - {entry.measurements.flatMap((measurement) => { - const byVariant = new Map( - measurement.comparisons.map((comparison) => [comparison.variant, comparison]), - ); - return measurement.variants.map((variant) => { - // The row is about this variant, so it needs the direction whose subject is the - // variant. The comparison itself holds the other one, the reference relative to it. - const againstReference = byVariant.get(variant.variant)?.versusReference; - const isReference = variant.variant === entry.reference; + {variants.map((variantName) => { + const samples: number[] = []; + let bytesSent: number | undefined; + + const cells = entry.measurements.map((measurement) => { + const found = measurement.variants.find( + (candidate) => shortNameOf(entry.name, candidate.variant) === variantName, + ); + const comparison = measurement.comparisons.find( + (candidate) => shortNameOf(entry.name, candidate.variant) === variantName, + ); + if (found) { + bytesSent = found.bytesSent; + samples.push(found.samples); + } + // The comparison holds the reference relative to this variant; a row about the + // variant needs the direction whose subject is the variant. + const againstReference = comparison?.versusReference; return ( - - {measurement.name} - {shortNameOf(entry.name, variant.variant)} - {formatMean(variant.meanMs)} + + + {found ? formatMean(found.meanMs) : —} + - {isReference ? ( - - reference - - ) : ( - againstReference && ( - - {againstReference.verdict} {formatPercent(againstReference.percentChange)} - - ) + {variantName === reference && reference} + {variantName !== reference && againstReference && ( + )} + {variantName !== reference && !againstReference && —} - {variant.samples} - {formatBytes(variant.bytesSent)} - + ); }); + + return ( + + {variantName} + {cells} + + {bytesSent === undefined ? — : formatBytes(bytesSent)} + + {formatSamples(samples)} + + ); })}
    -     + + ); +} + +/** + * A group of cases sharing a variant set, one row per case and measurement. + * + * Each variant gets a column of its own, followed by the difference of the reference against it — + * which is the direction a regression is stated in, and what the Δ heading names. + */ +function CaseGroupTable({ cases, variants }: { cases: SummarizedCase[]; variants: string[] }) { + const [reference, ...others] = variants; + + return ( + + + + + Case + Measurement + {variants.map((variantName) => ( + + {variantName} + + ))} + {others.map((variantName) => ( + Δ vs {variantName} + ))} + Samples + + + + {cases.flatMap((entry, caseIndex) => + entry.measurements.map((measurement, measurementIndex) => { + const byVariant = new Map( + measurement.variants.map((variant) => [ + shortNameOf(entry.name, variant.variant), + variant, + ]), + ); + const byComparison = new Map( + measurement.comparisons.map((comparison) => [ + shortNameOf(entry.name, comparison.variant), + comparison, + ]), + ); + // A rule where the next case starts, since its name only appears on its first row. + const startsCase = measurementIndex === 0 && caseIndex > 0; + + return ( + + {measurementIndex === 0 ? entry.name : ''} + {measurement.name} + {variants.map((variantName) => { + const found = byVariant.get(variantName); + return ( + + {found ? formatMean(found.meanMs) : —} + + ); + })} + {others.map((variantName) => { + const comparison = byComparison.get(variantName); + return ( + + {comparison ? ( + + ) : ( + — + )} + + ); + })} + + {formatSamples(measurement.variants.map((variant) => variant.samples))} + + + ); + }), + )} + +
    + + Δ is tachometer's confidence interval on the difference, {reference} relative to the + other variant{others.length > 1 ? 's' : ''} — negative is faster. + +     ); } @@ -155,10 +295,17 @@ export default function TachometerDetails() { const report = upload?.report; const summarized = (report?.cases ?? []).filter( - (entry): entry is TachometerCaseResult & { measurements: TachometerMeasurementResult[] } => + (entry): entry is SummarizedCase => entry.measurements !== undefined && entry.measurements.length > 0, ); const failed = (report?.cases ?? []).filter((entry) => !entry.measurements?.length); + const groups = groupCasesByVariantSet(summarized); + + // Bundle weight is a property of the variant's page, not of a measurement, so it goes in a note + // rather than down every row — except where the variant table already gives it a column. + const withBytesColumn = new Set( + groups.filter((group) => group.variants.length > 2).flatMap((group) => group.cases), + ); // A tachometer report carries its own comparison, so the baseline is one of its own refs rather // than a separately fetched report. @@ -196,16 +343,50 @@ export default function TachometerDetails() { ))} - {summarized.map((entry) => ( - - ))} + {groups.map((group) => + group.variants.length > 2 ? ( + group.cases.map((entry) => ( + + )) + ) : ( + + ), + )} {report && ( - - Each cell is a 95% confidence interval for the mean; one sample is one page load. - "unsure" means the interval still straddles zero — the expected result for two - equivalent builds. Measured on the production bundle, installed from a packed tarball. - + + + Each cell is a 95% confidence interval for the mean, in milliseconds; one sample is + one page load. "unsure" means the interval still straddles zero: the + difference did not resolve within the case's sampling budget. + + {summarized + .filter((entry) => !withBytesColumn.has(entry)) + .map((entry) => ( + + {entry.name} transferred:{' '} + {bytesPerVariant(entry) + .map(([variantName, bytes]) => `${variantName} ${formatBytes(bytes)}`) + .join(' · ')} + + ))} + + Builds:{' '} + {report.refs + .map( + (ref) => `${ref.id} = ${ref.label}${ref.sha ? ` (${ref.sha.slice(0, 9)})` : ''}`, + ) + .join(' · ')} + + + head: {report.head.sha.slice(0, 9)} ({report.head.branch || '?'}) · measured on the + production bundle, installed from a packed tarball. + + )} From 825742ccab49e5a458d997325c50140c3e72d5e4 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:44:47 +0200 Subject: [PATCH 014/173] [dashboard] Format the tachometer view --- apps/code-infra-dashboard/src/views/TachometerDetails.tsx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx index ee896f744b..43aebd9d84 100644 --- a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx +++ b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx @@ -16,11 +16,7 @@ import TableRow from '@mui/material/TableRow'; import Typography from '@mui/material/Typography'; import { fetchCiReport } from '@/utils/fetchCiReport'; import type { ConfidenceInterval, TachometerUpload, Verdict } from '@/lib/tachometer/types'; -import { - bytesPerVariant, - groupCasesByVariantSet, - shortNameOf, -} from '@/lib/tachometer/groupCases'; +import { bytesPerVariant, groupCasesByVariantSet, shortNameOf } from '@/lib/tachometer/groupCases'; import type { SummarizedCase } from '@/lib/tachometer/groupCases'; import Heading from '../components/Heading'; import ReportHeader from '../components/ReportHeader'; From f34a71c1a4edf6c967eed2ea410a83d3684b81fc Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:13:52 +0200 Subject: [PATCH 015/173] [code-infra] Share the tachometer helpers each file had written itself `run`, `pathExists`, and the dashboard's name and number formatting had all been written out two or three times, and one report's two renderers could drift on precision without anything noticing. The upload and the comment sync each carried their own copy of the same authenticated POST. The transfer-size note in both renderers derived its cases from a set filled while printing, so the notes silently depended on the tables having rendered first. Along the way: packing caps its concurrency rather than starting a process per public package, tarballs are hashed as streams instead of read whole, a resolved git ref is remembered rather than re-resolved per case, and a ref's sandbox stops installing the browser and driver that only the runner resolves. --- .../src/lib/tachometer/buildMarkdownReport.ts | 40 +----- .../src/lib/tachometer/formatInterval.ts | 22 +++ .../src/lib/tachometer/groupCases.ts | 5 + .../src/views/TachometerDetails.tsx | 129 +++++++++--------- .../code-infra/src/tachometer/browser.mjs | 17 +-- .../code-infra/src/tachometer/buildPages.mjs | 30 ++-- .../code-infra/src/tachometer/ciReport.mjs | 84 ++++++------ .../src/tachometer/discoverCases.mjs | 18 +-- packages/code-infra/src/tachometer/refs.mjs | 22 +-- .../src/tachometer/renderReport.mjs | 12 +- .../src/tachometer/runTachometer.mjs | 24 ++-- .../src/{tachometer => utils}/exec.mjs | 0 .../code-infra/src/utils/packWorkspace.mjs | 95 ++++++------- packages/code-infra/src/utils/path.mjs | 16 +++ 14 files changed, 259 insertions(+), 255 deletions(-) create mode 100644 apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts rename packages/code-infra/src/{tachometer => utils}/exec.mjs (100%) diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts index 5451818cb9..cbd6cee5a6 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts @@ -1,9 +1,7 @@ -import type { - ConfidenceInterval, - TachometerCaseResult, - TachometerMeasurementResult, - TachometerReport, -} from './types'; +import type { ConfidenceInterval, TachometerReport } from './types'; +import { formatBytes, formatMean, formatPercent } from './formatInterval'; +import { isSummarized, shortNameOf } from './groupCases'; +import type { SummarizedCase } from './groupCases'; export const TACHOMETER_SECTION_TITLE = 'Tachometer'; @@ -20,40 +18,12 @@ interface BuildOptions { detailsUrl?: string; } -/** `+2.7% – +6.8%`, both bounds signed so the direction reads without the verdict. */ -function formatPercent(interval: ConfidenceInterval): string { - const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(1)}%`; - return `${signed(interval.low)} – ${signed(interval.high)}`; -} - -/** `+6.1 ms – +15.2 ms`. */ +/** `+6.1 ms – +15.2 ms`. Only a regression line states an absolute difference. */ function formatMs(interval: ConfidenceInterval): string { const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(2)} ms`; return `${signed(interval.low)} – ${signed(interval.high)}`; } -/** `20.50 – 22.67 ms`, with the unit written once. */ -function formatMean(interval: ConfidenceInterval): string { - return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; -} - -function formatBytes(bytes: number): string { - return `${(bytes / 1024).toFixed(1)} KiB`; -} - -/** - * Tachometer names each variant ` []`; the case is already its own heading here. - */ -function shortNameOf(caseName: string, variant: string): string { - return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; -} - -type SummarizedCase = TachometerCaseResult & { measurements: TachometerMeasurementResult[] }; - -function isSummarized(entry: TachometerCaseResult): entry is SummarizedCase { - return entry.measurements !== undefined && entry.measurements.length > 0; -} - /** * Collects the comparisons that mean "this pull request made something slower". * diff --git a/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts b/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts new file mode 100644 index 0000000000..765dea35d4 --- /dev/null +++ b/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts @@ -0,0 +1,22 @@ +import type { ConfidenceInterval } from './types'; + +/** + * How a tachometer report's numbers are written, shared by the pull request comment and the details + * page so the two never disagree about precision. + */ + +/** `20.50 – 22.67 ms`, with the unit written once. */ +export function formatMean(interval: ConfidenceInterval): string { + return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; +} + +/** `+2.7% – +6.8%`, both bounds signed so the direction reads without the verdict. */ +export function formatPercent(interval: ConfidenceInterval): string { + const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(1)}%`; + return `${signed(interval.low)} – ${signed(interval.high)}`; +} + +/** Kibibytes, matching how tachometer's own table reports `bytesSent`. */ +export function formatBytes(bytes: number): string { + return `${(bytes / 1024).toFixed(1)} KiB`; +} diff --git a/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts b/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts index 9829732ced..7825edb87f 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts @@ -20,6 +20,11 @@ export function shortNameOf(caseName: string, variant: string): string { return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; } +/** A case that produced results, as opposed to one whose run failed and carries only an error. */ +export function isSummarized(entry: TachometerCaseResult): entry is SummarizedCase { + return entry.measurements !== undefined && entry.measurements.length > 0; +} + /** The distinct variants a case reports, in order of first appearance. */ export function variantsOf(entry: SummarizedCase): string[] { const variants: string[] = []; diff --git a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx index 43aebd9d84..c00a7547dd 100644 --- a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx +++ b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx @@ -16,8 +16,14 @@ import TableRow from '@mui/material/TableRow'; import Typography from '@mui/material/Typography'; import { fetchCiReport } from '@/utils/fetchCiReport'; import type { ConfidenceInterval, TachometerUpload, Verdict } from '@/lib/tachometer/types'; -import { bytesPerVariant, groupCasesByVariantSet, shortNameOf } from '@/lib/tachometer/groupCases'; +import { + bytesPerVariant, + groupCasesByVariantSet, + isSummarized, + shortNameOf, +} from '@/lib/tachometer/groupCases'; import type { SummarizedCase } from '@/lib/tachometer/groupCases'; +import { formatBytes, formatMean, formatPercent } from '@/lib/tachometer/formatInterval'; import Heading from '../components/Heading'; import ReportHeader from '../components/ReportHeader'; import ErrorDisplay from '../components/ErrorDisplay'; @@ -31,19 +37,6 @@ import ErrorDisplay from '../components/ErrorDisplay'; * actually carries that. */ -function formatMean(interval: ConfidenceInterval): string { - return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; -} - -function formatPercent(interval: ConfidenceInterval): string { - const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(1)}%`; - return `${signed(interval.low)} – ${signed(interval.high)}`; -} - -function formatBytes(bytes: number): string { - return `${(bytes / 1024).toFixed(1)} KiB`; -} - /** Only a resolved difference is coloured; `unsure` is the expected result and stays neutral. */ function verdictColor(verdict: Verdict): 'error' | 'success' | 'text.secondary' { if (verdict === 'slower') { @@ -91,6 +84,15 @@ function Muted({ children }: { children: React.ReactNode }) { */ function VariantTable({ entry, variants }: { entry: SummarizedCase; variants: string[] }) { const [reference] = variants; + // Both are properties of the variant's page rather than of any one measurement, so they are read + // once per row instead of being accumulated while the measurement cells render. + const bytes = new Map(bytesPerVariant(entry)); + const samplesOf = (variantName: string) => + entry.measurements.flatMap((measurement) => + measurement.variants + .filter((variant) => shortNameOf(entry.name, variant.variant) === variantName) + .map((variant) => variant.samples), + ); return ( @@ -110,50 +112,46 @@ function VariantTable({ entry, variants }: { entry: SummarizedCase; variants: st {variants.map((variantName) => { - const samples: number[] = []; - let bytesSent: number | undefined; + const bytesSent = bytes.get(variantName); - const cells = entry.measurements.map((measurement) => { - const found = measurement.variants.find( - (candidate) => shortNameOf(entry.name, candidate.variant) === variantName, - ); - const comparison = measurement.comparisons.find( - (candidate) => shortNameOf(entry.name, candidate.variant) === variantName, - ); - if (found) { - bytesSent = found.bytesSent; - samples.push(found.samples); - } - // The comparison holds the reference relative to this variant; a row about the - // variant needs the direction whose subject is the variant. - const againstReference = comparison?.versusReference; - return ( - - - {found ? formatMean(found.meanMs) : —} - - - {variantName === reference && reference} - {variantName !== reference && againstReference && ( + return ( + + {variantName} + {entry.measurements.map((measurement) => { + const found = measurement.variants.find( + (candidate) => shortNameOf(entry.name, candidate.variant) === variantName, + ); + // The comparison holds the reference relative to this variant; a row about the + // variant needs the direction whose subject is the variant. + const againstReference = measurement.comparisons.find( + (candidate) => shortNameOf(entry.name, candidate.variant) === variantName, + )?.versusReference; + + let versus = —; + if (variantName === reference) { + versus = reference; + } else if (againstReference) { + versus = ( - )} - {variantName !== reference && !againstReference && —} - - - ); - }); + ); + } - return ( - - {variantName} - {cells} + return ( + + + {found ? formatMean(found.meanMs) : —} + + {versus} + + ); + })} {bytesSent === undefined ? — : formatBytes(bytesSent)} - {formatSamples(samples)} + {formatSamples(samplesOf(variantName))} ); })} @@ -290,18 +288,15 @@ export default function TachometerDetails() { } const report = upload?.report; - const summarized = (report?.cases ?? []).filter( - (entry): entry is SummarizedCase => - entry.measurements !== undefined && entry.measurements.length > 0, - ); - const failed = (report?.cases ?? []).filter((entry) => !entry.measurements?.length); + const summarized = (report?.cases ?? []).filter(isSummarized); + const failed = (report?.cases ?? []).filter((entry) => !isSummarized(entry)); const groups = groupCasesByVariantSet(summarized); // Bundle weight is a property of the variant's page, not of a measurement, so it goes in a note - // rather than down every row — except where the variant table already gives it a column. - const withBytesColumn = new Set( - groups.filter((group) => group.variants.length > 2).flatMap((group) => group.cases), - ); + // rather than down every row — except where a variant table already gives it a column. + const needBytesNote = groups + .filter((group) => group.variants.length <= 2) + .flatMap((group) => group.cases); // A tachometer report carries its own comparison, so the baseline is one of its own refs rather // than a separately fetched report. @@ -360,16 +355,14 @@ export default function TachometerDetails() { one page load. "unsure" means the interval still straddles zero: the difference did not resolve within the case's sampling budget. - {summarized - .filter((entry) => !withBytesColumn.has(entry)) - .map((entry) => ( - - {entry.name} transferred:{' '} - {bytesPerVariant(entry) - .map(([variantName, bytes]) => `${variantName} ${formatBytes(bytes)}`) - .join(' · ')} - - ))} + {needBytesNote.map((entry) => ( + + {entry.name} transferred:{' '} + {bytesPerVariant(entry) + .map(([variantName, bytes]) => `${variantName} ${formatBytes(bytes)}`) + .join(' · ')} + + ))} Builds:{' '} {report.refs diff --git a/packages/code-infra/src/tachometer/browser.mjs b/packages/code-infra/src/tachometer/browser.mjs index 550bab1ebe..e47b5d5b05 100644 --- a/packages/code-infra/src/tachometer/browser.mjs +++ b/packages/code-infra/src/tachometer/browser.mjs @@ -1,8 +1,8 @@ import * as path from 'node:path'; import { createRequire } from 'node:module'; -import { access } from 'node:fs/promises'; import { execFileSync } from 'node:child_process'; import chalk from 'chalk'; +import { pathExists } from '../utils/path.mjs'; /** * Which browser the benchmarks run on, and whether the driver can actually drive it. @@ -45,21 +45,6 @@ export async function resolveBrowserBinary(harnessDir) { return binary; } -/** - * True if `target` exists on disk. - * - * @param {string} target - Path to check - * @returns {Promise} - */ -async function pathExists(target) { - try { - await access(target); - return true; - } catch { - return false; - } -} - /** * Extracts the major version out of output like `Google Chrome for Testing 151.0.7922.34`. * diff --git a/packages/code-infra/src/tachometer/buildPages.mjs b/packages/code-infra/src/tachometer/buildPages.mjs index 427b7f5f56..7fbea70b15 100644 --- a/packages/code-infra/src/tachometer/buildPages.mjs +++ b/packages/code-infra/src/tachometer/buildPages.mjs @@ -4,8 +4,18 @@ import * as path from 'node:path'; import { cp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; import { parse, stringify } from 'yaml'; +import { run } from '../utils/exec.mjs'; import { tarballFor } from '../utils/packWorkspace.mjs'; -import { run } from './exec.mjs'; +import { readPackageJson } from '../utils/pnpm.mjs'; + +/** + * Dependencies the run needs but a page never imports, so a ref's sandbox does without them. + * + * All three are resolved from the harness itself — the browser and its driver by the runner, and + * tachometer by the command that samples — and every one of them is expensive: tachometer pulls a + * chromedriver, `@playwright/test` a browser download. + */ +const RUNNER_ONLY_DEPS = ['tachometer', 'chromedriver', '@playwright/test']; /** * Builds the benchmark pages for one ref, in an isolated install. @@ -49,7 +59,7 @@ async function findViteConfig(harnessDir) { * @param {string} outDir - Absolute output directory * @returns {Promise} */ -export async function runViteBuild(cwd, outDir) { +async function runViteBuild(cwd, outDir) { await run('pnpm', ['exec', 'vite', 'build', '--outDir', outDir], cwd); } @@ -63,16 +73,16 @@ export async function runViteBuild(cwd, outDir) { * libraries a cross-library case compares against — come along unchanged, so every page builds * against every ref. * - * @param {Record | undefined} deps - A `dependencies` or `devDependencies` map + * @param {Partial> | undefined} deps - A `dependencies` or `devDependencies` map * @param {PackedPackage[]} packages - The packed workspace packages * @param {string[]} [omit] - Dependency names to drop entirely * @returns {Record} */ -export function rewriteWorkspaceDeps(deps, packages, omit = []) { +function rewriteWorkspaceDeps(deps, packages, omit = []) { /** @type {Record} */ const out = {}; for (const [name, version] of Object.entries(deps ?? {})) { - if (omit.includes(name)) { + if (version === undefined || omit.includes(name)) { continue; } out[name] = version.startsWith('workspace:') ? `file:${tarballFor(packages, name)}` : version; @@ -150,7 +160,7 @@ export async function buildRefPages(options) { await cp(path.join(harnessDir, viteConfig), path.join(workDir, viteConfig)); await syncEnvFiles(harnessDir, workDir); - const harnessPkg = JSON.parse(await readFile(path.join(harnessDir, 'package.json'), 'utf8')); + const harnessPkg = await readPackageJson(harnessDir); await writeFile( path.join(workDir, 'package.json'), `${JSON.stringify( @@ -159,9 +169,11 @@ export async function buildRefPages(options) { private: true, type: harnessPkg.type, dependencies: rewriteWorkspaceDeps(harnessPkg.dependencies, packages), - // No page imports tachometer; installing it here would only slow every ref down, and it - // pulls a chromedriver besides. - devDependencies: rewriteWorkspaceDeps(harnessPkg.devDependencies, packages, ['tachometer']), + devDependencies: rewriteWorkspaceDeps( + harnessPkg.devDependencies, + packages, + RUNNER_ONLY_DEPS, + ), }, null, 2, diff --git a/packages/code-infra/src/tachometer/ciReport.mjs b/packages/code-infra/src/tachometer/ciReport.mjs index b7315e7dd9..50ec750c94 100644 --- a/packages/code-infra/src/tachometer/ciReport.mjs +++ b/packages/code-infra/src/tachometer/ciReport.mjs @@ -92,6 +92,7 @@ export const tachometerUploadSchema = z.object({ /** * @typedef {z.infer} TachometerUpload + * @typedef {z.infer} TachometerReport */ /** @@ -119,38 +120,60 @@ export async function getCiMetadata() { } /** - * Uploads a report to the CI report store. - * - * Validates before sending because the server stores `report` without inspecting it — a malformed - * report would be accepted here and only fail much later, when the dashboard tried to render it. + * The dashboard endpoint for `pathname`, honouring an override so a pull request's own preview + * deployment can be pointed at. * - * @param {TachometerUpload} upload - The envelope to send - * @returns {Promise} + * @param {string} pathname - The API path + * @returns {URL} */ -export async function uploadCiReport(upload) { - tachometerUploadSchema.parse(upload); - - const apiUrl = process.env.CI_REPORT_API_URL ?? DEFAULT_API_URL; - const url = new URL('/api/ci-reports/upload', apiUrl); +function endpoint(pathname) { + return new URL(pathname, process.env.CI_REPORT_API_URL ?? DEFAULT_API_URL); +} +/** + * Posts JSON to the dashboard, authenticated as this CI job. + * + * @param {URL} url - The endpoint to post to + * @param {unknown} body - The payload + * @param {string} purpose - What the token is needed for, named in the error when it is missing + * @param {string} failurePrefix - How to open the error when the server refuses + * @returns {Promise} The response body + */ +async function postJson(url, body, purpose, failurePrefix) { const oidcToken = process.env.CIRCLE_OIDC_TOKEN_V2; if (!oidcToken) { - throw new Error('CIRCLE_OIDC_TOKEN_V2 environment variable is required for uploads'); + throw new Error(`CIRCLE_OIDC_TOKEN_V2 environment variable is required for ${purpose}`); } - console.log(`Uploading tachometer report to ${url.href}`); - const response = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${oidcToken}` }, - body: JSON.stringify(upload), + body: JSON.stringify(body), }); const responseText = await response.text(); if (!response.ok) { - throw new Error(`Upload failed (${response.status}): ${responseText}`); + throw new Error(`${failurePrefix} (${response.status}): ${responseText}`); } + return responseText; +} +/** + * Uploads a report to the CI report store. + * + * Validates before sending because the server stores `report` without inspecting it — a malformed + * report would be accepted here and only fail much later, when the dashboard tried to render it. + * + * @param {TachometerUpload} upload - The envelope to send + * @returns {Promise} + */ +export async function uploadCiReport(upload) { + tachometerUploadSchema.parse(upload); + + const url = endpoint('/api/ci-reports/upload'); + console.log(`Uploading tachometer report to ${url.href}`); + + const responseText = await postJson(url, upload, 'uploads', 'Upload failed'); const result = JSON.parse(responseText); console.log(`Tachometer report uploaded. S3 key: ${result.key}`); } @@ -162,26 +185,11 @@ export async function uploadCiReport(upload) { * @returns {Promise<{ success: boolean, skipped?: boolean }>} */ export async function syncPrComment(repo) { - const oidcToken = process.env.CIRCLE_OIDC_TOKEN_V2; - if (!oidcToken) { - throw new Error('CIRCLE_OIDC_TOKEN_V2 environment variable is required for PR comment sync'); - } - - const apiUrl = process.env.CI_REPORT_API_URL ?? DEFAULT_API_URL; - const url = new URL('/api/ci-reports/sync-pr-comment', apiUrl); - - const response = await fetch(url, { - method: 'POST', - headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${oidcToken}` }, - body: JSON.stringify({ repo }), - }); - - if (!response.ok) { - const responseText = await response.text(); - throw new Error( - `Sync PR comment API returned ${response.status} ${response.statusText}: ${responseText}`, - ); - } - - return response.json(); + const responseText = await postJson( + endpoint('/api/ci-reports/sync-pr-comment'), + { repo }, + 'PR comment sync', + 'Sync PR comment API returned', + ); + return JSON.parse(responseText); } diff --git a/packages/code-infra/src/tachometer/discoverCases.mjs b/packages/code-infra/src/tachometer/discoverCases.mjs index bb82549242..0177f6d300 100644 --- a/packages/code-infra/src/tachometer/discoverCases.mjs +++ b/packages/code-infra/src/tachometer/discoverCases.mjs @@ -1,6 +1,7 @@ import * as path from 'node:path'; -import { access, readFile, readdir } from 'node:fs/promises'; +import { readFile, readdir } from 'node:fs/promises'; import { pathToFileURL } from 'node:url'; +import { pathExists } from '../utils/path.mjs'; import { parseRefToken } from './refs.mjs'; /** @@ -50,21 +51,6 @@ import { parseRefToken } from './refs.mjs'; /** The file that marks a directory under `src/` as a benchmark case. */ const CONFIG_NAME = 'tachometer.json'; -/** - * True if `target` exists on disk. - * - * @param {string} target - Path to check - * @returns {Promise} - */ -async function pathExists(target) { - try { - await access(target); - return true; - } catch { - return false; - } -} - /** * The name tachometer gives a measurement: an explicit `name`, else the expression, else the entry * name. diff --git a/packages/code-infra/src/tachometer/refs.mjs b/packages/code-infra/src/tachometer/refs.mjs index 65b448dd1b..e116d7d58a 100644 --- a/packages/code-infra/src/tachometer/refs.mjs +++ b/packages/code-infra/src/tachometer/refs.mjs @@ -34,7 +34,7 @@ import { execaSync } from 'execa'; */ /** The working tree — never cached, since it has no immutable identity. */ -export const WORKTREE_REF = /** @type {ResolvedRef} */ ({ +const WORKTREE_REF = /** @type {ResolvedRef} */ ({ kind: 'worktree', id: 'current', label: 'working tree', @@ -100,7 +100,7 @@ function gitCapture(args, cwd) { * @param {string} cwd - Repository to inspect * @returns {string} The base branch name */ -export function detectBaseBranch(cwd) { +function detectBaseBranch(cwd) { const result = execaSync('git', ['symbolic-ref', '--short', 'refs/remotes/origin/HEAD'], { cwd, reject: false, @@ -127,7 +127,7 @@ export function detectBaseBranch(cwd) { * @param {string} baseBranch - Base branch name, e.g. `master` * @returns {{ ref: string, mergeBase: string } | undefined} */ -export function closestBaseBranch(repoRoot, baseBranch) { +function closestBaseBranch(repoRoot, baseBranch) { // On ties (same merge base), prefer upstream's, then origin's, then a local base branch. const preference = [`upstream/${baseBranch}`, `origin/${baseBranch}`, baseBranch]; /** @@ -181,6 +181,9 @@ export function createRefResolver(options) { /** @type {ResolvedRef | undefined} */ let baselineRef; + // Several cases commonly pin the same commit, and resolving one spawns a git process. + /** @type {Map} */ + const gitRefs = new Map(); /** * Resolves a git committish to a `ResolvedRef` keyed by its immutable SHA. @@ -190,14 +193,20 @@ export function createRefResolver(options) { * @returns {ResolvedRef} */ function gitRef(committish, label) { + const cached = gitRefs.get(committish); + if (cached) { + return cached; + } const sha = gitCapture(['rev-parse', committish], repoRoot); - return { - kind: 'git', + const ref = { + kind: /** @type {const} */ ('git'), id: `git-${sha.slice(0, 9)}`, label: label ?? `${committish} (${sha.slice(0, 9)})`, sha, committish: sha, }; + gitRefs.set(committish, ref); + return ref; } /** @@ -209,9 +218,6 @@ export function createRefResolver(options) { */ function computeBaselineRef() { if (baselineOverride !== undefined) { - if (baselineOverride.trim() === 'baseline') { - throw new Error('The baseline cannot itself be "baseline" — that is the symbol it binds.'); - } const descriptor = parseRefToken(baselineOverride); if (descriptor.kind === 'baseline') { throw new Error('The baseline cannot itself be "baseline" — that is the symbol it binds.'); diff --git a/packages/code-infra/src/tachometer/renderReport.mjs b/packages/code-infra/src/tachometer/renderReport.mjs index c07fd9ba48..bb115fb552 100644 --- a/packages/code-infra/src/tachometer/renderReport.mjs +++ b/packages/code-infra/src/tachometer/renderReport.mjs @@ -389,9 +389,14 @@ export function renderTachometerReport(report) { groups.set(key, group); } - // Cases whose table carries the transfer size as a column, so the note below need not repeat it. - /** @type {Set} */ - const withBytesColumn = new Set(); + // Only a variant table carries the transfer size as a column, so only the other cases need the + // note below. Derived from the grouping rather than collected while printing, so the notes do not + // depend on the tables having been rendered first. + const withBytesColumn = new Set( + [...groups.values()] + .filter((group) => group.variants.length > 2) + .flatMap((group) => group.cases.map((entry) => entry.name)), + ); for (const [groupIndex, group] of [...groups.values()].entries()) { if (groupIndex > 0) { @@ -403,7 +408,6 @@ export function renderTachometerReport(report) { console.log(''); } printVariantTable(entry, group.variants); - withBytesColumn.add(entry.name); } continue; } diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index 3061b4f236..1274da7ef0 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -18,7 +18,7 @@ import { import { summarizeCase } from './summarizeCase.mjs'; import { renderTachometerReport } from './renderReport.mjs'; import { getCiMetadata, syncPrComment, uploadCiReport } from './ciReport.mjs'; -import { run } from './exec.mjs'; +import { run } from '../utils/exec.mjs'; /** * @typedef {Object} RunTachometerOptions @@ -122,15 +122,17 @@ export async function runTachometer(options) { buildCmd: workingTreeBuildCmd, }) : // packRef caches a ref's tarballs by SHA; a hit skips the checkout, install, and build. - // eslint-disable-next-line no-await-in-loop - await packRef({ - repoRoot, - ref: /** @type {string} */ (ref.committish), - outRoot: packedDir, - // An empty install command means "skip"; otherwise packRef's default install runs. - installCmd: install ? undefined : '', - buildCmd, - }).then((packed) => packed.packages); + ( + // eslint-disable-next-line no-await-in-loop + await packRef({ + repoRoot, + ref: /** @type {string} */ (ref.committish), + outRoot: packedDir, + // An empty install command means "skip"; otherwise packRef's default install runs. + installCmd: install ? undefined : '', + buildCmd, + }) + ).packages; // eslint-disable-next-line no-await-in-loop await buildRefPages({ @@ -235,7 +237,7 @@ export async function runTachometer(options) { * A failure to refresh the comment does not fail the run: the numbers are already measured, written * and uploaded, and losing them to a comment API hiccup would waste the whole benchmark. * - * @param {any} report - The report just written + * @param {import('./ciReport.mjs').TachometerReport} report - The report just written * @returns {Promise} */ async function publishReport(report) { diff --git a/packages/code-infra/src/tachometer/exec.mjs b/packages/code-infra/src/utils/exec.mjs similarity index 100% rename from packages/code-infra/src/tachometer/exec.mjs rename to packages/code-infra/src/utils/exec.mjs diff --git a/packages/code-infra/src/utils/packWorkspace.mjs b/packages/code-infra/src/utils/packWorkspace.mjs index 65cfffce22..6ac52ce984 100644 --- a/packages/code-infra/src/utils/packWorkspace.mjs +++ b/packages/code-infra/src/utils/packWorkspace.mjs @@ -3,9 +3,14 @@ import * as path from 'node:path'; import * as os from 'node:os'; import { createHash } from 'node:crypto'; -import { access, mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { createReadStream } from 'node:fs'; +import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { pipeline } from 'node:stream/promises'; import chalk from 'chalk'; import { execa, parseCommandString } from 'execa'; +import { mapConcurrently } from './build.mjs'; +import { run } from './exec.mjs'; +import { pathExists } from './path.mjs'; import { getWorkspacePackages } from './pnpm.mjs'; /** @@ -59,20 +64,6 @@ import { getWorkspacePackages } from './pnpm.mjs'; /** Name of the file written into each packed folder describing its contents. */ const MANIFEST = 'manifest.json'; -/** - * Runs a command in `cwd` with inherited stdio, throwing on a non-zero exit. - * - * @param {string} file - Executable to run - * @param {string[]} args - Arguments - * @param {string} cwd - Working directory - * @param {NodeJS.ProcessEnv} [env] - Extends the current environment - * @returns {Promise} - */ -async function run(file, args, cwd, env) { - console.log(chalk.dim(`$ ${file} ${args.join(' ')} (in ${cwd})`)); - await execa(file, args, { cwd, stdio: 'inherit', env }); -} - /** * Resolves a git ref to its commit SHA. * @@ -135,18 +126,18 @@ async function removeCheckout(repoRoot, checkout) { } /** - * True if `target` exists on disk. + * Content hash of a file, short enough to live in a filename. * - * @param {string} target - Path to check - * @returns {Promise} + * Streamed rather than read whole: these are tarballs of built packages, and every one of them + * would otherwise be resident at once. + * + * @param {string} file - File to hash + * @returns {Promise} */ -async function pathExists(target) { - try { - await access(target); - return true; - } catch { - return false; - } +async function hashFile(file) { + const hash = createHash('sha256'); + await pipeline(createReadStream(file), hash); + return hash.digest('hex').slice(0, 12); } /** @@ -227,13 +218,25 @@ export async function packBuiltPackages(checkoutDir, outDir) { if (packages.length === 0) { throw new Error(`No public workspace packages found in ${checkoutDir}.`); } - return Promise.all( - packages.map(async ({ path: pkgDir, name, version }) => { + // Each `pnpm pack` is its own node process, so a repository with a dozen public packages would + // otherwise start a dozen at once. + const packed = await mapConcurrently( + packages, + async ({ path: pkgDir, name, version }) => { const tarball = path.join(outDir, tarballName(name)); await run('pnpm', ['pack', '--out', tarball], pkgDir); return { name, version, tarball }; - }), + }, + os.availableParallelism(), ); + // A failed pack rejects rather than landing in the results, but the signature admits an Error and + // one reaching a manifest would be far more confusing than one thrown here. + return packed.map((entry) => { + if (entry instanceof Error) { + throw entry; + } + return entry; + }); } /** @@ -296,28 +299,23 @@ export async function packRef(options) { const packages = await packBuiltPackages(checkout, staging); // Store tarballs by basename so the folder is relocatable; record buildCmd so a later run can // tell whether the cache matches. Write the manifest last — it marks completeness. - await writeFile( - path.join(staging, MANIFEST), - `${JSON.stringify( - { - ref, - sha, - buildCmd, - packages: packages.map(({ name, version, tarball }) => ({ - name, - version, - tarball: path.basename(tarball), - })), - }, - null, - 2, - )}\n`, - ); + /** @type {RawManifest} */ + const manifest = { + ref, + sha, + buildCmd, + packages: packages.map(({ name, version, tarball }) => ({ + name, + version, + tarball: path.basename(tarball), + })), + }; + await writeFile(path.join(staging, MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`); // Replace any pre-existing folder — a stale/partial one, or a mismatched cache we chose to // rebuild — so the rename can't fail with ENOTEMPTY. await rm(dir, { recursive: true, force: true }); await rename(staging, dir); - return resolveManifest(await readRawManifest(dir), dir); + return resolveManifest(manifest, dir); } catch (error) { await rm(staging, { recursive: true, force: true }); throw error; @@ -363,10 +361,7 @@ export async function packWorkingTree(options) { await mkdir(outRoot, { recursive: true }); return await Promise.all( packed.map(async (pkg) => { - const hash = createHash('sha256') - .update(await readFile(pkg.tarball)) - .digest('hex') - .slice(0, 12); + const hash = await hashFile(pkg.tarball); const tarball = path.join(outRoot, `${path.basename(pkg.tarball, '.tgz')}-${hash}.tgz`); await rename(pkg.tarball, tarball); return { ...pkg, tarball }; diff --git a/packages/code-infra/src/utils/path.mjs b/packages/code-infra/src/utils/path.mjs index f58238a5e6..6a69dabfb5 100644 --- a/packages/code-infra/src/utils/path.mjs +++ b/packages/code-infra/src/utils/path.mjs @@ -1,4 +1,5 @@ import * as path from 'node:path'; +import { access } from 'node:fs/promises'; /** * Normalize a file path to use POSIX separators @@ -8,3 +9,18 @@ import * as path from 'node:path'; export function toPosixPath(filePath) { return filePath.split(path.sep).join('/'); } + +/** + * True if `target` exists on disk. + * + * @param {string} target - Path to check + * @returns {Promise} + */ +export async function pathExists(target) { + try { + await access(target); + return true; + } catch { + return false; + } +} From 455116c1b4796560e16d5f32ae3d655a40961c13 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:21:08 +0200 Subject: [PATCH 016/173] [code-infra] Let tachometer cases live in folders, and filter on the path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery walked one level, so every case had to sit directly under `src/`, and the filter matched a folder name. Both now work on the path: cases are found at any depth and a filter is a case-insensitive substring of the path, so naming a folder selects everything beneath it. That is the whole of the partitioning story — a quick set for pull requests, everything for a release — without the filter having to mean anything. A case takes its name from its benchmark's own `name` rather than its directory, which were previously two declarations of one identity kept in sync by hand: the ` []` prefix tachometer builds from that name is now strippable for display by construction, and moving a case between folders leaves its name in every report untouched. Two cases claiming the same name stop discovery, naming both files, since a duplicate would otherwise quietly overwrite the other's results. --- packages/code-infra/src/cli/cmdTacho.mjs | 4 +- .../src/tachometer/discoverCases.mjs | 81 +++++++++++++++---- .../src/tachometer/discoverCases.test.mjs | 57 +++++++++++++ .../src/tachometer/runTachometer.mjs | 60 ++++++++------ test/tachometer/README.md | 6 ++ 5 files changed, 166 insertions(+), 42 deletions(-) diff --git a/packages/code-infra/src/cli/cmdTacho.mjs b/packages/code-infra/src/cli/cmdTacho.mjs index 3d4e22d714..60a62e0267 100644 --- a/packages/code-infra/src/cli/cmdTacho.mjs +++ b/packages/code-infra/src/cli/cmdTacho.mjs @@ -2,7 +2,7 @@ /** * @typedef {Object} Args - * @property {string[]} [filters] - Only run cases whose folder name contains one of these substrings + * @property {string[]} [filters] - Only run cases whose path under `src` contains one of these substrings, case-insensitively * @property {string} [baseline] - Binds the `baseline` symbol, in the ref grammar * @property {string} [baseBranch] - Branch PRs fork from * @property {string} [buildCmd] - Command that builds the publishable packages of a checked-out ref @@ -21,7 +21,7 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ .positional('filters', { type: 'string', array: true, - describe: 'Only run cases whose folder name contains this substring', + describe: 'Only run cases whose path under src contains this substring', }) .option('baseline', { type: 'string', diff --git a/packages/code-infra/src/tachometer/discoverCases.mjs b/packages/code-infra/src/tachometer/discoverCases.mjs index 0177f6d300..b4aba7bd83 100644 --- a/packages/code-infra/src/tachometer/discoverCases.mjs +++ b/packages/code-infra/src/tachometer/discoverCases.mjs @@ -33,7 +33,7 @@ import { parseRefToken } from './refs.mjs'; /** * @typedef {Object} BenchmarkCase - * @property {string} name - The case folder name + * @property {string} name - The case name, as its benchmark declares it * @property {string} configPath - Absolute path to its `tachometer.json` * @property {any} config - The parsed config, mutated in place as urls are rewritten * @property {Leaf[]} leaves - Every node that selects a page @@ -44,7 +44,7 @@ import { parseRefToken } from './refs.mjs'; /** * @typedef {Object} DiscoverCasesOptions * @property {string} harnessDir - The harness package directory; cases live under its `src/` - * @property {string[]} [filters] - Only include cases whose folder name contains one of these substrings + * @property {string[]} [filters] - Only include cases whose path under `src` contains one of these substrings, case-insensitively * @property {(token?: string) => ResolvedRef} [resolveRef] - Resolves a ref token. Omit to skip resolution entirely (no git) */ @@ -139,8 +139,42 @@ async function parseLeafUrl(url, configDir, srcDir, resolveRef) { } /** - * Reads every `src//tachometer.json` (optionally filtered by name), expands the sugar for - * cases that declare no variants of their own, and resolves each leaf's page and ref. + * Every directory under `srcDir` that holds a `tachometer.json`, as a posix path relative to it. + * + * Nested rather than flat, so a harness can group its cases in folders — which is the whole of the + * partitioning story: the filter matches these paths, and nothing else has to understand what a + * folder means. + * + * @param {string} srcDir - The harness's `src` directory + * @returns {Promise} Case locations, e.g. `workload` or `libs/mount` + */ +async function findCaseLocations(srcDir) { + /** @type {string[]} */ + const locations = []; + + /** + * @param {string} location - Directory to scan, relative to `srcDir` + * @returns {Promise} + */ + async function walk(location) { + const entries = await readdir(path.join(srcDir, location), { withFileTypes: true }); + if (entries.some((entry) => entry.isFile() && entry.name === CONFIG_NAME)) { + locations.push(location); + } + await Promise.all( + entries + .filter((entry) => entry.isDirectory()) + .map((entry) => walk(location ? `${location}/${entry.name}` : entry.name)), + ); + } + + await walk(''); + return locations.sort(); +} + +/** + * Reads every `tachometer.json` under `src/` (optionally filtered by location), expands the sugar + * for cases that declare no variants of their own, and resolves each leaf's page and ref. * * A benchmark with no `expand` is the common regression case: it is expanded into `[current]` (the * working tree) versus `[baseline]`. A benchmark that declares its own `expand` owns its variant @@ -153,19 +187,16 @@ export async function discoverCases(options) { const { harnessDir, filters = [], resolveRef } = options; const srcDir = path.join(harnessDir, 'src'); - const dirEntries = (await readdir(srcDir, { withFileTypes: true })).filter((entry) => - entry.isDirectory(), - ); - const hasConfig = await Promise.all( - dirEntries.map((entry) => pathExists(path.join(srcDir, entry.name, CONFIG_NAME))), + const located = await findCaseLocations(srcDir); + // Matched case-insensitively against the location, the way vitest matches its file filters: a + // folder name selects everything under it without the filter having to know what a folder means. + const selected = located.filter( + (location) => + filters.length === 0 || + filters.some((filter) => location.toLowerCase().includes(filter.toLowerCase())), ); - const caseNames = dirEntries - .filter((_, index) => hasConfig[index]) - .map((entry) => entry.name) - .filter((name) => filters.length === 0 || filters.some((filter) => name.includes(filter))) - .sort(); - if (caseNames.length === 0) { + if (selected.length === 0) { throw new Error( filters.length > 0 ? `No benchmark case under ${srcDir} matches ${filters.map((filter) => `"${filter}"`).join(', ')}.` @@ -174,15 +205,31 @@ export async function discoverCases(options) { } const configs = await Promise.all( - caseNames.map(async (name) => { - const configPath = path.join(srcDir, name, CONFIG_NAME); + selected.map(async (location) => { + const configPath = path.join(srcDir, location, CONFIG_NAME); const config = JSON.parse(await readFile(configPath, 'utf8')); // `$schema` is for editors; tachometer rejects it in a config it is handed. delete config.$schema; + // The benchmark names itself, so a case keeps its identity wherever its folder is moved to, + // and the ` []` prefix that tachometer builds from that name is strippable for + // display by construction rather than by the folder happening to agree with it. + const name = config.benchmarks?.[0]?.name ?? path.basename(location); return { name, configPath, config }; }), ); + const byName = new Map(); + for (const entry of configs) { + const clash = byName.get(entry.name); + if (clash) { + throw new Error( + `Two benchmark cases are both named "${entry.name}": ${clash} and ${entry.configPath}. ` + + `Names identify a case in the report and in its results file, so they have to be unique.`, + ); + } + byName.set(entry.name, entry.configPath); + } + /** @type {BenchmarkCase[]} */ const cases = []; for (const { name, configPath, config } of configs) { diff --git a/packages/code-infra/src/tachometer/discoverCases.test.mjs b/packages/code-infra/src/tachometer/discoverCases.test.mjs index 9517736906..19e6389da7 100644 --- a/packages/code-infra/src/tachometer/discoverCases.test.mjs +++ b/packages/code-infra/src/tachometer/discoverCases.test.mjs @@ -62,6 +62,63 @@ describe('discoverCases', () => { expect(cases.map((entry) => entry.name)).toEqual(['grid-scroll']); }); + it('finds a case nested in a folder', async () => { + const harnessDir = await makeHarness({ + 'libs/mount': { config: config('libs-mount', './index.html'), pages: ['index.html'] }, + }); + + const cases = await discoverCases({ harnessDir }); + + expect(cases.map((entry) => entry.name)).toEqual(['libs-mount']); + }); + + it('takes a case name from its benchmark rather than its folder', async () => { + // The folder is where a case lives; the config is what it calls itself. Keeping them separate + // is what lets a case move between folders without being renamed in every report. + const harnessDir = await makeHarness({ + 'libs/mount': { config: config('libs-mount', './index.html'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.name).toBe('libs-mount'); + }); + + it('selects a whole folder by filtering on the path', async () => { + const harnessDir = await makeHarness({ + 'libs/mount': { config: config('libs-mount', './index.html'), pages: ['index.html'] }, + 'libs/scroll': { config: config('libs-scroll', './index.html'), pages: ['index.html'] }, + regression: { config: config('regression', './index.html'), pages: ['index.html'] }, + }); + + const cases = await discoverCases({ harnessDir, filters: ['libs'] }); + + expect(cases.map((entry) => entry.name)).toEqual(['libs-mount', 'libs-scroll']); + }); + + it('matches a filter case-insensitively', async () => { + const harnessDir = await makeHarness({ + GridScroll: { config: config('grid-scroll', './index.html'), pages: ['index.html'] }, + }); + + const cases = await discoverCases({ harnessDir, filters: ['gridscroll'] }); + + expect(cases.map((entry) => entry.name)).toEqual(['grid-scroll']); + }); + + it('refuses two cases that call themselves the same thing', async () => { + // The name identifies a case in the report and in the file its results are written to, so a + // duplicate would silently overwrite rather than fail. + const harnessDir = await makeHarness({ + 'libs/scroll': { config: config('scroll', './index.html'), pages: ['index.html'] }, + scroll: { config: config('scroll', './index.html'), pages: ['index.html'] }, + }); + + await expect(discoverCases({ harnessDir })).rejects.toThrow( + /Two benchmark cases are both named "scroll"/, + ); + }); + it('strips $schema, which tachometer rejects in a config it is handed', async () => { const harnessDir = await makeHarness({ alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index 1274da7ef0..8da8ad1d12 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -20,10 +20,21 @@ import { renderTachometerReport } from './renderReport.mjs'; import { getCiMetadata, syncPrComment, uploadCiReport } from './ciReport.mjs'; import { run } from '../utils/exec.mjs'; +/** + * A case's name as a filename component. Names come from the benchmark's own `name`, so they are + * prose rather than identifiers and may hold anything — a slash included. + * + * @param {string} name - The case name + * @returns {string} + */ +function fileSlugOf(name) { + return name.replace(/[^a-zA-Z0-9._-]+/g, '-'); +} + /** * @typedef {Object} RunTachometerOptions * @property {string} harnessDir - The harness package directory (where the command was run) - * @property {string[]} [filters] - Only run cases whose folder name contains one of these substrings + * @property {string[]} [filters] - Only run cases whose path under `src` contains one of these substrings, case-insensitively * @property {string} [baseline] - Binds the `baseline` symbol, in the ref grammar (e.g. `git:abc1234`) * @property {string} [baseBranch] - Branch PRs fork from. Defaults to detection via `origin/HEAD` * @property {string} [buildCmd] - Command that builds the publishable packages of a checked-out ref. Defaults to `pnpm release:build` @@ -113,26 +124,28 @@ export async function runTachometer(options) { // Build one variant per distinct ref, every one through its own isolated install so both sides // of a comparison resolve the library identically. for (const ref of refs.values()) { - const packages = - ref.kind === 'worktree' - ? // eslint-disable-next-line no-await-in-loop - await packWorkingTree({ - repoRoot, - outRoot: path.join(packedDir, 'current'), - buildCmd: workingTreeBuildCmd, - }) - : // packRef caches a ref's tarballs by SHA; a hit skips the checkout, install, and build. - ( - // eslint-disable-next-line no-await-in-loop - await packRef({ - repoRoot, - ref: /** @type {string} */ (ref.committish), - outRoot: packedDir, - // An empty install command means "skip"; otherwise packRef's default install runs. - installCmd: install ? undefined : '', - buildCmd, - }) - ).packages; + /** @type {import('../utils/packWorkspace.mjs').PackedPackage[]} */ + let packages; + if (ref.kind === 'worktree') { + // eslint-disable-next-line no-await-in-loop + packages = await packWorkingTree({ + repoRoot, + outRoot: path.join(packedDir, 'current'), + buildCmd: workingTreeBuildCmd, + }); + } else { + // packRef caches a ref's tarballs by SHA; a hit skips the checkout, install, and build. + // eslint-disable-next-line no-await-in-loop + const packed = await packRef({ + repoRoot, + ref: /** @type {string} */ (ref.committish), + outRoot: packedDir, + // An empty install command means "skip"; otherwise packRef's default install runs. + installCmd: install ? undefined : '', + buildCmd, + }); + packages = packed.packages; + } // eslint-disable-next-line no-await-in-loop await buildRefPages({ @@ -160,7 +173,8 @@ export async function runTachometer(options) { for (const benchmark of entry.config.benchmarks ?? []) { benchmark.browser = withBrowserDefaults(benchmark.browser, browserBinary, asRoot); } - const configPath = path.join(tmpBase, `tachometer-${entry.name}.json`); + const slug = fileSlugOf(entry.name); + const configPath = path.join(tmpBase, `tachometer-${slug}.json`); // eslint-disable-next-line no-await-in-loop await writeFile( configPath, @@ -168,7 +182,7 @@ export async function runTachometer(options) { ); console.log(chalk.cyan(`\nRunning "${entry.name}"…`)); - const jsonPath = path.join(tmpBase, `result-${entry.name}.json`); + const jsonPath = path.join(tmpBase, `result-${slug}.json`); // eslint-disable-next-line no-await-in-loop await run( 'pnpm', diff --git a/test/tachometer/README.md b/test/tachometer/README.md index 22d3fb3c40..d56db993f7 100644 --- a/test/tachometer/README.md +++ b/test/tachometer/README.md @@ -43,6 +43,12 @@ pnpm -F ./test/tachometer test:tacho large pnpm -F ./test/tachometer test:tacho --baseline git:HEAD ``` +A filter is a case-insensitive substring of a case's path under `src/`, the way vitest matches test +files. Cases may sit in folders, and since the path is what a filter matches, a folder name selects +everything beneath it — which is how a repository partitions its suite (a quick set for pull +requests, everything for a release) without the runner being taught what any of it means. A case is +named by its benchmark's own `name`, so moving one between folders does not rename it in the report. + The report lands in `results/report.json`. Cases run sequentially, and each auto-samples until its difference resolves or the (deliberately short) timeout is hit. Because both sides run identical code, the expected verdict is `unsure` — that is the "no change" outcome, not a failure. From 5d70a0d9eff67bf8b3200e47fa80996887e8396e Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 3 Sep 2026 12:33:18 +0200 Subject: [PATCH 017/173] [code-infra] Write every tachometer output under one .tachometer folder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The built pages, packed tarballs, isolated installs and reports each had their own directory in the harness, so a repository had three entries to ignore and had to learn which — mui-public's ignore file carried a comment explaining why they sat at the root, which is exactly the knowledge a second consumer has to rediscover. One directory means one entry, and deleting it is the whole reset. It also writes a `.gitignore` holding `*` inside, the way pytest marks its cache, so a harness that forgets the entry does not offer tarballs for commit. That does not replace the root entry here: ESLint is fed from the root ignore file and never reads nested ones. Tachometer serves the pages through koa-static, which refuses a dot-prefixed segment in a served path, so each case's config is now rooted at the output directory rather than at the harness — which keeps the folder out of the url. --- .gitignore | 10 +++---- packages/code-infra/src/cli/cmdTacho.mjs | 2 +- .../code-infra/src/tachometer/outputDir.mjs | 27 +++++++++++++++++++ packages/code-infra/src/tachometer/plugin.mjs | 9 ++++--- .../code-infra/src/tachometer/plugin.test.mjs | 4 +-- .../src/tachometer/runTachometer.mjs | 26 +++++++++++------- test/tachometer/README.md | 11 +++++--- 7 files changed, 63 insertions(+), 26 deletions(-) create mode 100644 packages/code-infra/src/tachometer/outputDir.mjs diff --git a/.gitignore b/.gitignore index 2a2e6b8555..bae08dce3b 100644 --- a/.gitignore +++ b/.gitignore @@ -61,9 +61,7 @@ next-env.d.ts .claude/worktrees/ .claude/cache/ -# Tachometer harness: built pages per ref, packed workspace tarballs, and run reports. -# Listed here rather than under test/tachometer so ESLint picks them up too: it is fed from -# the root .gitignore and never reads nested ones. -/test/tachometer/builds/ -/test/tachometer/.cache/ -/test/tachometer/results/ +# Everything the tachometer harness writes: built pages per ref, packed workspace tarballs, the +# isolated installs and the run reports. The directory ignores itself from the inside too, but the +# entry is listed here because ESLint is fed from the root .gitignore and never reads nested ones. +/test/tachometer/.tachometer/ diff --git a/packages/code-infra/src/cli/cmdTacho.mjs b/packages/code-infra/src/cli/cmdTacho.mjs index 60a62e0267..7bb5e17356 100644 --- a/packages/code-infra/src/cli/cmdTacho.mjs +++ b/packages/code-infra/src/cli/cmdTacho.mjs @@ -56,7 +56,7 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ }) .option('out', { type: 'string', - describe: 'Write the combined JSON report here. Default: results/report.json', + describe: 'Write the combined JSON report here. Default: .tachometer/results/report.json', }) .epilogue( 'Sampling (sampleSize, autoSampleConditions, timeout) is configured per case in its own tachometer.json — tachometer rejects those as CLI flags when a config file is used.', diff --git a/packages/code-infra/src/tachometer/outputDir.mjs b/packages/code-infra/src/tachometer/outputDir.mjs new file mode 100644 index 0000000000..e6875ef0b3 --- /dev/null +++ b/packages/code-infra/src/tachometer/outputDir.mjs @@ -0,0 +1,27 @@ +import * as path from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; + +/** + * Everything a run writes — packed tarballs, isolated installs, built pages, the report — lives + * under this one directory inside the harness, so a repository has a single thing to ignore and + * deleting it is the whole reset story. + */ +export const OUTPUT_DIR = '.tachometer'; + +/** + * Creates the output directory, marking it ignored from the inside. + * + * A `.gitignore` holding `*` ignores the directory's contents and itself, the way pytest marks its + * cache, so a harness that forgets to list it does not end up offering built pages and tarballs for + * commit. It does not replace an entry in the repository's own ignore file: tools that read only + * the root one — this repository's ESLint config among them — never see a nested `.gitignore`. + * + * @param {string} harnessDir - The harness package directory + * @returns {Promise} The output directory + */ +export async function prepareOutputDir(harnessDir) { + const outputDir = path.join(harnessDir, OUTPUT_DIR); + await mkdir(outputDir, { recursive: true }); + await writeFile(path.join(outputDir, '.gitignore'), '*\n'); + return outputDir; +} diff --git a/packages/code-infra/src/tachometer/plugin.mjs b/packages/code-infra/src/tachometer/plugin.mjs index d7ed688cd7..b468231e1c 100644 --- a/packages/code-infra/src/tachometer/plugin.mjs +++ b/packages/code-infra/src/tachometer/plugin.mjs @@ -1,6 +1,7 @@ import * as path from 'node:path'; import { readFile, realpath } from 'node:fs/promises'; import { discoverCases, pagesOf } from './discoverCases.mjs'; +import { OUTPUT_DIR } from './outputDir.mjs'; /** * Vite plugin for a tachometer benchmark harness. @@ -136,14 +137,14 @@ export function tachometer(options = {}) { // `//`, so absolute "/assets/…" paths would 404. base: './', build: { - // A plain `vite build` lands in `builds/manual`, next to the per-ref directories the - // runner writes. Without this it would default to `/dist` — inside `src/`, next to - // the case sources. + // A plain `vite build` lands in the run output directory, next to the per-ref + // directories the runner writes. Without this it would default to `/dist` — inside + // `src/`, next to the case sources. // // Only fill it in when the caller said nothing: a plugin's returned config is merged // *over* the inline config, so unconditionally setting it here would silently override // `vite build --outDir`, which is exactly how `tacho run` directs each ref's build. - outDir: userConfig.build?.outDir ?? path.join(harnessDir, 'builds', 'manual'), + outDir: userConfig.build?.outDir ?? path.join(harnessDir, OUTPUT_DIR, 'builds', 'manual'), // The output directory is outside `root` — and, for an isolated ref build, outside the // temporary package entirely; allow vite to clean it anyway. emptyOutDir: true, diff --git a/packages/code-infra/src/tachometer/plugin.test.mjs b/packages/code-infra/src/tachometer/plugin.test.mjs index 7cc12b30fb..7c692e02cf 100644 --- a/packages/code-infra/src/tachometer/plugin.test.mjs +++ b/packages/code-infra/src/tachometer/plugin.test.mjs @@ -82,13 +82,13 @@ describe('tachometer plugin', () => { expect(config.base).toBe('./'); }); - it('defaults the output directory to builds/manual, outside the source root', async () => { + it('defaults the output directory to the run output folder, outside the source root', async () => { // Vite would otherwise emit into /dist, i.e. inside src/ next to the case sources. const harnessDir = await makeHarness({ cases: oneCase }); const config = await callConfig(harnessDir, 'build'); - expect(config.build.outDir).toBe(path.join(harnessDir, 'builds', 'manual')); + expect(config.build.outDir).toBe(path.join(harnessDir, '.tachometer', 'builds', 'manual')); }); it('leaves an output directory the caller chose alone', async () => { diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index 8da8ad1d12..06809b478b 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -18,6 +18,7 @@ import { import { summarizeCase } from './summarizeCase.mjs'; import { renderTachometerReport } from './renderReport.mjs'; import { getCiMetadata, syncPrComment, uploadCiReport } from './ciReport.mjs'; +import { prepareOutputDir } from './outputDir.mjs'; import { run } from '../utils/exec.mjs'; /** @@ -40,7 +41,7 @@ function fileSlugOf(name) { * @property {string} [buildCmd] - Command that builds the publishable packages of a checked-out ref. Defaults to `pnpm release:build` * @property {string} [workingTreeBuildCmd] - Command that builds the working tree. Defaults to `buildCmd` * @property {boolean} [install] - Whether to install inside a ref's checkout. Defaults to true - * @property {string} [out] - Where to write the combined JSON report. Defaults to `results/report.json` + * @property {string} [out] - Where to write the combined JSON report. Defaults to `.tachometer/results/report.json` * @property {boolean} [upload] - Upload the report and refresh the pull request comment */ @@ -77,13 +78,15 @@ export async function runTachometer(options) { throw new Error(`Could not find a pnpm workspace root above ${harnessDir}.`); } - const buildsDir = path.join(harnessDir, 'builds'); - // Everything reusable between runs lives under `.cache`. `packed` holds tarballs — a ref's keyed - // by commit SHA, the working tree's by content hash — and is what CI should cache; `installs` - // holds each ref's isolated install, cheap to recreate. Both are gitignored and safe to delete. - const cacheDir = path.join(harnessDir, '.cache'); - const packedDir = path.join(cacheDir, 'packed'); - const installsDir = path.join(cacheDir, 'installs'); + // Everything a run writes goes under one directory, so a harness has one thing to ignore and + // deleting it is the whole reset story. `packed` holds tarballs — a ref's keyed by commit SHA, + // the working tree's by content hash — and is the one worth caching in CI; `installs` holds each + // ref's isolated install, cheap to recreate and not portable between containers, since its links + // point into a pnpm store; `builds` holds the pages built per ref; `results` the report. + const outputDir = await prepareOutputDir(harnessDir); + const buildsDir = path.join(outputDir, 'builds'); + const packedDir = path.join(outputDir, 'packed'); + const installsDir = path.join(outputDir, 'installs'); const resolver = createRefResolver({ repoRoot, baseBranch, baselineOverride: baseline }); const cases = await discoverCases({ harnessDir, filters, resolveRef: resolver.parse }); @@ -178,7 +181,10 @@ export async function runTachometer(options) { // eslint-disable-next-line no-await-in-loop await writeFile( configPath, - `${JSON.stringify({ ...entry.config, root: harnessDir }, null, 2)}\n`, + // Served from the output directory rather than the harness: tachometer's static server is + // koa-static with its default `hidden: false`, so any dot-prefixed segment in the *served* + // path is refused. Rooting here keeps `.tachometer` out of the url entirely. + `${JSON.stringify({ ...entry.config, root: outputDir }, null, 2)}\n`, ); console.log(chalk.cyan(`\nRunning "${entry.name}"…`)); @@ -195,7 +201,7 @@ export async function runTachometer(options) { results.push({ entry, json: JSON.parse(await readFile(jsonPath, 'utf8')) }); } - const outPath = out ? path.resolve(out) : path.join(harnessDir, 'results', 'report.json'); + const outPath = out ? path.resolve(out) : path.join(outputDir, 'results', 'report.json'); const report = { // Consumers render reports from several benchmark axes; the pair identifies which one this // is and how to read it. diff --git a/test/tachometer/README.md b/test/tachometer/README.md index d56db993f7..7a1d7ae162 100644 --- a/test/tachometer/README.md +++ b/test/tachometer/README.md @@ -49,9 +49,14 @@ everything beneath it — which is how a repository partitions its suite (a quic requests, everything for a release) without the runner being taught what any of it means. A case is named by its benchmark's own `name`, so moving one between folders does not rename it in the report. -The report lands in `results/report.json`. Cases run sequentially, and each auto-samples until its -difference resolves or the (deliberately short) timeout is hit. Because both sides run identical -code, the expected verdict is `unsure` — that is the "no change" outcome, not a failure. +Cases run sequentially, and each auto-samples until its difference resolves or the (deliberately +short) timeout is hit. Because both sides run identical code, the expected verdict is `unsure` — +that is the "no change" outcome, not a failure. + +Everything a run writes goes under `.tachometer/`: the report in `results/`, the pages built per ref +in `builds/`, the packed tarballs in `packed/` (the one worth caching in CI, keyed by commit SHA) +and each ref's isolated install in `installs/`. Deleting that directory resets the harness +completely, and it is the only thing a repository has to ignore. ### Prerequisites From 076fe56bb1eeea6a45dd36fe7ffa3acfc09bace1 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 3 Sep 2026 15:20:02 +0200 Subject: [PATCH 018/173] [code-infra] Lean on the platform and the existing helpers in the tachometer code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node ships an ANSI stripper, so the regex built from a char code to dodge `no-control-regex` goes; globby is already a dependency and replaces the hand-rolled recursive walk that found case configs; the dashboard's own markdown table helper replaces hand-concatenated pipe rows and separator lines. The terminal renderer had drifted from the dashboard's version of the same tables: it accumulated a row's transfer size and sample count as side effects of rendering that row's cells, formatted milliseconds through `Intl` only to strip the unit back off with a regex, carried a parallel array of column alignments restating the header layout, and partitioned cases on two conditions that had to agree — leaving a case with an empty `measurements` array rendered nowhere and reported nowhere. All four now match how the dashboard does it. The output directory names its own `builds/` subpath, so the runner and the plugin no longer compose it separately while having to agree; the base branch is detected only when the `baseline` symbol is actually resolved. --- .../src/lib/tachometer/buildMarkdownReport.ts | 81 ++++++------- .../src/lib/tachometer/formatInterval.ts | 16 ++- .../code-infra/src/tachometer/buildPages.mjs | 18 +-- .../code-infra/src/tachometer/ciReport.mjs | 14 +-- .../src/tachometer/discoverCases.mjs | 35 ++---- .../code-infra/src/tachometer/outputDir.mjs | 16 +++ packages/code-infra/src/tachometer/plugin.mjs | 4 +- packages/code-infra/src/tachometer/refs.mjs | 16 +-- .../src/tachometer/renderReport.mjs | 112 +++++++----------- .../src/tachometer/runTachometer.mjs | 4 +- .../src/tachometer/summarizeCase.mjs | 3 +- 11 files changed, 148 insertions(+), 171 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts index cbd6cee5a6..66b2e80852 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts @@ -1,5 +1,6 @@ +import { formatMarkdownTable } from '@/utils/formatters'; import type { ConfidenceInterval, TachometerReport } from './types'; -import { formatBytes, formatMean, formatPercent } from './formatInterval'; +import { formatBytes, formatMean, formatPercent, formatSignedMs } from './formatInterval'; import { isSummarized, shortNameOf } from './groupCases'; import type { SummarizedCase } from './groupCases'; @@ -18,12 +19,6 @@ interface BuildOptions { detailsUrl?: string; } -/** `+6.1 ms – +15.2 ms`. Only a regression line states an absolute difference. */ -function formatMs(interval: ConfidenceInterval): string { - const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(2)} ms`; - return `${signed(interval.low)} – ${signed(interval.high)}`; -} - /** * Collects the comparisons that mean "this pull request made something slower". * @@ -39,15 +34,11 @@ function comparisonsAcrossRefs(entry: SummarizedCase) { const found = []; for (const measurement of entry.measurements) { - const referenceRefId = measurement.variants.find( - (variant) => variant.variant === entry.reference, - )?.refId; + const refIds = new Map(measurement.variants.map((variant) => [variant.variant, variant.refId])); + const referenceRefId = refIds.get(entry.reference ?? ''); for (const comparison of measurement.comparisons) { - const comparedRefId = measurement.variants.find( - (variant) => variant.variant === comparison.variant, - )?.refId; - if (comparedRefId !== referenceRefId) { + if (refIds.get(comparison.variant) !== referenceRefId) { found.push({ measurement, comparison }); } } @@ -81,39 +72,43 @@ export function findRegressions(report: TachometerReport): Regression[] { /** One flat table per case: every measurement, every variant, every number the report holds. */ function renderCaseTable(entry: SummarizedCase): string { - const rows: string[] = []; - - for (const measurement of entry.measurements) { + const rows = entry.measurements.flatMap((measurement) => { const byVariant = new Map( measurement.comparisons.map((comparison) => [comparison.variant, comparison]), ); - for (const variant of measurement.variants) { - const comparison = byVariant.get(variant.variant); - const isReference = variant.variant === entry.reference; + return measurement.variants.map((variant) => { // Each row reads as "this variant, compared to the reference", so it needs the direction - // whose subject is the variant. `comparison` itself holds the opposite one — the reference + // whose subject is the variant. The comparison itself holds the opposite one — the reference // relative to the variant — which is what a regression is stated in, above the fold. - const againstReference = comparison?.versusReference; - const versus = isReference - ? '_reference_' - : (againstReference && - `${againstReference.verdict} \`${formatPercent(againstReference.percentChange)}\``) || - '—'; - rows.push( - `| ${measurement.name} | ${shortNameOf(entry.name, variant.variant)} | ${formatMean( - variant.meanMs, - )} | ${versus} | ${variant.samples} | ${formatBytes(variant.bytesSent)} |`, - ); - } - } - - return [ - `**${entry.name}**`, - '', - '| Measurement | Variant | Mean (95% CI) | vs reference | Samples | Transferred |', - '| :--- | :--- | ---: | :--- | ---: | ---: |', - ...rows, - ].join('\n'); + const againstReference = byVariant.get(variant.variant)?.versusReference; + let versus = '—'; + if (variant.variant === entry.reference) { + versus = '_reference_'; + } else if (againstReference) { + versus = `${againstReference.verdict} \`${formatPercent(againstReference.percentChange)}\``; + } + return { + measurement: measurement.name, + variant: shortNameOf(entry.name, variant.variant), + mean: formatMean(variant.meanMs), + versus, + samples: variant.samples, + transferred: formatBytes(variant.bytesSent), + }; + }); + }); + + return `**${entry.name}**\n\n${formatMarkdownTable( + [ + { field: 'measurement', header: 'Measurement', align: 'left' }, + { field: 'variant', header: 'Variant', align: 'left' }, + { field: 'mean', header: 'Mean (95% CI)', align: 'right' }, + { field: 'versus', header: 'vs reference', align: 'left' }, + { field: 'samples', header: 'Samples', align: 'right' }, + { field: 'transferred', header: 'Transferred', align: 'right' }, + ], + rows, + )}`; } /** @@ -139,7 +134,7 @@ export function buildTachometerMarkdownReport( lines.push( `⚠️ **${regression.caseName}** · ${regression.measurement} · slower than ` + `${regression.variant} \`${formatPercent(regression.percentChange)}\` ` + - `(\`${formatMs(regression.absoluteMs)}\`)`, + `(\`${formatSignedMs(regression.absoluteMs)}\`)`, ); } lines.push(''); diff --git a/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts b/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts index 765dea35d4..1366bf4474 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts @@ -10,12 +10,22 @@ export function formatMean(interval: ConfidenceInterval): string { return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; } -/** `+2.7% – +6.8%`, both bounds signed so the direction reads without the verdict. */ -export function formatPercent(interval: ConfidenceInterval): string { - const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(1)}%`; +/** Both bounds signed, so the direction of a difference reads without the verdict beside it. */ +function signedInterval(interval: ConfidenceInterval, unit: string, digits: number): string { + const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(digits)}${unit}`; return `${signed(interval.low)} – ${signed(interval.high)}`; } +/** `+2.7% – +6.8%`. */ +export function formatPercent(interval: ConfidenceInterval): string { + return signedInterval(interval, '%', 1); +} + +/** `+6.10 ms – +15.20 ms`. Only a regression line states an absolute difference. */ +export function formatSignedMs(interval: ConfidenceInterval): string { + return signedInterval(interval, ' ms', 2); +} + /** Kibibytes, matching how tachometer's own table reports `bytesSent`. */ export function formatBytes(bytes: number): string { return `${(bytes / 1024).toFixed(1)} KiB`; diff --git a/packages/code-infra/src/tachometer/buildPages.mjs b/packages/code-infra/src/tachometer/buildPages.mjs index 7fbea70b15..1522381e0d 100644 --- a/packages/code-infra/src/tachometer/buildPages.mjs +++ b/packages/code-infra/src/tachometer/buildPages.mjs @@ -49,20 +49,6 @@ async function findViteConfig(harnessDir) { return found[0]; } -/** - * Runs the harness's vite build, emitting into `outDir`. - * - * The page entries come from the tachometer plugin in the harness's own config, so nothing about - * which pages exist is passed here. - * - * @param {string} cwd - Directory holding the vite config and `src/` - * @param {string} outDir - Absolute output directory - * @returns {Promise} - */ -async function runViteBuild(cwd, outDir) { - await run('pnpm', ['exec', 'vite', 'build', '--outDir', outDir], cwd); -} - /** * Rewrites a dependency map, pointing every `workspace:`-protocol entry at its packed tarball and * copying everything else verbatim. @@ -203,5 +189,7 @@ export async function buildRefPages(options) { ['install', '--prefer-offline', '--ignore-scripts', '--config.engine-strict=false'], workDir, ); - await runViteBuild(workDir, outDir); + // The page entries come from the tachometer plugin in the harness's own config, so nothing about + // which pages exist is passed here. + await run('pnpm', ['exec', 'vite', 'build', '--outDir', outDir], workDir); } diff --git a/packages/code-infra/src/tachometer/ciReport.mjs b/packages/code-infra/src/tachometer/ciReport.mjs index 50ec750c94..920e7380c2 100644 --- a/packages/code-infra/src/tachometer/ciReport.mjs +++ b/packages/code-infra/src/tachometer/ciReport.mjs @@ -135,14 +135,13 @@ function endpoint(pathname) { * * @param {URL} url - The endpoint to post to * @param {unknown} body - The payload - * @param {string} purpose - What the token is needed for, named in the error when it is missing - * @param {string} failurePrefix - How to open the error when the server refuses + * @param {string} what - Names the call in both errors it can raise * @returns {Promise} The response body */ -async function postJson(url, body, purpose, failurePrefix) { +async function postJson(url, body, what) { const oidcToken = process.env.CIRCLE_OIDC_TOKEN_V2; if (!oidcToken) { - throw new Error(`CIRCLE_OIDC_TOKEN_V2 environment variable is required for ${purpose}`); + throw new Error(`CIRCLE_OIDC_TOKEN_V2 environment variable is required for ${what}`); } const response = await fetch(url, { @@ -153,7 +152,7 @@ async function postJson(url, body, purpose, failurePrefix) { const responseText = await response.text(); if (!response.ok) { - throw new Error(`${failurePrefix} (${response.status}): ${responseText}`); + throw new Error(`${what} failed (${response.status}): ${responseText}`); } return responseText; } @@ -173,7 +172,7 @@ export async function uploadCiReport(upload) { const url = endpoint('/api/ci-reports/upload'); console.log(`Uploading tachometer report to ${url.href}`); - const responseText = await postJson(url, upload, 'uploads', 'Upload failed'); + const responseText = await postJson(url, upload, 'The tachometer upload'); const result = JSON.parse(responseText); console.log(`Tachometer report uploaded. S3 key: ${result.key}`); } @@ -188,8 +187,7 @@ export async function syncPrComment(repo) { const responseText = await postJson( endpoint('/api/ci-reports/sync-pr-comment'), { repo }, - 'PR comment sync', - 'Sync PR comment API returned', + 'The pull request comment sync', ); return JSON.parse(responseText); } diff --git a/packages/code-infra/src/tachometer/discoverCases.mjs b/packages/code-infra/src/tachometer/discoverCases.mjs index b4aba7bd83..263cca6ecf 100644 --- a/packages/code-infra/src/tachometer/discoverCases.mjs +++ b/packages/code-infra/src/tachometer/discoverCases.mjs @@ -1,6 +1,7 @@ import * as path from 'node:path'; -import { readFile, readdir } from 'node:fs/promises'; +import { readFile } from 'node:fs/promises'; import { pathToFileURL } from 'node:url'; +import { globby } from 'globby'; import { pathExists } from '../utils/path.mjs'; import { parseRefToken } from './refs.mjs'; @@ -89,7 +90,7 @@ export function measurementNameOf(measurement) { * @param {Array<{ node: { url?: string, name?: string }, url: string }>} out - Collected leaves * @returns {void} */ -export function collectLeafNodes(node, inheritedUrl, out) { +function collectLeafNodes(node, inheritedUrl, out) { const url = node.url ?? inheritedUrl; if (Array.isArray(node.expand) && node.expand.length > 0) { for (const child of node.expand) { @@ -149,27 +150,15 @@ async function parseLeafUrl(url, configDir, srcDir, resolveRef) { * @returns {Promise} Case locations, e.g. `workload` or `libs/mount` */ async function findCaseLocations(srcDir) { - /** @type {string[]} */ - const locations = []; - - /** - * @param {string} location - Directory to scan, relative to `srcDir` - * @returns {Promise} - */ - async function walk(location) { - const entries = await readdir(path.join(srcDir, location), { withFileTypes: true }); - if (entries.some((entry) => entry.isFile() && entry.name === CONFIG_NAME)) { - locations.push(location); - } - await Promise.all( - entries - .filter((entry) => entry.isDirectory()) - .map((entry) => walk(location ? `${location}/${entry.name}` : entry.name)), - ); - } - - await walk(''); - return locations.sort(); + const configs = await globby(`**/${CONFIG_NAME}`, { cwd: srcDir }); + return ( + configs + // `dirname` gives `.` for a config sitting directly in `src`; that case's location is the root. + .map((config) => (path.posix.dirname(config) === '.' ? '' : path.posix.dirname(config))) + // Sorted so a run's case order — and therefore its report — does not depend on the order the + // filesystem happened to return. + .sort() + ); } /** diff --git a/packages/code-infra/src/tachometer/outputDir.mjs b/packages/code-infra/src/tachometer/outputDir.mjs index e6875ef0b3..c8ab500a32 100644 --- a/packages/code-infra/src/tachometer/outputDir.mjs +++ b/packages/code-infra/src/tachometer/outputDir.mjs @@ -25,3 +25,19 @@ export async function prepareOutputDir(harnessDir) { await writeFile(path.join(outputDir, '.gitignore'), '*\n'); return outputDir; } + +/** + * Where the pages built for `refId` go, or the root of every ref's pages when it is omitted. + * + * Named here rather than composed at each call site because the runner and the vite plugin have to + * agree on it: the runner points tachometer's static server at the output directory and rewrites + * each url under this path, and a plain `vite build` has to land somewhere that does not collide. + * + * @param {string} harnessDir - The harness package directory + * @param {string} [refId] - A ref's build directory name, e.g. `current` + * @returns {string} + */ +export function buildsDirOf(harnessDir, refId) { + const builds = path.join(harnessDir, OUTPUT_DIR, 'builds'); + return refId ? path.join(builds, refId) : builds; +} diff --git a/packages/code-infra/src/tachometer/plugin.mjs b/packages/code-infra/src/tachometer/plugin.mjs index b468231e1c..24b7dee033 100644 --- a/packages/code-infra/src/tachometer/plugin.mjs +++ b/packages/code-infra/src/tachometer/plugin.mjs @@ -1,7 +1,7 @@ import * as path from 'node:path'; import { readFile, realpath } from 'node:fs/promises'; import { discoverCases, pagesOf } from './discoverCases.mjs'; -import { OUTPUT_DIR } from './outputDir.mjs'; +import { buildsDirOf } from './outputDir.mjs'; /** * Vite plugin for a tachometer benchmark harness. @@ -144,7 +144,7 @@ export function tachometer(options = {}) { // Only fill it in when the caller said nothing: a plugin's returned config is merged // *over* the inline config, so unconditionally setting it here would silently override // `vite build --outDir`, which is exactly how `tacho run` directs each ref's build. - outDir: userConfig.build?.outDir ?? path.join(harnessDir, OUTPUT_DIR, 'builds', 'manual'), + outDir: userConfig.build?.outDir ?? buildsDirOf(harnessDir, 'manual'), // The output directory is outside `root` — and, for an isolated ref build, outside the // temporary package entirely; allow vite to clean it anyway. emptyOutDir: true, diff --git a/packages/code-infra/src/tachometer/refs.mjs b/packages/code-infra/src/tachometer/refs.mjs index e116d7d58a..76dbe2d7f3 100644 --- a/packages/code-infra/src/tachometer/refs.mjs +++ b/packages/code-infra/src/tachometer/refs.mjs @@ -58,12 +58,11 @@ export function parseRefToken(token) { return { kind: 'baseline' }; } - const git = /^git:(.+)$/.exec(token); - if (git) { - return { kind: 'git', committish: git[1] }; + if (token.startsWith('git:') && token.length > 'git:'.length) { + return { kind: 'git', committish: token.slice('git:'.length) }; } - if (/^github:/.test(token) || /^preview:/.test(token)) { + if (token.startsWith('github:') || token.startsWith('preview:')) { const scheme = token.split(':', 1)[0]; throw new Error( `Ref scheme "${scheme}:" is recognised but not implemented yet (in "${token}"). ` + @@ -173,11 +172,13 @@ function closestBaseBranch(repoRoot, baseBranch) { * `baseline` symbol so it is computed at most once per run. * * @param {RefResolverOptions} options - Repository and baseline policy - * @returns {{ resolve: (descriptor: RefDescriptor) => ResolvedRef, parse: (token?: string) => ResolvedRef }} + * @returns {{ parse: (token?: string) => ResolvedRef }} */ export function createRefResolver(options) { const { repoRoot, baselineOverride } = options; - const baseBranch = options.baseBranch ?? detectBaseBranch(repoRoot); + // Detected on demand: only the `baseline` symbol needs it, so a run whose cases all pin `git:` + // refs — or none at all — never shells out for it. + let baseBranch = options.baseBranch; /** @type {ResolvedRef | undefined} */ let baselineRef; @@ -225,6 +226,8 @@ export function createRefResolver(options) { return resolve(descriptor); } + baseBranch ??= detectBaseBranch(repoRoot); + if (gitCapture(['rev-parse', '--abbrev-ref', 'HEAD'], repoRoot) === baseBranch) { return gitRef('HEAD~1', 'HEAD~1'); } @@ -260,7 +263,6 @@ export function createRefResolver(options) { } return { - resolve, parse: (token) => resolve(parseRefToken(token)), }; } diff --git a/packages/code-infra/src/tachometer/renderReport.mjs b/packages/code-infra/src/tachometer/renderReport.mjs index bb115fb552..2d0797046e 100644 --- a/packages/code-infra/src/tachometer/renderReport.mjs +++ b/packages/code-infra/src/tachometer/renderReport.mjs @@ -1,5 +1,7 @@ /* eslint-disable no-console -- a reporter: printing the report is its output. */ +// Column widths have to measure visible characters, so chalk's escapes come off first. +import { stripVTControlCharacters as stripAnsi } from 'node:util'; import chalk from 'chalk'; /** @@ -58,43 +60,15 @@ import chalk from 'chalk'; * @property {CaseResult[]} cases - One entry per case */ -/** Milliseconds, to the precision tachometer's own table uses. */ -const MS_FORMAT = /** @type {Intl.NumberFormatOptions} */ ({ - style: 'unit', - unit: 'millisecond', - maximumFractionDigits: 2, - // Pad to the declared precision so a column lines up on the decimal point (`29.0 ms` under - // `30.9 ms`, not `29 ms`). - minimumFractionDigits: 2, -}); - -const msFormatter = new Intl.NumberFormat('en-US', MS_FORMAT); - -/** - * Matches the escape sequences chalk emits, so column widths measure visible characters. - * - * Built from a char code rather than written as a literal: an escape character inside a regular - * expression source is invisible in a diff and trips `no-control-regex`. - */ -const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, 'g'); - -/** - * @param {string} text - Possibly coloured text - * @returns {string} The same text with colour escapes removed - */ -function stripAnsi(text) { - return text.replace(ANSI, ''); -} - /** * Prints a table, padding each column to its widest visible cell. * * @param {string[]} headers - Column headers * @param {string[][]} rows - Row cells, in column order - * @param {boolean[]} alignLeft - Whether each column is left-aligned + * @param {number} leftColumns - How many leading columns are left-aligned; the rest are right * @returns {void} */ -function printTable(headers, rows, alignLeft) { +function printTable(headers, rows, leftColumns) { const widths = headers.map((header, index) => Math.max(stripAnsi(header).length, ...rows.map((row) => stripAnsi(row[index] ?? '').length)), ); @@ -114,7 +88,7 @@ function printTable(headers, rows, alignLeft) { */ const line = (cells) => cells - .map((cell, index) => pad(cell ?? '', widths[index], alignLeft[index])) + .map((cell, index) => pad(cell ?? '', widths[index], index < leftColumns)) .join(' ') .trimEnd(); @@ -132,7 +106,9 @@ function printTable(headers, rows, alignLeft) { * @returns {string} */ function formatInterval(interval) { - return `${msFormatter.format(interval.low).replace(/\s*ms$/, '')} – ${msFormatter.format(interval.high)}`; + // Fixed to two places so a column lines up on the decimal point, and written the same way the + // dashboard writes it, so the terminal and the pull request comment agree digit for digit. + return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; } /** @@ -187,6 +163,16 @@ function shortNameOf(caseName, variant) { return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; } +/** + * Whether a case produced results, as opposed to carrying only the error that stopped it. + * + * @param {CaseResult} entry - A case from the report + * @returns {boolean} + */ +function isSummarized(entry) { + return entry.measurements !== undefined && entry.measurements.length > 0; +} + /** * The distinct variants a case reports, in order of first appearance. * @@ -227,24 +213,25 @@ function printVariantTable(entry, variants) { 'transferred', 'Samples', ]; - const alignLeft = headers.map((_, index) => index === 0); const rows = variants.map((variant) => { - /** @type {number | undefined} */ - let bytesSent; - /** @type {Set} */ - const sampleCounts = new Set(); - const cells = entry.measurements.flatMap((measurement) => { - const found = measurement.variants.find( + // Looked up once per row rather than accumulated while the measurement cells render, so the + // last two columns do not depend on the cell loop having run. + const perMeasurement = entry.measurements.map((measurement) => ({ + found: measurement.variants.find( (candidate) => shortNameOf(entry.name, candidate.variant) === variant, - ); - const comparison = measurement.comparisons.find( + ), + comparison: measurement.comparisons.find( (candidate) => shortNameOf(entry.name, candidate.variant) === variant, - ); - if (found) { - bytesSent = found.bytesSent; - sampleCounts.add(found.samples); - } + ), + })); + // Transfer size is a property of the variant's page, so every measurement reports the same one. + const bytesSent = perMeasurement.find(({ found }) => found)?.found?.bytesSent; + const sampleCounts = new Set( + perMeasurement.flatMap(({ found }) => (found ? [found.samples] : [])), + ); + + const cells = perMeasurement.flatMap(({ found, comparison }) => { let delta = chalk.dim('—'); if (variant === reference) { delta = chalk.dim('reference'); @@ -254,6 +241,7 @@ function printVariantTable(entry, variants) { } return [found ? formatInterval(found.meanMs) : chalk.dim('—'), delta]; }); + return [ variant, ...cells, @@ -262,7 +250,7 @@ function printVariantTable(entry, variants) { ]; }); - printTable(headers, rows, alignLeft); + printTable(headers, rows, 1); console.log( chalk.dim( `vs ${reference} is tachometer's confidence interval on the difference, that variant ` + @@ -287,7 +275,6 @@ function printCaseTable(cases, variants) { ...others.map((variant) => `Δ vs ${variant}`), 'Samples', ]; - const alignLeft = [true, true, ...variants.map(() => false), ...others.map(() => false), false]; /** @type {string[][]} */ const rows = []; @@ -333,7 +320,7 @@ function printCaseTable(cases, variants) { } } - printTable(headers, rows, alignLeft); + printTable(headers, rows, 2); console.log( chalk.dim( `Δ is tachometer's confidence interval on the difference, ${reference ?? 'the reference'} ` + @@ -360,12 +347,12 @@ function printCaseTable(cases, variants) { * @returns {void} */ export function renderTachometerReport(report) { - const failed = report.cases.filter((entry) => !entry.measurements); + // One predicate for both halves: partitioning on two conditions that have to agree left a case + // with an empty `measurements` array rendered nowhere and reported nowhere. const usable = /** @type {Array} */ ( - report.cases.filter( - (entry) => entry.measurements !== undefined && entry.measurements.length > 0, - ) + report.cases.filter(isSummarized) ); + const failed = report.cases.filter((entry) => !isSummarized(entry)); if (usable.length === 0) { console.log(chalk.dim('No tachometer results to show.')); @@ -389,15 +376,6 @@ export function renderTachometerReport(report) { groups.set(key, group); } - // Only a variant table carries the transfer size as a column, so only the other cases need the - // note below. Derived from the grouping rather than collected while printing, so the notes do not - // depend on the tables having been rendered first. - const withBytesColumn = new Set( - [...groups.values()] - .filter((group) => group.variants.length > 2) - .flatMap((group) => group.cases.map((entry) => entry.name)), - ); - for (const [groupIndex, group] of [...groups.values()].entries()) { if (groupIndex > 0) { console.log(''); @@ -421,11 +399,11 @@ export function renderTachometerReport(report) { // Bundle weight is a property of the variant's page, not of a measurement, so it belongs in a // note rather than repeated down every row. It comes free with the run and is the number that - // explains a cold-start difference that mount time alone does not. - for (const entry of usable) { - if (withBytesColumn.has(entry.name)) { - continue; - } + // explains a cold-start difference that mount time alone does not. A variant table already gives + // it a column, so only the cases in the smaller groups need the note. + for (const entry of [...groups.values()] + .filter((group) => group.variants.length <= 2) + .flatMap((group) => group.cases)) { /** @type {Map} */ const bytes = new Map(); for (const measurement of entry.measurements) { diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index 06809b478b..739f161a85 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -18,7 +18,7 @@ import { import { summarizeCase } from './summarizeCase.mjs'; import { renderTachometerReport } from './renderReport.mjs'; import { getCiMetadata, syncPrComment, uploadCiReport } from './ciReport.mjs'; -import { prepareOutputDir } from './outputDir.mjs'; +import { buildsDirOf, prepareOutputDir } from './outputDir.mjs'; import { run } from '../utils/exec.mjs'; /** @@ -84,7 +84,7 @@ export async function runTachometer(options) { // ref's isolated install, cheap to recreate and not portable between containers, since its links // point into a pnpm store; `builds` holds the pages built per ref; `results` the report. const outputDir = await prepareOutputDir(harnessDir); - const buildsDir = path.join(outputDir, 'builds'); + const buildsDir = buildsDirOf(harnessDir); const packedDir = path.join(outputDir, 'packed'); const installsDir = path.join(outputDir, 'installs'); diff --git a/packages/code-infra/src/tachometer/summarizeCase.mjs b/packages/code-infra/src/tachometer/summarizeCase.mjs index 18a3ea5e03..dca500e016 100644 --- a/packages/code-infra/src/tachometer/summarizeCase.mjs +++ b/packages/code-infra/src/tachometer/summarizeCase.mjs @@ -107,11 +107,12 @@ export function summarizeCase(entry, json) { const byMeasurement = indexByMeasurement(json); const measurements = entry.measurements.map((measurement) => { + const byVariant = byMeasurement.get(measurement); /** * @param {string} variant - Variant name to look up * @returns {{ benchmark: TachometerBenchmark, index: number } | undefined} */ - const lookup = (variant) => byMeasurement.get(measurement)?.get(variant); + const lookup = (variant) => byVariant?.get(variant); const referenceResult = lookup(reference.name); if (!referenceResult) { From aa0597ccba1d5ce9ad75bbecb693be7ab10a5d7b Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 8 Sep 2026 13:47:14 +0200 Subject: [PATCH 019/173] [code-infra] Build a ref's benchmark pages in place, resolving through its own tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each ref used to be built in a copy of the harness — `src/`, one vite config and `.env*`, and nothing else. Every other file a harness owns was invisible to that build: `tsconfig.json`, which vite reads for its esbuild settings, and any postcss or tailwind config, and any file the vite config itself imports. The build still succeeded, just configured differently from the harness's own, and the difference landed in the measurement. Now the pages are built where they live, from the harness's own directory and config, and only *resolution* differs: a plugin sends every bare import into a small tree installed for that ref, where the library under test is that ref's packed build. Resolution is handed back to vite with the importer rewritten, so conditions and `exports` maps still apply — an alias to a directory would skip both. The tree carries the harness's own dependencies too, so the pages and the library share one copy of React. The tree no longer installs this package either. It is the harness's vite config that loads the plugin, and that config is read from the harness — which takes a ref's tree from 976 packages to 80. --- .../code-infra/src/tachometer/buildPages.mjs | 189 ++++++++++-------- .../code-infra/src/tachometer/outputDir.mjs | 2 +- .../src/tachometer/runTachometer.mjs | 13 +- test/tachometer/README.md | 6 +- 4 files changed, 118 insertions(+), 92 deletions(-) diff --git a/packages/code-infra/src/tachometer/buildPages.mjs b/packages/code-infra/src/tachometer/buildPages.mjs index 1522381e0d..eaa86909fc 100644 --- a/packages/code-infra/src/tachometer/buildPages.mjs +++ b/packages/code-infra/src/tachometer/buildPages.mjs @@ -1,7 +1,9 @@ /* eslint-disable no-console */ import * as path from 'node:path'; -import { cp, mkdir, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { createRequire } from 'node:module'; +import { readFile, mkdir, writeFile } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; import chalk from 'chalk'; import { parse, stringify } from 'yaml'; import { run } from '../utils/exec.mjs'; @@ -9,21 +11,34 @@ import { tarballFor } from '../utils/packWorkspace.mjs'; import { readPackageJson } from '../utils/pnpm.mjs'; /** - * Dependencies the run needs but a page never imports, so a ref's sandbox does without them. + * Dependencies the run needs but a page never imports, so a ref's tree does without them. * - * All three are resolved from the harness itself — the browser and its driver by the runner, and - * tachometer by the command that samples — and every one of them is expensive: tachometer pulls a - * chromedriver, `@playwright/test` a browser download. + * Every one is resolved from the harness rather than from here: the browser and its driver by the + * runner, tachometer by the command that samples, and this package by the harness's own vite config + * — which is loaded from the harness, since the build happens in place. Every one is also expensive + * to install: tachometer pulls a chromedriver, `@playwright/test` a browser, and this package a + * dependency tree many times the size of a harness. */ -const RUNNER_ONLY_DEPS = ['tachometer', 'chromedriver', '@playwright/test']; +const RUNNER_ONLY_DEPS = [ + 'tachometer', + 'chromedriver', + '@playwright/test', + '@mui/internal-code-infra', +]; /** - * Builds the benchmark pages for one ref, in an isolated install. + * Builds the benchmark pages for one ref. * - * Every ref goes through this, the working tree included, so both sides of a comparison resolve the - * library the way a consumer installs it: from a tarball, through its own `exports` map and its own - * dependency ranges. Building one side through a workspace link instead would compare two different - * resolution paths, and that difference lands in the measurement rather than in the library. + * The pages are built **where they live**, from the harness's own directory and its own vite + * config, so a `tacho run` build sees exactly what a plain `vite build` sees — the same tsconfig, + * the same postcss config, the same everything. Only *resolution* differs: a plugin redirects every + * bare import into a small tree installed for this ref, where the library under test is that ref's + * packed build. + * + * That tree holds the harness's own dependencies too, so the pages and the library resolve React + * (and everything else) to one copy. Building one side through a workspace link instead would + * compare two different resolution paths, and that difference lands in the measurement rather than + * in the library. */ /** @@ -31,30 +46,12 @@ const RUNNER_ONLY_DEPS = ['tachometer', 'chromedriver', '@playwright/test']; * @typedef {import('../utils/packWorkspace.mjs').PackedPackage} PackedPackage */ -/** Matches any filename vite accepts as its config. */ -const VITE_CONFIG = /^vite\.config\.(?:[cm]?[jt]s)$/; - -/** - * Finds the harness's vite config file. - * - * @param {string} harnessDir - The harness package directory - * @returns {Promise} The config's basename - */ -async function findViteConfig(harnessDir) { - const entries = await readdir(harnessDir); - const found = entries.filter((entry) => VITE_CONFIG.test(entry)).sort(); - if (found.length === 0) { - throw new Error(`No vite config found in ${harnessDir}.`); - } - return found[0]; -} - /** * Rewrites a dependency map, pointing every `workspace:`-protocol entry at its packed tarball and * copying everything else verbatim. * - * This is what lets the isolated install be derived rather than configured: the harness already - * declares the library under test as `workspace:*`, so there is no library name to hardcode and no + * This is what lets the tree be derived rather than configured: the harness already declares the + * library under test as `workspace:*`, so there is no library name to hardcode and no * hand-maintained list of versions to pin. Third-party dependencies — including the competitor * libraries a cross-library case compares against — come along unchanged, so every page builds * against every ref. @@ -77,13 +74,13 @@ function rewriteWorkspaceDeps(deps, packages, omit = []) { } /** - * Reads the repository's `overrides`, so the isolated install resolves the same dependency graph - * the repository does. + * Reads the repository's `overrides`, so the ref's tree resolves the same dependency graph the + * repository does. * - * The throwaway package is its own workspace root and inherits nothing. Without this, a repository - * that pins a transitive dependency through an override would get a *different* graph in the - * sandbox than in its real install — quietly changing what is being benchmarked — and any policy - * the override exists to satisfy (a blocked resolution, a security pin) would fail there too. + * The tree is its own workspace root and inherits nothing. Without this, a repository that pins a + * transitive dependency through an override would get a *different* graph here than in its real + * install — quietly changing what is being benchmarked — and any policy the override exists to + * satisfy (a blocked resolution, a security pin) would fail here too. * * @param {string} repoRoot - The workspace root to copy overrides from * @returns {Promise>} @@ -97,63 +94,58 @@ async function readOverrides(repoRoot) { } } +/** Name of the file resolution pretends to come from. It has to exist; see `resolveFromTree`. */ +const RESOLVE_STUB = '__resolve__.js'; + /** - * Replaces the install directory's `.env*` files with the harness's. + * A vite plugin that resolves every bare import from `treeDir` instead of from the harness. * - * Vite reads `VITE_*` variables from `.env*` files sitting next to the config, and a harness may - * need one (a licence key, say). Stale files are cleared first, so removing one from the harness - * removes it here too. + * Bare specifiers only: a relative or absolute id belongs to the harness's own sources and stays + * where it is. Resolution is handed back to vite with the importer rewritten, so vite's own + * conditions (`browser`, `import`) and the target package's `exports` map still decide the answer — + * which is what an alias to a directory would skip. * - * @param {string} harnessDir - The harness package directory - * @param {string} workDir - The isolated install directory - * @returns {Promise} + * @param {string} treeDir - The ref's installed tree + * @returns {import('vite').Plugin} */ -async function syncEnvFiles(harnessDir, workDir) { - const existing = (await readdir(workDir)).filter((name) => name.startsWith('.env')); - await Promise.all(existing.map((name) => rm(path.join(workDir, name), { force: true }))); +function resolveFromTree(treeDir) { + // Vite falls back to the project root when the importer does not exist on disk, which silently + // resolves the harness's copy of a dependency instead of this ref's. + const importer = path.join(treeDir, RESOLVE_STUB); - const incoming = (await readdir(harnessDir)).filter((name) => name.startsWith('.env')); - await Promise.all( - incoming.map((name) => cp(path.join(harnessDir, name), path.join(workDir, name))), - ); + return { + name: 'mui-code-infra:tachometer-resolve', + enforce: 'pre', + async resolveId(id, _importer, options) { + if (id.startsWith('.') || path.isAbsolute(id) || id.startsWith('\0') || id.includes(':')) { + return null; + } + return this.resolve(id, importer, { ...options, skipSelf: true }); + }, + }; } /** - * Builds the benchmark pages against one ref's packed build, in an isolated install. - * - * `workDir` persists between runs and only its inputs are refreshed. Combined with tarball paths - * that change only when their content does — a ref's embeds its commit SHA, the working tree's a - * hash of its bytes — an unchanged run is a no-op install and a changed one swaps a single package. + * Installs one ref's packed build, with the harness's own dependencies around it. * - * @param {Object} options - Build inputs - * @param {string} options.harnessDir - The harness package directory, copied from - * @param {string} options.repoRoot - The workspace root, whose overrides the sandbox inherits - * @param {ResolvedRef} options.ref - The ref being built + * @param {Object} options - Install inputs + * @param {string} options.harnessDir - The harness package directory, whose dependencies are copied + * @param {string} options.repoRoot - The workspace root, whose overrides the tree inherits * @param {PackedPackage[]} options.packages - That ref's packed workspace packages - * @param {string} options.workDir - Persistent directory holding this ref's isolated install - * @param {string} options.outDir - Absolute output directory for the built pages + * @param {string} options.treeDir - Where to install * @returns {Promise} */ -export async function buildRefPages(options) { - const { harnessDir, repoRoot, ref, packages, workDir, outDir } = options; - console.log(chalk.cyan(`\nBuilding benchmark pages for "${ref.label}"…`)); - - const viteConfig = await findViteConfig(harnessDir); - await mkdir(workDir, { recursive: true }); - // Sources are replaced rather than overlaid, so a page deleted from the harness cannot linger. - await rm(path.join(workDir, 'src'), { recursive: true, force: true }); - await cp(path.join(harnessDir, 'src'), path.join(workDir, 'src'), { recursive: true }); - await cp(path.join(harnessDir, viteConfig), path.join(workDir, viteConfig)); - await syncEnvFiles(harnessDir, workDir); - +async function installRefTree({ harnessDir, repoRoot, packages, treeDir }) { const harnessPkg = await readPackageJson(harnessDir); + await mkdir(treeDir, { recursive: true }); + await writeFile( - path.join(workDir, 'package.json'), + path.join(treeDir, 'package.json'), `${JSON.stringify( { - name: `tacho-bench-${ref.id}`, + name: 'tacho-resolve-tree', private: true, - type: harnessPkg.type, + version: '0.0.0', dependencies: rewriteWorkspaceDeps(harnessPkg.dependencies, packages), devDependencies: rewriteWorkspaceDeps( harnessPkg.devDependencies, @@ -172,7 +164,7 @@ export async function buildRefPages(options) { // version) resolves to a local build rather than 404ing on the registry. The packed pins win, // since they are the whole point of this install. await writeFile( - path.join(workDir, 'pnpm-workspace.yaml'), + path.join(treeDir, 'pnpm-workspace.yaml'), stringify({ overrides: { ...(await readOverrides(repoRoot)), @@ -181,15 +173,48 @@ export async function buildRefPages(options) { }), ); + await writeFile(path.join(treeDir, RESOLVE_STUB), ''); + // `--ignore-scripts`: this workspace has no build-script approvals, and pnpm fails an install // over unapproved ones rather than warning. Nothing installed here needs its scripts either — the // pages are built from the packages exactly as packed. await run( 'pnpm', ['install', '--prefer-offline', '--ignore-scripts', '--config.engine-strict=false'], - workDir, + treeDir, ); - // The page entries come from the tachometer plugin in the harness's own config, so nothing about - // which pages exist is passed here. - await run('pnpm', ['exec', 'vite', 'build', '--outDir', outDir], workDir); +} + +/** + * Builds the benchmark pages against one ref's packed build. + * + * @param {Object} options - Build inputs + * @param {string} options.harnessDir - The harness package directory, built in place + * @param {string} options.repoRoot - The workspace root, whose overrides the tree inherits + * @param {ResolvedRef} options.ref - The ref being built + * @param {PackedPackage[]} options.packages - That ref's packed workspace packages + * @param {string} options.treeDir - Persistent directory holding this ref's installed tree + * @param {string} options.outDir - Absolute output directory for the built pages + * @returns {Promise} + */ +export async function buildRefPages(options) { + const { harnessDir, repoRoot, ref, packages, treeDir, outDir } = options; + console.log(chalk.cyan(`\nBuilding benchmark pages for "${ref.label}"…`)); + + await installRefTree({ harnessDir, repoRoot, packages, treeDir }); + + // vite is the harness's own, not this package's: the harness declares the version its pages are + // built with, and the config being run is the harness's. + const requireFromHarness = createRequire(path.join(harnessDir, 'package.json')); + /** @type {typeof import('vite')} */ + const vite = await import(pathToFileURL(requireFromHarness.resolve('vite')).href); + + // `root` only points vite's config discovery at the harness; the harness's own plugin then moves + // it to `src/`, the same as for a plain `vite build`. + await vite.build({ + root: harnessDir, + logLevel: 'warn', + plugins: [resolveFromTree(treeDir)], + build: { outDir }, + }); } diff --git a/packages/code-infra/src/tachometer/outputDir.mjs b/packages/code-infra/src/tachometer/outputDir.mjs index c8ab500a32..47b1652d56 100644 --- a/packages/code-infra/src/tachometer/outputDir.mjs +++ b/packages/code-infra/src/tachometer/outputDir.mjs @@ -2,7 +2,7 @@ import * as path from 'node:path'; import { mkdir, writeFile } from 'node:fs/promises'; /** - * Everything a run writes — packed tarballs, isolated installs, built pages, the report — lives + * Everything a run writes — packed tarballs, per-ref installs, built pages, the report — lives * under this one directory inside the harness, so a repository has a single thing to ignore and * deleting it is the whole reset story. */ diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index 739f161a85..b8c2b24d0a 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -80,13 +80,14 @@ export async function runTachometer(options) { // Everything a run writes goes under one directory, so a harness has one thing to ignore and // deleting it is the whole reset story. `packed` holds tarballs — a ref's keyed by commit SHA, - // the working tree's by content hash — and is the one worth caching in CI; `installs` holds each - // ref's isolated install, cheap to recreate and not portable between containers, since its links - // point into a pnpm store; `builds` holds the pages built per ref; `results` the report. + // the working tree's by content hash — and is the one worth caching in CI; `trees` holds the + // install each ref's pages resolve through, cheap to recreate and not portable between + // containers, since its links point into a pnpm store; `builds` holds the pages built per ref; + // `results` the report. const outputDir = await prepareOutputDir(harnessDir); const buildsDir = buildsDirOf(harnessDir); const packedDir = path.join(outputDir, 'packed'); - const installsDir = path.join(outputDir, 'installs'); + const treesDir = path.join(outputDir, 'trees'); const resolver = createRefResolver({ repoRoot, baseBranch, baselineOverride: baseline }); const cases = await discoverCases({ harnessDir, filters, resolveRef: resolver.parse }); @@ -124,7 +125,7 @@ export async function runTachometer(options) { const tmpBase = await mkdtemp(path.join(os.tmpdir(), 'tacho-')); try { - // Build one variant per distinct ref, every one through its own isolated install so both sides + // Build one variant per distinct ref, each resolving through its own install so both sides // of a comparison resolve the library identically. for (const ref of refs.values()) { /** @type {import('../utils/packWorkspace.mjs').PackedPackage[]} */ @@ -156,7 +157,7 @@ export async function runTachometer(options) { repoRoot, ref, packages, - workDir: path.join(installsDir, ref.id), + treeDir: path.join(treesDir, ref.id), outDir: path.join(buildsDir, ref.id), }); } diff --git a/test/tachometer/README.md b/test/tachometer/README.md index 7a1d7ae162..c74e75e148 100644 --- a/test/tachometer/README.md +++ b/test/tachometer/README.md @@ -25,8 +25,8 @@ rewrite, that `?ref=` is appended with `&` when a url already has a query, and t referencing one page build it only once. Both pages import `@mui/internal-test-utils` and put its value on screen. That is the load-bearing -part: every ref — the working tree included — resolves it from a packed tarball in an isolated -install, so both sides of a comparison consume the library the way a consumer does. If that path +part: every ref — the working tree included — resolves it from a packed tarball in that ref's own +tree, so both sides of a comparison consume the library the way a consumer does. If that path breaks the page fails to build, rather than quietly measuring nothing. ## Running @@ -55,7 +55,7 @@ that is the "no change" outcome, not a failure. Everything a run writes goes under `.tachometer/`: the report in `results/`, the pages built per ref in `builds/`, the packed tarballs in `packed/` (the one worth caching in CI, keyed by commit SHA) -and each ref's isolated install in `installs/`. Deleting that directory resets the harness +and the install each ref resolves through in `trees/`. Deleting that directory resets the harness completely, and it is the only thing a repository has to ignore. ### Prerequisites From b584810223f216d19893577bbd8a29bc664b1a2b Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 8 Sep 2026 14:35:37 +0200 Subject: [PATCH 020/173] [code-infra] Give vite preview the harness's output directory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `vite preview` runs as `serve`, so the plugin's early return skipped `build.outDir` and preview looked for `/dist` — inside `src/` — failing with "The directory \"dist\" does not exist" instead of serving the pages that had just been built. --- packages/code-infra/src/tachometer/plugin.mjs | 22 +++++++++++-------- .../code-infra/src/tachometer/plugin.test.mjs | 12 +++++++++- test/tachometer/README.md | 3 ++- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/packages/code-infra/src/tachometer/plugin.mjs b/packages/code-infra/src/tachometer/plugin.mjs index 24b7dee033..e0ae54ccd9 100644 --- a/packages/code-infra/src/tachometer/plugin.mjs +++ b/packages/code-infra/src/tachometer/plugin.mjs @@ -117,8 +117,17 @@ export function tachometer(options = {}) { ...pkg.devDependencies, }); + // Resolved for every command, not just `build`: `vite preview` runs as `serve` but serves + // `build.outDir`, so leaving it unset there sends preview looking for `/dist` — inside + // `src/` — and it exits rather than serving the pages that were just built. + // + // Only fill it in when the caller said nothing: a plugin's returned config is merged *over* + // the inline config, so unconditionally setting it here would silently override + // `vite build --outDir`, which is exactly how `tacho run` directs each ref's build. + const outDir = userConfig.build?.outDir ?? buildsDirOf(harnessDir, 'manual'); + if (env.command !== 'build') { - return { root: srcDir, appType: 'mpa' }; + return { root: srcDir, appType: 'mpa', build: { outDir } }; } const cases = await discoverCases({ harnessDir }); @@ -137,14 +146,9 @@ export function tachometer(options = {}) { // `//`, so absolute "/assets/…" paths would 404. base: './', build: { - // A plain `vite build` lands in the run output directory, next to the per-ref - // directories the runner writes. Without this it would default to `/dist` — inside - // `src/`, next to the case sources. - // - // Only fill it in when the caller said nothing: a plugin's returned config is merged - // *over* the inline config, so unconditionally setting it here would silently override - // `vite build --outDir`, which is exactly how `tacho run` directs each ref's build. - outDir: userConfig.build?.outDir ?? buildsDirOf(harnessDir, 'manual'), + // A plain `vite build` lands in the run output directory, next to the per-ref directories + // the runner writes. + outDir, // The output directory is outside `root` — and, for an isolated ref build, outside the // temporary package entirely; allow vite to clean it anyway. emptyOutDir: true, diff --git a/packages/code-infra/src/tachometer/plugin.test.mjs b/packages/code-infra/src/tachometer/plugin.test.mjs index 7c692e02cf..773e8b8689 100644 --- a/packages/code-infra/src/tachometer/plugin.test.mjs +++ b/packages/code-infra/src/tachometer/plugin.test.mjs @@ -69,7 +69,17 @@ describe('tachometer plugin', () => { const config = await callConfig(harnessDir, 'serve'); - expect(config.build).toBeUndefined(); + expect(config.build?.rollupOptions).toBeUndefined(); + }); + + it('points serving at the output directory too, so preview finds the built pages', async () => { + // `vite preview` runs as `serve` but serves `build.outDir`. Leaving it unset there sends + // preview to `/dist` — inside `src/` — and it exits instead of serving the build. + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'serve'); + + expect(config.build?.outDir).toBe(path.join(harnessDir, '.tachometer', 'builds', 'manual')); }); it('uses relative asset urls for a build', async () => { diff --git a/test/tachometer/README.md b/test/tachometer/README.md index c74e75e148..b8ff980786 100644 --- a/test/tachometer/README.md +++ b/test/tachometer/README.md @@ -88,4 +88,5 @@ pnpm -F ./test/tachometer dev The dev server lists every discovered case at `/`, including `workload-large`, which owns no page and would otherwise be invisible. `pnpm -F ./test/tachometer build:pages` and `preview` produce and -serve the production bundle, which is what actually gets measured. +serve the production bundle, which is what actually gets measured. That index is a dev-server +convenience, so `preview` has nothing at `/` — open a case's page directly. From dedbdbfd8a638e9d6093f6320b647c9121afb15c Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:20:16 +0200 Subject: [PATCH 021/173] [code-infra] Match tachometer's own browser and measurement naming MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four ways a legal `tachometer.json` was mishandled, none of which the smoke harness exercises: - A case that declares no `browser` had `{ binary }` injected, which tachometer's schema rejects outright — every browser variant requires `name` and forbids unknown keys — so the run died before sampling. A case using the string shorthand ("chrome-headless") was spread into one key per character, failing the same way. - Browser defaults were applied only to the top-level benchmark. `expand` shallow-overrides, so a variant declaring its own `browser` lost the binary and, as root, the flag Chrome needs to start. - The `global` measurement shorthand expands to an expression measurement named by the expression, so pairing on "global" never matched and the case was reported as failed. - A benchmark that names neither itself nor its variants produced "undefined [current]", and a name set partway down an `expand` tree was skipped in favour of the benchmark's. --- .../code-infra/src/tachometer/browser.mjs | 68 ++++++++++++++- .../src/tachometer/browser.test.mjs | 86 ++++++++++++++++++- .../src/tachometer/discoverCases.mjs | 60 +++++++------ .../src/tachometer/discoverCases.test.mjs | 53 ++++++++++++ .../src/tachometer/runTachometer.mjs | 6 +- 5 files changed, 240 insertions(+), 33 deletions(-) diff --git a/packages/code-infra/src/tachometer/browser.mjs b/packages/code-infra/src/tachometer/browser.mjs index e47b5d5b05..75ec1f90ac 100644 --- a/packages/code-infra/src/tachometer/browser.mjs +++ b/packages/code-infra/src/tachometer/browser.mjs @@ -115,6 +115,45 @@ export function assertDriverMatchesBrowser(harnessDir, binary) { /** Chrome cannot enter its sandbox as root, and refusing to start is all it does about it. */ const NO_SANDBOX = '--no-sandbox'; +/** Tachometer's default when a benchmark names no browser. */ +const DEFAULT_BROWSER = 'chrome'; + +/** The suffix tachometer's browser shorthand uses to ask for a headless run. */ +const HEADLESS_SUFFIX = '-headless'; + +/** + * Normalises whatever a case put in `browser` into the object form, the way tachometer does. + * + * Tachometer accepts three things there: nothing at all, the string shorthand + * `"[-headless][@]"`, and an object. Only the object form can carry the binary and + * the arguments a run has to add — and tachometer validates the whole config against its own schema + * before reading any of it, where every browser variant requires `name` and forbids unknown keys. + * So merging into the other two forms without converting first yields a config tachometer rejects + * outright: `{ binary }` has no `name`, and spreading a string produces one key per character. + * + * `headless` and `windowSize` are left out on purpose. Tachometer's own `parseBrowserObject` fills + * both from the defaults it would otherwise have applied, so naming the browser is all it takes to + * reproduce what an absent `browser` did. + * + * @param {string | BrowserConfig | undefined} browser - Whatever the case declared, if anything + * @returns {BrowserConfig} + */ +function asBrowserObject(browser) { + if (browser === undefined) { + return { name: DEFAULT_BROWSER }; + } + if (typeof browser !== 'string') { + // An object of its own may still leave out `name`, which the schema requires. + return { name: DEFAULT_BROWSER, ...browser }; + } + const at = browser.indexOf('@'); + const remoteUrl = at === -1 ? undefined : browser.slice(at + 1); + const named = at === -1 ? browser : browser.slice(0, at); + const headless = named.endsWith(HEADLESS_SUFFIX); + const name = headless ? named.slice(0, -HEADLESS_SUFFIX.length) : named; + return remoteUrl === undefined ? { name, headless } : { name, headless, remoteUrl }; +} + /** * Merges what the run decides into a case's own `browser` config. * @@ -124,16 +163,41 @@ const NO_SANDBOX = '--no-sandbox'; * `session not created: Chrome instance exited`. Containerised CI runs as root, so the flag goes * on there and nowhere else — a local run keeps the sandbox. * - * @param {BrowserConfig | undefined} browser - The case's own browser config, if it has one + * @param {string | BrowserConfig | undefined} browser - The case's own browser config, if it has one * @param {string} binary - Binary to drive, unless the case pins its own * @param {boolean} asRoot - Whether the run is happening as root * @returns {BrowserConfig} */ export function withBrowserDefaults(browser, binary, asRoot) { - const merged = { ...browser, binary: browser?.binary ?? binary }; + const base = asBrowserObject(browser); + const merged = { ...base, binary: base.binary ?? binary }; const addArguments = merged.addArguments ?? []; if (asRoot && !addArguments.includes(NO_SANDBOX)) { merged.addArguments = [...addArguments, NO_SANDBOX]; } return merged; } + +/** + * Applies the run's browser defaults to a benchmark and to every `expand` node that declares a + * browser of its own. + * + * A node only inherits its parent's browser until it names one: tachometer merges an expansion over + * its parent with `Object.assign`, which replaces `browser` wholesale rather than merging into it. + * So a variant that sets, say, its own `addArguments` would otherwise drop the binary this run + * picked — and, in a containerised CI, the flag without which Chrome refuses to start as root. + * + * @param {any} node - A benchmark, or a node in its `expand` tree + * @param {string} binary - Binary to drive, unless the node pins its own + * @param {boolean} asRoot - Whether the run is happening as root + * @param {boolean} [isRoot] - Whether this is the benchmark itself, which always gets a browser + * @returns {void} + */ +export function applyBrowserDefaults(node, binary, asRoot, isRoot = true) { + if (isRoot || node.browser !== undefined) { + node.browser = withBrowserDefaults(node.browser, binary, asRoot); + } + for (const child of node.expand ?? []) { + applyBrowserDefaults(child, binary, asRoot, false); + } +} diff --git a/packages/code-infra/src/tachometer/browser.test.mjs b/packages/code-infra/src/tachometer/browser.test.mjs index 0b5aed23cf..a41a17e4de 100644 --- a/packages/code-infra/src/tachometer/browser.test.mjs +++ b/packages/code-infra/src/tachometer/browser.test.mjs @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { majorVersionOf, withBrowserDefaults } from './browser.mjs'; +import { applyBrowserDefaults, majorVersionOf, withBrowserDefaults } from './browser.mjs'; describe('majorVersionOf', () => { it('reads the major from Chrome for Testing output', () => { @@ -73,10 +73,92 @@ describe('withBrowserDefaults', () => { expect(merged.addArguments).toEqual(['--no-sandbox']); }); - it('handles a case with no browser config at all', () => { + it('names the browser for a case that declares none', () => { + // Tachometer defaults an absent `browser` to chrome, but validates the config against a schema + // where every browser variant requires `name` — so injecting the binary alone is rejected + // before a single sample is taken. expect(withBrowserDefaults(undefined, BINARY, true)).toEqual({ + name: 'chrome', binary: BINARY, addArguments: ['--no-sandbox'], }); }); + + it('expands the string shorthand instead of spreading it into characters', () => { + expect(withBrowserDefaults('chrome-headless', BINARY, false)).toEqual({ + name: 'chrome', + headless: true, + binary: BINARY, + }); + }); + + it('keeps the remote url out of the browser name', () => { + expect(withBrowserDefaults('firefox@http://localhost:4444', BINARY, false)).toEqual({ + name: 'firefox', + headless: false, + remoteUrl: 'http://localhost:4444', + binary: BINARY, + }); + }); + + it('names the browser for a case whose object leaves it out', () => { + expect(withBrowserDefaults({ headless: true }, BINARY, false).name).toBe('chrome'); + }); +}); + +describe('applyBrowserDefaults', () => { + const BINARY = '/path/to/Chrome for Testing'; + + it('gives a benchmark a browser even when it declares none', () => { + /** @type {any} */ + const benchmark = { name: 'x', expand: [{ name: 'x [current]', url: './a.html' }] }; + + applyBrowserDefaults(benchmark, BINARY, false); + + expect(benchmark.browser).toEqual({ name: 'chrome', binary: BINARY }); + }); + + it('leaves an expand node that inherits its browser alone', () => { + // Tachometer merges an expansion over its parent, so a node with no browser of its own already + // gets the parent's — adding one here would only duplicate it. + /** @type {any} */ + const benchmark = { name: 'x', expand: [{ name: 'x [current]', url: './a.html' }] }; + + applyBrowserDefaults(benchmark, BINARY, false); + + expect(benchmark.expand[0]).not.toHaveProperty('browser'); + }); + + it('merges into an expand node that declares its own browser', () => { + // `expand` shallow-overrides: a node naming `browser` replaces the parent's wholesale, so + // without this the variant loses the binary and, as root, the flag Chrome needs to start. + /** @type {any} */ + const benchmark = { + name: 'x', + browser: { name: 'chrome' }, + expand: [ + { name: 'x [slow]', url: './a.html', browser: { addArguments: ['--cpu-throttle'] } }, + ], + }; + + applyBrowserDefaults(benchmark, BINARY, true); + + expect(benchmark.expand[0].browser).toEqual({ + name: 'chrome', + binary: BINARY, + addArguments: ['--cpu-throttle', '--no-sandbox'], + }); + }); + + it('reaches a browser declared deeper in the expand tree', () => { + /** @type {any} */ + const benchmark = { + name: 'x', + expand: [{ expand: [{ name: 'x [a]', url: './a.html', browser: { name: 'firefox' } }] }], + }; + + applyBrowserDefaults(benchmark, BINARY, false); + + expect(benchmark.expand[0].expand[0].browser).toEqual({ name: 'firefox', binary: BINARY }); + }); }); diff --git a/packages/code-infra/src/tachometer/discoverCases.mjs b/packages/code-infra/src/tachometer/discoverCases.mjs index 263cca6ecf..fe348b427a 100644 --- a/packages/code-infra/src/tachometer/discoverCases.mjs +++ b/packages/code-infra/src/tachometer/discoverCases.mjs @@ -52,6 +52,9 @@ import { parseRefToken } from './refs.mjs'; /** The file that marks a directory under `src/` as a benchmark case. */ const CONFIG_NAME = 'tachometer.json'; +/** Tachometer's expression when a `global` measurement does not name one. */ +const DEFAULT_MEASUREMENT_EXPRESSION = 'window.tachometerResult'; + /** * The name tachometer gives a measurement: an explicit `name`, else the expression, else the entry * name. @@ -61,11 +64,18 @@ const CONFIG_NAME = 'tachometer.json'; * unrelated measurements. * * @param {any} measurement - A tachometer `measurement` entry + * @param {string} [measurementExpression] - The benchmark's own `measurementExpression`, which the `global` shorthand is named by * @returns {string} */ -export function measurementNameOf(measurement) { +export function measurementNameOf(measurement, measurementExpression) { if (typeof measurement === 'string') { - return measurement; + // Tachometer's string shorthands. `callback` and `fcp` name their results the same way the + // shorthand reads, but `global` expands to an *expression* measurement that is named by the + // expression itself — so taking the shorthand at face value would look for a result that is + // never emitted under that name. + return measurement === 'global' + ? (measurementExpression ?? DEFAULT_MEASUREMENT_EXPRESSION) + : measurement; } if (measurement.name) { return measurement.name; @@ -82,26 +92,30 @@ export function measurementNameOf(measurement) { /** * Walks a benchmark's `expand` tree and collects its leaves — the nodes that actually select a page. * - * `expand` is recursive and a child inherits its parent's `url` unless it overrides it, so the - * effective url is threaded down and only nodes without further `expand` are leaves. + * `expand` is recursive and a child inherits both its parent's `url` and its `name` unless it + * overrides them, so each is threaded down and only nodes without further `expand` are leaves. + * Tachometer merges an expansion over its parent, so the *nearest* name wins — reading a leaf's own + * name and otherwise jumping straight to the benchmark's would miss a name set in between, and the + * variant would then be looked up in the results under a name tachometer never reported. * * @param {{ url?: string, name?: string, expand?: any[] }} node - The node to walk - * @param {string | undefined} inheritedUrl - The effective url from the parent - * @param {Array<{ node: { url?: string, name?: string }, url: string }>} out - Collected leaves + * @param {{ url?: string, name: string }} inherited - The effective url and name from the parent; the walk is always seeded with the benchmark's, so a leaf always has one + * @param {Array<{ node: { url?: string, name?: string }, url: string, name: string }>} out - Collected leaves * @returns {void} */ -function collectLeafNodes(node, inheritedUrl, out) { - const url = node.url ?? inheritedUrl; +function collectLeafNodes(node, inherited, out) { + const url = node.url ?? inherited.url; + const name = node.name ?? inherited.name; if (Array.isArray(node.expand) && node.expand.length > 0) { for (const child of node.expand) { - collectLeafNodes(child, url, out); + collectLeafNodes(child, { url, name }, out); } return; } if (url === undefined) { throw new Error('A benchmark variant has no "url" to resolve.'); } - out.push({ node, url }); + out.push({ node, url, name }); } /** @@ -224,39 +238,35 @@ export async function discoverCases(options) { for (const { name, configPath, config } of configs) { const configDir = path.dirname(configPath); - /** @type {Array<{ node: { url?: string, name?: string }, url: string, benchmarkName: string }>} */ + /** @type {Array<{ node: { url?: string, name?: string }, url: string, name: string }>} */ const nodes = []; /** @type {Set} */ const measurements = new Set(); for (const benchmark of config.benchmarks ?? []) { + // A benchmark need not name itself, and the case name is the same fallback its own name gets + // — without it the auto-expanded variants would read "undefined [current]". + const benchmarkName = benchmark.name ?? name; // `measurement` may be a single entry or a list; a page with several is exactly the case // whose results have to be paired by measurement name rather than by position. for (const measurement of [benchmark.measurement ?? 'callback'].flat()) { - measurements.add(measurementNameOf(measurement)); + measurements.add(measurementNameOf(measurement, benchmark.measurementExpression)); } if (!Array.isArray(benchmark.expand) || benchmark.expand.length === 0) { const base = benchmark.url; if (base === undefined) { - throw new Error(`Benchmark "${benchmark.name}" in ${configPath} has no "url".`); + throw new Error(`Benchmark "${benchmarkName}" in ${configPath} has no "url".`); } const separator = base.includes('?') ? '&' : '?'; benchmark.expand = [ - { name: `${benchmark.name} [current]`, url: base }, - { name: `${benchmark.name} [baseline]`, url: `${base}${separator}ref=baseline` }, + { name: `${benchmarkName} [current]`, url: base }, + { name: `${benchmarkName} [baseline]`, url: `${base}${separator}ref=baseline` }, ]; // The variants carry the url now; leaving the un-rewritten source url on the parent would // let tachometer inherit a page that was never built. delete benchmark.url; } - /** @type {Array<{ node: { url?: string, name?: string }, url: string }>} */ - const benchmarkNodes = []; - collectLeafNodes(benchmark, undefined, benchmarkNodes); - // A leaf's own name is what tachometer reports; only an unexpanded benchmark falls back to - // the benchmark's. - for (const entry of benchmarkNodes) { - nodes.push({ ...entry, benchmarkName: benchmark.name }); - } + collectLeafNodes(benchmark, { name: benchmarkName }, nodes); } // eslint-disable-next-line no-await-in-loop @@ -264,8 +274,8 @@ export async function discoverCases(options) { nodes.map(({ url }) => parseLeafUrl(url, configDir, srcDir, resolveRef)), ); const leaves = nodes.map(({ node }, index) => ({ node, ...resolved[index] })); - const variants = nodes.map(({ node, benchmarkName }, index) => ({ - name: node.name ?? benchmarkName, + const variants = nodes.map(({ name: variantName }, index) => ({ + name: variantName, refId: resolved[index].ref?.id ?? null, })); diff --git a/packages/code-infra/src/tachometer/discoverCases.test.mjs b/packages/code-infra/src/tachometer/discoverCases.test.mjs index 19e6389da7..8667120db2 100644 --- a/packages/code-infra/src/tachometer/discoverCases.test.mjs +++ b/packages/code-infra/src/tachometer/discoverCases.test.mjs @@ -379,6 +379,13 @@ describe('measurementNameOf', () => { expect(measurementNameOf('callback')).toBe('callback'); }); + it('names the "global" shorthand by the expression tachometer expands it to', () => { + // `global` becomes an expression measurement, and a result is named by the expression — not by + // the shorthand — so taking the shorthand at face value would never match its own result. + expect(measurementNameOf('global')).toBe('window.tachometerResult'); + expect(measurementNameOf('global', 'window.myResult')).toBe('window.myResult'); + }); + it('falls back to the entry name', () => { expect(measurementNameOf({ mode: 'performance', entryName: 'mount' })).toBe('mount'); }); @@ -416,6 +423,52 @@ describe('case variants and measurements', () => { expect(entry.measurements).toEqual(['mount']); }); + it('falls back to the case name when a benchmark names neither itself nor its variants', async () => { + // The case name is already the fallback for a nameless benchmark, so the variants read + // "alpha [current]" rather than "undefined [current]". + const harnessDir = await makeHarness({ + alpha: { + config: { benchmarks: [{ url: './index.html' }] }, + pages: ['index.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.variants.map((variant) => variant.name)).toEqual([ + 'alpha [current]', + 'alpha [baseline]', + ]); + }); + + it('takes a variant name from the nearest node that sets one', async () => { + // Tachometer merges an expansion over its parent, so a name set partway down the tree is what + // it reports — reading only the leaf and the benchmark would look up a name it never used. + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [ + { + name: 'alpha', + expand: [ + { name: 'alpha [ours]', expand: [{ url: './index.html' }] }, + { name: 'alpha [theirs]', expand: [{ url: './other.html' }] }, + ], + }, + ], + }, + pages: ['index.html', 'other.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.variants.map((variant) => variant.name)).toEqual([ + 'alpha [ours]', + 'alpha [theirs]', + ]); + }); + it('collects every measurement a benchmark declares', async () => { // A page with several measurements is exactly the case whose results must be paired by name. const harnessDir = await makeHarness({ diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index b8c2b24d0a..56b348f01a 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -11,9 +11,9 @@ import { createRefResolver } from './refs.mjs'; import { discoverCases, pagesOf } from './discoverCases.mjs'; import { buildRefPages } from './buildPages.mjs'; import { + applyBrowserDefaults, assertDriverMatchesBrowser, resolveBrowserBinary, - withBrowserDefaults, } from './browser.mjs'; import { summarizeCase } from './summarizeCase.mjs'; import { renderTachometerReport } from './renderReport.mjs'; @@ -172,10 +172,8 @@ export async function runTachometer(options) { const refId = leaf.ref ? leaf.ref.id : 'current'; leaf.node.url = `${path.join(buildsDir, refId, leaf.page)}${leaf.suffix}`; } - // `browser` is inherited down the `expand` tree, so setting it per benchmark covers every - // variant. for (const benchmark of entry.config.benchmarks ?? []) { - benchmark.browser = withBrowserDefaults(benchmark.browser, browserBinary, asRoot); + applyBrowserDefaults(benchmark, browserBinary, asRoot); } const slug = fileSlugOf(entry.name); const configPath = path.join(tmpBase, `tachometer-${slug}.json`); From e59c22328c988f95523fc53d68ba6b38565a21e7 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:20:22 +0200 Subject: [PATCH 022/173] [dashboard] Label the tachometer baseline with the baseline's own ref `baseRef` labels `baseSha` in the header, so passing the run's own branch read "comparing against ()". The baseline ref's label describes that commit. --- apps/code-infra-dashboard/src/views/TachometerDetails.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx index c00a7547dd..b73ce3f351 100644 --- a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx +++ b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx @@ -311,7 +311,9 @@ export default function TachometerDetails() { sha={sha} baseSha={baselineRef?.sha ?? null} prNumber={prNumber ? Number(prNumber) : undefined} - baseRef={upload?.branch ?? undefined} + // Labels `baseSha`, so it has to describe the baseline — the run's own branch would read + // "comparing against ()". + baseRef={baselineRef?.label} /> From 2103bbe01650cb1b940ed254ed722910dec369c9 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:37:24 +0200 Subject: [PATCH 023/173] [code-infra] Flatten a benchmark's expand tree once, the way tachometer does MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inheritance down `expand` was being re-derived per field: `url` and `name` threaded through the leaf walk, `browser` through a second recursive walk with an `isRoot` flag, and `measurementExpression` not at all. Each is a separate mirror of one rule — tachometer's `Object.assign` merge — and is only worth anything while it agrees with that rule exactly. Flattening once at discovery collapses all of it. The leaf walk and the browser walk both disappear, every benchmark handed to tachometer carries its own effective fields, and inheritance for fields nobody has needed yet comes for free. `expand` is dropped on the way out: tachometer leaves the parent's on the merged object where it is inert, but the config written here is parsed afresh and would otherwise expand a second time. --- .../code-infra/src/tachometer/browser.mjs | 36 ++------- .../src/tachometer/browser.test.mjs | 59 +------------- .../src/tachometer/discoverCases.mjs | 80 ++++++++++--------- .../src/tachometer/discoverCases.test.mjs | 49 ++++++++++-- .../src/tachometer/runTachometer.mjs | 6 +- 5 files changed, 97 insertions(+), 133 deletions(-) diff --git a/packages/code-infra/src/tachometer/browser.mjs b/packages/code-infra/src/tachometer/browser.mjs index 75ec1f90ac..2335b6bea7 100644 --- a/packages/code-infra/src/tachometer/browser.mjs +++ b/packages/code-infra/src/tachometer/browser.mjs @@ -131,9 +131,10 @@ const HEADLESS_SUFFIX = '-headless'; * So merging into the other two forms without converting first yields a config tachometer rejects * outright: `{ binary }` has no `name`, and spreading a string produces one key per character. * - * `headless` and `windowSize` are left out on purpose. Tachometer's own `parseBrowserObject` fills - * both from the defaults it would otherwise have applied, so naming the browser is all it takes to - * reproduce what an absent `browser` did. + * Only `name` is filled in for the other two forms: tachometer's own `parseBrowserObject` supplies + * `headless` and `windowSize` from the defaults it would otherwise have applied, so naming the + * browser is all it takes to reproduce what an absent `browser` did. The string shorthand is the + * exception — it encodes `headless` itself, so that is read out of it. * * @param {string | BrowserConfig | undefined} browser - Whatever the case declared, if anything * @returns {BrowserConfig} @@ -147,11 +148,10 @@ function asBrowserObject(browser) { return { name: DEFAULT_BROWSER, ...browser }; } const at = browser.indexOf('@'); - const remoteUrl = at === -1 ? undefined : browser.slice(at + 1); const named = at === -1 ? browser : browser.slice(0, at); const headless = named.endsWith(HEADLESS_SUFFIX); - const name = headless ? named.slice(0, -HEADLESS_SUFFIX.length) : named; - return remoteUrl === undefined ? { name, headless } : { name, headless, remoteUrl }; + const base = { name: headless ? named.slice(0, -HEADLESS_SUFFIX.length) : named, headless }; + return at === -1 ? base : { ...base, remoteUrl: browser.slice(at + 1) }; } /** @@ -177,27 +177,3 @@ export function withBrowserDefaults(browser, binary, asRoot) { } return merged; } - -/** - * Applies the run's browser defaults to a benchmark and to every `expand` node that declares a - * browser of its own. - * - * A node only inherits its parent's browser until it names one: tachometer merges an expansion over - * its parent with `Object.assign`, which replaces `browser` wholesale rather than merging into it. - * So a variant that sets, say, its own `addArguments` would otherwise drop the binary this run - * picked — and, in a containerised CI, the flag without which Chrome refuses to start as root. - * - * @param {any} node - A benchmark, or a node in its `expand` tree - * @param {string} binary - Binary to drive, unless the node pins its own - * @param {boolean} asRoot - Whether the run is happening as root - * @param {boolean} [isRoot] - Whether this is the benchmark itself, which always gets a browser - * @returns {void} - */ -export function applyBrowserDefaults(node, binary, asRoot, isRoot = true) { - if (isRoot || node.browser !== undefined) { - node.browser = withBrowserDefaults(node.browser, binary, asRoot); - } - for (const child of node.expand ?? []) { - applyBrowserDefaults(child, binary, asRoot, false); - } -} diff --git a/packages/code-infra/src/tachometer/browser.test.mjs b/packages/code-infra/src/tachometer/browser.test.mjs index a41a17e4de..8734e274de 100644 --- a/packages/code-infra/src/tachometer/browser.test.mjs +++ b/packages/code-infra/src/tachometer/browser.test.mjs @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { applyBrowserDefaults, majorVersionOf, withBrowserDefaults } from './browser.mjs'; +import { majorVersionOf, withBrowserDefaults } from './browser.mjs'; describe('majorVersionOf', () => { it('reads the major from Chrome for Testing output', () => { @@ -105,60 +105,3 @@ describe('withBrowserDefaults', () => { expect(withBrowserDefaults({ headless: true }, BINARY, false).name).toBe('chrome'); }); }); - -describe('applyBrowserDefaults', () => { - const BINARY = '/path/to/Chrome for Testing'; - - it('gives a benchmark a browser even when it declares none', () => { - /** @type {any} */ - const benchmark = { name: 'x', expand: [{ name: 'x [current]', url: './a.html' }] }; - - applyBrowserDefaults(benchmark, BINARY, false); - - expect(benchmark.browser).toEqual({ name: 'chrome', binary: BINARY }); - }); - - it('leaves an expand node that inherits its browser alone', () => { - // Tachometer merges an expansion over its parent, so a node with no browser of its own already - // gets the parent's — adding one here would only duplicate it. - /** @type {any} */ - const benchmark = { name: 'x', expand: [{ name: 'x [current]', url: './a.html' }] }; - - applyBrowserDefaults(benchmark, BINARY, false); - - expect(benchmark.expand[0]).not.toHaveProperty('browser'); - }); - - it('merges into an expand node that declares its own browser', () => { - // `expand` shallow-overrides: a node naming `browser` replaces the parent's wholesale, so - // without this the variant loses the binary and, as root, the flag Chrome needs to start. - /** @type {any} */ - const benchmark = { - name: 'x', - browser: { name: 'chrome' }, - expand: [ - { name: 'x [slow]', url: './a.html', browser: { addArguments: ['--cpu-throttle'] } }, - ], - }; - - applyBrowserDefaults(benchmark, BINARY, true); - - expect(benchmark.expand[0].browser).toEqual({ - name: 'chrome', - binary: BINARY, - addArguments: ['--cpu-throttle', '--no-sandbox'], - }); - }); - - it('reaches a browser declared deeper in the expand tree', () => { - /** @type {any} */ - const benchmark = { - name: 'x', - expand: [{ expand: [{ name: 'x [a]', url: './a.html', browser: { name: 'firefox' } }] }], - }; - - applyBrowserDefaults(benchmark, BINARY, false); - - expect(benchmark.expand[0].expand[0].browser).toEqual({ name: 'firefox', binary: BINARY }); - }); -}); diff --git a/packages/code-infra/src/tachometer/discoverCases.mjs b/packages/code-infra/src/tachometer/discoverCases.mjs index fe348b427a..c8c99f2159 100644 --- a/packages/code-infra/src/tachometer/discoverCases.mjs +++ b/packages/code-infra/src/tachometer/discoverCases.mjs @@ -90,32 +90,28 @@ export function measurementNameOf(measurement, measurementExpression) { } /** - * Walks a benchmark's `expand` tree and collects its leaves — the nodes that actually select a page. + * Flattens a benchmark's `expand` tree into the list of benchmarks it stands for, the way tachometer + * does before it reads anything else. * - * `expand` is recursive and a child inherits both its parent's `url` and its `name` unless it - * overrides them, so each is threaded down and only nodes without further `expand` are leaves. - * Tachometer merges an expansion over its parent, so the *nearest* name wins — reading a leaf's own - * name and otherwise jumping straight to the benchmark's would miss a name set in between, and the - * variant would then be looked up in the results under a name tachometer never reported. + * `expand` is recursive and an expansion is merged over its parent, so the nearest value of every + * field wins — `url`, `name`, `measurement`, `browser` alike. Doing that merge once, here, is what + * keeps the rest of discovery dealing in whole benchmarks: per-field inheritance is a rule that only + * has value while it agrees with tachometer's exactly, and re-deriving it once per field is how the + * two drift apart. * - * @param {{ url?: string, name?: string, expand?: any[] }} node - The node to walk - * @param {{ url?: string, name: string }} inherited - The effective url and name from the parent; the walk is always seeded with the benchmark's, so a leaf always has one - * @param {Array<{ node: { url?: string, name?: string }, url: string, name: string }>} out - Collected leaves - * @returns {void} + * `expand` itself is dropped. Tachometer leaves the parent's on the merged object, where it is inert + * because expansion has already happened — but the config written from this is parsed afresh, and a + * benchmark still carrying `expand` would be expanded a second time. + * + * @param {any} benchmark - A benchmark, or a node within its `expand` tree + * @returns {any[]} The benchmarks it expands to, in tachometer's order */ -function collectLeafNodes(node, inherited, out) { - const url = node.url ?? inherited.url; - const name = node.name ?? inherited.name; - if (Array.isArray(node.expand) && node.expand.length > 0) { - for (const child of node.expand) { - collectLeafNodes(child, { url, name }, out); - } - return; - } - if (url === undefined) { - throw new Error('A benchmark variant has no "url" to resolve.'); +function flattenExpansions(benchmark) { + const { expand, ...rest } = benchmark; + if (!Array.isArray(expand) || expand.length === 0) { + return [rest]; } - out.push({ node, url, name }); + return expand.flatMap((child) => flattenExpansions(child).map((leaf) => ({ ...rest, ...leaf }))); } /** @@ -238,14 +234,15 @@ export async function discoverCases(options) { for (const { name, configPath, config } of configs) { const configDir = path.dirname(configPath); - /** @type {Array<{ node: { url?: string, name?: string }, url: string, name: string }>} */ - const nodes = []; /** @type {Set} */ const measurements = new Set(); + /** @type {any[]} */ + const benchmarks = []; for (const benchmark of config.benchmarks ?? []) { // A benchmark need not name itself, and the case name is the same fallback its own name gets - // — without it the auto-expanded variants would read "undefined [current]". - const benchmarkName = benchmark.name ?? name; + // — without it the auto-expanded variants would read "undefined [current]". Set before + // flattening so every benchmark it expands to inherits it. + benchmark.name ??= name; // `measurement` may be a single entry or a list; a page with several is exactly the case // whose results have to be paired by measurement name rather than by position. for (const measurement of [benchmark.measurement ?? 'callback'].flat()) { @@ -254,28 +251,37 @@ export async function discoverCases(options) { if (!Array.isArray(benchmark.expand) || benchmark.expand.length === 0) { const base = benchmark.url; if (base === undefined) { - throw new Error(`Benchmark "${benchmarkName}" in ${configPath} has no "url".`); + throw new Error(`Benchmark "${benchmark.name}" in ${configPath} has no "url".`); } const separator = base.includes('?') ? '&' : '?'; benchmark.expand = [ - { name: `${benchmarkName} [current]`, url: base }, - { name: `${benchmarkName} [baseline]`, url: `${base}${separator}ref=baseline` }, + { name: `${benchmark.name} [current]`, url: base }, + { name: `${benchmark.name} [baseline]`, url: `${base}${separator}ref=baseline` }, ]; - // The variants carry the url now; leaving the un-rewritten source url on the parent would - // let tachometer inherit a page that was never built. - delete benchmark.url; } + benchmarks.push(...flattenExpansions(benchmark)); + } - collectLeafNodes(benchmark, { name: benchmarkName }, nodes); + // A declared `expand` tree may bottom out in a variant that names no url and inherits none. + for (const benchmark of benchmarks) { + if (benchmark.url === undefined) { + throw new Error( + `Benchmark "${benchmark.name}" in ${configPath} expands to a variant with no "url".`, + ); + } } + // The flattened list is what tachometer is handed: it expands to exactly this, and every later + // step — the url rewrite, the browser defaults — then addresses one whole benchmark at a time. + config.benchmarks = benchmarks; + // eslint-disable-next-line no-await-in-loop const resolved = await Promise.all( - nodes.map(({ url }) => parseLeafUrl(url, configDir, srcDir, resolveRef)), + benchmarks.map((benchmark) => parseLeafUrl(benchmark.url, configDir, srcDir, resolveRef)), ); - const leaves = nodes.map(({ node }, index) => ({ node, ...resolved[index] })); - const variants = nodes.map(({ name: variantName }, index) => ({ - name: variantName, + const leaves = benchmarks.map((node, index) => ({ node, ...resolved[index] })); + const variants = benchmarks.map((node, index) => ({ + name: node.name, refId: resolved[index].ref?.id ?? null, })); diff --git a/packages/code-infra/src/tachometer/discoverCases.test.mjs b/packages/code-infra/src/tachometer/discoverCases.test.mjs index 8667120db2..70806062ba 100644 --- a/packages/code-infra/src/tachometer/discoverCases.test.mjs +++ b/packages/code-infra/src/tachometer/discoverCases.test.mjs @@ -147,22 +147,24 @@ describe('discoverCases', () => { const [entry] = await discoverCases({ harnessDir }); - expect(entry.config.benchmarks[0].expand).toEqual([ + expect(entry.config.benchmarks).toEqual([ { name: 'alpha [current]', url: './index.html' }, { name: 'alpha [baseline]', url: './index.html?ref=baseline' }, ]); }); - it('removes the parent url once variants carry it', async () => { - // Leaving the un-rewritten source url on the parent would let tachometer inherit a page that - // was never built. + it('hands tachometer benchmarks it cannot expand again', async () => { + // The config is parsed afresh, so a benchmark that still carried `expand` would be expanded a + // second time — and an un-rewritten parent url would point at a page that was never built. const harnessDir = await makeHarness({ alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, }); const [entry] = await discoverCases({ harnessDir }); - expect(entry.config.benchmarks[0]).not.toHaveProperty('url'); + for (const benchmark of entry.config.benchmarks) { + expect(benchmark).not.toHaveProperty('expand'); + } }); it('joins the ref with an ampersand when the url already has a query', async () => { @@ -172,7 +174,42 @@ describe('discoverCases', () => { const [entry] = await discoverCases({ harnessDir }); - expect(entry.config.benchmarks[0].expand[1].url).toBe('./index.html?rows=100&ref=baseline'); + expect(entry.config.benchmarks[1].url).toBe('./index.html?rows=100&ref=baseline'); + }); + + it('merges an expansion over its parent, nearest value winning', async () => { + // Tachometer merges an expansion over its parent before it reads anything, so a field set + // deeper wins and one set only on the parent is inherited. Flattening here has to match, or + // the config handed over would describe different benchmarks than the ones discovered. + const harnessDir = await makeHarness({ + libs: { + config: { + benchmarks: [ + { + name: 'libs', + measurement: { mode: 'performance', entryName: 'mount' }, + expand: [{ name: 'libs [ours]', url: './ours.html' }, { url: './theirs.html' }], + }, + ], + }, + pages: ['ours.html', 'theirs.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.config.benchmarks).toEqual([ + { + name: 'libs [ours]', + url: './ours.html', + measurement: { mode: 'performance', entryName: 'mount' }, + }, + { + name: 'libs', + url: './theirs.html', + measurement: { mode: 'performance', entryName: 'mount' }, + }, + ]); }); it('leaves a benchmark that declares its own variants alone', async () => { diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/code-infra/src/tachometer/runTachometer.mjs index 56b348f01a..292ba417a6 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/code-infra/src/tachometer/runTachometer.mjs @@ -11,9 +11,9 @@ import { createRefResolver } from './refs.mjs'; import { discoverCases, pagesOf } from './discoverCases.mjs'; import { buildRefPages } from './buildPages.mjs'; import { - applyBrowserDefaults, assertDriverMatchesBrowser, resolveBrowserBinary, + withBrowserDefaults, } from './browser.mjs'; import { summarizeCase } from './summarizeCase.mjs'; import { renderTachometerReport } from './renderReport.mjs'; @@ -172,8 +172,10 @@ export async function runTachometer(options) { const refId = leaf.ref ? leaf.ref.id : 'current'; leaf.node.url = `${path.join(buildsDir, refId, leaf.page)}${leaf.suffix}`; } + // Discovery flattened `expand` away, so every benchmark already carries its own effective + // browser and there is no inheritance left to walk. for (const benchmark of entry.config.benchmarks ?? []) { - applyBrowserDefaults(benchmark, browserBinary, asRoot); + benchmark.browser = withBrowserDefaults(benchmark.browser, browserBinary, asRoot); } const slug = fileSlugOf(entry.name); const configPath = path.join(tmpBase, `tachometer-${slug}.json`); From 41726a352d09bceeda941a17560dfcd519d83230 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:37:31 +0200 Subject: [PATCH 024/173] [code-infra] Resolve a ref's tree through its own manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vite needs a base directory and derives one from the importer's path, falling back to the project root unless that path exists on disk. That was satisfied by writing an empty `__resolve__.js` into every ref tree — inventing a file to make a path lookup succeed. The tree's `package.json` is already there and names the same directory, which is how `createRequire` is pointed at one. Also reuse `writePackageJson` for the generated manifest, and read this package's own name rather than spelling it out in `RUNNER_ONLY_DEPS` — a rename would otherwise quietly restore a multi-minute install per ref. --- .../code-infra/src/tachometer/buildPages.mjs | 70 +++++++------------ 1 file changed, 27 insertions(+), 43 deletions(-) diff --git a/packages/code-infra/src/tachometer/buildPages.mjs b/packages/code-infra/src/tachometer/buildPages.mjs index eaa86909fc..c85dd5154a 100644 --- a/packages/code-infra/src/tachometer/buildPages.mjs +++ b/packages/code-infra/src/tachometer/buildPages.mjs @@ -8,7 +8,7 @@ import chalk from 'chalk'; import { parse, stringify } from 'yaml'; import { run } from '../utils/exec.mjs'; import { tarballFor } from '../utils/packWorkspace.mjs'; -import { readPackageJson } from '../utils/pnpm.mjs'; +import { readPackageJson, writePackageJson } from '../utils/pnpm.mjs'; /** * Dependencies the run needs but a page never imports, so a ref's tree does without them. @@ -23,24 +23,11 @@ const RUNNER_ONLY_DEPS = [ 'tachometer', 'chromedriver', '@playwright/test', - '@mui/internal-code-infra', + // Read rather than spelled out: this is the one entry naming *this* package, and a rename that + // left a literal behind would quietly restore a multi-minute install per ref. + createRequire(import.meta.url)('../../package.json').name, ]; -/** - * Builds the benchmark pages for one ref. - * - * The pages are built **where they live**, from the harness's own directory and its own vite - * config, so a `tacho run` build sees exactly what a plain `vite build` sees — the same tsconfig, - * the same postcss config, the same everything. Only *resolution* differs: a plugin redirects every - * bare import into a small tree installed for this ref, where the library under test is that ref's - * packed build. - * - * That tree holds the harness's own dependencies too, so the pages and the library resolve React - * (and everything else) to one copy. Building one side through a workspace link instead would - * compare two different resolution paths, and that difference lands in the measurement rather than - * in the library. - */ - /** * @typedef {import('./refs.mjs').ResolvedRef} ResolvedRef * @typedef {import('../utils/packWorkspace.mjs').PackedPackage} PackedPackage @@ -94,9 +81,6 @@ async function readOverrides(repoRoot) { } } -/** Name of the file resolution pretends to come from. It has to exist; see `resolveFromTree`. */ -const RESOLVE_STUB = '__resolve__.js'; - /** * A vite plugin that resolves every bare import from `treeDir` instead of from the harness. * @@ -109,9 +93,11 @@ const RESOLVE_STUB = '__resolve__.js'; * @returns {import('vite').Plugin} */ function resolveFromTree(treeDir) { - // Vite falls back to the project root when the importer does not exist on disk, which silently - // resolves the harness's copy of a dependency instead of this ref's. - const importer = path.join(treeDir, RESOLVE_STUB); + // Resolution needs a base directory, and vite derives one from the importer's path — falling back + // to the project root unless that path exists on disk, which would silently resolve the harness's + // copy of a dependency instead of this ref's. The tree's own manifest is the file that names this + // directory, the same way `createRequire` is pointed at one. + const importer = path.join(treeDir, 'package.json'); return { name: 'mui-code-infra:tachometer-resolve', @@ -139,24 +125,13 @@ async function installRefTree({ harnessDir, repoRoot, packages, treeDir }) { const harnessPkg = await readPackageJson(harnessDir); await mkdir(treeDir, { recursive: true }); - await writeFile( - path.join(treeDir, 'package.json'), - `${JSON.stringify( - { - name: 'tacho-resolve-tree', - private: true, - version: '0.0.0', - dependencies: rewriteWorkspaceDeps(harnessPkg.dependencies, packages), - devDependencies: rewriteWorkspaceDeps( - harnessPkg.devDependencies, - packages, - RUNNER_ONLY_DEPS, - ), - }, - null, - 2, - )}\n`, - ); + await writePackageJson(treeDir, { + name: 'tacho-resolve-tree', + private: true, + version: '0.0.0', + dependencies: rewriteWorkspaceDeps(harnessPkg.dependencies, packages), + devDependencies: rewriteWorkspaceDeps(harnessPkg.devDependencies, packages, RUNNER_ONLY_DEPS), + }); // Marks this folder as its own workspace root, so pnpm does not walk up into the monorepo. The // overrides carry the repository's own, then pin every packed workspace package on top — so a @@ -173,8 +148,6 @@ async function installRefTree({ harnessDir, repoRoot, packages, treeDir }) { }), ); - await writeFile(path.join(treeDir, RESOLVE_STUB), ''); - // `--ignore-scripts`: this workspace has no build-script approvals, and pnpm fails an install // over unapproved ones rather than warning. Nothing installed here needs its scripts either — the // pages are built from the packages exactly as packed. @@ -188,6 +161,17 @@ async function installRefTree({ harnessDir, repoRoot, packages, treeDir }) { /** * Builds the benchmark pages against one ref's packed build. * + * The pages are built **where they live**, from the harness's own directory and its own vite config, + * so a `tacho run` build sees exactly what a plain `vite build` sees — the same tsconfig, the same + * postcss config, the same everything. Only *resolution* differs: a plugin redirects every bare + * import into a small tree installed for this ref, where the library under test is that ref's packed + * build. + * + * That tree holds the harness's own dependencies too, so the pages and the library resolve React + * (and everything else) to one copy. Building one side through a workspace link instead would + * compare two different resolution paths, and that difference would land in the measurement rather + * than in the library. + * * @param {Object} options - Build inputs * @param {string} options.harnessDir - The harness package directory, built in place * @param {string} options.repoRoot - The workspace root, whose overrides the tree inherits From 71cd328c906dcd1143e7f47afb615754ea4b58be Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 8 Sep 2026 15:37:40 +0200 Subject: [PATCH 025/173] [code-infra] Emit the case index into the build, and use vite's usual script names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The index listing every case was a dev-server behaviour, so a built directory could only be entered by typing page urls from memory — and a case that owns no page, reachable only through the query in its config, could not be entered at all. Emitting it as a real asset makes the build navigable under `preview`, under any static server, or opened from disk. Links are relative, which `base: './'` needs and which resolves the same on the dev server. The harness's `build:pages` becomes `build`, restoring vite's familiar dev/build/preview trio. Nothing swept it up: this repository's `release:build` is scoped to `./packages/*`. --- packages/code-infra/src/tachometer/plugin.mjs | 44 ++++++++++++++----- test/tachometer/README.md | 10 +++-- test/tachometer/package.json | 2 +- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/packages/code-infra/src/tachometer/plugin.mjs b/packages/code-infra/src/tachometer/plugin.mjs index e0ae54ccd9..6d38340710 100644 --- a/packages/code-infra/src/tachometer/plugin.mjs +++ b/packages/code-infra/src/tachometer/plugin.mjs @@ -58,16 +58,17 @@ async function assertWorkspaceDepsBuilt(harnessDir, deps) { * its page with. * * @param {import('./discoverCases.mjs').BenchmarkCase[]} cases - Discovered cases + * @param {string} note - What the pages behind these links are, which differs per command * @returns {string} An HTML document */ -function renderIndex(cases) { +function renderIndex(cases, note) { const items = cases .map((entry) => { // A case's variants can point at several pages (a cross-library comparison) or at one page // with different queries (a parameterised case), so dedupe on the pair. const targets = [...new Set(entry.leaves.map((leaf) => `${leaf.page}${leaf.suffix}`))].sort(); const links = targets - .map((target) => `
  • ${target}
    • `) + .map((target) => `
  • ${target}
    • `) .join('\n '); return `
  • \n ${entry.name}\n
      \n ${links}\n
    \n
    • `; }) @@ -88,7 +89,7 @@ function renderIndex(cases) {

    Tachometer benchmark cases

    -

    Pages are served from the working tree's build of the library. Rebuild it and reload to pick up a source change.

    +

    ${note}

      ${items}
    @@ -106,6 +107,8 @@ ${items} export function tachometer(options = {}) { const harnessDir = options.harnessDir ?? process.cwd(); const srcDir = path.join(harnessDir, 'src'); + /** @type {import('./discoverCases.mjs').BenchmarkCase[]} */ + let buildCases = []; return { name: 'mui-code-infra:tachometer', @@ -117,9 +120,12 @@ export function tachometer(options = {}) { ...pkg.devDependencies, }); + // The run output directory, next to the per-ref directories the runner writes — vite would + // otherwise default to `/dist`, i.e. inside `src/` next to the case sources. + // // Resolved for every command, not just `build`: `vite preview` runs as `serve` but serves - // `build.outDir`, so leaving it unset there sends preview looking for `/dist` — inside - // `src/` — and it exits rather than serving the pages that were just built. + // `build.outDir`, so leaving it unset there sends preview looking for `/dist` and it + // exits rather than serving the pages that were just built. // // Only fill it in when the caller said nothing: a plugin's returned config is merged *over* // the inline config, so unconditionally setting it here would silently override @@ -131,6 +137,7 @@ export function tachometer(options = {}) { } const cases = await discoverCases({ harnessDir }); + buildCases = cases; // Key the input map by the page path without its extension. Vite emits each page at its path // relative to `root` regardless, so `/index.html` lands at `//index.html`; // a distinct key per page is what lets one case folder hold several pages — a cross-library @@ -146,11 +153,8 @@ export function tachometer(options = {}) { // `//`, so absolute "/assets/…" paths would 404. base: './', build: { - // A plain `vite build` lands in the run output directory, next to the per-ref directories - // the runner writes. outDir, - // The output directory is outside `root` — and, for an isolated ref build, outside the - // temporary package entirely; allow vite to clean it anyway. + // The output directory sits outside `root`, which is `src/`; allow vite to clean it anyway. emptyOutDir: true, chunkSizeWarningLimit: 9999, rollupOptions: { input }, @@ -158,6 +162,21 @@ export function tachometer(options = {}) { }; }, + // The same index the dev server renders, emitted as a real file so the built directory is + // navigable too — under `vite preview`, under any static server, or opened from disk. A page-less + // case is only reachable through this list, so a build without it can only be entered by typing + // urls from memory. + generateBundle() { + this.emitFile({ + type: 'asset', + fileName: 'index.html', + source: renderIndex( + buildCases, + 'These are production bundles — the same build tachometer measures.', + ), + }); + }, + configureServer(server) { server.middlewares.use(async (req, res, next) => { const url = (req.url ?? '/').split('?')[0]; @@ -168,7 +187,12 @@ export function tachometer(options = {}) { try { const cases = await discoverCases({ harnessDir }); res.setHeader('Content-Type', 'text/html; charset=utf-8'); - res.end(renderIndex(cases)); + res.end( + renderIndex( + cases, + "Pages are served from the working tree's build of the library. Rebuild it and reload to pick up a source change.", + ), + ); } catch (error) { next(error); } diff --git a/test/tachometer/README.md b/test/tachometer/README.md index b8ff980786..6aa0a1caaf 100644 --- a/test/tachometer/README.md +++ b/test/tachometer/README.md @@ -86,7 +86,9 @@ current commit while that is still true. pnpm -F ./test/tachometer dev ``` -The dev server lists every discovered case at `/`, including `workload-large`, which owns no page -and would otherwise be invisible. `pnpm -F ./test/tachometer build:pages` and `preview` produce and -serve the production bundle, which is what actually gets measured. That index is a dev-server -convenience, so `preview` has nothing at `/` — open a case's page directly. +Every discovered case is listed at `/`, including `workload-large`, which owns no page of its own and +would otherwise be unreachable — its `?size=` query is in the config, not the file tree. The list is +generated from the case configs rather than written by hand, so it cannot go stale. + +`pnpm -F ./test/tachometer build` and `preview` produce and serve the production bundle, which is +what actually gets measured. The index is emitted into that build too, so the same list is there. diff --git a/test/tachometer/package.json b/test/tachometer/package.json index 2c865cf133..ed984b090c 100644 --- a/test/tachometer/package.json +++ b/test/tachometer/package.json @@ -3,7 +3,7 @@ "description": "Tachometer smoke-test harness for the code-infra tacho tooling", "scripts": { "dev": "vite", - "build:pages": "vite build", + "build": "vite build", "preview": "vite preview", "test:tacho": "code-infra tacho run" }, From ff3007f8f111432bd34f1aa0e967c231e723f108 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 9 Sep 2026 10:54:09 +0200 Subject: [PATCH 026/173] [code-infra] Pin a ref's tree to the versions the harness resolved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tree is its own workspace root and installs without the repository's lockfile, so copying a declared range let it float to whatever was newest at run time. base-ui-mosaic's CI hit the visible half of this: the harness declares `@base-ui/react: ^1.6.0`, the repository resolves 1.7.0, and the tree installed the freshly released 1.8.0 — which no longer exports the `./internals/*` paths the library imports, so every ref build failed. The quiet half is worse. A range that floats to a merely *slower* release measures that release and attributes the difference to our own code, which is the one thing a benchmark must not do. Direct dependencies are now pinned to what the harness actually has installed, read from its own node_modules. --- .../code-infra/src/tachometer/buildPages.mjs | 65 +++++++++++++++---- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/packages/code-infra/src/tachometer/buildPages.mjs b/packages/code-infra/src/tachometer/buildPages.mjs index c85dd5154a..ef3e08e111 100644 --- a/packages/code-infra/src/tachometer/buildPages.mjs +++ b/packages/code-infra/src/tachometer/buildPages.mjs @@ -40,24 +40,56 @@ const RUNNER_ONLY_DEPS = [ * This is what lets the tree be derived rather than configured: the harness already declares the * library under test as `workspace:*`, so there is no library name to hardcode and no * hand-maintained list of versions to pin. Third-party dependencies — including the competitor - * libraries a cross-library case compares against — come along unchanged, so every page builds - * against every ref. + * libraries a cross-library case compares against — come along, so every page builds against every + * ref. * + * Everything else is pinned to the version the harness *resolved*, not the range it declares. The + * tree is its own workspace root and installs without the repository's lockfile, so a range would + * float to whatever is newest at run time: a different library than the repository builds against, + * quietly measured as if it were ours. A release that also drops an export fails the build outright, + * which is the visible half of the same problem. + * + * @param {string} harnessDir - The harness package directory, whose installed versions are read * @param {Partial> | undefined} deps - A `dependencies` or `devDependencies` map * @param {PackedPackage[]} packages - The packed workspace packages * @param {string[]} [omit] - Dependency names to drop entirely - * @returns {Record} + * @returns {Promise>} + */ +async function rewriteWorkspaceDeps(harnessDir, deps, packages, omit = []) { + const entries = /** @type {Array<[string, string]>} */ ( + Object.entries(deps ?? {}).filter( + ([name, version]) => version !== undefined && !omit.includes(name), + ) + ); + const resolved = await Promise.all( + entries.map(async ([name, version]) => { + if (version.startsWith('workspace:')) { + return [name, `file:${tarballFor(packages, name)}`]; + } + const installed = await installedVersionOf(harnessDir, name); + return [name, installed ?? version]; + }), + ); + return Object.fromEntries(resolved); +} + +/** + * The version of `name` the harness actually has installed, or `undefined` when it has none. + * + * Read from the harness's own `node_modules` rather than a lockfile: pnpm links every direct + * dependency there, and these maps only ever hold direct dependencies. + * + * @param {string} harnessDir - The harness package directory + * @param {string} name - A dependency name + * @returns {Promise} */ -function rewriteWorkspaceDeps(deps, packages, omit = []) { - /** @type {Record} */ - const out = {}; - for (const [name, version] of Object.entries(deps ?? {})) { - if (version === undefined || omit.includes(name)) { - continue; - } - out[name] = version.startsWith('workspace:') ? `file:${tarballFor(packages, name)}` : version; +async function installedVersionOf(harnessDir, name) { + try { + const manifest = path.join(harnessDir, 'node_modules', ...name.split('/'), 'package.json'); + return JSON.parse(await readFile(manifest, 'utf8')).version; + } catch { + return undefined; } - return out; } /** @@ -129,8 +161,13 @@ async function installRefTree({ harnessDir, repoRoot, packages, treeDir }) { name: 'tacho-resolve-tree', private: true, version: '0.0.0', - dependencies: rewriteWorkspaceDeps(harnessPkg.dependencies, packages), - devDependencies: rewriteWorkspaceDeps(harnessPkg.devDependencies, packages, RUNNER_ONLY_DEPS), + dependencies: await rewriteWorkspaceDeps(harnessDir, harnessPkg.dependencies, packages), + devDependencies: await rewriteWorkspaceDeps( + harnessDir, + harnessPkg.devDependencies, + packages, + RUNNER_ONLY_DEPS, + ), }); // Marks this folder as its own workspace root, so pnpm does not walk up into the monorepo. The From 23ca0ca18eea5c67fb42b9d7b8ed27f3beed4042 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:31:33 +0200 Subject: [PATCH 027/173] [code-infra] Add `tacho report` to print a saved report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `tacho run` prints its table and writes the JSON, but reading that JSON afterwards meant either sampling again or wiring a renderer into the consuming repository. The renderer already lives here, so the command that produced the report can print it: a report kept from an earlier run, or downloaded from a CI artifact, is now readable in place. Rejects a report from the other benchmark axis rather than rendering it — the envelope is the same but the statistics are not, and its numbers would be read as confidence intervals on a difference. --- .../code-infra/src/cli/cmdTachoReport.mjs | 53 +++++++++++++++++++ packages/code-infra/src/cli/index.mjs | 2 + test/tachometer/README.md | 3 ++ 3 files changed, 58 insertions(+) create mode 100644 packages/code-infra/src/cli/cmdTachoReport.mjs diff --git a/packages/code-infra/src/cli/cmdTachoReport.mjs b/packages/code-infra/src/cli/cmdTachoReport.mjs new file mode 100644 index 0000000000..d1d10e9e79 --- /dev/null +++ b/packages/code-infra/src/cli/cmdTachoReport.mjs @@ -0,0 +1,53 @@ +#!/usr/bin/env node + +import * as path from 'node:path'; +import { readFile } from 'node:fs/promises'; +import { OUTPUT_DIR } from '../tachometer/outputDir.mjs'; + +/** + * @typedef {Object} Args + * @property {string} file - Path to a JSON report written by `tacho run` + */ + +const DEFAULT_REPORT = path.join(OUTPUT_DIR, 'results', 'report.json'); + +export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ + command: 'tacho report [file]', + describe: + 'Print the table for a JSON report written by `tacho run`. Reads the last run of the harness in the current directory when no path is given.', + builder: (yargs) => { + return ( + yargs + // Not named `report`: yargs binds a positional whose name repeats a command word to the + // literal token, so `tacho report` would resolve the path "report". + .positional('file', { + type: 'string', + default: DEFAULT_REPORT, + describe: 'Path to the JSON report', + }) + .epilogue( + 'The same table `tacho run` prints when it finishes, so a report kept from an earlier run — or downloaded from CI — can be read without sampling again.', + ) + ); + }, + handler: async (argv) => { + const reportPath = path.resolve(argv.file); + const raw = await readFile(reportPath, 'utf8').catch(() => { + throw new Error( + `No report at ${reportPath}. Pass a path, or run \`tacho run\` from a harness first.`, + ); + }); + + const report = JSON.parse(raw); + // Reports from the other benchmark axis have the same envelope but different statistics, and + // rendering one here would silently read its numbers as confidence intervals on a difference. + if (report?.reportType !== 'tachometer') { + throw new Error( + `${reportPath} is not a tachometer report (its "reportType" is ${JSON.stringify(report?.reportType)}).`, + ); + } + + const { renderTachometerReport } = await import('../tachometer/renderReport.mjs'); + renderTachometerReport(report); + }, +}); diff --git a/packages/code-infra/src/cli/index.mjs b/packages/code-infra/src/cli/index.mjs index cd692b6111..cbc73ced4c 100644 --- a/packages/code-infra/src/cli/index.mjs +++ b/packages/code-infra/src/cli/index.mjs @@ -17,6 +17,7 @@ import cmdPublishNewPackage from './cmdPublishNewPackage.mjs'; import cmdRenameScope from './cmdRenameScope.mjs'; import cmdSetVersionOverrides from './cmdSetVersionOverrides.mjs'; import cmdTacho from './cmdTacho.mjs'; +import cmdTachoReport from './cmdTachoReport.mjs'; import cmdVale from './cmdVale.mjs'; import cmdValidateBuiltTypes from './cmdValidateBuiltTypes.mjs'; @@ -52,6 +53,7 @@ await yargs(hideBin(process.argv)) .command(cmdRenameScope) .command(cmdSetVersionOverrides) .command(cmdTacho) + .command(cmdTachoReport) .command(cmdVale) .command(cmdValidateBuiltTypes) .fail((msg, err, yargsInstance) => { diff --git a/test/tachometer/README.md b/test/tachometer/README.md index 6aa0a1caaf..74b5cf2525 100644 --- a/test/tachometer/README.md +++ b/test/tachometer/README.md @@ -53,6 +53,9 @@ Cases run sequentially, and each auto-samples until its difference resolves or t short) timeout is hit. Because both sides run identical code, the expected verdict is `unsure` — that is the "no change" outcome, not a failure. +`code-infra tacho report` prints that table again from the saved JSON, so a run kept from earlier — +or a report downloaded from CI — can be read without sampling again. + Everything a run writes goes under `.tachometer/`: the report in `results/`, the pages built per ref in `builds/`, the packed tarballs in `packed/` (the one worth caching in CI, keyed by commit SHA) and the install each ref resolves through in `trees/`. Deleting that directory resets the harness From 480e486a68dc4bb26f4741c96983794fddb9555f Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 9 Sep 2026 15:09:24 +0200 Subject: [PATCH 028/173] [dashboard] Drop the cross-library aside from comparisonsAcrossRefs --- .../src/lib/tachometer/buildMarkdownReport.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts index 66b2e80852..47f6315aed 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts @@ -24,11 +24,6 @@ interface BuildOptions { * * `comparisons` express the reference relative to each other variant, so a `slower` verdict means * the reference — `[current]` for an auto-expanded case — is the slow side. - * - * Only comparisons **across refs** count. A cross-library case runs every variant from the same - * build, so a `slower` verdict there says this library is slower than a competitor: true, worth - * knowing, and completely unrelated to what the pull request changed. Flagging those would put a - * warning on every comment until nobody read it. */ function comparisonsAcrossRefs(entry: SummarizedCase) { const found = []; From 45a735a352e4e4e5d9e5ea2c761e177340dc907f Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:36:35 +0200 Subject: [PATCH 029/173] [benchmark] Move the tachometer tooling out of code-infra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tachometer axis is a benchmark concern, not a build-tooling one, and living inside the code-infra CLI tied its release cadence to a package it has nothing else to do with. Everything moves: the ten modules and their tests, `packWorkspace` (which nothing else imported), and the CLI, now behind a `benchmark` bin — `benchmark tacho run`, `benchmark tacho report`. Sources become TypeScript rather than JSDoc-annotated `.mjs`, because this package compiles to `build/` and publishes from there; babel only transforms `.js`/`.ts`/`.tsx`, so `.mjs` sources would have shipped as nothing at all. The move pays for itself immediately. `tachometer/ciReport` carried a comment explaining that its upload transport was deliberately copied rather than shared, since depending on code-infra would push ~900 transitive packages onto every consumer of this one. That reason is gone, so it now uses the package's own `getCiMetadata`, `syncPrComment` and `ciReportUploadSchema` — the envelope factory was already byte-identical. --- .../src/lib/tachometer/types.ts | 2 +- packages/benchmark/package.json | 31 ++- packages/benchmark/src/ciReport.ts | 2 +- packages/benchmark/src/cli/cmdTacho.ts | 23 ++ packages/benchmark/src/cli/index.ts | 44 ++++ .../src/cli/tachoReport.ts} | 29 +-- .../src/cli/tachoRun.ts} | 45 ++-- .../src/tachometer/browser.test.ts} | 2 +- .../src/tachometer/browser.ts} | 58 ++--- .../src/tachometer/buildPages.ts} | 134 ++++++----- .../src/tachometer/ciReport.test.ts} | 2 +- packages/benchmark/src/tachometer/ciReport.ts | 123 ++++++++++ .../src/tachometer/discoverCases.test.ts} | 31 ++- .../src/tachometer/discoverCases.ts} | 135 +++++------ .../src/tachometer/outputDir.ts} | 11 +- .../src/tachometer/plugin.test.ts} | 45 ++-- .../src/tachometer/plugin.ts} | 50 ++-- .../src/tachometer/refs.test.ts} | 2 +- .../src/tachometer/refs.ts} | 134 +++++------ .../src/tachometer/renderReport.ts} | 217 +++++++----------- .../src/tachometer/runTachometer.ts} | 99 ++++---- .../src/tachometer/summarizeCase.test.ts} | 37 +-- .../src/tachometer/summarizeCase.ts} | 128 ++++++----- packages/benchmark/src/utils/build.ts | 31 +++ packages/benchmark/src/utils/exec.ts | 20 ++ .../src/utils/packWorkspace.test.ts} | 29 +-- .../src/utils/packWorkspace.ts} | 188 +++++++-------- packages/benchmark/src/utils/path.ts | 11 + packages/benchmark/src/utils/pnpm.ts | 63 +++++ packages/benchmark/src/utils/testUtils.ts | 13 ++ packages/code-infra/package.json | 11 +- packages/code-infra/src/cli/index.mjs | 4 - .../code-infra/src/tachometer/ciReport.mjs | 193 ---------------- pnpm-lock.yaml | 31 ++- test/tachometer/README.md | 31 ++- test/tachometer/package.json | 6 +- test/tachometer/src/libs/sort/alpha.html | 11 + test/tachometer/src/libs/sort/alpha.ts | 9 + test/tachometer/src/libs/sort/beta.html | 11 + test/tachometer/src/libs/sort/beta.ts | 34 +++ test/tachometer/src/libs/sort/ours.html | 11 + test/tachometer/src/libs/sort/ours.ts | 9 + test/tachometer/src/libs/sort/shared.ts | 42 ++++ test/tachometer/src/libs/sort/tachometer.json | 22 ++ test/tachometer/src/workload/index.ts | 2 +- test/tachometer/vite.config.mjs | 2 +- 46 files changed, 1180 insertions(+), 988 deletions(-) create mode 100644 packages/benchmark/src/cli/cmdTacho.ts create mode 100644 packages/benchmark/src/cli/index.ts rename packages/{code-infra/src/cli/cmdTachoReport.mjs => benchmark/src/cli/tachoReport.ts} (78%) rename packages/{code-infra/src/cli/cmdTacho.mjs => benchmark/src/cli/tachoRun.ts} (73%) rename packages/{code-infra/src/tachometer/browser.test.mjs => benchmark/src/tachometer/browser.test.ts} (98%) rename packages/{code-infra/src/tachometer/browser.mjs => benchmark/src/tachometer/browser.ts} (78%) rename packages/{code-infra/src/tachometer/buildPages.mjs => benchmark/src/tachometer/buildPages.ts} (73%) rename packages/{code-infra/src/tachometer/ciReport.test.mjs => benchmark/src/tachometer/ciReport.test.ts} (98%) create mode 100644 packages/benchmark/src/tachometer/ciReport.ts rename packages/{code-infra/src/tachometer/discoverCases.test.mjs => benchmark/src/tachometer/discoverCases.test.ts} (97%) rename packages/{code-infra/src/tachometer/discoverCases.mjs => benchmark/src/tachometer/discoverCases.ts} (73%) rename packages/{code-infra/src/tachometer/outputDir.mjs => benchmark/src/tachometer/outputDir.ts} (80%) rename packages/{code-infra/src/tachometer/plugin.test.mjs => benchmark/src/tachometer/plugin.test.ts} (87%) rename packages/{code-infra/src/tachometer/plugin.mjs => benchmark/src/tachometer/plugin.ts} (84%) rename packages/{code-infra/src/tachometer/refs.test.mjs => benchmark/src/tachometer/refs.test.ts} (97%) rename packages/{code-infra/src/tachometer/refs.mjs => benchmark/src/tachometer/refs.ts} (68%) rename packages/{code-infra/src/tachometer/renderReport.mjs => benchmark/src/tachometer/renderReport.ts} (67%) rename packages/{code-infra/src/tachometer/runTachometer.mjs => benchmark/src/tachometer/runTachometer.ts} (80%) rename packages/{code-infra/src/tachometer/summarizeCase.test.mjs => benchmark/src/tachometer/summarizeCase.test.ts} (89%) rename packages/{code-infra/src/tachometer/summarizeCase.mjs => benchmark/src/tachometer/summarizeCase.ts} (58%) create mode 100644 packages/benchmark/src/utils/build.ts create mode 100644 packages/benchmark/src/utils/exec.ts rename packages/{code-infra/src/utils/packWorkspace.test.mjs => benchmark/src/utils/packWorkspace.test.ts} (94%) rename packages/{code-infra/src/utils/packWorkspace.mjs => benchmark/src/utils/packWorkspace.ts} (70%) create mode 100644 packages/benchmark/src/utils/path.ts create mode 100644 packages/benchmark/src/utils/pnpm.ts create mode 100644 packages/benchmark/src/utils/testUtils.ts delete mode 100644 packages/code-infra/src/tachometer/ciReport.mjs create mode 100644 test/tachometer/src/libs/sort/alpha.html create mode 100644 test/tachometer/src/libs/sort/alpha.ts create mode 100644 test/tachometer/src/libs/sort/beta.html create mode 100644 test/tachometer/src/libs/sort/beta.ts create mode 100644 test/tachometer/src/libs/sort/ours.html create mode 100644 test/tachometer/src/libs/sort/ours.ts create mode 100644 test/tachometer/src/libs/sort/shared.ts create mode 100644 test/tachometer/src/libs/sort/tachometer.json diff --git a/apps/code-infra-dashboard/src/lib/tachometer/types.ts b/apps/code-infra-dashboard/src/lib/tachometer/types.ts index 29e68f68e7..c49e4eadc2 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/types.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/types.ts @@ -1,5 +1,5 @@ /** - * Types mirroring @mui/internal-code-infra/tachometerReport. + * Types mirroring @mui/internal-benchmark/tachometerReport. * * Mirrored rather than imported on purpose: these describe artifacts already sitting in S3, written * by whatever version of the tooling a repository had at the time. Importing the current definitions diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index e87b68bc78..0e300f0a5b 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -12,7 +12,9 @@ ".": "./src/index.tsx", "./ciReport": "./src/ciReport.ts", "./reporter": "./src/reporter.ts", - "./vitest": "./src/vitest.ts" + "./vitest": "./src/vitest.ts", + "./tachometerPlugin": "./src/tachometer/plugin.ts", + "./tachometerReport": "./src/tachometer/renderReport.ts" }, "scripts": { "build": "code-infra build --flat --tsgo --bundle esm", @@ -20,25 +22,52 @@ }, "dependencies": { "@babel/runtime": "^7.29.7", + "@pnpm/find-workspace-dir": "^1000.1.0", "@vitejs/plugin-react": "^6.1.1", + "chalk": "^5.6.2", "env-ci": "^11.2.0", "execa": "^10.0.1", + "globby": "^16.2.4", + "yaml": "^2.9.0", + "yargs": "^18.1.0", "zod": "^4.5.4" }, "peerDependencies": { + "@playwright/test": ">=1.50.0", "@vitest/browser-playwright": ">=4.1", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0", + "vite": "^7.0.0 || ^8.0.0", "vitest": ">=4.1" }, "devDependencies": { + "@playwright/test": "1.62.1", "@types/env-ci": "3.1.4", "@types/react": "19.2.18", "@types/react-dom": "19.2.7", + "@types/yargs": "17.0.35", + "vite": "^8.2.2", "vitest": "4.1.11" }, "publishConfig": { "access": "public", "directory": "build" + }, + "bin": { + "benchmark": "./src/cli/index.ts" + }, + "files": [ + "build", + "src", + "README.md", + "LICENSE" + ], + "peerDependenciesMeta": { + "@playwright/test": { + "optional": true + }, + "vite": { + "optional": true + } } } diff --git a/packages/benchmark/src/ciReport.ts b/packages/benchmark/src/ciReport.ts index 8e2d8cd1f4..f2951db67a 100644 --- a/packages/benchmark/src/ciReport.ts +++ b/packages/benchmark/src/ciReport.ts @@ -29,7 +29,7 @@ async function getGitInfo(): Promise { * Creates a CI report upload schema for a specific report type. * Common fields (commitSha, repo, branch, prNumber) are shared across all report types. */ -function ciReportUploadSchema(type: string, version: number, reportSchema: z.ZodType) { +export function ciReportUploadSchema(type: string, version: number, reportSchema: z.ZodType) { return z.object({ version: z.literal(version), timestamp: z.number(), diff --git a/packages/benchmark/src/cli/cmdTacho.ts b/packages/benchmark/src/cli/cmdTacho.ts new file mode 100644 index 0000000000..58c855e13c --- /dev/null +++ b/packages/benchmark/src/cli/cmdTacho.ts @@ -0,0 +1,23 @@ +import type { CommandModule } from 'yargs'; +import tachoReport from './tachoReport'; +import tachoRun from './tachoRun'; + +/** + * The `tacho` command group. + * + * Its subcommands are registered here rather than as separate top-level commands: yargs keys a + * command on its first word, so `tacho run …` and `tacho report …` registered side by side would + * both claim `tacho` and the later one would answer for both. + */ +const command: CommandModule<{}, {}> = { + command: 'tacho ', + describe: 'Benchmark a tachometer harness, and read the reports it writes.', + builder: (yargs) => + yargs + .command(tachoRun) + .command(tachoReport) + .demandCommand(1, 'Specify a tacho subcommand.') as any, + handler: () => {}, +}; + +export default command; diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts new file mode 100644 index 0000000000..e1240624b1 --- /dev/null +++ b/packages/benchmark/src/cli/index.ts @@ -0,0 +1,44 @@ +#!/usr/bin/env node + +import { createRequire } from 'node:module'; +import chalk from 'chalk'; +import yargs from 'yargs'; +import { hideBin } from 'yargs/helpers'; + +import cmdTacho from './cmdTacho'; + +const pkgJson = createRequire(import.meta.url)('../../package.json'); + +let globalArgv: { verbose?: boolean } = {}; + +await yargs(hideBin(process.argv)) + .scriptName('benchmark') + .usage('$0 [args]') + .option('verbose', { + alias: 'v', + type: 'boolean', + default: false, + describe: 'Increase output verbosity', + global: true, + }) + .middleware((argv) => { + globalArgv = argv; + }, true) + .command(cmdTacho) + .fail((msg, err, yargsInstance) => { + if (msg) { + yargsInstance.showHelp(); + console.error(`\n${msg}`); + } else if (err) { + console.error(err.message); + if (globalArgv.verbose) { + console.error(chalk.dim(err.stack)); + } + } + process.exit(1); + }) + .demandCommand(1, 'You need at least one command before moving on') + .strict() + .help() + .version(pkgJson.version) + .parseAsync(); diff --git a/packages/code-infra/src/cli/cmdTachoReport.mjs b/packages/benchmark/src/cli/tachoReport.ts similarity index 78% rename from packages/code-infra/src/cli/cmdTachoReport.mjs rename to packages/benchmark/src/cli/tachoReport.ts index d1d10e9e79..aa533182a2 100644 --- a/packages/code-infra/src/cli/cmdTachoReport.mjs +++ b/packages/benchmark/src/cli/tachoReport.ts @@ -1,25 +1,24 @@ -#!/usr/bin/env node - import * as path from 'node:path'; import { readFile } from 'node:fs/promises'; -import { OUTPUT_DIR } from '../tachometer/outputDir.mjs'; +import type { CommandModule } from 'yargs'; +import { OUTPUT_DIR } from '../tachometer/outputDir'; -/** - * @typedef {Object} Args - * @property {string} file - Path to a JSON report written by `tacho run` - */ +interface Args { + /** Path to a JSON report written by `tacho run`. */ + file: string; +} const DEFAULT_REPORT = path.join(OUTPUT_DIR, 'results', 'report.json'); -export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ - command: 'tacho report [file]', +const command: CommandModule<{}, Args> = { + command: 'report [file]', describe: 'Print the table for a JSON report written by `tacho run`. Reads the last run of the harness in the current directory when no path is given.', builder: (yargs) => { return ( yargs - // Not named `report`: yargs binds a positional whose name repeats a command word to the - // literal token, so `tacho report` would resolve the path "report". + // Not named `report`: yargs binds a positional whose name repeats its own subcommand word to + // the literal token, so `tacho report` would resolve the path "report". .positional('file', { type: 'string', default: DEFAULT_REPORT, @@ -27,7 +26,7 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ }) .epilogue( 'The same table `tacho run` prints when it finishes, so a report kept from an earlier run — or downloaded from CI — can be read without sampling again.', - ) + ) as any ); }, handler: async (argv) => { @@ -47,7 +46,9 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ ); } - const { renderTachometerReport } = await import('../tachometer/renderReport.mjs'); + const { renderTachometerReport } = await import('../tachometer/renderReport'); renderTachometerReport(report); }, -}); +}; + +export default command; diff --git a/packages/code-infra/src/cli/cmdTacho.mjs b/packages/benchmark/src/cli/tachoRun.ts similarity index 73% rename from packages/code-infra/src/cli/cmdTacho.mjs rename to packages/benchmark/src/cli/tachoRun.ts index 7bb5e17356..ea38392246 100644 --- a/packages/code-infra/src/cli/cmdTacho.mjs +++ b/packages/benchmark/src/cli/tachoRun.ts @@ -1,19 +1,28 @@ -#!/usr/bin/env node +import type { CommandModule } from 'yargs'; -/** - * @typedef {Object} Args - * @property {string[]} [filters] - Only run cases whose path under `src` contains one of these substrings, case-insensitively - * @property {string} [baseline] - Binds the `baseline` symbol, in the ref grammar - * @property {string} [baseBranch] - Branch PRs fork from - * @property {string} [buildCmd] - Command that builds the publishable packages of a checked-out ref - * @property {string} [workingTreeBuildCmd] - Command that builds the working tree - * @property {boolean} [install] - Whether to install inside a ref's checkout - * @property {string} [out] - Where to write the combined JSON report - * @property {boolean} [upload] - Upload the report and refresh the pull request comment - */ +interface Args { + /** + * Only run cases whose path under `src` contains one of these substrings, case-insensitively. + */ + filters?: string[]; + /** Binds the `baseline` symbol, in the ref grammar. */ + baseline?: string; + /** Branch PRs fork from. */ + baseBranch?: string; + /** Command that builds the publishable packages of a checked-out ref. */ + buildCmd?: string; + /** Command that builds the working tree. */ + workingTreeBuildCmd?: string; + /** Whether to install inside a ref's checkout. */ + install?: boolean; + /** Where to write the combined JSON report. */ + out?: string; + /** Upload the report and refresh the pull request comment. */ + upload?: boolean; +} -export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ - command: 'tacho run [filters...]', +const command: CommandModule<{}, Args> = { + command: 'run [filters...]', describe: 'Benchmark a tachometer harness across builds of this workspace and write a JSON report. Run from the harness package.', builder: (yargs) => { @@ -60,10 +69,10 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ }) .epilogue( 'Sampling (sampleSize, autoSampleConditions, timeout) is configured per case in its own tachometer.json — tachometer rejects those as CLI flags when a config file is used.', - ); + ) as any; }, handler: async (argv) => { - const { runTachometer } = await import('../tachometer/runTachometer.mjs'); + const { runTachometer } = await import('../tachometer/runTachometer'); await runTachometer({ harnessDir: process.cwd(), filters: argv.filters ?? [], @@ -76,4 +85,6 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ upload: argv.upload, }); }, -}); +}; + +export default command; diff --git a/packages/code-infra/src/tachometer/browser.test.mjs b/packages/benchmark/src/tachometer/browser.test.ts similarity index 98% rename from packages/code-infra/src/tachometer/browser.test.mjs rename to packages/benchmark/src/tachometer/browser.test.ts index 8734e274de..8fc1d592f6 100644 --- a/packages/code-infra/src/tachometer/browser.test.mjs +++ b/packages/benchmark/src/tachometer/browser.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { majorVersionOf, withBrowserDefaults } from './browser.mjs'; +import { majorVersionOf, withBrowserDefaults } from './browser'; describe('majorVersionOf', () => { it('reads the major from Chrome for Testing output', () => { diff --git a/packages/code-infra/src/tachometer/browser.mjs b/packages/benchmark/src/tachometer/browser.ts similarity index 78% rename from packages/code-infra/src/tachometer/browser.mjs rename to packages/benchmark/src/tachometer/browser.ts index 2335b6bea7..1364da74d2 100644 --- a/packages/code-infra/src/tachometer/browser.mjs +++ b/packages/benchmark/src/tachometer/browser.ts @@ -2,7 +2,8 @@ import * as path from 'node:path'; import { createRequire } from 'node:module'; import { execFileSync } from 'node:child_process'; import chalk from 'chalk'; -import { pathExists } from '../utils/path.mjs'; +import type * as PlaywrightTest from '@playwright/test'; +import { pathExists } from '../utils/path'; /** * Which browser the benchmarks run on, and whether the driver can actually drive it. @@ -16,16 +17,12 @@ import { pathExists } from '../utils/path.mjs'; * whatever Chrome the machine has auto-updated to. * * The path is machine-specific, so callers inject it into each config instead of committing it. - * - * @param {string} harnessDir - The harness package directory, which `@playwright/test` resolves from - * @returns {Promise} Absolute path to the browser binary */ -export async function resolveBrowserBinary(harnessDir) { +export async function resolveBrowserBinary(harnessDir: string): Promise { const require = createRequire(path.join(harnessDir, 'package.json')); // `@playwright/test` is CommonJS, so `require` hands back its exports directly; a dynamic // `import()` would bury them under `default`. - /** @type {typeof import('@playwright/test')} */ - let playwright; + let playwright: typeof PlaywrightTest; try { playwright = require('@playwright/test'); } catch { @@ -45,13 +42,8 @@ export async function resolveBrowserBinary(harnessDir) { return binary; } -/** - * Extracts the major version out of output like `Google Chrome for Testing 151.0.7922.34`. - * - * @param {string} versionOutput - A version string or `--version` output - * @returns {number | undefined} - */ -export function majorVersionOf(versionOutput) { +/** Extracts the major version out of output like `Google Chrome for Testing 151.0.7922.34`. */ +export function majorVersionOf(versionOutput: string): number | undefined { const match = /(\d+)\.\d+\.\d+/.exec(versionOutput); return match ? Number(match[1]) : undefined; } @@ -62,18 +54,13 @@ export function majorVersionOf(versionOutput) { * A chromedriver only drives its own Chrome major. The driver is resolved the way tachometer * resolves it — from tachometer's own package root — so this checks the pair that will actually be * used rather than whatever else is installed. - * - * @param {string} harnessDir - The harness package directory, which tachometer resolves from - * @param {string} binary - The browser binary that will be driven - * @returns {void} */ -export function assertDriverMatchesBrowser(harnessDir, binary) { +export function assertDriverMatchesBrowser(harnessDir: string, binary: string): void { const browserMajor = majorVersionOf( execFileSync(binary, ['--version'], { encoding: 'utf8' }).trim(), ); - /** @type {string | undefined} */ - let driverVersion; + let driverVersion: string | undefined; try { const fromHarness = createRequire(path.join(harnessDir, 'package.json')); const fromTachometer = createRequire(fromHarness.resolve('tachometer/package.json')); @@ -108,9 +95,11 @@ export function assertDriverMatchesBrowser(harnessDir, binary) { /** * A benchmark's `browser` config. Only the two fields a run touches are named; everything else * tachometer accepts there (`name`, `headless`, `windowSize`, …) passes through untouched. - * - * @typedef {{ binary?: string, addArguments?: string[] } & Record} BrowserConfig */ +export type BrowserConfig = { + binary?: string; + addArguments?: string[]; +} & Record; /** Chrome cannot enter its sandbox as root, and refusing to start is all it does about it. */ const NO_SANDBOX = '--no-sandbox'; @@ -135,11 +124,8 @@ const HEADLESS_SUFFIX = '-headless'; * `headless` and `windowSize` from the defaults it would otherwise have applied, so naming the * browser is all it takes to reproduce what an absent `browser` did. The string shorthand is the * exception — it encodes `headless` itself, so that is read out of it. - * - * @param {string | BrowserConfig | undefined} browser - Whatever the case declared, if anything - * @returns {BrowserConfig} */ -function asBrowserObject(browser) { +function asBrowserObject(browser: string | BrowserConfig | undefined): BrowserConfig { if (browser === undefined) { return { name: DEFAULT_BROWSER }; } @@ -150,7 +136,10 @@ function asBrowserObject(browser) { const at = browser.indexOf('@'); const named = at === -1 ? browser : browser.slice(0, at); const headless = named.endsWith(HEADLESS_SUFFIX); - const base = { name: headless ? named.slice(0, -HEADLESS_SUFFIX.length) : named, headless }; + const base: BrowserConfig = { + name: headless ? named.slice(0, -HEADLESS_SUFFIX.length) : named, + headless, + }; return at === -1 ? base : { ...base, remoteUrl: browser.slice(at + 1) }; } @@ -162,15 +151,14 @@ function asBrowserObject(browser) { * exits during launch without explaining itself, leaving the driver to report nothing more than * `session not created: Chrome instance exited`. Containerised CI runs as root, so the flag goes * on there and nowhere else — a local run keeps the sandbox. - * - * @param {string | BrowserConfig | undefined} browser - The case's own browser config, if it has one - * @param {string} binary - Binary to drive, unless the case pins its own - * @param {boolean} asRoot - Whether the run is happening as root - * @returns {BrowserConfig} */ -export function withBrowserDefaults(browser, binary, asRoot) { +export function withBrowserDefaults( + browser: string | BrowserConfig | undefined, + binary: string, + asRoot: boolean, +): BrowserConfig { const base = asBrowserObject(browser); - const merged = { ...base, binary: base.binary ?? binary }; + const merged: BrowserConfig = { ...base, binary: base.binary ?? binary }; const addArguments = merged.addArguments ?? []; if (asRoot && !addArguments.includes(NO_SANDBOX)) { merged.addArguments = [...addArguments, NO_SANDBOX]; diff --git a/packages/code-infra/src/tachometer/buildPages.mjs b/packages/benchmark/src/tachometer/buildPages.ts similarity index 73% rename from packages/code-infra/src/tachometer/buildPages.mjs rename to packages/benchmark/src/tachometer/buildPages.ts index ef3e08e111..3c43a9cbb1 100644 --- a/packages/code-infra/src/tachometer/buildPages.mjs +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -5,10 +5,14 @@ import { createRequire } from 'node:module'; import { readFile, mkdir, writeFile } from 'node:fs/promises'; import { pathToFileURL } from 'node:url'; import chalk from 'chalk'; +import type * as Vite from 'vite'; +import type { Plugin } from 'vite'; import { parse, stringify } from 'yaml'; -import { run } from '../utils/exec.mjs'; -import { tarballFor } from '../utils/packWorkspace.mjs'; -import { readPackageJson, writePackageJson } from '../utils/pnpm.mjs'; +import { run } from '../utils/exec'; +import { tarballFor } from '../utils/packWorkspace'; +import type { PackedPackage } from '../utils/packWorkspace'; +import { readPackageJson, writePackageJson } from '../utils/pnpm'; +import type { ResolvedRef } from './refs'; /** * Dependencies the run needs but a page never imports, so a ref's tree does without them. @@ -25,13 +29,23 @@ const RUNNER_ONLY_DEPS = [ '@playwright/test', // Read rather than spelled out: this is the one entry naming *this* package, and a rename that // left a literal behind would quietly restore a multi-minute install per ref. - createRequire(import.meta.url)('../../package.json').name, + createRequire(import.meta.url)('../../package.json').name as string, ]; /** - * @typedef {import('./refs.mjs').ResolvedRef} ResolvedRef - * @typedef {import('../utils/packWorkspace.mjs').PackedPackage} PackedPackage + * The version of `name` the harness actually has installed, or `undefined` when it has none. + * + * Read from the harness's own `node_modules` rather than a lockfile: pnpm links every direct + * dependency there, and these maps only ever hold direct dependencies. */ +async function installedVersionOf(harnessDir: string, name: string): Promise { + try { + const manifest = path.join(harnessDir, 'node_modules', ...name.split('/'), 'package.json'); + return JSON.parse(await readFile(manifest, 'utf8')).version; + } catch { + return undefined; + } +} /** * Rewrites a dependency map, pointing every `workspace:`-protocol entry at its packed tarball and @@ -48,21 +62,18 @@ const RUNNER_ONLY_DEPS = [ * float to whatever is newest at run time: a different library than the repository builds against, * quietly measured as if it were ours. A release that also drops an export fails the build outright, * which is the visible half of the same problem. - * - * @param {string} harnessDir - The harness package directory, whose installed versions are read - * @param {Partial> | undefined} deps - A `dependencies` or `devDependencies` map - * @param {PackedPackage[]} packages - The packed workspace packages - * @param {string[]} [omit] - Dependency names to drop entirely - * @returns {Promise>} */ -async function rewriteWorkspaceDeps(harnessDir, deps, packages, omit = []) { - const entries = /** @type {Array<[string, string]>} */ ( - Object.entries(deps ?? {}).filter( - ([name, version]) => version !== undefined && !omit.includes(name), - ) - ); +async function rewriteWorkspaceDeps( + harnessDir: string, + deps: Partial> | undefined, + packages: PackedPackage[], + omit: string[] = [], +): Promise> { + const entries = Object.entries(deps ?? {}).filter( + ([name, version]) => version !== undefined && !omit.includes(name), + ) as Array<[string, string]>; const resolved = await Promise.all( - entries.map(async ([name, version]) => { + entries.map(async ([name, version]): Promise<[string, string]> => { if (version.startsWith('workspace:')) { return [name, `file:${tarballFor(packages, name)}`]; } @@ -73,25 +84,6 @@ async function rewriteWorkspaceDeps(harnessDir, deps, packages, omit = []) { return Object.fromEntries(resolved); } -/** - * The version of `name` the harness actually has installed, or `undefined` when it has none. - * - * Read from the harness's own `node_modules` rather than a lockfile: pnpm links every direct - * dependency there, and these maps only ever hold direct dependencies. - * - * @param {string} harnessDir - The harness package directory - * @param {string} name - A dependency name - * @returns {Promise} - */ -async function installedVersionOf(harnessDir, name) { - try { - const manifest = path.join(harnessDir, 'node_modules', ...name.split('/'), 'package.json'); - return JSON.parse(await readFile(manifest, 'utf8')).version; - } catch { - return undefined; - } -} - /** * Reads the repository's `overrides`, so the ref's tree resolves the same dependency graph the * repository does. @@ -100,11 +92,8 @@ async function installedVersionOf(harnessDir, name) { * transitive dependency through an override would get a *different* graph here than in its real * install — quietly changing what is being benchmarked — and any policy the override exists to * satisfy (a blocked resolution, a security pin) would fail here too. - * - * @param {string} repoRoot - The workspace root to copy overrides from - * @returns {Promise>} */ -async function readOverrides(repoRoot) { +async function readOverrides(repoRoot: string): Promise> { try { const root = parse(await readFile(path.join(repoRoot, 'pnpm-workspace.yaml'), 'utf8')); return root?.overrides ?? {}; @@ -120,11 +109,8 @@ async function readOverrides(repoRoot) { * where it is. Resolution is handed back to vite with the importer rewritten, so vite's own * conditions (`browser`, `import`) and the target package's `exports` map still decide the answer — * which is what an alias to a directory would skip. - * - * @param {string} treeDir - The ref's installed tree - * @returns {import('vite').Plugin} */ -function resolveFromTree(treeDir) { +function resolveFromTree(treeDir: string): Plugin { // Resolution needs a base directory, and vite derives one from the importer's path — falling back // to the project root unless that path exists on disk, which would silently resolve the harness's // copy of a dependency instead of this ref's. The tree's own manifest is the file that names this @@ -132,7 +118,7 @@ function resolveFromTree(treeDir) { const importer = path.join(treeDir, 'package.json'); return { - name: 'mui-code-infra:tachometer-resolve', + name: 'mui-benchmark:tachometer-resolve', enforce: 'pre', async resolveId(id, _importer, options) { if (id.startsWith('.') || path.isAbsolute(id) || id.startsWith('\0') || id.includes(':')) { @@ -143,17 +129,22 @@ function resolveFromTree(treeDir) { }; } -/** - * Installs one ref's packed build, with the harness's own dependencies around it. - * - * @param {Object} options - Install inputs - * @param {string} options.harnessDir - The harness package directory, whose dependencies are copied - * @param {string} options.repoRoot - The workspace root, whose overrides the tree inherits - * @param {PackedPackage[]} options.packages - That ref's packed workspace packages - * @param {string} options.treeDir - Where to install - * @returns {Promise} - */ -async function installRefTree({ harnessDir, repoRoot, packages, treeDir }) { +/** Installs one ref's packed build, with the harness's own dependencies around it. */ +async function installRefTree({ + harnessDir, + repoRoot, + packages, + treeDir, +}: { + /** The harness package directory, whose dependencies are copied. */ + harnessDir: string; + /** The workspace root, whose overrides the tree inherits. */ + repoRoot: string; + /** That ref's packed workspace packages. */ + packages: PackedPackage[]; + /** Where to install. */ + treeDir: string; +}): Promise { const harnessPkg = await readPackageJson(harnessDir); await mkdir(treeDir, { recursive: true }); @@ -208,17 +199,21 @@ async function installRefTree({ harnessDir, repoRoot, packages, treeDir }) { * (and everything else) to one copy. Building one side through a workspace link instead would * compare two different resolution paths, and that difference would land in the measurement rather * than in the library. - * - * @param {Object} options - Build inputs - * @param {string} options.harnessDir - The harness package directory, built in place - * @param {string} options.repoRoot - The workspace root, whose overrides the tree inherits - * @param {ResolvedRef} options.ref - The ref being built - * @param {PackedPackage[]} options.packages - That ref's packed workspace packages - * @param {string} options.treeDir - Persistent directory holding this ref's installed tree - * @param {string} options.outDir - Absolute output directory for the built pages - * @returns {Promise} */ -export async function buildRefPages(options) { +export async function buildRefPages(options: { + /** The harness package directory, built in place. */ + harnessDir: string; + /** The workspace root, whose overrides the tree inherits. */ + repoRoot: string; + /** The ref being built. */ + ref: ResolvedRef; + /** That ref's packed workspace packages. */ + packages: PackedPackage[]; + /** Persistent directory holding this ref's installed tree. */ + treeDir: string; + /** Absolute output directory for the built pages. */ + outDir: string; +}): Promise { const { harnessDir, repoRoot, ref, packages, treeDir, outDir } = options; console.log(chalk.cyan(`\nBuilding benchmark pages for "${ref.label}"…`)); @@ -227,8 +222,7 @@ export async function buildRefPages(options) { // vite is the harness's own, not this package's: the harness declares the version its pages are // built with, and the config being run is the harness's. const requireFromHarness = createRequire(path.join(harnessDir, 'package.json')); - /** @type {typeof import('vite')} */ - const vite = await import(pathToFileURL(requireFromHarness.resolve('vite')).href); + const vite: typeof Vite = await import(pathToFileURL(requireFromHarness.resolve('vite')).href); // `root` only points vite's config discovery at the harness; the harness's own plugin then moves // it to `src/`, the same as for a plain `vite build`. diff --git a/packages/code-infra/src/tachometer/ciReport.test.mjs b/packages/benchmark/src/tachometer/ciReport.test.ts similarity index 98% rename from packages/code-infra/src/tachometer/ciReport.test.mjs rename to packages/benchmark/src/tachometer/ciReport.test.ts index f210f05ce2..96e99fac7e 100644 --- a/packages/code-infra/src/tachometer/ciReport.test.mjs +++ b/packages/benchmark/src/tachometer/ciReport.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { tachometerUploadSchema } from './ciReport.mjs'; +import { tachometerUploadSchema } from './ciReport'; /** * A report of the shape `runTachometer` writes, trimmed to one case. diff --git a/packages/benchmark/src/tachometer/ciReport.ts b/packages/benchmark/src/tachometer/ciReport.ts new file mode 100644 index 0000000000..85c1ff3a85 --- /dev/null +++ b/packages/benchmark/src/tachometer/ciReport.ts @@ -0,0 +1,123 @@ +/* eslint-disable no-console -- progress belongs in the CI log. */ + +import { z } from 'zod/v4'; +import { ciReportUploadSchema } from '../ciReport'; + +/** + * Uploading a tachometer report to the CI report store. + * + * The envelope, the CI metadata and the pull request sync are the package's own — this axis differs + * from the Vitest one only in the shape of `report`. Consumers reach the shared halves through + * {@link getCiMetadata} and {@link syncPrComment}, re-exported here so a runner has one import. + */ + +export { getCiMetadata } from '../ciReport'; +export { syncPrComment } from '../syncPrComment'; + +const DEFAULT_API_URL = 'https://frontend-public.mui.com'; + +const confidenceIntervalSchema = z.object({ + low: z.number(), + high: z.number(), +}); + +const differenceSchema = z.object({ + verdict: z.enum(['faster', 'slower', 'unsure']), + absoluteMs: confidenceIntervalSchema, + percentChange: confidenceIntervalSchema, +}); + +const variantResultSchema = z.object({ + variant: z.string(), + refId: z.string().nullable(), + meanMs: confidenceIntervalSchema, + samples: z.number(), + bytesSent: z.number(), +}); + +const comparisonSchema = differenceSchema.extend({ + variant: z.string(), + versusReference: differenceSchema.optional(), +}); + +const measurementResultSchema = z.object({ + name: z.string(), + variants: z.array(variantResultSchema), + comparisons: z.array(comparisonSchema), +}); + +// A case that failed to summarize carries an `error` and no measurements. It is kept in the report +// rather than dropped, so the comment can say a case produced nothing instead of quietly omitting it. +const caseResultSchema = z.object({ + name: z.string(), + reference: z.string().optional(), + measurements: z.array(measurementResultSchema).optional(), + error: z.string().optional(), +}); + +const tachometerReportSchema = z.object({ + version: z.number(), + reportType: z.literal('tachometer'), + generatedAt: z.string(), + head: z.object({ ref: z.string(), sha: z.string(), branch: z.string().optional() }), + browser: z.string().optional(), + refs: z.array( + z.object({ + id: z.string(), + kind: z.string(), + label: z.string(), + sha: z.string().optional(), + }), + ), + cases: z.array(caseResultSchema), + // Every sample tachometer produced. Not read by the comment, but kept so a richer view can be + // built later without changing what CI uploads. + raw: z.record(z.string(), z.unknown()), +}); + +/** + * The upload envelope. Built from the same factory the Vitest axis uses, so both report types reach + * the dashboard's upload route in the shape it expects. + */ +export const tachometerUploadSchema = ciReportUploadSchema( + 'tachometer', + 1, + tachometerReportSchema, +).extend({ + report: tachometerReportSchema, +}); + +export type TachometerUpload = z.infer; +export type TachometerReport = z.infer; + +/** + * Uploads a report to the CI report store. + * + * Validates before sending because the server stores `report` without inspecting it — a malformed + * report would be accepted here and only fail much later, when the dashboard tried to render it. + */ +export async function uploadCiReport(upload: TachometerUpload): Promise { + tachometerUploadSchema.parse(upload); + + const oidcToken = process.env.CIRCLE_OIDC_TOKEN_V2; + if (!oidcToken) { + throw new Error('CIRCLE_OIDC_TOKEN_V2 environment variable is required for uploads'); + } + + const url = new URL('/api/ci-reports/upload', process.env.CI_REPORT_API_URL ?? DEFAULT_API_URL); + console.log(`Uploading tachometer report to ${url.href}`); + + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${oidcToken}` }, + body: JSON.stringify(upload), + }); + + const responseText = await response.text(); + if (!response.ok) { + throw new Error(`The tachometer upload failed (${response.status}): ${responseText}`); + } + + const result = JSON.parse(responseText); + console.log(`Tachometer report uploaded. S3 key: ${result.key}`); +} diff --git a/packages/code-infra/src/tachometer/discoverCases.test.mjs b/packages/benchmark/src/tachometer/discoverCases.test.ts similarity index 97% rename from packages/code-infra/src/tachometer/discoverCases.test.mjs rename to packages/benchmark/src/tachometer/discoverCases.test.ts index 70806062ba..4f799ecb62 100644 --- a/packages/code-infra/src/tachometer/discoverCases.test.mjs +++ b/packages/benchmark/src/tachometer/discoverCases.test.ts @@ -1,16 +1,17 @@ import * as path from 'node:path'; import { mkdir, writeFile } from 'node:fs/promises'; import { describe, expect, it } from 'vitest'; -import { makeTempDir } from '../utils/testUtils.mjs'; -import { discoverCases, measurementNameOf, pagesOf } from './discoverCases.mjs'; - -/** - * Builds a throwaway harness: `src//tachometer.json` plus whatever pages each case owns. - * - * @param {Record} cases - Case folders to create - * @returns {Promise} The harness directory - */ -async function makeHarness(cases) { +import { makeTempDir } from '../utils/testUtils'; +import { discoverCases, measurementNameOf, pagesOf } from './discoverCases'; + +/** A case folder to create in a throwaway harness. */ +interface CaseFixture { + config: any; + pages?: string[]; +} + +/** Builds a throwaway harness: `src//tachometer.json` plus whatever pages each case owns. */ +async function makeHarness(cases: Record): Promise { const harnessDir = await makeTempDir(); await Promise.all( Object.entries(cases).map(async ([name, { config, pages = [] }]) => { @@ -25,14 +26,8 @@ async function makeHarness(cases) { return harnessDir; } -/** - * A minimal single-benchmark config. - * - * @param {string} name - Benchmark name - * @param {string} url - Page url - * @returns {any} - */ -function config(name, url) { +/** A minimal single-benchmark config. */ +function config(name: string, url: string): any { return { $schema: '../../node_modules/tachometer/config.schema.json', benchmarks: [{ name, url }], diff --git a/packages/code-infra/src/tachometer/discoverCases.mjs b/packages/benchmark/src/tachometer/discoverCases.ts similarity index 73% rename from packages/code-infra/src/tachometer/discoverCases.mjs rename to packages/benchmark/src/tachometer/discoverCases.ts index c8c99f2159..587f03aca7 100644 --- a/packages/code-infra/src/tachometer/discoverCases.mjs +++ b/packages/benchmark/src/tachometer/discoverCases.ts @@ -2,8 +2,9 @@ import * as path from 'node:path'; import { readFile } from 'node:fs/promises'; import { pathToFileURL } from 'node:url'; import { globby } from 'globby'; -import { pathExists } from '../utils/path.mjs'; -import { parseRefToken } from './refs.mjs'; +import { pathExists } from '../utils/path'; +import { parseRefToken } from './refs'; +import type { ResolvedRef } from './refs'; /** * Case discovery: the `tachometer.json` files under a harness's `src/` are the source of truth for @@ -14,40 +15,50 @@ import { parseRefToken } from './refs.mjs'; * plain `vite build` must not require a checkout with full history. */ -/** - * @typedef {import('./refs.mjs').ResolvedRef} ResolvedRef - */ - -/** - * @typedef {Object} Leaf - * @property {{ url?: string }} node - The config node whose `url` this is; rewritten in place once builds exist - * @property {string} page - Page path relative to `src`, e.g. `data-grid-init/index.html` - * @property {ResolvedRef | null} ref - The resolved ref, or null when discovery ran without `resolveRef` - * @property {string} suffix - Everything after the page path once `ref` was consumed: remaining query plus any fragment - */ +export interface Leaf { + /** The config node whose `url` this is; rewritten in place once builds exist. */ + node: { url?: string }; + /** Page path relative to `src`, e.g. `data-grid-init/index.html`. */ + page: string; + /** The resolved ref, or null when discovery ran without `resolveRef`. */ + ref: ResolvedRef | null; + /** Everything after the page path once `ref` was consumed: remaining query plus any fragment. */ + suffix: string; +} -/** - * @typedef {Object} CaseVariant - * @property {string} name - The variant's name, as tachometer will report it - * @property {string | null} refId - The ref this variant loads, or null when refs were not resolved - */ +export interface CaseVariant { + /** The variant's name, as tachometer will report it. */ + name: string; + /** The ref this variant loads, or null when refs were not resolved. */ + refId: string | null; +} -/** - * @typedef {Object} BenchmarkCase - * @property {string} name - The case name, as its benchmark declares it - * @property {string} configPath - Absolute path to its `tachometer.json` - * @property {any} config - The parsed config, mutated in place as urls are rewritten - * @property {Leaf[]} leaves - Every node that selects a page - * @property {CaseVariant[]} variants - Declared variants in order; the first is every comparison's reference - * @property {string[]} measurements - Measurement names, as tachometer will name them in its output - */ +export interface BenchmarkCase { + /** The case name, as its benchmark declares it. */ + name: string; + /** Absolute path to its `tachometer.json`. */ + configPath: string; + /** The parsed config, mutated in place as urls are rewritten. */ + config: any; + /** Every node that selects a page. */ + leaves: Leaf[]; + /** Declared variants in order; the first is every comparison's reference. */ + variants: CaseVariant[]; + /** Measurement names, as tachometer will name them in its output. */ + measurements: string[]; +} -/** - * @typedef {Object} DiscoverCasesOptions - * @property {string} harnessDir - The harness package directory; cases live under its `src/` - * @property {string[]} [filters] - Only include cases whose path under `src` contains one of these substrings, case-insensitively - * @property {(token?: string) => ResolvedRef} [resolveRef] - Resolves a ref token. Omit to skip resolution entirely (no git) - */ +export interface DiscoverCasesOptions { + /** The harness package directory; cases live under its `src/`. */ + harnessDir: string; + /** + * Only include cases whose path under `src` contains one of these substrings, + * case-insensitively. + */ + filters?: string[]; + /** Resolves a ref token. Omit to skip resolution entirely (no git). */ + resolveRef?: (token?: string) => ResolvedRef; +} /** The file that marks a directory under `src/` as a benchmark case. */ const CONFIG_NAME = 'tachometer.json'; @@ -62,12 +73,8 @@ const DEFAULT_MEASUREMENT_EXPRESSION = 'window.tachometerResult'; * With more than one measurement on a page, tachometer appends ` []` to each benchmark's * name, and results have to be paired back by that name — pairing by position would compare * unrelated measurements. - * - * @param {any} measurement - A tachometer `measurement` entry - * @param {string} [measurementExpression] - The benchmark's own `measurementExpression`, which the `global` shorthand is named by - * @returns {string} */ -export function measurementNameOf(measurement, measurementExpression) { +export function measurementNameOf(measurement: any, measurementExpression?: string): string { if (typeof measurement === 'string') { // Tachometer's string shorthands. `callback` and `fcp` name their results the same way the // shorthand reads, but `global` expands to an *expression* measurement that is named by the @@ -102,11 +109,8 @@ export function measurementNameOf(measurement, measurementExpression) { * `expand` itself is dropped. Tachometer leaves the parent's on the merged object, where it is inert * because expansion has already happened — but the config written from this is parsed afresh, and a * benchmark still carrying `expand` would be expanded a second time. - * - * @param {any} benchmark - A benchmark, or a node within its `expand` tree - * @returns {any[]} The benchmarks it expands to, in tachometer's order */ -function flattenExpansions(benchmark) { +function flattenExpansions(benchmark: any): any[] { const { expand, ...rest } = benchmark; if (!Array.isArray(expand) || expand.length === 0) { return [rest]; @@ -119,14 +123,13 @@ function flattenExpansions(benchmark) { * * `ref` selects the build and is consumed here; everything else belongs to the page and has to * survive the rewrite, or a parameterised benchmark would silently run its defaults. - * - * @param {string} url - The leaf's url - * @param {string} configDir - Directory the config lives in; relative urls resolve against it - * @param {string} srcDir - The harness `src/` directory - * @param {((token?: string) => ResolvedRef) | undefined} resolveRef - Ref resolver, or undefined to skip resolution - * @returns {Promise<{ page: string, ref: ResolvedRef | null, suffix: string }>} */ -async function parseLeafUrl(url, configDir, srcDir, resolveRef) { +async function parseLeafUrl( + url: string, + configDir: string, + srcDir: string, + resolveRef: ((token?: string) => ResolvedRef) | undefined, +): Promise<{ page: string; ref: ResolvedRef | null; suffix: string }> { const parsed = new URL(url, pathToFileURL(path.join(configDir, path.sep))); const token = parsed.searchParams.get('ref') ?? undefined; // Validate the grammar even when not resolving, so a typo fails the build rather than silently @@ -155,11 +158,8 @@ async function parseLeafUrl(url, configDir, srcDir, resolveRef) { * Nested rather than flat, so a harness can group its cases in folders — which is the whole of the * partitioning story: the filter matches these paths, and nothing else has to understand what a * folder means. - * - * @param {string} srcDir - The harness's `src` directory - * @returns {Promise} Case locations, e.g. `workload` or `libs/mount` */ -async function findCaseLocations(srcDir) { +async function findCaseLocations(srcDir: string): Promise { const configs = await globby(`**/${CONFIG_NAME}`, { cwd: srcDir }); return ( configs @@ -178,11 +178,8 @@ async function findCaseLocations(srcDir) { * A benchmark with no `expand` is the common regression case: it is expanded into `[current]` (the * working tree) versus `[baseline]`. A benchmark that declares its own `expand` owns its variant * axis — only the refs its leaves reference are resolved. - * - * @param {DiscoverCasesOptions} options - Where to look and how to resolve refs - * @returns {Promise} */ -export async function discoverCases(options) { +export async function discoverCases(options: DiscoverCasesOptions): Promise { const { harnessDir, filters = [], resolveRef } = options; const srcDir = path.join(harnessDir, 'src'); @@ -212,12 +209,12 @@ export async function discoverCases(options) { // The benchmark names itself, so a case keeps its identity wherever its folder is moved to, // and the ` []` prefix that tachometer builds from that name is strippable for // display by construction rather than by the folder happening to agree with it. - const name = config.benchmarks?.[0]?.name ?? path.basename(location); + const name: string = config.benchmarks?.[0]?.name ?? path.basename(location); return { name, configPath, config }; }), ); - const byName = new Map(); + const byName = new Map(); for (const entry of configs) { const clash = byName.get(entry.name); if (clash) { @@ -229,15 +226,12 @@ export async function discoverCases(options) { byName.set(entry.name, entry.configPath); } - /** @type {BenchmarkCase[]} */ - const cases = []; + const cases: BenchmarkCase[] = []; for (const { name, configPath, config } of configs) { const configDir = path.dirname(configPath); - /** @type {Set} */ - const measurements = new Set(); - /** @type {any[]} */ - const benchmarks = []; + const measurements = new Set(); + const benchmarks: any[] = []; for (const benchmark of config.benchmarks ?? []) { // A benchmark need not name itself, and the case name is the same fallback its own name gets // — without it the auto-expanded variants would read "undefined [current]". Set before @@ -281,7 +275,7 @@ export async function discoverCases(options) { ); const leaves = benchmarks.map((node, index) => ({ node, ...resolved[index] })); const variants = benchmarks.map((node, index) => ({ - name: node.name, + name: node.name as string, refId: resolved[index].ref?.id ?? null, })); @@ -297,12 +291,7 @@ export async function discoverCases(options) { return cases; } -/** - * The distinct pages referenced by a set of cases, as paths relative to `src`, sorted. - * - * @param {BenchmarkCase[]} cases - Discovered cases - * @returns {string[]} - */ -export function pagesOf(cases) { +/** The distinct pages referenced by a set of cases, as paths relative to `src`, sorted. */ +export function pagesOf(cases: BenchmarkCase[]): string[] { return [...new Set(cases.flatMap((entry) => entry.leaves.map((leaf) => leaf.page)))].sort(); } diff --git a/packages/code-infra/src/tachometer/outputDir.mjs b/packages/benchmark/src/tachometer/outputDir.ts similarity index 80% rename from packages/code-infra/src/tachometer/outputDir.mjs rename to packages/benchmark/src/tachometer/outputDir.ts index 47b1652d56..fee71d0e3e 100644 --- a/packages/code-infra/src/tachometer/outputDir.mjs +++ b/packages/benchmark/src/tachometer/outputDir.ts @@ -15,11 +15,8 @@ export const OUTPUT_DIR = '.tachometer'; * cache, so a harness that forgets to list it does not end up offering built pages and tarballs for * commit. It does not replace an entry in the repository's own ignore file: tools that read only * the root one — this repository's ESLint config among them — never see a nested `.gitignore`. - * - * @param {string} harnessDir - The harness package directory - * @returns {Promise} The output directory */ -export async function prepareOutputDir(harnessDir) { +export async function prepareOutputDir(harnessDir: string): Promise { const outputDir = path.join(harnessDir, OUTPUT_DIR); await mkdir(outputDir, { recursive: true }); await writeFile(path.join(outputDir, '.gitignore'), '*\n'); @@ -32,12 +29,8 @@ export async function prepareOutputDir(harnessDir) { * Named here rather than composed at each call site because the runner and the vite plugin have to * agree on it: the runner points tachometer's static server at the output directory and rewrites * each url under this path, and a plain `vite build` has to land somewhere that does not collide. - * - * @param {string} harnessDir - The harness package directory - * @param {string} [refId] - A ref's build directory name, e.g. `current` - * @returns {string} */ -export function buildsDirOf(harnessDir, refId) { +export function buildsDirOf(harnessDir: string, refId?: string): string { const builds = path.join(harnessDir, OUTPUT_DIR, 'builds'); return refId ? path.join(builds, refId) : builds; } diff --git a/packages/code-infra/src/tachometer/plugin.test.mjs b/packages/benchmark/src/tachometer/plugin.test.ts similarity index 87% rename from packages/code-infra/src/tachometer/plugin.test.mjs rename to packages/benchmark/src/tachometer/plugin.test.ts index 773e8b8689..49175b5582 100644 --- a/packages/code-infra/src/tachometer/plugin.test.mjs +++ b/packages/benchmark/src/tachometer/plugin.test.ts @@ -1,18 +1,23 @@ import * as path from 'node:path'; import { mkdir, symlink, writeFile } from 'node:fs/promises'; import { describe, expect, it } from 'vitest'; -import { makeTempDir } from '../utils/testUtils.mjs'; -import { tachometer } from './plugin.mjs'; - -/** - * Builds a throwaway harness with a package.json, so the plugin can read its dependencies. - * - * @param {Object} options - Harness contents - * @param {Record} options.cases - Case folders to create - * @param {Record} [options.dependencies] - The harness's dependencies - * @returns {Promise} The harness directory - */ -async function makeHarness({ cases, dependencies = {} }) { +import { makeTempDir } from '../utils/testUtils'; +import { tachometer } from './plugin'; + +/** A case folder to create in a throwaway harness. */ +interface CaseFixture { + config: any; + pages?: string[]; +} + +/** Builds a throwaway harness with a package.json, so the plugin can read its dependencies. */ +async function makeHarness({ + cases, + dependencies = {}, +}: { + cases: Record; + dependencies?: Record; +}): Promise { const harnessDir = await makeTempDir(); await writeFile( path.join(harnessDir, 'package.json'), @@ -31,16 +36,14 @@ async function makeHarness({ cases, dependencies = {} }) { return harnessDir; } -/** - * Invokes the plugin's `config` hook. - * - * @param {string} harnessDir - The harness directory - * @param {'build' | 'serve'} command - Which vite command is running - * @returns {Promise} - */ -async function callConfig(harnessDir, command, userConfig = {}) { +/** Invokes the plugin's `config` hook. */ +async function callConfig( + harnessDir: string, + command: 'build' | 'serve', + userConfig: any = {}, +): Promise { const plugin = tachometer({ harnessDir }); - const hook = /** @type {any} */ (plugin.config); + const hook = plugin.config as any; return hook(userConfig, { command, mode: command === 'build' ? 'production' : 'development' }); } diff --git a/packages/code-infra/src/tachometer/plugin.mjs b/packages/benchmark/src/tachometer/plugin.ts similarity index 84% rename from packages/code-infra/src/tachometer/plugin.mjs rename to packages/benchmark/src/tachometer/plugin.ts index 6d38340710..a90bf00eca 100644 --- a/packages/code-infra/src/tachometer/plugin.mjs +++ b/packages/benchmark/src/tachometer/plugin.ts @@ -1,7 +1,9 @@ import * as path from 'node:path'; import { readFile, realpath } from 'node:fs/promises'; -import { discoverCases, pagesOf } from './discoverCases.mjs'; -import { buildsDirOf } from './outputDir.mjs'; +import type { Plugin } from 'vite'; +import { discoverCases, pagesOf } from './discoverCases'; +import type { BenchmarkCase } from './discoverCases'; +import { buildsDirOf } from './outputDir'; /** * Vite plugin for a tachometer benchmark harness. @@ -19,10 +21,13 @@ import { buildsDirOf } from './outputDir.mjs'; * It deliberately never resolves refs, so a plain `vite build` needs no git history. */ -/** - * @typedef {Object} TachometerPluginOptions - * @property {string} [harnessDir] - The harness package directory. Defaults to the current working directory, which is where vite is run from - */ +export interface TachometerPluginOptions { + /** + * The harness package directory. Defaults to the current working directory, which is where vite + * is run from. + */ + harnessDir?: string; +} /** * Checks that the workspace packages the harness links to have actually been built. @@ -31,12 +36,11 @@ import { buildsDirOf } from './outputDir.mjs'; * makes that link point at the package's build output — which does not exist until the library is * built. The symlink is then dangling, and vite's failure ("failed to resolve import") does not * say why. - * - * @param {string} harnessDir - The harness package directory - * @param {Record} deps - The harness's combined dependency map - * @returns {Promise} */ -async function assertWorkspaceDepsBuilt(harnessDir, deps) { +async function assertWorkspaceDepsBuilt( + harnessDir: string, + deps: Record, +): Promise { const linked = Object.entries(deps).filter(([, version]) => version.startsWith('workspace:')); await Promise.all( linked.map(async ([name]) => { @@ -54,14 +58,10 @@ async function assertWorkspaceDepsBuilt(harnessDir, deps) { } /** - * Renders the dev-server index: one link per case, carrying whatever query the case parameterises - * its page with. - * - * @param {import('./discoverCases.mjs').BenchmarkCase[]} cases - Discovered cases - * @param {string} note - What the pages behind these links are, which differs per command - * @returns {string} An HTML document + * Renders the case index: one link per case, carrying whatever query the case parameterises its + * page with. */ -function renderIndex(cases, note) { +function renderIndex(cases: BenchmarkCase[], note: string): string { const items = cases .map((entry) => { // A case's variants can point at several pages (a cross-library comparison) or at one page @@ -98,20 +98,14 @@ ${items} `; } -/** - * Creates the tachometer harness plugin. - * - * @param {TachometerPluginOptions} [options] - Plugin options - * @returns {import('vite').Plugin} - */ -export function tachometer(options = {}) { +/** Creates the tachometer harness plugin. */ +export function tachometer(options: TachometerPluginOptions = {}): Plugin { const harnessDir = options.harnessDir ?? process.cwd(); const srcDir = path.join(harnessDir, 'src'); - /** @type {import('./discoverCases.mjs').BenchmarkCase[]} */ - let buildCases = []; + let buildCases: BenchmarkCase[] = []; return { - name: 'mui-code-infra:tachometer', + name: 'mui-benchmark:tachometer', async config(userConfig, env) { const pkg = JSON.parse(await readFile(path.join(harnessDir, 'package.json'), 'utf8')); diff --git a/packages/code-infra/src/tachometer/refs.test.mjs b/packages/benchmark/src/tachometer/refs.test.ts similarity index 97% rename from packages/code-infra/src/tachometer/refs.test.mjs rename to packages/benchmark/src/tachometer/refs.test.ts index 28d7825671..428cc82bc3 100644 --- a/packages/code-infra/src/tachometer/refs.test.mjs +++ b/packages/benchmark/src/tachometer/refs.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { parseRefToken } from './refs.mjs'; +import { parseRefToken } from './refs'; describe('parseRefToken', () => { it('treats an absent ref as the working tree', () => { diff --git a/packages/code-infra/src/tachometer/refs.mjs b/packages/benchmark/src/tachometer/refs.ts similarity index 68% rename from packages/code-infra/src/tachometer/refs.mjs rename to packages/benchmark/src/tachometer/refs.ts index 76dbe2d7f3..545c8bb5a8 100644 --- a/packages/code-infra/src/tachometer/refs.mjs +++ b/packages/benchmark/src/tachometer/refs.ts @@ -11,45 +11,49 @@ import { execaSync } from 'execa'; * - {@link createRefResolver} turns a descriptor into a {@link ResolvedRef} with an immutable SHA. */ -/** - * @typedef {Object} RefDescriptor - * @property {'worktree'|'baseline'|'git'} kind - Which scheme the token used - * @property {string} [committish] - The revision, for `git:` refs - */ +export interface RefDescriptor { + /** Which scheme the token used. */ + kind: 'worktree' | 'baseline' | 'git'; + /** The revision, for `git:` refs. */ + committish?: string; +} -/** - * @typedef {Object} ResolvedRef - * @property {'worktree'|'git'} kind - Whether this is the working tree or a committed revision - * @property {string} id - Build directory name, e.g. `current` or `git-230342ee2`. Doubles as the dedupe identity - * @property {string} label - Human-readable label for logs and the report - * @property {string} [sha] - The immutable commit this ref denotes. Absent only for the working tree - * @property {string} [committish] - The committish to hand to `packRef`. Absent only for the working tree - */ +export interface ResolvedRef { + /** Whether this is the working tree or a committed revision. */ + kind: 'worktree' | 'git'; + /** Build directory name, e.g. `current` or `git-230342ee2`. Doubles as the dedupe identity. */ + id: string; + /** Human-readable label for logs and the report. */ + label: string; + /** The immutable commit this ref denotes. Absent only for the working tree. */ + sha?: string; + /** The committish to hand to `packRef`. Absent only for the working tree. */ + committish?: string; +} -/** - * @typedef {Object} RefResolverOptions - * @property {string} repoRoot - Repository to resolve revisions in - * @property {string} [baseBranch] - Branch PRs fork from. Defaults to detection via `origin/HEAD`, then `master` - * @property {string} [baselineOverride] - Binds the `baseline` symbol, in the ref grammar (e.g. `git:abc1234`) - */ +export interface RefResolverOptions { + /** Repository to resolve revisions in. */ + repoRoot: string; + /** Branch PRs fork from. Defaults to detection via `origin/HEAD`, then `master`. */ + baseBranch?: string; + /** Binds the `baseline` symbol, in the ref grammar (e.g. `git:abc1234`). */ + baselineOverride?: string; +} /** The working tree — never cached, since it has no immutable identity. */ -const WORKTREE_REF = /** @type {ResolvedRef} */ ({ +const WORKTREE_REF: ResolvedRef = { kind: 'worktree', id: 'current', label: 'working tree', -}); +}; /** * Parses a `?ref=` token into a descriptor, validating the grammar. Makes no git calls. * * Bare values are never auto-prefixed: an absent ref already means "working tree", so letting a * bare value mean a git revision would give one token two meanings depending on where it appears. - * - * @param {string} [token] - The raw `ref` query value - * @returns {RefDescriptor} */ -export function parseRefToken(token) { +export function parseRefToken(token?: string): RefDescriptor { if (token === undefined || token === '') { return { kind: 'worktree' }; } @@ -78,28 +82,17 @@ export function parseRefToken(token) { ); } -/** - * Runs a git command, throwing on a non-zero exit. - * - * @param {string[]} args - Arguments to git - * @param {string} cwd - Working directory - * @returns {string} Trimmed stdout - */ -function gitCapture(args, cwd) { +/** Runs a git command, throwing on a non-zero exit. */ +function gitCapture(args: string[], cwd: string): string { const result = execaSync('git', args, { cwd, reject: false }); if (result.exitCode !== 0) { - throw new Error(`git ${args.join(' ')} failed: ${result.stderr.trim()}`); + throw new Error(`git ${args.join(' ')} failed: ${String(result.stderr).trim()}`); } return result.stdout.trim(); } -/** - * Detects the branch PRs fork from, from `origin`'s default branch, falling back to `master`. - * - * @param {string} cwd - Repository to inspect - * @returns {string} The base branch name - */ -function detectBaseBranch(cwd) { +/** Detects the branch PRs fork from, from `origin`'s default branch, falling back to `master`. */ +function detectBaseBranch(cwd: string): string { const result = execaSync('git', ['symbolic-ref', '--short', 'refs/remotes/origin/HEAD'], { cwd, reject: false, @@ -121,19 +114,14 @@ function detectBaseBranch(cwd) { * Picking by most-recent merge base prefers an up-to-date upstream over a stale fork's base branch * without hardcoding which remote is authoritative: locally `origin` may be a months-behind fork * while `upstream` tracks the real repo, yet in CI `origin` *is* that repo. - * - * @param {string} repoRoot - Repository to inspect - * @param {string} baseBranch - Base branch name, e.g. `master` - * @returns {{ ref: string, mergeBase: string } | undefined} */ -function closestBaseBranch(repoRoot, baseBranch) { +function closestBaseBranch( + repoRoot: string, + baseBranch: string, +): { ref: string; mergeBase: string } | undefined { // On ties (same merge base), prefer upstream's, then origin's, then a local base branch. const preference = [`upstream/${baseBranch}`, `origin/${baseBranch}`, baseBranch]; - /** - * @param {string} ref - Candidate ref name - * @returns {number} Sort priority, lower first - */ - const priority = (ref) => { + const priority = (ref: string): number => { const index = preference.indexOf(ref); return index === -1 ? preference.length : index; }; @@ -150,8 +138,7 @@ function closestBaseBranch(repoRoot, baseBranch) { .map((ref) => ref.replace(/^refs\/(remotes|heads)\//, '')) .sort((a, b) => priority(a) - priority(b)); - /** @type {{ ref: string, mergeBase: string, when: number } | undefined} */ - let best; + let best: { ref: string; mergeBase: string; when: number } | undefined; for (const ref of candidates) { const result = execaSync('git', ['merge-base', 'HEAD', ref], { cwd: repoRoot, reject: false }); if (result.exitCode !== 0) { @@ -170,37 +157,28 @@ function closestBaseBranch(repoRoot, baseBranch) { /** * Creates a resolver that turns {@link RefDescriptor}s into {@link ResolvedRef}s, memoizing the * `baseline` symbol so it is computed at most once per run. - * - * @param {RefResolverOptions} options - Repository and baseline policy - * @returns {{ parse: (token?: string) => ResolvedRef }} */ -export function createRefResolver(options) { +export function createRefResolver(options: RefResolverOptions): { + parse: (token?: string) => ResolvedRef; +} { const { repoRoot, baselineOverride } = options; // Detected on demand: only the `baseline` symbol needs it, so a run whose cases all pin `git:` // refs — or none at all — never shells out for it. let baseBranch = options.baseBranch; - /** @type {ResolvedRef | undefined} */ - let baselineRef; + let baselineRef: ResolvedRef | undefined; // Several cases commonly pin the same commit, and resolving one spawns a git process. - /** @type {Map} */ - const gitRefs = new Map(); + const gitRefs = new Map(); - /** - * Resolves a git committish to a `ResolvedRef` keyed by its immutable SHA. - * - * @param {string} committish - Revision to resolve - * @param {string} [label] - Override for the human-readable label - * @returns {ResolvedRef} - */ - function gitRef(committish, label) { + /** Resolves a git committish to a `ResolvedRef` keyed by its immutable SHA. */ + function gitRef(committish: string, label?: string): ResolvedRef { const cached = gitRefs.get(committish); if (cached) { return cached; } const sha = gitCapture(['rev-parse', committish], repoRoot); - const ref = { - kind: /** @type {const} */ ('git'), + const ref: ResolvedRef = { + kind: 'git', id: `git-${sha.slice(0, 9)}`, label: label ?? `${committish} (${sha.slice(0, 9)})`, sha, @@ -214,10 +192,8 @@ export function createRefResolver(options) { * Computes the `baseline` symbol: the override when given, otherwise the sensible default — on * the base branch the previous commit, on any other branch the fork point from the closest base * branch, which isolates what this branch changed. - * - * @returns {ResolvedRef} */ - function computeBaselineRef() { + function computeBaselineRef(): ResolvedRef { if (baselineOverride !== undefined) { const descriptor = parseRefToken(baselineOverride); if (descriptor.kind === 'baseline') { @@ -244,11 +220,7 @@ export function createRefResolver(options) { return gitRef(base.mergeBase, `merge-base with ${base.ref}`); } - /** - * @param {RefDescriptor} descriptor - A parsed ref token - * @returns {ResolvedRef} - */ - function resolve(descriptor) { + function resolve(descriptor: RefDescriptor): ResolvedRef { switch (descriptor.kind) { case 'worktree': return WORKTREE_REF; @@ -256,13 +228,13 @@ export function createRefResolver(options) { baselineRef ??= computeBaselineRef(); return baselineRef; case 'git': - return gitRef(/** @type {string} */ (descriptor.committish)); + return gitRef(descriptor.committish as string); default: throw new Error(`Unhandled ref kind "${descriptor.kind}".`); } } return { - parse: (token) => resolve(parseRefToken(token)), + parse: (token?: string) => resolve(parseRefToken(token)), }; } diff --git a/packages/code-infra/src/tachometer/renderReport.mjs b/packages/benchmark/src/tachometer/renderReport.ts similarity index 67% rename from packages/code-infra/src/tachometer/renderReport.mjs rename to packages/benchmark/src/tachometer/renderReport.ts index 2d0797046e..10c8eab12a 100644 --- a/packages/code-infra/src/tachometer/renderReport.mjs +++ b/packages/benchmark/src/tachometer/renderReport.ts @@ -3,6 +3,7 @@ // Column widths have to measure visible characters, so chalk's escapes come off first. import { stripVTControlCharacters as stripAnsi } from 'node:util'; import chalk from 'chalk'; +import type { ConfidenceInterval, Verdict } from './summarizeCase'; /** * Renders the report `tacho run` produces. @@ -12,81 +13,72 @@ import chalk from 'chalk'; * breaks the other. */ -/** - * @typedef {import('./summarizeCase.mjs').ConfidenceInterval} ConfidenceInterval - * @typedef {import('./summarizeCase.mjs').Verdict} Verdict - */ +export interface VariantResult { + variant: string; + /** The ref this variant loaded. */ + refId: string | null; + /** Mean duration, as a 95% confidence interval. */ + meanMs: ConfidenceInterval; + samples: number; + bytesSent: number; +} -/** - * @typedef {Object} VariantResult - * @property {string} variant - Variant name - * @property {string | null} refId - The ref this variant loaded - * @property {ConfidenceInterval} meanMs - Mean duration, as a 95% confidence interval - * @property {number} samples - Sample count - * @property {number} bytesSent - Bytes transferred - */ +export interface Comparison { + /** The variant compared against the reference. */ + variant: string; + /** Whether the reference is faster, slower, or unresolved. */ + verdict: Verdict; + absoluteMs: ConfidenceInterval; + percentChange: ConfidenceInterval; + /** The same pair the other way round. */ + versusReference?: { + verdict: Verdict; + absoluteMs: ConfidenceInterval; + percentChange: ConfidenceInterval; + }; +} -/** - * @typedef {Object} Comparison - * @property {string} variant - The variant compared against the reference - * @property {Verdict} verdict - Whether the reference is faster, slower, or unresolved - * @property {ConfidenceInterval} absoluteMs - Difference in milliseconds - * @property {ConfidenceInterval} percentChange - Difference as a percentage - * @property {{ verdict: Verdict, absoluteMs: ConfidenceInterval, percentChange: ConfidenceInterval }} [versusReference] - The same pair the other way round - */ +export interface MeasurementResult { + name: string; + variants: VariantResult[]; + /** The reference against every other variant. */ + comparisons: Comparison[]; +} -/** - * @typedef {Object} MeasurementResult - * @property {string} name - Measurement name - * @property {VariantResult[]} variants - One entry per variant - * @property {Comparison[]} comparisons - The reference against every other variant - */ +export interface CaseResult { + name: string; + /** The variant every comparison is expressed against. Absent when the case failed to summarize. */ + reference?: string; + measurements?: MeasurementResult[]; + /** Why the case produced no summary. */ + error?: string; +} -/** - * @typedef {Object} CaseResult - * @property {string} name - Case name - * @property {string} [reference] - The variant every comparison is expressed against. Absent when the case failed to summarize - * @property {MeasurementResult[]} [measurements] - One entry per measurement - * @property {string} [error] - Why the case produced no summary - */ +export interface TachometerReport { + version: number; + /** Which benchmark axis produced this. */ + reportType: 'tachometer'; + generatedAt: string; + /** The commit measured. */ + head: { ref: string; sha: string; branch?: string }; + /** The builds compared. */ + refs: Array<{ id: string; kind: string; label: string; sha?: string }>; + cases: CaseResult[]; +} -/** - * @typedef {Object} TachometerReport - * @property {number} version - Report format version - * @property {'tachometer'} reportType - Which benchmark axis produced this - * @property {string} generatedAt - ISO timestamp - * @property {{ ref: string, sha: string, branch?: string }} head - The commit measured - * @property {Array<{ id: string, kind: string, label: string, sha?: string }>} refs - The builds compared - * @property {CaseResult[]} cases - One entry per case - */ +/** A case that produced results, as opposed to one carrying only the error that stopped it. */ +type SummarizedCase = CaseResult & { measurements: MeasurementResult[] }; -/** - * Prints a table, padding each column to its widest visible cell. - * - * @param {string[]} headers - Column headers - * @param {string[][]} rows - Row cells, in column order - * @param {number} leftColumns - How many leading columns are left-aligned; the rest are right - * @returns {void} - */ -function printTable(headers, rows, leftColumns) { +/** Prints a table, padding each column to its widest visible cell. */ +function printTable(headers: string[], rows: string[][], leftColumns: number): void { const widths = headers.map((header, index) => Math.max(stripAnsi(header).length, ...rows.map((row) => stripAnsi(row[index] ?? '').length)), ); - /** - * @param {string} text - Cell contents - * @param {number} width - Target width - * @param {boolean} left - Whether to left-align - * @returns {string} - */ - const pad = (text, width, left) => { + const pad = (text: string, width: number, left: boolean) => { const fill = ' '.repeat(Math.max(0, width - stripAnsi(text).length)); return left ? text + fill : fill + text; }; - /** - * @param {string[]} cells - One row - * @returns {string} - */ - const line = (cells) => + const line = (cells: string[]) => cells .map((cell, index) => pad(cell ?? '', widths[index], index < leftColumns)) .join(' ') @@ -99,37 +91,20 @@ function printTable(headers, rows, leftColumns) { } } -/** - * `low – high`, with the unit written once. - * - * @param {ConfidenceInterval} interval - The interval to format - * @returns {string} - */ -function formatInterval(interval) { +/** `low – high`, with the unit written once. */ +function formatInterval(interval: ConfidenceInterval): string { // Fixed to two places so a column lines up on the decimal point, and written the same way the // dashboard writes it, so the terminal and the pull request comment agree digit for digit. return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; } -/** - * @param {ConfidenceInterval} interval - The interval to format - * @returns {string} Both bounds as signed percentages - */ -function formatPercentInterval(interval) { - /** - * @param {number} value - A percentage - * @returns {string} - */ - const signed = (value) => `${value >= 0 ? '+' : ''}${value.toFixed(1)}%`; +/** Both bounds as signed percentages. */ +function formatPercentInterval(interval: ConfidenceInterval): string { + const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(1)}%`; return `${signed(interval.low)} – ${signed(interval.high)}`; } -/** - * @param {Verdict} verdict - The verdict to colour by - * @param {string} text - Text to colour - * @returns {string} - */ -function colorVerdict(verdict, text) { +function colorVerdict(verdict: Verdict, text: string): string { if (verdict === 'faster') { return chalk.green(text); } @@ -139,13 +114,8 @@ function colorVerdict(verdict, text) { return chalk.dim(text); } -/** - * Kibibytes, matching how tachometer's own table reports `bytesSent`. - * - * @param {number} bytes - Byte count - * @returns {string} - */ -function formatBytes(bytes) { +/** Kibibytes, matching how tachometer's own table reports `bytesSent`. */ +function formatBytes(bytes: number): string { return `${(bytes / 1024).toFixed(1)} KiB`; } @@ -154,34 +124,19 @@ function formatBytes(bytes) { * the prefix is dropped for display. Dropping it is also what makes columns line up across cases: * every auto-expanded case then contributes the same `[current]`/`[baseline]` pair rather than a * column of its own. - * - * @param {string} caseName - The case's name - * @param {string} variant - The variant's full name - * @returns {string} */ -function shortNameOf(caseName, variant) { +function shortNameOf(caseName: string, variant: string): string { return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; } -/** - * Whether a case produced results, as opposed to carrying only the error that stopped it. - * - * @param {CaseResult} entry - A case from the report - * @returns {boolean} - */ -function isSummarized(entry) { +/** Whether a case produced results, as opposed to carrying only the error that stopped it. */ +function isSummarized(entry: CaseResult): entry is SummarizedCase { return entry.measurements !== undefined && entry.measurements.length > 0; } -/** - * The distinct variants a case reports, in order of first appearance. - * - * @param {CaseResult & { measurements: MeasurementResult[] }} entry - A summarized case - * @returns {string[]} - */ -function variantsOf(entry) { - /** @type {string[]} */ - const variants = []; +/** The distinct variants a case reports, in order of first appearance. */ +function variantsOf(entry: SummarizedCase): string[] { + const variants: string[] = []; for (const measurement of entry.measurements) { for (const variant of measurement.variants) { const short = shortNameOf(entry.name, variant.variant); @@ -200,12 +155,8 @@ function variantsOf(entry) { * pair would run off the screen. Each measurement then shows the variant's interval next to its * difference *relative to the reference* — the direction a row about that library reads in — taken * from the variant's own row of tachometer's matrix. - * - * @param {CaseResult & { measurements: MeasurementResult[] }} entry - A summarized case - * @param {string[]} variants - The variant set, reference first - * @returns {void} */ -function printVariantTable(entry, variants) { +function printVariantTable(entry: SummarizedCase, variants: string[]): void { const [reference] = variants; const headers = [ entry.name, @@ -259,14 +210,8 @@ function printVariantTable(entry, variants) { ); } -/** - * Renders a group of cases sharing a variant set, one row per case and measurement. - * - * @param {Array} cases - Cases in the group - * @param {string[]} variants - The shared variant set, reference first - * @returns {void} - */ -function printCaseTable(cases, variants) { +/** Renders a group of cases sharing a variant set, one row per case and measurement. */ +function printCaseTable(cases: SummarizedCase[], variants: string[]): void { const [reference, ...others] = variants; const headers = [ 'Case', @@ -276,8 +221,7 @@ function printCaseTable(cases, variants) { 'Samples', ]; - /** @type {string[][]} */ - const rows = []; + const rows: string[][] = []; for (const [caseIndex, entry] of cases.entries()) { if (caseIndex > 0) { rows.push(headers.map(() => '')); @@ -342,16 +286,11 @@ function printCaseTable(cases, variants) { * * Signs are the reference relative to the variant, so a faster reference reads negative; the * verdict word carries the direction so the sign never has to be read twice. - * - * @param {TachometerReport} report - The report to render - * @returns {void} */ -export function renderTachometerReport(report) { +export function renderTachometerReport(report: TachometerReport): void { // One predicate for both halves: partitioning on two conditions that have to agree left a case // with an empty `measurements` array rendered nowhere and reported nowhere. - const usable = /** @type {Array} */ ( - report.cases.filter(isSummarized) - ); + const usable = report.cases.filter(isSummarized); const failed = report.cases.filter((entry) => !isSummarized(entry)); if (usable.length === 0) { @@ -366,8 +305,7 @@ export function renderTachometerReport(report) { // `[current]`/`[baseline]`, a cross-library case `[ours]`/`[theirs]`, and each set has its own // reference (the first variant). A single table over the union of the columns would put a blank // cell in every row of every case, and could name only one reference for all of them. - /** @type {Map }>} */ - const groups = new Map(); + const groups = new Map(); for (const entry of usable) { const variants = variantsOf(entry); const key = JSON.stringify(variants); @@ -404,8 +342,7 @@ export function renderTachometerReport(report) { for (const entry of [...groups.values()] .filter((group) => group.variants.length <= 2) .flatMap((group) => group.cases)) { - /** @type {Map} */ - const bytes = new Map(); + const bytes = new Map(); for (const measurement of entry.measurements) { for (const variant of measurement.variants) { bytes.set(shortNameOf(entry.name, variant.variant), variant.bytesSent); diff --git a/packages/code-infra/src/tachometer/runTachometer.mjs b/packages/benchmark/src/tachometer/runTachometer.ts similarity index 80% rename from packages/code-infra/src/tachometer/runTachometer.mjs rename to packages/benchmark/src/tachometer/runTachometer.ts index 292ba417a6..f4fc03c553 100644 --- a/packages/code-infra/src/tachometer/runTachometer.mjs +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -6,44 +6,54 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; import { execaSync } from 'execa'; import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; -import { packRef, packWorkingTree } from '../utils/packWorkspace.mjs'; -import { createRefResolver } from './refs.mjs'; -import { discoverCases, pagesOf } from './discoverCases.mjs'; -import { buildRefPages } from './buildPages.mjs'; -import { - assertDriverMatchesBrowser, - resolveBrowserBinary, - withBrowserDefaults, -} from './browser.mjs'; -import { summarizeCase } from './summarizeCase.mjs'; -import { renderTachometerReport } from './renderReport.mjs'; -import { getCiMetadata, syncPrComment, uploadCiReport } from './ciReport.mjs'; -import { buildsDirOf, prepareOutputDir } from './outputDir.mjs'; -import { run } from '../utils/exec.mjs'; +import { packRef, packWorkingTree } from '../utils/packWorkspace'; +import type { PackedPackage } from '../utils/packWorkspace'; +import { createRefResolver } from './refs'; +import type { ResolvedRef } from './refs'; +import { discoverCases, pagesOf } from './discoverCases'; +import type { BenchmarkCase } from './discoverCases'; +import { buildRefPages } from './buildPages'; +import { assertDriverMatchesBrowser, resolveBrowserBinary, withBrowserDefaults } from './browser'; +import { summarizeCase } from './summarizeCase'; +import { renderTachometerReport } from './renderReport'; +import { getCiMetadata, syncPrComment, uploadCiReport } from './ciReport'; +import type { TachometerReport } from './ciReport'; +import { buildsDirOf, prepareOutputDir } from './outputDir'; +import { run } from '../utils/exec'; /** * A case's name as a filename component. Names come from the benchmark's own `name`, so they are * prose rather than identifiers and may hold anything — a slash included. - * - * @param {string} name - The case name - * @returns {string} */ -function fileSlugOf(name) { +function fileSlugOf(name: string): string { return name.replace(/[^a-zA-Z0-9._-]+/g, '-'); } -/** - * @typedef {Object} RunTachometerOptions - * @property {string} harnessDir - The harness package directory (where the command was run) - * @property {string[]} [filters] - Only run cases whose path under `src` contains one of these substrings, case-insensitively - * @property {string} [baseline] - Binds the `baseline` symbol, in the ref grammar (e.g. `git:abc1234`) - * @property {string} [baseBranch] - Branch PRs fork from. Defaults to detection via `origin/HEAD` - * @property {string} [buildCmd] - Command that builds the publishable packages of a checked-out ref. Defaults to `pnpm release:build` - * @property {string} [workingTreeBuildCmd] - Command that builds the working tree. Defaults to `buildCmd` - * @property {boolean} [install] - Whether to install inside a ref's checkout. Defaults to true - * @property {string} [out] - Where to write the combined JSON report. Defaults to `.tachometer/results/report.json` - * @property {boolean} [upload] - Upload the report and refresh the pull request comment - */ +export interface RunTachometerOptions { + /** The harness package directory (where the command was run). */ + harnessDir: string; + /** + * Only run cases whose path under `src` contains one of these substrings, case-insensitively. + */ + filters?: string[]; + /** Binds the `baseline` symbol, in the ref grammar (e.g. `git:abc1234`). */ + baseline?: string; + /** Branch PRs fork from. Defaults to detection via `origin/HEAD`. */ + baseBranch?: string; + /** + * Command that builds the publishable packages of a checked-out ref. Defaults to + * `pnpm release:build`. + */ + buildCmd?: string; + /** Command that builds the working tree. Defaults to `buildCmd`. */ + workingTreeBuildCmd?: string; + /** Whether to install inside a ref's checkout. Defaults to true. */ + install?: boolean; + /** Where to write the combined JSON report. Defaults to `.tachometer/results/report.json`. */ + out?: string; + /** Upload the report and refresh the pull request comment. */ + upload?: boolean; +} /** * Benchmarks the harness's pages across one or more builds of the workspace with tachometer, and @@ -53,11 +63,8 @@ function fileSlugOf(name) { * build, one variant is built per distinct ref, each url is then rewritten to that ref's built page, * and each case runs as its own tachometer invocation so its result is a clean table for that * scenario rather than a grid comparing unrelated scenarios against each other. - * - * @param {RunTachometerOptions} options - What to run - * @returns {Promise} */ -export async function runTachometer(options) { +export async function runTachometer(options: RunTachometerOptions): Promise { const { harnessDir, filters = [], @@ -92,8 +99,7 @@ export async function runTachometer(options) { const resolver = createRefResolver({ repoRoot, baseBranch, baselineOverride: baseline }); const cases = await discoverCases({ harnessDir, filters, resolveRef: resolver.parse }); - /** @type {Map} */ - const refs = new Map(); + const refs = new Map(); for (const entry of cases) { for (const leaf of entry.leaves) { if (leaf.ref) { @@ -105,7 +111,7 @@ export async function runTachometer(options) { // Show the resolved commit next to each ref — a symbolic baseline (e.g. "merge-base with // upstream/master") otherwise hides which commit it actually chose, which matters when several // base branches exist and only the closest fork point is used. - const describeRef = (/** @type {import('./refs.mjs').ResolvedRef} */ ref) => + const describeRef = (ref: ResolvedRef) => ref.sha ? `${ref.label} (${ref.sha.slice(0, 9)})` : ref.label; // Before any build: a driver that cannot open the browser fails the run either way, and finding @@ -128,8 +134,7 @@ export async function runTachometer(options) { // Build one variant per distinct ref, each resolving through its own install so both sides // of a comparison resolve the library identically. for (const ref of refs.values()) { - /** @type {import('../utils/packWorkspace.mjs').PackedPackage[]} */ - let packages; + let packages: PackedPackage[]; if (ref.kind === 'worktree') { // eslint-disable-next-line no-await-in-loop packages = await packWorkingTree({ @@ -142,7 +147,7 @@ export async function runTachometer(options) { // eslint-disable-next-line no-await-in-loop const packed = await packRef({ repoRoot, - ref: /** @type {string} */ (ref.committish), + ref: ref.committish as string, outRoot: packedDir, // An empty install command means "skip"; otherwise packRef's default install runs. installCmd: install ? undefined : '', @@ -165,8 +170,7 @@ export async function runTachometer(options) { // Rewrite each leaf url to its ref's built page, then run each case's own config. Tachometer // resolves relative urls against the config file's directory and these configs are written to a // temp dir, so the rewritten urls are absolute. - /** @type {Array<{ entry: import('./discoverCases.mjs').BenchmarkCase, json: any }>} */ - const results = []; + const results: Array<{ entry: BenchmarkCase; json: any }> = []; for (const entry of cases) { for (const leaf of entry.leaves) { const refId = leaf.ref ? leaf.ref.id : 'current'; @@ -207,7 +211,7 @@ export async function runTachometer(options) { // Consumers render reports from several benchmark axes; the pair identifies which one this // is and how to read it. version: 1, - reportType: /** @type {const} */ ('tachometer'), + reportType: 'tachometer' as const, generatedAt: new Date().toISOString(), head: { ref: 'HEAD', @@ -241,11 +245,11 @@ export async function runTachometer(options) { await mkdir(path.dirname(outPath), { recursive: true }); await writeFile(outPath, `${JSON.stringify(report, null, 2)}\n`); console.log(''); - renderTachometerReport(report); + renderTachometerReport(report as unknown as TachometerReport); console.log(chalk.green(`\nWrote JSON report to ${outPath}`)); if (upload) { - await publishReport(report); + await publishReport(report as unknown as TachometerReport); } } finally { await rm(tmpBase, { recursive: true, force: true }); @@ -257,11 +261,8 @@ export async function runTachometer(options) { * * A failure to refresh the comment does not fail the run: the numbers are already measured, written * and uploaded, and losing them to a comment API hiccup would waste the whole benchmark. - * - * @param {import('./ciReport.mjs').TachometerReport} report - The report just written - * @returns {Promise} */ -async function publishReport(report) { +async function publishReport(report: TachometerReport): Promise { const metadata = await getCiMetadata(); if (!metadata.repo) { console.warn( diff --git a/packages/code-infra/src/tachometer/summarizeCase.test.mjs b/packages/benchmark/src/tachometer/summarizeCase.test.ts similarity index 89% rename from packages/code-infra/src/tachometer/summarizeCase.test.mjs rename to packages/benchmark/src/tachometer/summarizeCase.test.ts index cedbb32a7f..31b471f0cd 100644 --- a/packages/code-infra/src/tachometer/summarizeCase.test.mjs +++ b/packages/benchmark/src/tachometer/summarizeCase.test.ts @@ -1,16 +1,20 @@ import { describe, expect, it, vi } from 'vitest'; -import { summarizeCase, verdictOf } from './summarizeCase.mjs'; - -/** - * A tachometer benchmark entry. - * - * @param {Object} options - Entry fields - * @param {string} options.name - Entry name, as tachometer reports it - * @param {string} [options.measurement] - Measurement name - * @param {Array<{ ms: [number, number], pct: [number, number] } | null>} [options.differences] - Differences against every entry, by flat index - * @returns {any} - */ -function benchmark({ name, measurement, differences = [] }) { +import { summarizeCase, verdictOf } from './summarizeCase'; + +type FakeDifference = { ms: [number, number]; pct: [number, number] } | null; + +/** A tachometer benchmark entry. */ +function benchmark({ + name, + measurement, + differences = [], +}: { + /** Entry name, as tachometer reports it. */ + name: string; + measurement?: string; + /** Differences against every entry, by flat index. */ + differences?: FakeDifference[]; +}): any { return { name, bytesSent: 100, @@ -29,13 +33,10 @@ function benchmark({ name, measurement, differences = [] }) { } /** - * A discovered case, as `discoverCases` would return it. - * - * @param {string[]} variants - Variant names in order; the first is the reference - * @param {string[]} measurements - Measurement names - * @returns {any} + * A discovered case, as `discoverCases` would return it. `variants` is in order; the first is the + * reference. */ -function discovered(variants, measurements) { +function discovered(variants: string[], measurements: string[]): any { return { name: 'example', configPath: '/tmp/tachometer.json', diff --git a/packages/code-infra/src/tachometer/summarizeCase.mjs b/packages/benchmark/src/tachometer/summarizeCase.ts similarity index 58% rename from packages/code-infra/src/tachometer/summarizeCase.mjs rename to packages/benchmark/src/tachometer/summarizeCase.ts index dca500e016..75046083cb 100644 --- a/packages/code-infra/src/tachometer/summarizeCase.mjs +++ b/packages/benchmark/src/tachometer/summarizeCase.ts @@ -1,43 +1,71 @@ import chalk from 'chalk'; +import type { BenchmarkCase } from './discoverCases'; -/** - * Reduces tachometer's raw JSON output to a flat, interpretable summary. - */ +/** Reduces tachometer's raw JSON output to a flat, interpretable summary. */ -/** - * @typedef {import('./discoverCases.mjs').BenchmarkCase} BenchmarkCase - */ +export interface ConfidenceInterval { + low: number; + high: number; +} -/** - * @typedef {Object} ConfidenceInterval - * @property {number} low - Lower bound - * @property {number} high - Upper bound - */ +export interface TachometerDifference { + /** Absolute difference, in milliseconds. */ + absolute: ConfidenceInterval; + /** Relative difference, as a percentage. */ + percentChange: ConfidenceInterval; +} -/** - * @typedef {Object} TachometerDifference - * @property {ConfidenceInterval} absolute - Absolute difference, in milliseconds - * @property {ConfidenceInterval} percentChange - Relative difference, as a percentage - */ +export interface TachometerBenchmark { + /** Variant name, suffixed with ` []` when a page has several. */ + name: string; + bytesSent: number; + /** Mean duration, as a 95% confidence interval. */ + mean: ConfidenceInterval; + samples: number[]; + /** The measurement this entry reports. */ + measurement?: { name?: string }; + /** This entry against every other, by flat index. */ + differences: Array; +} -/** - * @typedef {Object} TachometerBenchmark - * @property {string} name - Variant name, suffixed with ` []` when a page has several - * @property {number} bytesSent - Bytes transferred - * @property {ConfidenceInterval} mean - Mean duration, as a 95% confidence interval - * @property {number[]} samples - Individual sample durations - * @property {{ name?: string }} [measurement] - The measurement this entry reports - * @property {Array} differences - This entry against every other, by flat index - */ +export interface TachometerJson { + /** One entry per variant × measurement. */ + benchmarks: TachometerBenchmark[]; +} -/** - * @typedef {Object} TachometerJson - * @property {TachometerBenchmark[]} benchmarks - One entry per variant × measurement - */ +export type Verdict = 'faster' | 'slower' | 'unsure'; -/** - * @typedef {'faster'|'slower'|'unsure'} Verdict - */ +export interface VariantSummary { + variant: string; + refId: string | null; + meanMs: ConfidenceInterval; + samples: number; + bytesSent: number; +} + +export interface ComparisonSummary { + variant: string; + verdict: Verdict; + absoluteMs: ConfidenceInterval; + percentChange: ConfidenceInterval; + versusReference?: { + verdict: Verdict; + absoluteMs: ConfidenceInterval; + percentChange: ConfidenceInterval; + }; +} + +export interface MeasurementSummary { + name: string; + variants: VariantSummary[]; + comparisons: ComparisonSummary[]; +} + +export interface CaseSummary { + name: string; + reference: string; + measurements: MeasurementSummary[]; +} /** * Judges a difference. @@ -46,11 +74,8 @@ import chalk from 'chalk'; * interval both exclude zero on the same side. They come from the same samples but are computed * separately, so near zero one can exclude it while the other does not — and a verdict that rests * on only one of them flips depending on which is read. - * - * @param {TachometerDifference} difference - A difference from tachometer's matrix - * @returns {Verdict} */ -export function verdictOf(difference) { +export function verdictOf(difference: TachometerDifference): Verdict { if (difference.absolute.low > 0 && difference.percentChange.low > 0) { return 'slower'; } @@ -63,13 +88,14 @@ export function verdictOf(difference) { /** * Indexes tachometer's flat benchmark list by measurement and then by variant, keeping each entry's * position so the difference matrix can be read back. - * - * @param {TachometerJson} json - Raw tachometer output - * @returns {Map>} */ -function indexByMeasurement(json) { - /** @type {Map>} */ - const byMeasurement = new Map(); +function indexByMeasurement( + json: TachometerJson, +): Map> { + const byMeasurement = new Map< + string, + Map + >(); json.benchmarks.forEach((benchmark, index) => { const measurement = benchmark.measurement?.name ?? ''; const suffix = ` [${measurement}]`; @@ -93,12 +119,8 @@ function indexByMeasurement(json) { * * The first declared variant is the reference, because that is the one under test: for the * auto-expanded regression case it is `[current]`. - * - * @param {BenchmarkCase} entry - The discovered case, which declares the variants and measurements - * @param {TachometerJson} json - Raw tachometer output for that case - * @returns {{ name: string, reference: string, measurements: Array<{ name: string, variants: Array<{ variant: string, refId: string | null, meanMs: ConfidenceInterval, samples: number, bytesSent: number }>, comparisons: Array<{ variant: string, verdict: Verdict, absoluteMs: ConfidenceInterval, percentChange: ConfidenceInterval, versusReference?: { verdict: Verdict, absoluteMs: ConfidenceInterval, percentChange: ConfidenceInterval } }> }> }} */ -export function summarizeCase(entry, json) { +export function summarizeCase(entry: BenchmarkCase, json: TachometerJson): CaseSummary { const [reference] = entry.variants; if (!reference) { throw new Error(`Case "${entry.name}" declares no variants.`); @@ -106,13 +128,9 @@ export function summarizeCase(entry, json) { const byMeasurement = indexByMeasurement(json); - const measurements = entry.measurements.map((measurement) => { + const measurements = entry.measurements.map((measurement): MeasurementSummary => { const byVariant = byMeasurement.get(measurement); - /** - * @param {string} variant - Variant name to look up - * @returns {{ benchmark: TachometerBenchmark, index: number } | undefined} - */ - const lookup = (variant) => byVariant?.get(variant); + const lookup = (variant: string) => byVariant?.get(variant); const referenceResult = lookup(reference.name); if (!referenceResult) { @@ -121,8 +139,8 @@ export function summarizeCase(entry, json) { ); } - const variants = []; - const comparisons = []; + const variants: VariantSummary[] = []; + const comparisons: ComparisonSummary[] = []; for (const variant of entry.variants) { const found = lookup(variant.name); if (!found) { diff --git a/packages/benchmark/src/utils/build.ts b/packages/benchmark/src/utils/build.ts new file mode 100644 index 0000000000..64d87df98c --- /dev/null +++ b/packages/benchmark/src/utils/build.ts @@ -0,0 +1,31 @@ +/** + * Maps `mapper` over `items` with at most `concurrency` in flight. + * + * Each worker takes the next item as it frees up, so a slow item does not hold back the rest — + * which matters when the items are `pnpm pack` invocations and each is its own node process. + */ +export async function mapConcurrently( + items: T[], + mapper: (item: T) => Promise, + concurrency: number, +): Promise<(R | Error)[]> { + if (!items.length) { + return []; + } + const itemIterator = items.entries(); + const count = Math.min(concurrency, items.length); + const results: (R | Error)[] = new Array(items.length); + const workers: Promise[] = []; + for (let index = 0; index < count; index += 1) { + workers.push( + Promise.resolve().then(async () => { + for (const [itemIndex, item] of itemIterator) { + // eslint-disable-next-line no-await-in-loop -- a worker processes its items in sequence + results[itemIndex] = await mapper(item); + } + }), + ); + } + await Promise.all(workers); + return results; +} diff --git a/packages/benchmark/src/utils/exec.ts b/packages/benchmark/src/utils/exec.ts new file mode 100644 index 0000000000..8412dd38f5 --- /dev/null +++ b/packages/benchmark/src/utils/exec.ts @@ -0,0 +1,20 @@ +/* eslint-disable no-console */ + +import chalk from 'chalk'; +import { execa } from 'execa'; + +/** + * Runs a command in `cwd` with inherited stdio, throwing on a non-zero exit. + * + * Echoes the command first: a benchmark run shells out a lot, and being able to copy a failing step + * out of the log and re-run it by hand is worth the noise. + */ +export async function run( + file: string, + args: string[], + cwd: string, + env?: NodeJS.ProcessEnv, +): Promise { + console.log(chalk.dim(`$ ${file} ${args.join(' ')} (in ${cwd})`)); + await execa(file, args, { cwd, stdio: 'inherit', env }); +} diff --git a/packages/code-infra/src/utils/packWorkspace.test.mjs b/packages/benchmark/src/utils/packWorkspace.test.ts similarity index 94% rename from packages/code-infra/src/utils/packWorkspace.test.mjs rename to packages/benchmark/src/utils/packWorkspace.test.ts index 73e0020efb..659952d57d 100644 --- a/packages/code-infra/src/utils/packWorkspace.test.mjs +++ b/packages/benchmark/src/utils/packWorkspace.test.ts @@ -2,14 +2,8 @@ import * as path from 'node:path'; import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises'; import { execa } from 'execa'; import { describe, expect, it } from 'vitest'; -import { makeTempDir } from './testUtils.mjs'; -import { - packRef, - packWorkingTree, - readFreshCache, - tarballFor, - tarballName, -} from './packWorkspace.mjs'; +import { makeTempDir } from './testUtils'; +import { packRef, packWorkingTree, readFreshCache, tarballFor, tarballName } from './packWorkspace'; describe('tarballName', () => { it('flattens a scoped name into a filesystem-safe basename', () => { @@ -42,15 +36,16 @@ describe('tarballFor', () => { }); describe('readFreshCache', () => { - /** - * Writes a packed folder with a manifest and its tarballs. - * - * @param {Object} options - Folder contents - * @param {string} options.buildCmd - The build command to record - * @param {boolean} [options.withTarball] - Whether to actually create the tarball file - * @returns {Promise} The folder - */ - async function makePackedDir({ buildCmd, withTarball = true }) { + /** Writes a packed folder with a manifest and its tarballs. */ + async function makePackedDir({ + buildCmd, + withTarball = true, + }: { + /** The build command to record. */ + buildCmd: string; + /** Whether to actually create the tarball file. */ + withTarball?: boolean; + }): Promise { const dir = await makeTempDir(); await writeFile( path.join(dir, 'manifest.json'), diff --git a/packages/code-infra/src/utils/packWorkspace.mjs b/packages/benchmark/src/utils/packWorkspace.ts similarity index 70% rename from packages/code-infra/src/utils/packWorkspace.mjs rename to packages/benchmark/src/utils/packWorkspace.ts index 6ac52ce984..9f19637134 100644 --- a/packages/code-infra/src/utils/packWorkspace.mjs +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -8,10 +8,10 @@ import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promise import { pipeline } from 'node:stream/promises'; import chalk from 'chalk'; import { execa, parseCommandString } from 'execa'; -import { mapConcurrently } from './build.mjs'; -import { run } from './exec.mjs'; -import { pathExists } from './path.mjs'; -import { getWorkspacePackages } from './pnpm.mjs'; +import { mapConcurrently } from './build'; +import { run } from './exec'; +import { pathExists } from './path'; +import { getWorkspacePackages } from './pnpm'; /** * Packs the public workspace packages at a given git ref into a folder of tarballs. @@ -22,73 +22,75 @@ import { getWorkspacePackages } from './pnpm.mjs'; * is what benchmark harnesses and end-to-end install tests need. */ -/** - * @typedef {Object} PackedPackage - * @property {string} name - Package name from its package.json - * @property {string} version - Package version - * @property {string} tarball - Absolute path to the packed `.tgz` - */ +export interface PackedPackage { + /** Package name from its package.json. */ + name: string; + version: string; + /** Absolute path to the packed `.tgz`. */ + tarball: string; +} -/** - * @typedef {Object} PackedWorkspace - * @property {string} ref - The ref that was packed (verbatim input) - * @property {string} sha - The resolved commit SHA - * @property {string} dir - Absolute path to the folder holding the tarballs and `manifest.json` - * @property {PackedPackage[]} packages - One entry per packed package - */ +export interface PackedWorkspace { + /** The ref that was packed, verbatim as given. */ + ref: string; + /** The resolved commit SHA. */ + sha: string; + /** Absolute path to the folder holding the tarballs and `manifest.json`. */ + dir: string; + packages: PackedPackage[]; +} -/** - * @typedef {Object} RawManifestPackage - * @property {string} name - Package name - * @property {string} version - Package version - * @property {string} tarball - Tarball name, relative to the folder, so the cache is relocatable - */ +interface RawManifestPackage { + name: string; + version: string; + /** Tarball name, relative to the folder, so the cache is relocatable. */ + tarball: string; +} -/** - * @typedef {Object} RawManifest - * @property {string} ref - The ref that was packed - * @property {string} sha - The resolved commit SHA - * @property {string} buildCmd - The build command used, part of the cache key - * @property {RawManifestPackage[]} packages - The packed packages - */ +interface RawManifest { + ref: string; + sha: string; + /** The build command used, part of the cache key. */ + buildCmd: string; + packages: RawManifestPackage[]; +} -/** - * @typedef {Object} PackRefOptions - * @property {string} repoRoot - The repository the ref lives in; also where the worktree is added from - * @property {string} ref - A git ref (SHA, branch, or tag) to pack - * @property {string} outRoot - Cache root. The packed folder lands at `/` and is reused on the next call - * @property {string} [installCmd] - Command run in the checkout to install before building. Defaults to a frozen-lockfile `pnpm install`; pass `''` to skip installing - * @property {string} [buildCmd] - Command run in the checkout to build before packing. Defaults to `pnpm release:build`. Part of the cache key - */ +export interface PackRefOptions { + /** The repository the ref lives in; also where the worktree is added from. */ + repoRoot: string; + /** A git ref (SHA, branch, or tag) to pack. */ + ref: string; + /** Cache root. The packed folder lands at `/` and is reused on the next call. */ + outRoot: string; + /** + * Command run in the checkout to install before building. Defaults to a frozen-lockfile + * `pnpm install`; pass `''` to skip installing. + */ + installCmd?: string; + /** + * Command run in the checkout to build before packing. Defaults to `pnpm release:build`. Part of + * the cache key. + */ + buildCmd?: string; +} /** Name of the file written into each packed folder describing its contents. */ const MANIFEST = 'manifest.json'; -/** - * Resolves a git ref to its commit SHA. - * - * @param {string} repoRoot - Repository to resolve in - * @param {string} ref - The ref to resolve - * @returns {Promise} The commit SHA - */ -async function resolveCommit(repoRoot, ref) { +/** Resolves a git ref to its commit SHA. */ +async function resolveCommit(repoRoot: string, ref: string): Promise { const result = await execa('git', ['rev-parse', '--verify', `${ref}^{commit}`], { cwd: repoRoot, reject: false, }); if (result.exitCode !== 0) { - throw new Error(`Could not resolve git ref "${ref}": ${result.stderr.trim()}`); + throw new Error(`Could not resolve git ref "${ref}": ${String(result.stderr).trim()}`); } return result.stdout.trim(); } -/** - * Turns a package name into a filesystem-safe tarball basename (`@scope/pkg` → `scope-pkg.tgz`). - * - * @param {string} pkgName - The package name - * @returns {string} The tarball basename - */ -export function tarballName(pkgName) { +/** Turns a package name into a filesystem-safe tarball basename (`@scope/pkg` → `scope-pkg.tgz`). */ +export function tarballName(pkgName: string): string { return `${pkgName.replace(/^@/, '').replace(/\//g, '-')}.tgz`; } @@ -99,12 +101,8 @@ export function tarballName(pkgName) { * repopulates the tree while git is deleting it; fall back to force-removing the directory and * pruning git's bookkeeping. Never throws — by the time this runs the packed tarballs are already * safe, so a leftover temp checkout must not fail the run. - * - * @param {string} repoRoot - Repository the checkout belongs to - * @param {string} checkout - Checkout directory to remove - * @returns {Promise} */ -async function removeCheckout(repoRoot, checkout) { +async function removeCheckout(repoRoot: string, checkout: string): Promise { const removed = await execa('git', ['worktree', 'remove', '--force', checkout], { cwd: repoRoot, reject: false, @@ -130,34 +128,20 @@ async function removeCheckout(repoRoot, checkout) { * * Streamed rather than read whole: these are tarballs of built packages, and every one of them * would otherwise be resident at once. - * - * @param {string} file - File to hash - * @returns {Promise} */ -async function hashFile(file) { +async function hashFile(file: string): Promise { const hash = createHash('sha256'); await pipeline(createReadStream(file), hash); return hash.digest('hex').slice(0, 12); } -/** - * Reads a folder's raw `manifest.json`. - * - * @param {string} dir - The packed folder - * @returns {Promise} - */ -async function readRawManifest(dir) { +/** Reads a folder's raw `manifest.json`. */ +async function readRawManifest(dir: string): Promise { return JSON.parse(await readFile(path.join(dir, MANIFEST), 'utf8')); } -/** - * Resolves a raw manifest's stored (relative) tarball names to absolute paths under `dir`. - * - * @param {RawManifest} raw - The parsed manifest - * @param {string} dir - The folder it describes - * @returns {PackedWorkspace} - */ -function resolveManifest(raw, dir) { +/** Resolves a raw manifest's stored (relative) tarball names to absolute paths under `dir`. */ +function resolveManifest(raw: RawManifest, dir: string): PackedWorkspace { return { ref: raw.ref, sha: raw.sha, @@ -177,12 +161,8 @@ function resolveManifest(raw, dir) { * Returning the parsed manifest lets the caller reuse it without a second read. Guards against * reusing a folder built with a different build script, or one whose tarballs were partially * evicted or deleted. - * - * @param {string} dir - Candidate cache folder - * @param {string} buildCmd - The build command the caller wants - * @returns {Promise} */ -export async function readFreshCache(dir, buildCmd) { +export async function readFreshCache(dir: string, buildCmd: string): Promise { if (!(await pathExists(path.join(dir, MANIFEST)))) { return null; } @@ -207,12 +187,11 @@ export async function readFreshCache(dir, buildCmd) { * by a consumer — the private package is never packed, so `pnpm install` looks for it on the * registry and 404s. That is a real packaging bug in the ref, not something to paper over here: * make the internal dependency public so it ships alongside the package that needs it. - * - * @param {string} checkoutDir - The (already built, already installed) checkout to pack - * @param {string} outDir - Folder to write tarballs into - * @returns {Promise} The packed packages, with absolute tarball paths */ -export async function packBuiltPackages(checkoutDir, outDir) { +export async function packBuiltPackages( + checkoutDir: string, + outDir: string, +): Promise { await mkdir(outDir, { recursive: true }); const packages = await getWorkspacePackages({ cwd: checkoutDir, publicOnly: true }); if (packages.length === 0) { @@ -253,11 +232,8 @@ export async function packBuiltPackages(checkoutDir, outDir) { * assembled in a staging directory and atomically renamed into place, and its `manifest.json` * stores tarball names relative to the folder — so the cache is safe to move between machines (for * example restored from a CI cache under a different absolute path). - * - * @param {PackRefOptions} options - What to pack, and where - * @returns {Promise} */ -export async function packRef(options) { +export async function packRef(options: PackRefOptions): Promise { const { repoRoot, ref, @@ -299,8 +275,7 @@ export async function packRef(options) { const packages = await packBuiltPackages(checkout, staging); // Store tarballs by basename so the folder is relocatable; record buildCmd so a later run can // tell whether the cache matches. Write the manifest last — it marks completeness. - /** @type {RawManifest} */ - const manifest = { + const manifest: RawManifest = { ref, sha, buildCmd, @@ -335,14 +310,15 @@ export async function packRef(options) { * * The tree is mutable, so this always builds; a good `buildCmd` is cached (nx, turbo) and an * unchanged tree rebuilds in seconds. - * - * @param {Object} options - What to pack, and where - * @param {string} options.repoRoot - The workspace to build and pack - * @param {string} options.outRoot - Directory to hold the hashed tarballs. Replaced on each call - * @param {string} [options.buildCmd] - Command that builds the publishable packages. Defaults to `pnpm release:build` - * @returns {Promise} */ -export async function packWorkingTree(options) { +export async function packWorkingTree(options: { + /** The workspace to build and pack. */ + repoRoot: string; + /** Directory to hold the hashed tarballs. Replaced on each call. */ + outRoot: string; + /** Command that builds the publishable packages. Defaults to `pnpm release:build`. */ + buildCmd?: string; +}): Promise { const { repoRoot, outRoot, buildCmd = 'pnpm release:build' } = options; console.log(chalk.cyan(`\nBuilding workspace packages for "working tree" (${buildCmd})…`)); @@ -372,14 +348,8 @@ export async function packWorkingTree(options) { } } -/** - * Looks up one packed package's tarball by name, throwing a clear error if it was not packed. - * - * @param {PackedPackage[]} packages - The packed packages - * @param {string} name - Package name to find - * @returns {string} Absolute path to the tarball - */ -export function tarballFor(packages, name) { +/** Looks up one packed package's tarball by name, throwing a clear error if it was not packed. */ +export function tarballFor(packages: PackedPackage[], name: string): string { const found = packages.find((pkg) => pkg.name === name); if (!found) { const available = packages.map((pkg) => pkg.name).join(', ') || '(none)'; diff --git a/packages/benchmark/src/utils/path.ts b/packages/benchmark/src/utils/path.ts new file mode 100644 index 0000000000..7a601e7e10 --- /dev/null +++ b/packages/benchmark/src/utils/path.ts @@ -0,0 +1,11 @@ +import { access } from 'node:fs/promises'; + +/** True if `target` exists on disk. */ +export async function pathExists(target: string): Promise { + try { + await access(target); + return true; + } catch { + return false; + } +} diff --git a/packages/benchmark/src/utils/pnpm.ts b/packages/benchmark/src/utils/pnpm.ts new file mode 100644 index 0000000000..2def2cd578 --- /dev/null +++ b/packages/benchmark/src/utils/pnpm.ts @@ -0,0 +1,63 @@ +import * as path from 'node:path'; +import * as fs from 'node:fs/promises'; +import { $ } from 'execa'; + +/** A workspace package, as `pnpm ls` reports it. */ +export interface WorkspacePackage { + name: string; + version: string; + /** Absolute path to the package directory. */ + path: string; + /** Whether the package is excluded from publishing. */ + isPrivate: boolean; +} + +interface ListedPackage { + name?: string; + version?: string; + path: string; + private?: boolean; +} + +/** + * The workspace packages of `cwd`, as `pnpm ls` resolves them. + * + * Asks pnpm rather than scanning directories, so the set follows `pnpm-workspace.yaml` and matches + * exactly what a release would publish. A package counts as private when it says so, or when it + * lacks the name or version that publishing requires. + */ +export async function getWorkspacePackages( + options: { cwd?: string; publicOnly?: boolean } = {}, +): Promise { + const { cwd, publicOnly = false } = options; + + const result = await $({ cwd })`pnpm ls -r --json --depth -1`; + const listed: ListedPackage[] = JSON.parse(result.stdout); + + return listed.flatMap((pkg) => { + const isPrivate = Boolean(pkg.private) || !pkg.name || !pkg.version; + if (publicOnly && isPrivate) { + return []; + } + return [ + { + name: pkg.name ?? '', + version: pkg.version ?? '', + path: pkg.path, + isPrivate, + }, + ]; + }); +} + +/** Reads a directory's `package.json`. */ +export async function readPackageJson(packagePath: string): Promise> { + const content = await fs.readFile(path.join(packagePath, 'package.json'), 'utf8'); + return JSON.parse(content); +} + +/** Writes a directory's `package.json`, formatted the way the repository writes manifests. */ +export async function writePackageJson(packagePath: string, packageJson: object): Promise { + const content = `${JSON.stringify(packageJson, null, 2)}\n`; + await fs.writeFile(path.join(packagePath, 'package.json'), content); +} diff --git a/packages/benchmark/src/utils/testUtils.ts b/packages/benchmark/src/utils/testUtils.ts new file mode 100644 index 0000000000..e4712f8492 --- /dev/null +++ b/packages/benchmark/src/utils/testUtils.ts @@ -0,0 +1,13 @@ +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { onTestFinished } from 'vitest'; + +/** Creates a temporary directory and removes it when the current test ends, even if it throws. */ +export async function makeTempDir(): Promise { + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'benchmark-test-')); + onTestFinished(async () => { + await fs.rm(tmpDir, { recursive: true, force: true }); + }); + return tmpDir; +} diff --git a/packages/code-infra/package.json b/packages/code-infra/package.json index 1225be09f5..459d330720 100644 --- a/packages/code-infra/package.json +++ b/packages/code-infra/package.json @@ -45,14 +45,6 @@ "types": "./build/utils/packWorkspace.d.mts", "default": "./src/utils/packWorkspace.mjs" }, - "./tachometerPlugin": { - "types": "./build/tachometer/plugin.d.mts", - "default": "./src/tachometer/plugin.mjs" - }, - "./tachometerReport": { - "types": "./build/tachometer/renderReport.d.mts", - "default": "./src/tachometer/renderReport.mjs" - }, "./brokenLinksChecker": { "types": "./build/brokenLinksChecker/index.d.mts", "default": "./src/brokenLinksChecker/index.mjs" @@ -156,8 +148,7 @@ "unified-lint-rule": "^3.0.1", "unist-util-visit": "^5.1.0", "yaml": "^2.9.0", - "yargs": "^18.1.0", - "zod": "^4.4.3" + "yargs": "^18.1.0" }, "peerDependencies": { "@next/eslint-plugin-next": "*", diff --git a/packages/code-infra/src/cli/index.mjs b/packages/code-infra/src/cli/index.mjs index cbc73ced4c..58982e6ccf 100644 --- a/packages/code-infra/src/cli/index.mjs +++ b/packages/code-infra/src/cli/index.mjs @@ -16,8 +16,6 @@ import cmdPublishCanary from './cmdPublishCanary.mjs'; import cmdPublishNewPackage from './cmdPublishNewPackage.mjs'; import cmdRenameScope from './cmdRenameScope.mjs'; import cmdSetVersionOverrides from './cmdSetVersionOverrides.mjs'; -import cmdTacho from './cmdTacho.mjs'; -import cmdTachoReport from './cmdTachoReport.mjs'; import cmdVale from './cmdVale.mjs'; import cmdValidateBuiltTypes from './cmdValidateBuiltTypes.mjs'; @@ -52,8 +50,6 @@ await yargs(hideBin(process.argv)) .command(cmdPublishNewPackage) .command(cmdRenameScope) .command(cmdSetVersionOverrides) - .command(cmdTacho) - .command(cmdTachoReport) .command(cmdVale) .command(cmdValidateBuiltTypes) .fail((msg, err, yargsInstance) => { diff --git a/packages/code-infra/src/tachometer/ciReport.mjs b/packages/code-infra/src/tachometer/ciReport.mjs deleted file mode 100644 index 920e7380c2..0000000000 --- a/packages/code-infra/src/tachometer/ciReport.mjs +++ /dev/null @@ -1,193 +0,0 @@ -/* eslint-disable no-console -- progress belongs in the CI log. */ - -import { execa } from 'execa'; -import envCi from 'env-ci'; -import { z } from 'zod/v4'; - -/** - * Uploading a tachometer report to the CI report store, and asking the dashboard to refresh the - * pull request comment. - * - * The transport is deliberately a small copy of `@mui/internal-benchmark`'s rather than a shared - * import: code-infra carries a large dependency tree, and having the benchmark package depend on it - * would push all of that onto every consumer. What is duplicated — a bearer-token POST to one - * endpoint — is small and does not move. - */ - -const DEFAULT_API_URL = 'https://frontend-public.mui.com'; - -const confidenceIntervalSchema = z.object({ - low: z.number(), - high: z.number(), -}); - -const differenceSchema = z.object({ - verdict: z.enum(['faster', 'slower', 'unsure']), - absoluteMs: confidenceIntervalSchema, - percentChange: confidenceIntervalSchema, -}); - -const variantResultSchema = z.object({ - variant: z.string(), - refId: z.string().nullable(), - meanMs: confidenceIntervalSchema, - samples: z.number(), - bytesSent: z.number(), -}); - -const comparisonSchema = differenceSchema.extend({ - variant: z.string(), - versusReference: differenceSchema.optional(), -}); - -const measurementResultSchema = z.object({ - name: z.string(), - variants: z.array(variantResultSchema), - comparisons: z.array(comparisonSchema), -}); - -// A case that failed to summarize carries an `error` and no measurements. It is kept in the report -// rather than dropped, so the comment can say a case produced nothing instead of quietly omitting it. -const caseResultSchema = z.object({ - name: z.string(), - reference: z.string().optional(), - measurements: z.array(measurementResultSchema).optional(), - error: z.string().optional(), -}); - -const tachometerReportSchema = z.object({ - version: z.number(), - reportType: z.literal('tachometer'), - generatedAt: z.string(), - head: z.object({ ref: z.string(), sha: z.string(), branch: z.string().optional() }), - browser: z.string().optional(), - refs: z.array( - z.object({ - id: z.string(), - kind: z.string(), - label: z.string(), - sha: z.string().optional(), - }), - ), - cases: z.array(caseResultSchema), - // Every sample tachometer produced. Not read by the comment, but kept so a richer view can be - // built later without changing what CI uploads. - raw: z.record(z.string(), z.unknown()), -}); - -/** - * The upload envelope, matching what `@mui/internal-benchmark` sends and what the dashboard's - * upload route expects. - */ -export const tachometerUploadSchema = z.object({ - version: z.literal(1), - timestamp: z.number(), - commitSha: z.string().regex(/^[0-9a-f]{40}$/, 'Must be a 40-character hex string'), - repo: z.string().includes('/', 'Must be in owner/repo format'), - reportType: z.literal('tachometer'), - prNumber: z.number().int().positive().optional(), - branch: z.string(), - report: tachometerReportSchema, -}); - -/** - * @typedef {z.infer} TachometerUpload - * @typedef {z.infer} TachometerReport - */ - -/** - * Resolves the commit, branch, repository and pull request this run belongs to. - * - * The commit and branch come from git rather than the CI provider: a provider reports the branch it - * was triggered for, which on a pull request build is not necessarily the checkout's own. - * - * @returns {Promise<{ timestamp: number, repo: string, prNumber: number | undefined, branch: string, commitSha: string }>} - */ -export async function getCiMetadata() { - const ciInfo = /** @type {any} */ (envCi()); - const [commit, branch] = await Promise.all([ - execa('git', ['rev-parse', 'HEAD']), - execa('git', ['branch', '--show-current']), - ]); - - return { - timestamp: Date.now(), - repo: ciInfo.slug ?? '', - prNumber: ciInfo.pr ? Number(ciInfo.pr) : undefined, - branch: branch.stdout.trim() || (ciInfo.isPr ? ciInfo.prBranch : ciInfo.branch) || '', - commitSha: commit.stdout.trim(), - }; -} - -/** - * The dashboard endpoint for `pathname`, honouring an override so a pull request's own preview - * deployment can be pointed at. - * - * @param {string} pathname - The API path - * @returns {URL} - */ -function endpoint(pathname) { - return new URL(pathname, process.env.CI_REPORT_API_URL ?? DEFAULT_API_URL); -} - -/** - * Posts JSON to the dashboard, authenticated as this CI job. - * - * @param {URL} url - The endpoint to post to - * @param {unknown} body - The payload - * @param {string} what - Names the call in both errors it can raise - * @returns {Promise} The response body - */ -async function postJson(url, body, what) { - const oidcToken = process.env.CIRCLE_OIDC_TOKEN_V2; - if (!oidcToken) { - throw new Error(`CIRCLE_OIDC_TOKEN_V2 environment variable is required for ${what}`); - } - - const response = await fetch(url, { - method: 'POST', - headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${oidcToken}` }, - body: JSON.stringify(body), - }); - - const responseText = await response.text(); - if (!response.ok) { - throw new Error(`${what} failed (${response.status}): ${responseText}`); - } - return responseText; -} - -/** - * Uploads a report to the CI report store. - * - * Validates before sending because the server stores `report` without inspecting it — a malformed - * report would be accepted here and only fail much later, when the dashboard tried to render it. - * - * @param {TachometerUpload} upload - The envelope to send - * @returns {Promise} - */ -export async function uploadCiReport(upload) { - tachometerUploadSchema.parse(upload); - - const url = endpoint('/api/ci-reports/upload'); - console.log(`Uploading tachometer report to ${url.href}`); - - const responseText = await postJson(url, upload, 'The tachometer upload'); - const result = JSON.parse(responseText); - console.log(`Tachometer report uploaded. S3 key: ${result.key}`); -} - -/** - * Asks the dashboard to regenerate the pull request comment for `repo`. - * - * @param {string} repo - The repository, as `owner/name` - * @returns {Promise<{ success: boolean, skipped?: boolean }>} - */ -export async function syncPrComment(repo) { - const responseText = await postJson( - endpoint('/api/ci-reports/sync-pr-comment'), - { repo }, - 'The pull request comment sync', - ); - return JSON.parse(responseText); -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index af931fcdda..6b6778aca2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -487,28 +487,46 @@ importers: '@babel/runtime': specifier: ^7.29.7 version: 7.29.7 + '@pnpm/find-workspace-dir': + specifier: ^1000.1.0 + version: 1000.1.5 '@vitejs/plugin-react': specifier: ^6.1.1 version: 6.1.1(babel-plugin-react-compiler@1.0.0)(vite@8.2.2(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0)) '@vitest/browser-playwright': specifier: '>=4.1' version: 4.1.11(playwright@1.63.0)(vite@8.2.2(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0))(vitest@4.1.11) + chalk: + specifier: ^5.6.2 + version: 5.6.2 env-ci: specifier: ^11.2.0 version: 11.2.0 execa: specifier: ^10.0.1 version: 10.0.1 + globby: + specifier: ^16.2.4 + version: 16.2.4 react: specifier: ^18.0.0 || ^19.0.0 version: 19.2.8 react-dom: specifier: ^18.0.0 || ^19.0.0 version: 19.2.8(react@19.2.8) + yaml: + specifier: ^2.9.0 + version: 2.9.0 + yargs: + specifier: ^18.1.0 + version: 18.1.0 zod: specifier: ^4.5.4 version: 4.5.4 devDependencies: + '@playwright/test': + specifier: 1.62.1 + version: 1.62.1 '@types/env-ci': specifier: 3.1.4 version: 3.1.4 @@ -518,6 +536,12 @@ importers: '@types/react-dom': specifier: 19.2.7 version: 19.2.7(@types/react@19.2.18) + '@types/yargs': + specifier: 17.0.35 + version: 17.0.35 + vite: + specifier: ^8.2.2 + version: 8.2.2(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0) vitest: specifier: 4.1.11 version: 4.1.11(@types/node@22.20.1)(@vitest/browser-playwright@4.1.11)(@vitest/coverage-v8@4.1.11)(jsdom@30.0.1)(vite@8.2.2(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0)) @@ -832,9 +856,6 @@ importers: yargs: specifier: ^18.1.0 version: 18.1.0 - zod: - specifier: ^4.4.3 - version: 4.5.4 devDependencies: '@octokit/types': specifier: 17.0.0 @@ -1233,9 +1254,9 @@ importers: specifier: 19.2.8 version: 19.2.8 devDependencies: - '@mui/internal-code-infra': + '@mui/internal-benchmark': specifier: workspace:* - version: link:../../packages/code-infra + version: link:../../packages/benchmark/build '@playwright/test': specifier: 1.62.1 version: 1.62.1 diff --git a/test/tachometer/README.md b/test/tachometer/README.md index 74b5cf2525..b819696bc2 100644 --- a/test/tachometer/README.md +++ b/test/tachometer/README.md @@ -1,6 +1,6 @@ # Tachometer smoke test -A working harness for `code-infra tacho run`, used to exercise that tooling end to end. +A working harness for `benchmark tacho run`, used to exercise that tooling end to end. This is **not** a meaningful benchmark, and it is not trying to be. This repository ships build and test tooling, not a browser library, so there is nothing here whose render time is worth tracking. @@ -13,21 +13,30 @@ For the real thing, see the consumers this tooling was extracted from — `base- ## What it covers -The two cases are chosen to walk the parts of the pipeline most likely to break: +The cases are chosen to walk the parts of the pipeline most likely to break: | Case | Covers | | :--------------- | :------------------------------------------------------------------------------- | | `workload` | The ordinary regression shape: one page, auto-expanded into current vs baseline | | `workload-large` | A case owning **no page of its own**, parameterising its sibling's with `?size=` | +| `libs-sort` | Three variants from **one build**: the cross-library shape, with no ref in sight | -`workload-large` is the interesting one. It exercises that a case's query parameters survive the url -rewrite, that `?ref=` is appended with `&` when a url already has a query, and that two cases -referencing one page build it only once. +`workload-large` exercises that a case's query parameters survive the url rewrite, that `?ref=` is +appended with `&` when a url already has a query, and that two cases referencing one page build it +only once. -Both pages import `@mui/internal-test-utils` and put its value on screen. That is the load-bearing -part: every ref — the working tree included — resolves it from a packed tarball in that ref's own -tree, so both sides of a comparison consume the library the way a consumer does. If that path -breaks the page fails to build, rather than quietly measuring nothing. +`libs-sort` covers the other axis, where nothing is compared across commits. Its three variants are +built from the same commit and differ only in how they sort, standing in for the competitor +libraries this repository has no reason to install. It exercises three things the two-variant cases +cannot: a case with more than two variants, which the report draws as a table of variants rather +than a row per case; a second variant set, which gets a table of its own rather than sharing one +full of blank cells; and a verdict that is genuinely `slower` yet must never be reported as a +regression, because both sides came from the same build. + +`workload` and `workload-large` import `@mui/internal-test-utils` and put its value on screen. That +is the load-bearing part: every ref — the working tree included — resolves it from a packed tarball +in that ref's own tree, so both sides of a comparison consume the library the way a consumer does. +If that path breaks the page fails to build, rather than quietly measuring nothing. ## Running @@ -53,7 +62,7 @@ Cases run sequentially, and each auto-samples until its difference resolves or t short) timeout is hit. Because both sides run identical code, the expected verdict is `unsure` — that is the "no change" outcome, not a failure. -`code-infra tacho report` prints that table again from the saved JSON, so a run kept from earlier — +`benchmark tacho report` prints that table again from the saved JSON, so a run kept from earlier — or a report downloaded from CI — can be read without sampling again. Everything a run writes goes under `.tachometer/`: the report in `results/`, the pages built per ref @@ -79,7 +88,7 @@ copy wins over the root pin. If the check reports a driver you did not install, ### A note on the baseline The default baseline is the fork point from the base branch, so a ref older than the introduction of -`code-infra tacho` will fail to build its pages: this harness's vite config imports a plugin that +`benchmark tacho` will fail to build its pages: this harness's vite config imports a plugin that did not exist at that commit. Pass `--baseline git:HEAD` to compare the working tree against the current commit while that is still true. diff --git a/test/tachometer/package.json b/test/tachometer/package.json index ed984b090c..08e0cd0600 100644 --- a/test/tachometer/package.json +++ b/test/tachometer/package.json @@ -1,18 +1,18 @@ { "private": true, - "description": "Tachometer smoke-test harness for the code-infra tacho tooling", + "description": "Tachometer smoke-test harness for the @mui/internal-benchmark tacho tooling", "scripts": { "dev": "vite", "build": "vite build", "preview": "vite preview", - "test:tacho": "code-infra tacho run" + "test:tacho": "benchmark tacho run" }, "dependencies": { "@mui/internal-test-utils": "workspace:*", "react": "19.2.8" }, "devDependencies": { - "@mui/internal-code-infra": "workspace:*", + "@mui/internal-benchmark": "workspace:*", "@playwright/test": "1.62.1", "@types/react": "19.2.18", "tachometer": "^0.7.2", diff --git a/test/tachometer/src/libs/sort/alpha.html b/test/tachometer/src/libs/sort/alpha.html new file mode 100644 index 0000000000..5de8ad125c --- /dev/null +++ b/test/tachometer/src/libs/sort/alpha.html @@ -0,0 +1,11 @@ + + + + + libs-sort [alpha] + + +
    + + + diff --git a/test/tachometer/src/libs/sort/alpha.ts b/test/tachometer/src/libs/sort/alpha.ts new file mode 100644 index 0000000000..2cde400413 --- /dev/null +++ b/test/tachometer/src/libs/sort/alpha.ts @@ -0,0 +1,9 @@ +import { measureMount } from '../../_shared/measure'; +import { SIZE, checksum, makeValues } from './shared'; + +/** Sorts the boxed array in place, paying a comparator call per comparison. */ +measureMount(() => { + const sorted = makeValues(); + sorted.sort((left, right) => left - right); + return `libs-sort [alpha] size=${SIZE} checksum=${checksum(sorted).toFixed(3)}`; +}); diff --git a/test/tachometer/src/libs/sort/beta.html b/test/tachometer/src/libs/sort/beta.html new file mode 100644 index 0000000000..f0b543d18e --- /dev/null +++ b/test/tachometer/src/libs/sort/beta.html @@ -0,0 +1,11 @@ + + + + + libs-sort [beta] + + +
    + + + diff --git a/test/tachometer/src/libs/sort/beta.ts b/test/tachometer/src/libs/sort/beta.ts new file mode 100644 index 0000000000..fdef66d77c --- /dev/null +++ b/test/tachometer/src/libs/sort/beta.ts @@ -0,0 +1,34 @@ +import { measureMount } from '../../_shared/measure'; +import { SIZE, checksum, makeValues } from './shared'; + +/** A merge sort written in JavaScript, allocating as it goes — the slow variant of the three. */ +function mergeSort(values: number[]): number[] { + if (values.length <= 1) { + return values; + } + const middle = Math.floor(values.length / 2); + const left = mergeSort(values.slice(0, middle)); + const right = mergeSort(values.slice(middle)); + + const merged: number[] = new Array(values.length); + let leftIndex = 0; + let rightIndex = 0; + for (let index = 0; index < merged.length; index += 1) { + if ( + leftIndex < left.length && + (rightIndex >= right.length || left[leftIndex] <= right[rightIndex]) + ) { + merged[index] = left[leftIndex]; + leftIndex += 1; + } else { + merged[index] = right[rightIndex]; + rightIndex += 1; + } + } + return merged; +} + +measureMount(() => { + const sorted = mergeSort(makeValues()); + return `libs-sort [beta] size=${SIZE} checksum=${checksum(sorted).toFixed(3)}`; +}); diff --git a/test/tachometer/src/libs/sort/ours.html b/test/tachometer/src/libs/sort/ours.html new file mode 100644 index 0000000000..68b8517ce4 --- /dev/null +++ b/test/tachometer/src/libs/sort/ours.html @@ -0,0 +1,11 @@ + + + + + libs-sort [ours] + + +
    + + + diff --git a/test/tachometer/src/libs/sort/ours.ts b/test/tachometer/src/libs/sort/ours.ts new file mode 100644 index 0000000000..3a083cae2a --- /dev/null +++ b/test/tachometer/src/libs/sort/ours.ts @@ -0,0 +1,9 @@ +import { measureMount } from '../../_shared/measure'; +import { SIZE, checksum, makeValues } from './shared'; + +/** Copies into a typed array first, so the engine sorts unboxed doubles with no comparator calls. */ +measureMount(() => { + const sorted = Float64Array.from(makeValues()); + sorted.sort(); + return `libs-sort [ours] size=${SIZE} checksum=${checksum(Array.from(sorted)).toFixed(3)}`; +}); diff --git a/test/tachometer/src/libs/sort/shared.ts b/test/tachometer/src/libs/sort/shared.ts new file mode 100644 index 0000000000..5fea325024 --- /dev/null +++ b/test/tachometer/src/libs/sort/shared.ts @@ -0,0 +1,42 @@ +/** + * The workload the three variants share, so they differ only in how they sort. + * + * This case stands in for a cross-library comparison: the shape where every variant is built from + * the *same* commit, and the difference worth knowing is between implementations rather than + * between two builds of ours. This repository has no competitor libraries to install, so the + * "libraries" here are three sorts with genuinely different constant factors. + */ + +/** Values per sample. Large enough that the sort, not the page load, dominates the measurement. */ +export const SIZE = 100_000; + +/** A small linear congruential generator, so every sample sorts identical input. */ +function createRandom(seed: number): () => number { + let state = seed; + return () => { + state = (state * 1664525 + 1013904223) % 0x100000000; + return state / 0x100000000; + }; +} + +/** A fresh unsorted array. Fresh per sample: re-sorting a sorted array measures something else. */ +export function makeValues(): number[] { + const random = createRandom(42); + const values: number[] = new Array(SIZE); + for (let index = 0; index < SIZE; index += 1) { + values[index] = random(); + } + return values; +} + +/** + * A checksum that depends on the order, so a variant that returned an unsorted or truncated array + * could not quietly report the same number as one that sorted correctly. + */ +export function checksum(values: readonly number[]): number { + let total = 0; + for (let index = 0; index < values.length; index += 1) { + total += values[index] * index; + } + return total; +} diff --git a/test/tachometer/src/libs/sort/tachometer.json b/test/tachometer/src/libs/sort/tachometer.json new file mode 100644 index 0000000000..1f87085259 --- /dev/null +++ b/test/tachometer/src/libs/sort/tachometer.json @@ -0,0 +1,22 @@ +{ + "$schema": "../../../node_modules/tachometer/config.schema.json", + "sampleSize": 10, + "timeout": 1, + "autoSampleConditions": ["10%"], + "benchmarks": [ + { + "name": "libs-sort", + "measurement": { "mode": "performance", "entryName": "mount" }, + "browser": { + "name": "chrome", + "headless": true, + "windowSize": { "width": 1024, "height": 768 } + }, + "expand": [ + { "name": "libs-sort [ours]", "url": "./ours.html" }, + { "name": "libs-sort [alpha]", "url": "./alpha.html" }, + { "name": "libs-sort [beta]", "url": "./beta.html" } + ] + } + ] +} diff --git a/test/tachometer/src/workload/index.ts b/test/tachometer/src/workload/index.ts index 23977b00f5..76cbad5521 100644 --- a/test/tachometer/src/workload/index.ts +++ b/test/tachometer/src/workload/index.ts @@ -6,7 +6,7 @@ import { measureMount, numberParam } from '../_shared/measure'; * * The point of this page is not the number it produces — this repository ships build tooling, not a * browser library, so there is nothing here whose render time is worth tracking. It exists to give - * `code-infra tacho run` something real to drive end to end. + * `benchmark tacho run` something real to drive end to end. * * It does import a workspace package (`@mui/internal-test-utils`) and put its value on screen. That * is the part that matters: the working tree resolves it through the workspace link to the diff --git a/test/tachometer/vite.config.mjs b/test/tachometer/vite.config.mjs index cbee30cf55..98416fbdc9 100644 --- a/test/tachometer/vite.config.mjs +++ b/test/tachometer/vite.config.mjs @@ -1,5 +1,5 @@ import { defineConfig } from 'vite'; -import { tachometer } from '@mui/internal-code-infra/tachometerPlugin'; +import { tachometer } from '@mui/internal-benchmark/tachometerPlugin'; // The plugin contributes everything derived from the benchmark cases: the `src/` root, the // multi-page app type, the build's entry points, and the dev-server case index. From e9b2a14c03bf44591a7cd7caead68e6183280559 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:39:59 +0200 Subject: [PATCH 030/173] [code-infra] Leave code-infra untouched What was left after the move was residue, and one piece of it was broken: a `./packWorkspace` export pointing at a file that now lives in the benchmark package, plus `vite` and `@playwright/test` peer dependencies and two utility modules that nothing here imports any more. This package no longer changes in this branch at all, so it needs no release for the tachometer tooling to ship. --- packages/code-infra/package.json | 18 ++---------------- packages/code-infra/src/utils/exec.mjs | 21 --------------------- packages/code-infra/src/utils/path.mjs | 16 ---------------- 3 files changed, 2 insertions(+), 53 deletions(-) delete mode 100644 packages/code-infra/src/utils/exec.mjs diff --git a/packages/code-infra/package.json b/packages/code-infra/package.json index 459d330720..c0942911d4 100644 --- a/packages/code-infra/package.json +++ b/packages/code-infra/package.json @@ -41,10 +41,6 @@ "types": "./build/stylelint/index.d.mts", "default": "./src/stylelint/index.mjs" }, - "./packWorkspace": { - "types": "./build/utils/packWorkspace.d.mts", - "default": "./src/utils/packWorkspace.mjs" - }, "./brokenLinksChecker": { "types": "./build/brokenLinksChecker/index.d.mts", "default": "./src/brokenLinksChecker/index.mjs" @@ -152,26 +148,17 @@ }, "peerDependencies": { "@next/eslint-plugin-next": "*", - "@playwright/test": ">=1.50.0", "eslint": "^9.0.0 || ^10.0.0", "prettier": "^3.0.0", - "typescript": "^5.0.0 || ^6.0.0 || ^7.0.0", - "vite": "^7.0.0 || ^8.0.0" + "typescript": "^5.0.0 || ^6.0.0 || ^7.0.0" }, "peerDependenciesMeta": { "typescript": { "optional": true - }, - "vite": { - "optional": true - }, - "@playwright/test": { - "optional": true } }, "devDependencies": { "@octokit/types": "17.0.0", - "@playwright/test": "1.62.1", "@types/babel__core": "7.20.5", "@types/babel__preset-env": "7.10.0", "@types/env-ci": "3.1.4", @@ -191,8 +178,7 @@ "remark-mdx": "3.1.1", "serve": "14.2.6", "typescript": "npm:@typescript/typescript6@6.0.2", - "typescript-eslint": "8.69.0", - "vite": "^8.2.2" + "typescript-eslint": "8.69.0" }, "files": [ "bin", diff --git a/packages/code-infra/src/utils/exec.mjs b/packages/code-infra/src/utils/exec.mjs deleted file mode 100644 index 574c9d3123..0000000000 --- a/packages/code-infra/src/utils/exec.mjs +++ /dev/null @@ -1,21 +0,0 @@ -/* eslint-disable no-console */ - -import chalk from 'chalk'; -import { execa } from 'execa'; - -/** - * Runs a command in `cwd` with inherited stdio, throwing on a non-zero exit. - * - * Echoes the command first: a benchmark run shells out a lot, and being able to copy a failing step - * out of the log and re-run it by hand is worth the noise. - * - * @param {string} file - Executable to run - * @param {string[]} args - Arguments - * @param {string} cwd - Working directory - * @param {NodeJS.ProcessEnv} [env] - Extends the current environment - * @returns {Promise} - */ -export async function run(file, args, cwd, env) { - console.log(chalk.dim(`$ ${file} ${args.join(' ')} (in ${cwd})`)); - await execa(file, args, { cwd, stdio: 'inherit', env }); -} diff --git a/packages/code-infra/src/utils/path.mjs b/packages/code-infra/src/utils/path.mjs index 6a69dabfb5..f58238a5e6 100644 --- a/packages/code-infra/src/utils/path.mjs +++ b/packages/code-infra/src/utils/path.mjs @@ -1,5 +1,4 @@ import * as path from 'node:path'; -import { access } from 'node:fs/promises'; /** * Normalize a file path to use POSIX separators @@ -9,18 +8,3 @@ import { access } from 'node:fs/promises'; export function toPosixPath(filePath) { return filePath.split(path.sep).join('/'); } - -/** - * True if `target` exists on disk. - * - * @param {string} target - Path to check - * @returns {Promise} - */ -export async function pathExists(target) { - try { - await access(target); - return true; - } catch { - return false; - } -} From e7861e5af506b351884c520344f5d68a85a6041b Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:43:57 +0200 Subject: [PATCH 031/173] Drop code-infra's tachometer devDependencies from the lockfile --- pnpm-lock.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6b6778aca2..3eb3108d15 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -860,9 +860,6 @@ importers: '@octokit/types': specifier: 17.0.0 version: 17.0.0 - '@playwright/test': - specifier: 1.62.1 - version: 1.62.1 '@types/babel__core': specifier: 7.20.5 version: 7.20.5 @@ -920,9 +917,6 @@ importers: typescript: specifier: npm:@typescript/typescript6@6.0.2 version: '@typescript/typescript6@6.0.2' - vite: - specifier: ^8.2.2 - version: 8.2.2(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0) packages/docs-infra: dependencies: From 4e85d8635aefe219d65638734172e1521b79e460 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:04:20 +0200 Subject: [PATCH 032/173] [benchmark] Ship the whole package, and stop describing the report twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `files` field was copied from code-infra, which publishes from its root. This package publishes from inside `build/`, where no `build/` or `src/` exists, so the field reduced the published tarball to four entries — every export missing, the Vitest axis's included. Removed; the other two directory-published packages here carry none either. The report shape was declared both as a hand-written interface and as the zod schema CI validates against, and the two had already drifted: the interface lacked `browser` and `raw`, and it is the one `./tachometerReport` exports. The schema is now the single definition. Also drops every type escape the conversion left behind. `RefDescriptor` and `ResolvedRef` become discriminated unions, so a `git:` ref carries its revision in the type rather than in a comment and an unreachable arm goes; `mapConcurrently` no longer claims it can yield an `Error`, which retires the runtime guard written to satisfy that; the yargs builders keep the inference their annotations exist to check. The three dashboard POSTs — both axes' uploads and the comment sync — become one `ciApi` call each. --- packages/benchmark/package.json | 6 -- packages/benchmark/src/ciApi.ts | 45 +++++++++++++ packages/benchmark/src/ciReport.ts | 6 +- packages/benchmark/src/cli/cmdTacho.ts | 5 +- packages/benchmark/src/cli/tachoReport.ts | 2 +- packages/benchmark/src/cli/tachoRun.ts | 25 ++----- packages/benchmark/src/syncPrComment.ts | 34 +++------- packages/benchmark/src/tachometer/ciReport.ts | 49 ++++---------- packages/benchmark/src/tachometer/refs.ts | 44 ++++++------- .../benchmark/src/tachometer/renderReport.ts | 65 ++++--------------- .../benchmark/src/tachometer/runTachometer.ts | 10 +-- packages/benchmark/src/upload.ts | 47 +++++--------- packages/benchmark/src/utils/build.ts | 4 +- packages/benchmark/src/utils/packWorkspace.ts | 14 +--- packages/benchmark/src/utils/pnpm.ts | 41 +++++------- test/tachometer/src/libs/sort/alpha.ts | 9 +-- test/tachometer/src/libs/sort/beta.ts | 9 ++- test/tachometer/src/libs/sort/ours.ts | 10 ++- test/tachometer/src/libs/sort/shared.ts | 11 +++- 19 files changed, 178 insertions(+), 258 deletions(-) create mode 100644 packages/benchmark/src/ciApi.ts diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index 0e300f0a5b..253e74fc03 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -56,12 +56,6 @@ "bin": { "benchmark": "./src/cli/index.ts" }, - "files": [ - "build", - "src", - "README.md", - "LICENSE" - ], "peerDependenciesMeta": { "@playwright/test": { "optional": true diff --git a/packages/benchmark/src/ciApi.ts b/packages/benchmark/src/ciApi.ts new file mode 100644 index 0000000000..88bccfeec5 --- /dev/null +++ b/packages/benchmark/src/ciApi.ts @@ -0,0 +1,45 @@ +/* eslint-disable no-console -- progress belongs in the CI log. */ + +/** + * The dashboard's CI-report API, which both benchmark axes talk to. + * + * Every call is the same shape — a bearer-token POST of one JSON body — and the token is the OIDC + * one CircleCI mints for the job. Keeping that in one place is what stops the endpoint host, the + * environment variable and the error handling from drifting between the two axes, where a + * divergence would only ever show up in CI, on whichever one nobody edited. + */ + +const DEFAULT_API_URL = 'https://frontend-public.mui.com'; + +/** + * Posts `body` to the dashboard, authenticated as this CI job, and returns the response text. + * + * `what` names the call in both errors it can raise, so a failure says which request failed rather + * than only that one did. + */ +export async function postToDashboard( + pathname: string, + body: unknown, + what: string, +): Promise { + const oidcToken = process.env.CIRCLE_OIDC_TOKEN_V2; + if (!oidcToken) { + throw new Error(`CIRCLE_OIDC_TOKEN_V2 environment variable is required for ${what}`); + } + + // The override exists so a pull request can point at its own dashboard preview deployment. + const url = new URL(pathname, process.env.CI_REPORT_API_URL || DEFAULT_API_URL); + console.log(`${what} → ${url.href}`); + + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${oidcToken}` }, + body: JSON.stringify(body), + }); + + const responseText = await response.text(); + if (!response.ok) { + throw new Error(`${what} failed (${response.status}): ${responseText}`); + } + return responseText; +} diff --git a/packages/benchmark/src/ciReport.ts b/packages/benchmark/src/ciReport.ts index f2951db67a..63bc40f30c 100644 --- a/packages/benchmark/src/ciReport.ts +++ b/packages/benchmark/src/ciReport.ts @@ -29,7 +29,11 @@ async function getGitInfo(): Promise { * Creates a CI report upload schema for a specific report type. * Common fields (commitSha, repo, branch, prNumber) are shared across all report types. */ -export function ciReportUploadSchema(type: string, version: number, reportSchema: z.ZodType) { +export function ciReportUploadSchema( + type: string, + version: number, + reportSchema: ReportSchema, +) { return z.object({ version: z.literal(version), timestamp: z.number(), diff --git a/packages/benchmark/src/cli/cmdTacho.ts b/packages/benchmark/src/cli/cmdTacho.ts index 58c855e13c..1933bbd8e3 100644 --- a/packages/benchmark/src/cli/cmdTacho.ts +++ b/packages/benchmark/src/cli/cmdTacho.ts @@ -13,10 +13,7 @@ const command: CommandModule<{}, {}> = { command: 'tacho ', describe: 'Benchmark a tachometer harness, and read the reports it writes.', builder: (yargs) => - yargs - .command(tachoRun) - .command(tachoReport) - .demandCommand(1, 'Specify a tacho subcommand.') as any, + yargs.command(tachoRun).command(tachoReport).demandCommand(1, 'Specify a tacho subcommand.'), handler: () => {}, }; diff --git a/packages/benchmark/src/cli/tachoReport.ts b/packages/benchmark/src/cli/tachoReport.ts index aa533182a2..2f2486d9dc 100644 --- a/packages/benchmark/src/cli/tachoReport.ts +++ b/packages/benchmark/src/cli/tachoReport.ts @@ -26,7 +26,7 @@ const command: CommandModule<{}, Args> = { }) .epilogue( 'The same table `tacho run` prints when it finishes, so a report kept from an earlier run — or downloaded from CI — can be read without sampling again.', - ) as any + ) ); }, handler: async (argv) => { diff --git a/packages/benchmark/src/cli/tachoRun.ts b/packages/benchmark/src/cli/tachoRun.ts index ea38392246..6cf5e22a94 100644 --- a/packages/benchmark/src/cli/tachoRun.ts +++ b/packages/benchmark/src/cli/tachoRun.ts @@ -1,25 +1,8 @@ import type { CommandModule } from 'yargs'; +import type { RunTachometerOptions } from '../tachometer/runTachometer'; -interface Args { - /** - * Only run cases whose path under `src` contains one of these substrings, case-insensitively. - */ - filters?: string[]; - /** Binds the `baseline` symbol, in the ref grammar. */ - baseline?: string; - /** Branch PRs fork from. */ - baseBranch?: string; - /** Command that builds the publishable packages of a checked-out ref. */ - buildCmd?: string; - /** Command that builds the working tree. */ - workingTreeBuildCmd?: string; - /** Whether to install inside a ref's checkout. */ - install?: boolean; - /** Where to write the combined JSON report. */ - out?: string; - /** Upload the report and refresh the pull request comment. */ - upload?: boolean; -} +/** Everything `runTachometer` takes except the directory, which is where the command was run. */ +type Args = Omit; const command: CommandModule<{}, Args> = { command: 'run [filters...]', @@ -69,7 +52,7 @@ const command: CommandModule<{}, Args> = { }) .epilogue( 'Sampling (sampleSize, autoSampleConditions, timeout) is configured per case in its own tachometer.json — tachometer rejects those as CLI flags when a config file is used.', - ) as any; + ); }, handler: async (argv) => { const { runTachometer } = await import('../tachometer/runTachometer'); diff --git a/packages/benchmark/src/syncPrComment.ts b/packages/benchmark/src/syncPrComment.ts index 79b0d2dca1..210aff4a20 100644 --- a/packages/benchmark/src/syncPrComment.ts +++ b/packages/benchmark/src/syncPrComment.ts @@ -1,34 +1,16 @@ -const DEFAULT_API_URL = 'https://frontend-public.mui.com'; +import { postToDashboard } from './ciApi'; interface SyncPrCommentResult { success: boolean; skipped?: boolean; } -/** - * Syncs a PR comment via the dashboard API. - */ +/** Asks the dashboard to regenerate the pull request comment for `repo`. */ export async function syncPrComment(repo: string): Promise { - const oidcToken = process.env.CIRCLE_OIDC_TOKEN_V2; - if (!oidcToken) { - throw new Error('CIRCLE_OIDC_TOKEN_V2 environment variable is required for PR comment sync'); - } - - const apiUrl = process.env.CI_REPORT_API_URL || DEFAULT_API_URL; - const url = new URL('/api/ci-reports/sync-pr-comment', apiUrl); - - const response = await fetch(url, { - method: 'POST', - headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${oidcToken}` }, - body: JSON.stringify({ repo }), - }); - - if (!response.ok) { - const responseText = await response.text(); - throw new Error( - `Sync PR comment API returned ${response.status} ${response.statusText}: ${responseText}`, - ); - } - - return response.json(); + const responseText = await postToDashboard( + '/api/ci-reports/sync-pr-comment', + { repo }, + 'The pull request comment sync', + ); + return JSON.parse(responseText); } diff --git a/packages/benchmark/src/tachometer/ciReport.ts b/packages/benchmark/src/tachometer/ciReport.ts index 85c1ff3a85..4b779598e5 100644 --- a/packages/benchmark/src/tachometer/ciReport.ts +++ b/packages/benchmark/src/tachometer/ciReport.ts @@ -1,21 +1,14 @@ /* eslint-disable no-console -- progress belongs in the CI log. */ import { z } from 'zod/v4'; +import { postToDashboard } from '../ciApi'; import { ciReportUploadSchema } from '../ciReport'; /** - * Uploading a tachometer report to the CI report store. - * - * The envelope, the CI metadata and the pull request sync are the package's own — this axis differs - * from the Vitest one only in the shape of `report`. Consumers reach the shared halves through - * {@link getCiMetadata} and {@link syncPrComment}, re-exported here so a runner has one import. + * The tachometer axis's upload envelope. It differs from the Vitest one only in the shape of + * `report`; the transport, the CI metadata and the pull request sync are shared. */ -export { getCiMetadata } from '../ciReport'; -export { syncPrComment } from '../syncPrComment'; - -const DEFAULT_API_URL = 'https://frontend-public.mui.com'; - const confidenceIntervalSchema = z.object({ low: z.number(), high: z.number(), @@ -79,16 +72,14 @@ const tachometerReportSchema = z.object({ * The upload envelope. Built from the same factory the Vitest axis uses, so both report types reach * the dashboard's upload route in the shape it expects. */ -export const tachometerUploadSchema = ciReportUploadSchema( - 'tachometer', - 1, - tachometerReportSchema, -).extend({ - report: tachometerReportSchema, -}); +export const tachometerUploadSchema = ciReportUploadSchema('tachometer', 1, tachometerReportSchema); export type TachometerUpload = z.infer; export type TachometerReport = z.infer; +export type CaseResult = z.infer; +export type MeasurementResult = z.infer; +export type VariantResult = z.infer; +export type Comparison = z.infer; /** * Uploads a report to the CI report store. @@ -99,25 +90,11 @@ export type TachometerReport = z.infer; export async function uploadCiReport(upload: TachometerUpload): Promise { tachometerUploadSchema.parse(upload); - const oidcToken = process.env.CIRCLE_OIDC_TOKEN_V2; - if (!oidcToken) { - throw new Error('CIRCLE_OIDC_TOKEN_V2 environment variable is required for uploads'); - } - - const url = new URL('/api/ci-reports/upload', process.env.CI_REPORT_API_URL ?? DEFAULT_API_URL); - console.log(`Uploading tachometer report to ${url.href}`); - - const response = await fetch(url, { - method: 'POST', - headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${oidcToken}` }, - body: JSON.stringify(upload), - }); - - const responseText = await response.text(); - if (!response.ok) { - throw new Error(`The tachometer upload failed (${response.status}): ${responseText}`); - } - + const responseText = await postToDashboard( + '/api/ci-reports/upload', + upload, + 'The tachometer upload', + ); const result = JSON.parse(responseText); console.log(`Tachometer report uploaded. S3 key: ${result.key}`); } diff --git a/packages/benchmark/src/tachometer/refs.ts b/packages/benchmark/src/tachometer/refs.ts index 545c8bb5a8..39d54de80f 100644 --- a/packages/benchmark/src/tachometer/refs.ts +++ b/packages/benchmark/src/tachometer/refs.ts @@ -11,26 +11,26 @@ import { execaSync } from 'execa'; * - {@link createRefResolver} turns a descriptor into a {@link ResolvedRef} with an immutable SHA. */ -export interface RefDescriptor { - /** Which scheme the token used. */ - kind: 'worktree' | 'baseline' | 'git'; - /** The revision, for `git:` refs. */ - committish?: string; -} +/** Only a `git:` ref carries a revision, which the union states rather than leaving to a comment. */ +export type RefDescriptor = + { kind: 'worktree' } | { kind: 'baseline' } | { kind: 'git'; committish: string }; -export interface ResolvedRef { - /** Whether this is the working tree or a committed revision. */ - kind: 'worktree' | 'git'; +interface RefIdentity { /** Build directory name, e.g. `current` or `git-230342ee2`. Doubles as the dedupe identity. */ id: string; /** Human-readable label for logs and the report. */ label: string; - /** The immutable commit this ref denotes. Absent only for the working tree. */ - sha?: string; - /** The committish to hand to `packRef`. Absent only for the working tree. */ - committish?: string; } +/** + * A ref with its build identity. The working tree has no commit behind it and a `git:` ref always + * does, so the two carry different fields rather than sharing optional ones a reader has to pair + * with the right `kind` by hand. + */ +export type ResolvedRef = + | (RefIdentity & { kind: 'worktree'; sha?: undefined; committish?: undefined }) + | (RefIdentity & { kind: 'git'; sha: string; committish: string }); + export interface RefResolverOptions { /** Repository to resolve revisions in. */ repoRoot: string; @@ -221,17 +221,15 @@ export function createRefResolver(options: RefResolverOptions): { } function resolve(descriptor: RefDescriptor): ResolvedRef { - switch (descriptor.kind) { - case 'worktree': - return WORKTREE_REF; - case 'baseline': - baselineRef ??= computeBaselineRef(); - return baselineRef; - case 'git': - return gitRef(descriptor.committish as string); - default: - throw new Error(`Unhandled ref kind "${descriptor.kind}".`); + if (descriptor.kind === 'worktree') { + return WORKTREE_REF; + } + if (descriptor.kind === 'baseline') { + baselineRef ??= computeBaselineRef(); + return baselineRef; } + // The union has no fourth member, so this is the `git:` case and carries a revision. + return gitRef(descriptor.committish); } return { diff --git a/packages/benchmark/src/tachometer/renderReport.ts b/packages/benchmark/src/tachometer/renderReport.ts index 10c8eab12a..56e388a341 100644 --- a/packages/benchmark/src/tachometer/renderReport.ts +++ b/packages/benchmark/src/tachometer/renderReport.ts @@ -4,6 +4,7 @@ import { stripVTControlCharacters as stripAnsi } from 'node:util'; import chalk from 'chalk'; import type { ConfidenceInterval, Verdict } from './summarizeCase'; +import type { CaseResult, MeasurementResult, TachometerReport } from './ciReport'; /** * Renders the report `tacho run` produces. @@ -13,58 +14,18 @@ import type { ConfidenceInterval, Verdict } from './summarizeCase'; * breaks the other. */ -export interface VariantResult { - variant: string; - /** The ref this variant loaded. */ - refId: string | null; - /** Mean duration, as a 95% confidence interval. */ - meanMs: ConfidenceInterval; - samples: number; - bytesSent: number; -} - -export interface Comparison { - /** The variant compared against the reference. */ - variant: string; - /** Whether the reference is faster, slower, or unresolved. */ - verdict: Verdict; - absoluteMs: ConfidenceInterval; - percentChange: ConfidenceInterval; - /** The same pair the other way round. */ - versusReference?: { - verdict: Verdict; - absoluteMs: ConfidenceInterval; - percentChange: ConfidenceInterval; - }; -} - -export interface MeasurementResult { - name: string; - variants: VariantResult[]; - /** The reference against every other variant. */ - comparisons: Comparison[]; -} - -export interface CaseResult { - name: string; - /** The variant every comparison is expressed against. Absent when the case failed to summarize. */ - reference?: string; - measurements?: MeasurementResult[]; - /** Why the case produced no summary. */ - error?: string; -} - -export interface TachometerReport { - version: number; - /** Which benchmark axis produced this. */ - reportType: 'tachometer'; - generatedAt: string; - /** The commit measured. */ - head: { ref: string; sha: string; branch?: string }; - /** The builds compared. */ - refs: Array<{ id: string; kind: string; label: string; sha?: string }>; - cases: CaseResult[]; -} +/** + * The wire format is defined once, by the schema CI validates against and S3 stores. Re-declaring + * it here is how the exported type and the stored one drift apart — the render side would keep + * compiling while the report grew a field it never learned about. + */ +export type { + TachometerReport, + CaseResult, + MeasurementResult, + VariantResult, + Comparison, +} from './ciReport'; /** A case that produced results, as opposed to one carrying only the error that stopped it. */ type SummarizedCase = CaseResult & { measurements: MeasurementResult[] }; diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts index f4fc03c553..7e30f0f928 100644 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -16,7 +16,9 @@ import { buildRefPages } from './buildPages'; import { assertDriverMatchesBrowser, resolveBrowserBinary, withBrowserDefaults } from './browser'; import { summarizeCase } from './summarizeCase'; import { renderTachometerReport } from './renderReport'; -import { getCiMetadata, syncPrComment, uploadCiReport } from './ciReport'; +import { getCiMetadata } from '../ciReport'; +import { syncPrComment } from '../syncPrComment'; +import { uploadCiReport } from './ciReport'; import type { TachometerReport } from './ciReport'; import { buildsDirOf, prepareOutputDir } from './outputDir'; import { run } from '../utils/exec'; @@ -147,7 +149,7 @@ export async function runTachometer(options: RunTachometerOptions): Promise { +/** + * Uploads a benchmark report to the CI report store. + * + * Validated before sending: the server stores `report` without inspecting it, so a malformed one + * would be accepted here and only fail later, when the dashboard tried to render it. + */ +async function uploadCiReport(report: BenchmarkUpload): Promise { benchmarkUploadSchema.parse(report); - const apiUrl = - options?.apiUrl ?? process.env.CI_REPORT_API_URL ?? 'https://frontend-public.mui.com'; - - const url = new URL('/api/ci-reports/upload', apiUrl); - - const oidcToken = process.env.CIRCLE_OIDC_TOKEN_V2; - if (!oidcToken) { - throw new Error('CIRCLE_OIDC_TOKEN_V2 environment variable is required for uploads'); - } - - // eslint-disable-next-line no-console - console.log('Upload request body:', JSON.stringify({ ...report, report: '...' }, null, 2)); - // eslint-disable-next-line no-console - console.log(`Uploading benchmark to ${url.href}`); - - const response = await fetch(url, { - method: 'POST', - headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${oidcToken}` }, - body: JSON.stringify(report), - }); - - const responseText = await response.text(); - - if (!response.ok) { - throw new Error(`Upload failed (${response.status}): ${responseText}`); - } - + const responseText = await postToDashboard( + '/api/ci-reports/upload', + report, + 'The benchmark upload', + ); const result = JSON.parse(responseText); - // eslint-disable-next-line no-console console.log(`Benchmark results uploaded successfully. S3 key: ${result.key}`); } diff --git a/packages/benchmark/src/utils/build.ts b/packages/benchmark/src/utils/build.ts index 64d87df98c..6900a559d3 100644 --- a/packages/benchmark/src/utils/build.ts +++ b/packages/benchmark/src/utils/build.ts @@ -8,13 +8,13 @@ export async function mapConcurrently( items: T[], mapper: (item: T) => Promise, concurrency: number, -): Promise<(R | Error)[]> { +): Promise { if (!items.length) { return []; } const itemIterator = items.entries(); const count = Math.min(concurrency, items.length); - const results: (R | Error)[] = new Array(items.length); + const results: R[] = new Array(items.length); const workers: Promise[] = []; for (let index = 0; index < count; index += 1) { workers.push( diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index 9f19637134..dbfa9bf3a8 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -11,7 +11,7 @@ import { execa, parseCommandString } from 'execa'; import { mapConcurrently } from './build'; import { run } from './exec'; import { pathExists } from './path'; -import { getWorkspacePackages } from './pnpm'; +import { listPublishablePackages } from './pnpm'; /** * Packs the public workspace packages at a given git ref into a folder of tarballs. @@ -193,13 +193,13 @@ export async function packBuiltPackages( outDir: string, ): Promise { await mkdir(outDir, { recursive: true }); - const packages = await getWorkspacePackages({ cwd: checkoutDir, publicOnly: true }); + const packages = await listPublishablePackages(checkoutDir); if (packages.length === 0) { throw new Error(`No public workspace packages found in ${checkoutDir}.`); } // Each `pnpm pack` is its own node process, so a repository with a dozen public packages would // otherwise start a dozen at once. - const packed = await mapConcurrently( + return mapConcurrently( packages, async ({ path: pkgDir, name, version }) => { const tarball = path.join(outDir, tarballName(name)); @@ -208,14 +208,6 @@ export async function packBuiltPackages( }, os.availableParallelism(), ); - // A failed pack rejects rather than landing in the results, but the signature admits an Error and - // one reaching a manifest would be far more confusing than one thrown here. - return packed.map((entry) => { - if (entry instanceof Error) { - throw entry; - } - return entry; - }); } /** diff --git a/packages/benchmark/src/utils/pnpm.ts b/packages/benchmark/src/utils/pnpm.ts index 2def2cd578..0d8d0f7154 100644 --- a/packages/benchmark/src/utils/pnpm.ts +++ b/packages/benchmark/src/utils/pnpm.ts @@ -2,14 +2,12 @@ import * as path from 'node:path'; import * as fs from 'node:fs/promises'; import { $ } from 'execa'; -/** A workspace package, as `pnpm ls` reports it. */ -export interface WorkspacePackage { +/** A publishable workspace package, as `pnpm ls` reports it. */ +export interface PublishablePackage { name: string; version: string; /** Absolute path to the package directory. */ path: string; - /** Whether the package is excluded from publishing. */ - isPrivate: boolean; } interface ListedPackage { @@ -20,34 +18,25 @@ interface ListedPackage { } /** - * The workspace packages of `cwd`, as `pnpm ls` resolves them. + * The publishable workspace packages of `cwd`, as `pnpm ls` resolves them. * - * Asks pnpm rather than scanning directories, so the set follows `pnpm-workspace.yaml` and matches - * exactly what a release would publish. A package counts as private when it says so, or when it + * Asks pnpm rather than scanning directories, so the set follows `pnpm-workspace.yaml` and mirrors + * exactly what a release publishes. A package is skipped when it says it is private, or when it * lacks the name or version that publishing requires. + * + * Deliberately narrower than the equivalent in `@mui/internal-code-infra`, which also filters by + * pattern and by what changed since a ref: those need git and pnpm-filter semantics nothing here + * wants. Named for what it does so the two cannot be mistaken for each other. */ -export async function getWorkspacePackages( - options: { cwd?: string; publicOnly?: boolean } = {}, -): Promise { - const { cwd, publicOnly = false } = options; - +export async function listPublishablePackages(cwd: string): Promise { const result = await $({ cwd })`pnpm ls -r --json --depth -1`; const listed: ListedPackage[] = JSON.parse(result.stdout); - return listed.flatMap((pkg) => { - const isPrivate = Boolean(pkg.private) || !pkg.name || !pkg.version; - if (publicOnly && isPrivate) { - return []; - } - return [ - { - name: pkg.name ?? '', - version: pkg.version ?? '', - path: pkg.path, - isPrivate, - }, - ]; - }); + return listed.flatMap((pkg) => + pkg.private || !pkg.name || !pkg.version + ? [] + : [{ name: pkg.name, version: pkg.version, path: pkg.path }], + ); } /** Reads a directory's `package.json`. */ diff --git a/test/tachometer/src/libs/sort/alpha.ts b/test/tachometer/src/libs/sort/alpha.ts index 2cde400413..425c67e246 100644 --- a/test/tachometer/src/libs/sort/alpha.ts +++ b/test/tachometer/src/libs/sort/alpha.ts @@ -1,9 +1,10 @@ import { measureMount } from '../../_shared/measure'; -import { SIZE, checksum, makeValues } from './shared'; +import { makeValues, report } from './shared'; + +const values = makeValues(); /** Sorts the boxed array in place, paying a comparator call per comparison. */ measureMount(() => { - const sorted = makeValues(); - sorted.sort((left, right) => left - right); - return `libs-sort [alpha] size=${SIZE} checksum=${checksum(sorted).toFixed(3)}`; + values.sort((left, right) => left - right); + return report('alpha', values); }); diff --git a/test/tachometer/src/libs/sort/beta.ts b/test/tachometer/src/libs/sort/beta.ts index fdef66d77c..bd68fad1b3 100644 --- a/test/tachometer/src/libs/sort/beta.ts +++ b/test/tachometer/src/libs/sort/beta.ts @@ -1,5 +1,7 @@ import { measureMount } from '../../_shared/measure'; -import { SIZE, checksum, makeValues } from './shared'; +import { makeValues, report } from './shared'; + +const values = makeValues(); /** A merge sort written in JavaScript, allocating as it goes — the slow variant of the three. */ function mergeSort(values: number[]): number[] { @@ -28,7 +30,4 @@ function mergeSort(values: number[]): number[] { return merged; } -measureMount(() => { - const sorted = mergeSort(makeValues()); - return `libs-sort [beta] size=${SIZE} checksum=${checksum(sorted).toFixed(3)}`; -}); +measureMount(() => report('beta', mergeSort(values))); diff --git a/test/tachometer/src/libs/sort/ours.ts b/test/tachometer/src/libs/sort/ours.ts index 3a083cae2a..15adf702f1 100644 --- a/test/tachometer/src/libs/sort/ours.ts +++ b/test/tachometer/src/libs/sort/ours.ts @@ -1,9 +1,13 @@ import { measureMount } from '../../_shared/measure'; -import { SIZE, checksum, makeValues } from './shared'; +import { makeValues, report } from './shared'; + +// Built outside the measured window: every sample is a fresh page load, so the input is still +// generated once per sample — it just no longer counts as part of the sort being measured. +const values = makeValues(); /** Copies into a typed array first, so the engine sorts unboxed doubles with no comparator calls. */ measureMount(() => { - const sorted = Float64Array.from(makeValues()); + const sorted = Float64Array.from(values); sorted.sort(); - return `libs-sort [ours] size=${SIZE} checksum=${checksum(Array.from(sorted)).toFixed(3)}`; + return report('ours', sorted); }); diff --git a/test/tachometer/src/libs/sort/shared.ts b/test/tachometer/src/libs/sort/shared.ts index 5fea325024..56e7dc3ba8 100644 --- a/test/tachometer/src/libs/sort/shared.ts +++ b/test/tachometer/src/libs/sort/shared.ts @@ -32,11 +32,20 @@ export function makeValues(): number[] { /** * A checksum that depends on the order, so a variant that returned an unsorted or truncated array * could not quietly report the same number as one that sorted correctly. + * + * Takes an `ArrayLike` so a typed array can be checked without being copied back into a boxed one — + * that copy would have landed inside the measured window of the very variant whose point is to + * avoid boxing. */ -export function checksum(values: readonly number[]): number { +export function checksum(values: ArrayLike): number { let total = 0; for (let index = 0; index < values.length; index += 1) { total += values[index] * index; } return total; } + +/** The line each variant puts on screen, so the three differ only in how they sort. */ +export function report(variant: string, sorted: ArrayLike): string { + return `libs-sort [${variant}] size=${SIZE} checksum=${checksum(sorted).toFixed(3)}`; +} From 67cd039336944248c38056e205705ca760ea15d1 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:20:05 +0200 Subject: [PATCH 033/173] [benchmark] Run the harness CLI without the bin shim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving the CLI into a package that publishes from `build/` broke the smoke test: pnpm creates a bin shim from the linked package's manifest, the workspace link points at `build/`, and that directory does not exist when install runs. A later build cannot repair it either — install has already decided there is no bin to link. CI failed with `benchmark: not found`. This repository already had the answer: `docs/scripts/runDocsInfra.mjs` wraps the same problem for `@mui/internal-docs-infra`, whose package has the same shape. The harness now imports the CLI through a `./cli` export instead. A consumer installing the published package gets a self-contained directory and can use the bin as normal; only a workspace link needs this. The job also builds first, the way `test_benchmark` does — the harness resolves the library through its build output, and now the CLI it runs comes from build output too. --- .circleci/config.yml | 5 +++++ packages/benchmark/package.json | 3 ++- test/tachometer/package.json | 2 +- test/tachometer/scripts/runBenchmark.mjs | 12 ++++++++++++ 4 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 test/tachometer/scripts/runBenchmark.mjs diff --git a/.circleci/config.yml b/.circleci/config.yml index bdd25cefaa..e19ecf0e29 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,6 +112,11 @@ jobs: steps: - checkout - code-infra/install-deps + - run: + # The harness resolves the library under test through its build output, and the CLI it + # runs is itself a workspace package that publishes from `build/`. + name: Build packages + command: pnpm release:build - run: name: Run the tachometer smoke test # Packing a ref installs and builds it in a quiet worktree; sampling then prints per case. diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index 253e74fc03..c05b86ecd2 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -14,7 +14,8 @@ "./reporter": "./src/reporter.ts", "./vitest": "./src/vitest.ts", "./tachometerPlugin": "./src/tachometer/plugin.ts", - "./tachometerReport": "./src/tachometer/renderReport.ts" + "./tachometerReport": "./src/tachometer/renderReport.ts", + "./cli": "./src/cli/index.ts" }, "scripts": { "build": "code-infra build --flat --tsgo --bundle esm", diff --git a/test/tachometer/package.json b/test/tachometer/package.json index 08e0cd0600..9dbe39fa24 100644 --- a/test/tachometer/package.json +++ b/test/tachometer/package.json @@ -5,7 +5,7 @@ "dev": "vite", "build": "vite build", "preview": "vite preview", - "test:tacho": "benchmark tacho run" + "test:tacho": "node ./scripts/runBenchmark.mjs tacho run" }, "dependencies": { "@mui/internal-test-utils": "workspace:*", diff --git a/test/tachometer/scripts/runBenchmark.mjs b/test/tachometer/scripts/runBenchmark.mjs new file mode 100644 index 0000000000..63da240419 --- /dev/null +++ b/test/tachometer/scripts/runBenchmark.mjs @@ -0,0 +1,12 @@ +// Thin wrapper that runs the `@mui/internal-benchmark` CLI from the workspace package's build +// output. Importing the package by name (rather than relying on its `benchmark` bin) avoids +// depending on pnpm linking the bin shim: the package publishes from `build/`, so the workspace +// link points at a directory that does not exist until the first build — and a later build cannot +// create a shim that install already skipped. The same wrapper exists in `docs/scripts` for the +// same reason. +// +// A consumer installing the published package gets a self-contained directory and can use the bin +// directly; only this repository, where the dependency is a workspace link, needs this. +// +// The CLI parses `process.argv` itself on import. +import '@mui/internal-benchmark/cli'; From 90c3e1bf04d601ea63fce20db43c506016068423 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:26:57 +0200 Subject: [PATCH 034/173] [benchmark] Import vite's types once Two `import type` statements from the same module trip `import/no-duplicates`, which CI counts as a failure because it lints with `--max-warnings 0`. The namespace import already carries `Plugin`. --- packages/benchmark/src/tachometer/buildPages.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts index 3c43a9cbb1..70f9a7ea04 100644 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -6,7 +6,6 @@ import { readFile, mkdir, writeFile } from 'node:fs/promises'; import { pathToFileURL } from 'node:url'; import chalk from 'chalk'; import type * as Vite from 'vite'; -import type { Plugin } from 'vite'; import { parse, stringify } from 'yaml'; import { run } from '../utils/exec'; import { tarballFor } from '../utils/packWorkspace'; @@ -110,7 +109,7 @@ async function readOverrides(repoRoot: string): Promise> * conditions (`browser`, `import`) and the target package's `exports` map still decide the answer — * which is what an alias to a directory would skip. */ -function resolveFromTree(treeDir: string): Plugin { +function resolveFromTree(treeDir: string): Vite.Plugin { // Resolution needs a base directory, and vite derives one from the importer's path — falling back // to the project root unless that path exists on disk, which would silently resolve the harness's // copy of a dependency instead of this ref's. The tree's own manifest is the file that names this From 10421ba857cbe4a34d43045221d6d277b4da607a Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:42:57 +0200 Subject: [PATCH 035/173] [benchmark] Find the manifest from the published layout too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CLI read its own version through `../../package.json`, which is the package root from `src/cli/` but escapes into `node_modules/@mui/` from `cli/` — where the file lands, since this package publishes from `build/`. A workspace link hides it: `import.meta.url` resolves through the symlink to the real `build/` directory, so the relative path happens to reach the manifest. Only a real install breaks, with `Cannot find module '../../package.json'`. Self-reference through the package name instead; the build declares the `./package.json` subpath in the published exports. --- packages/benchmark/src/cli/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index e1240624b1..89cc88a495 100644 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -7,7 +7,10 @@ import { hideBin } from 'yargs/helpers'; import cmdTacho from './cmdTacho'; -const pkgJson = createRequire(import.meta.url)('../../package.json'); +// Self-referencing, not a relative path: this file sits at `src/cli/` in the repository but at +// `cli/` in the published package, which is built from `build/`, so no single relative path +// reaches the manifest in both layouts. +const pkgJson = createRequire(import.meta.url)('@mui/internal-benchmark/package.json'); let globalArgv: { verbose?: boolean } = {}; From 05404886a47e2b6dbe20c7d59c9654f0a976940b Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 9 Sep 2026 20:50:55 +0200 Subject: [PATCH 036/173] [benchmark] Read the manifest from one place `buildPages` resolved this package's own name through `../../package.json` too, so the previous commit fixed only half the problem and a real run still died on import. Give both callers a single `ownPackage` module instead of repeating the self-reference, so there is one place to get this wrong rather than one per caller. --- packages/benchmark/src/cli/index.ts | 9 ++------- packages/benchmark/src/tachometer/buildPages.ts | 3 ++- packages/benchmark/src/utils/ownPackage.ts | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 packages/benchmark/src/utils/ownPackage.ts diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index 89cc88a495..41b00e3eb7 100644 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -1,16 +1,11 @@ #!/usr/bin/env node -import { createRequire } from 'node:module'; import chalk from 'chalk'; import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; import cmdTacho from './cmdTacho'; - -// Self-referencing, not a relative path: this file sits at `src/cli/` in the repository but at -// `cli/` in the published package, which is built from `build/`, so no single relative path -// reaches the manifest in both layouts. -const pkgJson = createRequire(import.meta.url)('@mui/internal-benchmark/package.json'); +import { ownPackage } from '../utils/ownPackage'; let globalArgv: { verbose?: boolean } = {}; @@ -43,5 +38,5 @@ await yargs(hideBin(process.argv)) .demandCommand(1, 'You need at least one command before moving on') .strict() .help() - .version(pkgJson.version) + .version(ownPackage.version) .parseAsync(); diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts index 70f9a7ea04..11ee62fd5b 100644 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -11,6 +11,7 @@ import { run } from '../utils/exec'; import { tarballFor } from '../utils/packWorkspace'; import type { PackedPackage } from '../utils/packWorkspace'; import { readPackageJson, writePackageJson } from '../utils/pnpm'; +import { ownPackage } from '../utils/ownPackage'; import type { ResolvedRef } from './refs'; /** @@ -28,7 +29,7 @@ const RUNNER_ONLY_DEPS = [ '@playwright/test', // Read rather than spelled out: this is the one entry naming *this* package, and a rename that // left a literal behind would quietly restore a multi-minute install per ref. - createRequire(import.meta.url)('../../package.json').name as string, + ownPackage.name, ]; /** diff --git a/packages/benchmark/src/utils/ownPackage.ts b/packages/benchmark/src/utils/ownPackage.ts new file mode 100644 index 0000000000..9dc9fe0c52 --- /dev/null +++ b/packages/benchmark/src/utils/ownPackage.ts @@ -0,0 +1,14 @@ +import { createRequire } from 'node:module'; + +/** + * This package's own manifest. + * + * Self-referencing, not a relative path: a module sits one directory deeper in the repository + * (`src/…`) than in the published package, which is built from `build/`, so no relative path + * reaches the manifest from both layouts — and the one that works in the repository silently keeps + * working through a workspace symlink, so only a real install shows the break. Resolving by name + * is independent of where the importing file sits. + */ +export const ownPackage: { name: string; version: string } = createRequire(import.meta.url)( + '@mui/internal-benchmark/package.json', +); From 04b47a0a9aafbd875b3bfe12f60820fd17aeefad Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 9 Sep 2026 21:05:59 +0200 Subject: [PATCH 037/173] [benchmark] Import the manifest instead of resolving it by hand Declaring the `./package.json` subpath lets both callers just import it, which drops the wrapper module and the `createRequire` call, and types `name` and `version` from the file rather than from a hand-written annotation. It also moves the failure: a specifier that cannot resolve is now a typecheck error rather than a crash in a published artifact. Import attributes need `module: esnext`, which is in any case the setting `moduleResolution: bundler` expects. The build warns that `./package.json` is missing from the output, because it generates the manifest after rewriting the exports. Harmless, and it already injects that same subpath into the published manifest. --- packages/benchmark/package.json | 1 + packages/benchmark/src/cli/index.ts | 7 +++++-- packages/benchmark/src/tachometer/buildPages.ts | 7 +++++-- packages/benchmark/src/utils/ownPackage.ts | 14 -------------- packages/benchmark/tsconfig.json | 2 +- 5 files changed, 12 insertions(+), 19 deletions(-) delete mode 100644 packages/benchmark/src/utils/ownPackage.ts diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index c05b86ecd2..7b57964cce 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -9,6 +9,7 @@ "directory": "packages/benchmark" }, "exports": { + "./package.json": "./package.json", ".": "./src/index.tsx", "./ciReport": "./src/ciReport.ts", "./reporter": "./src/reporter.ts", diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index 41b00e3eb7..e0c07d0dec 100644 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -4,8 +4,11 @@ import chalk from 'chalk'; import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; +// Self-referencing rather than `../../package.json`: this file is one directory deeper in the +// repository than in the published package, which is built from `build/`, so a relative path +// cannot reach the manifest from both — and the repository's workspace symlink hides the break. +import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; import cmdTacho from './cmdTacho'; -import { ownPackage } from '../utils/ownPackage'; let globalArgv: { verbose?: boolean } = {}; @@ -38,5 +41,5 @@ await yargs(hideBin(process.argv)) .demandCommand(1, 'You need at least one command before moving on') .strict() .help() - .version(ownPackage.version) + .version(pkgJson.version) .parseAsync(); diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts index 11ee62fd5b..21357e52f1 100644 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -7,11 +7,14 @@ import { pathToFileURL } from 'node:url'; import chalk from 'chalk'; import type * as Vite from 'vite'; import { parse, stringify } from 'yaml'; +// Self-referencing rather than `../../package.json`: this file is one directory deeper in the +// repository than in the published package, which is built from `build/`, so a relative path +// cannot reach the manifest from both — and the repository's workspace symlink hides the break. +import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; import { run } from '../utils/exec'; import { tarballFor } from '../utils/packWorkspace'; import type { PackedPackage } from '../utils/packWorkspace'; import { readPackageJson, writePackageJson } from '../utils/pnpm'; -import { ownPackage } from '../utils/ownPackage'; import type { ResolvedRef } from './refs'; /** @@ -29,7 +32,7 @@ const RUNNER_ONLY_DEPS = [ '@playwright/test', // Read rather than spelled out: this is the one entry naming *this* package, and a rename that // left a literal behind would quietly restore a multi-minute install per ref. - ownPackage.name, + pkgJson.name, ]; /** diff --git a/packages/benchmark/src/utils/ownPackage.ts b/packages/benchmark/src/utils/ownPackage.ts deleted file mode 100644 index 9dc9fe0c52..0000000000 --- a/packages/benchmark/src/utils/ownPackage.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { createRequire } from 'node:module'; - -/** - * This package's own manifest. - * - * Self-referencing, not a relative path: a module sits one directory deeper in the repository - * (`src/…`) than in the published package, which is built from `build/`, so no relative path - * reaches the manifest from both layouts — and the one that works in the repository silently keeps - * working through a workspace symlink, so only a real install shows the break. Resolving by name - * is independent of where the importing file sits. - */ -export const ownPackage: { name: string; version: string } = createRequire(import.meta.url)( - '@mui/internal-benchmark/package.json', -); diff --git a/packages/benchmark/tsconfig.json b/packages/benchmark/tsconfig.json index 8d73451597..2e10aaaadc 100644 --- a/packages/benchmark/tsconfig.json +++ b/packages/benchmark/tsconfig.json @@ -3,7 +3,7 @@ "compilerOptions": { "jsx": "react-jsx", "skipLibCheck": true, - "module": "es2022", + "module": "esnext", "lib": ["es2024"], "moduleResolution": "bundler", "resolveJsonModule": true, From 3b68e8812b4102e0bda30c3d76f6277ecc4b0123 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 06:33:01 +0200 Subject: [PATCH 038/173] [benchmark] Resolve and name a ref correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three faults in ref resolution, all of which produce a plausible-looking run: `rev-parse ` returns the tag *object* for an annotated tag, not a commit. That SHA became the ref's id, so the same commit reached by a tag and by its SHA looked like two builds — packed, installed and built twice — and the report recorded a SHA that is not a commit. `packRef` already peeled tags for its cache key; the two now share the argv rather than each having an opinion. A base branch was matched with a regex built from its own name, so `--base-branch v6.x` made `.` a wildcard and admitted `origin/v6-x` as a baseline candidate — which can win the closest-fork-point comparison and silently benchmark against the wrong commit. Compared segment by segment instead, like the local-branch case just below it. The default label embedded the short SHA that `id` already carries and both renderers then appended again, so a `git:` ref read `git-abc123def = v1.2.3 (abc123def) (abc123def)`. The label is now just the name the run was given, and the renderer no longer repeats the id. Also: the comment on `detectBaseBranch` described the opposite rule to the code, and would have had a reader break `release/7.x`. --- packages/benchmark/src/tachometer/refs.ts | 29 ++++++++++++++----- .../benchmark/src/tachometer/renderReport.ts | 6 ++-- packages/benchmark/src/utils/git.ts | 14 +++++++++ packages/benchmark/src/utils/packWorkspace.ts | 3 +- 4 files changed, 40 insertions(+), 12 deletions(-) create mode 100644 packages/benchmark/src/utils/git.ts diff --git a/packages/benchmark/src/tachometer/refs.ts b/packages/benchmark/src/tachometer/refs.ts index 39d54de80f..3c1c316396 100644 --- a/packages/benchmark/src/tachometer/refs.ts +++ b/packages/benchmark/src/tachometer/refs.ts @@ -1,5 +1,6 @@ import chalk from 'chalk'; import { execaSync } from 'execa'; +import { revParseCommitArgs } from '../utils/git'; /** * The `?ref=` grammar: which build of the workspace a benchmark page should load. @@ -82,6 +83,9 @@ export function parseRefToken(token?: string): RefDescriptor { ); } +/** Refname prefix every remote-tracking branch carries. */ +const REMOTES_PREFIX = 'refs/remotes/'; + /** Runs a git command, throwing on a non-zero exit. */ function gitCapture(args: string[], cwd: string): string { const result = execaSync('git', args, { cwd, reject: false }); @@ -100,7 +104,8 @@ function detectBaseBranch(cwd: string): string { if (result.exitCode !== 0) { return 'master'; } - // `origin/main` → `main`. Only the last segment is the branch name. + // `origin/main` → `main`. Everything after the remote name is the branch name, so a default + // branch like `release/7.x` keeps its slash. const shortRef = result.stdout.trim(); const slash = shortRef.indexOf('/'); return slash === -1 ? shortRef : shortRef.slice(slash + 1); @@ -128,13 +133,21 @@ function closestBaseBranch( // Match only real base branches from full refnames: a remote's `/` (exactly one // segment before it) or the local ``. Filtering short names on `/` would also catch a // local branch literally named e.g. `wip/master`. - const remoteBase = new RegExp(`^refs/remotes/[^/]+/${baseBranch}$`); + // + // Compared segment by segment rather than by a regex built from `baseBranch`: the branch name is + // arbitrary, and a real one like `v6.x` would make `.` match any character — quietly admitting + // `origin/v6-x` as a baseline candidate. + const isRemoteBase = (refname: string): boolean => { + const tail = refname.startsWith(REMOTES_PREFIX) ? refname.slice(REMOTES_PREFIX.length) : ''; + const slash = tail.indexOf('/'); + return slash !== -1 && tail.slice(slash + 1) === baseBranch; + }; const candidates = gitCapture( ['for-each-ref', '--format=%(refname)', 'refs/remotes', 'refs/heads'], repoRoot, ) .split('\n') - .filter((ref) => remoteBase.test(ref) || ref === `refs/heads/${baseBranch}`) + .filter((ref) => isRemoteBase(ref) || ref === `refs/heads/${baseBranch}`) .map((ref) => ref.replace(/^refs\/(remotes|heads)\//, '')) .sort((a, b) => priority(a) - priority(b)); @@ -176,11 +189,11 @@ export function createRefResolver(options: RefResolverOptions): { if (cached) { return cached; } - const sha = gitCapture(['rev-parse', committish], repoRoot); + const sha = gitCapture(revParseCommitArgs(committish), repoRoot); const ref: ResolvedRef = { kind: 'git', id: `git-${sha.slice(0, 9)}`, - label: label ?? `${committish} (${sha.slice(0, 9)})`, + label: label ?? committish, sha, committish: sha, }; @@ -205,17 +218,17 @@ export function createRefResolver(options: RefResolverOptions): { baseBranch ??= detectBaseBranch(repoRoot); if (gitCapture(['rev-parse', '--abbrev-ref', 'HEAD'], repoRoot) === baseBranch) { - return gitRef('HEAD~1', 'HEAD~1'); + return gitRef('HEAD~1'); } const base = closestBaseBranch(repoRoot, baseBranch); if (!base) { console.warn(chalk.yellow(`No ${baseBranch} branch found; using HEAD~1 as the baseline.`)); - return gitRef('HEAD~1', 'HEAD~1'); + return gitRef('HEAD~1'); } // Fork point is HEAD itself (HEAD already contained in the base branch) — nothing to diff. if (base.mergeBase === gitCapture(['rev-parse', 'HEAD'], repoRoot)) { - return gitRef('HEAD~1', 'HEAD~1'); + return gitRef('HEAD~1'); } return gitRef(base.mergeBase, `merge-base with ${base.ref}`); } diff --git a/packages/benchmark/src/tachometer/renderReport.ts b/packages/benchmark/src/tachometer/renderReport.ts index 56e388a341..901ba8ca8c 100644 --- a/packages/benchmark/src/tachometer/renderReport.ts +++ b/packages/benchmark/src/tachometer/renderReport.ts @@ -315,9 +315,9 @@ export function renderTachometerReport(report: TachometerReport): void { } } - const refLabels = report.refs.map( - (ref) => `${ref.id} = ${ref.label}${ref.sha ? ` (${ref.sha.slice(0, 9)})` : ''}`, - ); + // No short SHA appended: `id` is already `git-`, and the label is the name the run was + // given (a committish, or how a symbolic baseline resolved). + const refLabels = report.refs.map((ref) => `${ref.id} = ${ref.label}`); if (refLabels.length > 0) { notes.push(`Builds: ${refLabels.join(' · ')}`); } diff --git a/packages/benchmark/src/utils/git.ts b/packages/benchmark/src/utils/git.ts new file mode 100644 index 0000000000..12e192d0f2 --- /dev/null +++ b/packages/benchmark/src/utils/git.ts @@ -0,0 +1,14 @@ +/** + * Args that resolve a committish to the commit it names. + * + * `^{commit}` peels an annotated tag to its commit: `git rev-parse v1.2.3` returns the tag *object*, + * whose SHA is not the commit's, so the same commit reached by a tag and by its SHA would look like + * two different builds. `--verify` turns an unknown ref into a non-zero exit rather than an echoed + * argument. + * + * Shared because two callers resolve a ref — the ref resolver, and `packRef` to key its cache — and + * they disagreed about this. + */ +export function revParseCommitArgs(ref: string): string[] { + return ['rev-parse', '--verify', `${ref}^{commit}`]; +} diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index dbfa9bf3a8..8ff13cc6f7 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -11,6 +11,7 @@ import { execa, parseCommandString } from 'execa'; import { mapConcurrently } from './build'; import { run } from './exec'; import { pathExists } from './path'; +import { revParseCommitArgs } from './git'; import { listPublishablePackages } from './pnpm'; /** @@ -79,7 +80,7 @@ const MANIFEST = 'manifest.json'; /** Resolves a git ref to its commit SHA. */ async function resolveCommit(repoRoot: string, ref: string): Promise { - const result = await execa('git', ['rev-parse', '--verify', `${ref}^{commit}`], { + const result = await execa('git', revParseCommitArgs(ref), { cwd: repoRoot, reject: false, }); From dff0e4760ff03ab8169ce0397d50e37edfcc4846 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 06:33:11 +0200 Subject: [PATCH 039/173] [benchmark] Set the baseline's ref through URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Appending `?ref=baseline` to the url text only considered an existing `?`, not a `#`. A url with a fragment got `ref` placed inside that fragment, where `searchParams` never sees it — so the baseline variant resolved to the working tree, both sides shared a ref, every comparison was filtered out as same-ref, and the case reported as unchanged forever with no error. The query and fragment now go through `URL`. The path is still carried as written: these urls are relative to their own config's directory and are allowed to traverse out of it, and `URL` would collapse that traversal against whatever base it was handed. --- .../src/tachometer/discoverCases.test.ts | 26 +++++++++++++++++++ .../benchmark/src/tachometer/discoverCases.ts | 22 ++++++++++++++-- 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/packages/benchmark/src/tachometer/discoverCases.test.ts b/packages/benchmark/src/tachometer/discoverCases.test.ts index 4f799ecb62..f6846f4f73 100644 --- a/packages/benchmark/src/tachometer/discoverCases.test.ts +++ b/packages/benchmark/src/tachometer/discoverCases.test.ts @@ -172,6 +172,32 @@ describe('discoverCases', () => { expect(entry.config.benchmarks[1].url).toBe('./index.html?rows=100&ref=baseline'); }); + it('keeps the ref in the query when the url has a fragment', async () => { + // Appending to the url text put `ref` after the `#`, where it is part of the fragment and + // `searchParams` never sees it — so the baseline variant silently loaded the working tree and + // the case could never report a regression. + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html#state'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.config.benchmarks[1].url).toBe('./index.html?ref=baseline#state'); + }); + + it('leaves a traversing path untouched', async () => { + // The url is relative to its own config's directory and is allowed to leave it; resolving it + // against a base to edit the query would collapse the `..` segments. + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + beta: { config: config('beta', '../alpha/index.html?rows=100') }, + }); + + const [entry] = await discoverCases({ harnessDir, filters: ['beta'] }); + + expect(entry.config.benchmarks[1].url).toBe('../alpha/index.html?rows=100&ref=baseline'); + }); + it('merges an expansion over its parent, nearest value winning', async () => { // Tachometer merges an expansion over its parent before it reads anything, so a field set // deeper wins and one set only on the parent is inherited. Flattening here has to match, or diff --git a/packages/benchmark/src/tachometer/discoverCases.ts b/packages/benchmark/src/tachometer/discoverCases.ts index 587f03aca7..1d075d92dd 100644 --- a/packages/benchmark/src/tachometer/discoverCases.ts +++ b/packages/benchmark/src/tachometer/discoverCases.ts @@ -118,6 +118,25 @@ function flattenExpansions(benchmark: any): any[] { return expand.flatMap((child) => flattenExpansions(child).map((leaf) => ({ ...rest, ...leaf }))); } +/** Base for parsing a benchmark url's query and fragment. Its path is never read. */ +const URL_PARTS_BASE = 'http://case.invalid/'; + +/** + * The same url with `?ref=` set. + * + * The path is carried through as written, not resolved: these urls are relative to their own + * config's directory and traverse out of it (`../row-updates/index.html?throttle=16`), and `URL` + * would clamp that traversal against whatever base it was given. Only the query and fragment go + * through `URL`, which is where the syntax rules are — appending text instead put `ref` *inside* a + * fragment for a url that had one, where nothing reads it. + */ +function withRef(url: string, ref: string): string { + const parsed = new URL(url, URL_PARTS_BASE); + parsed.searchParams.set('ref', ref); + const [pathPart] = url.split(/[?#]/); + return `${pathPart}${parsed.search}${parsed.hash}`; +} + /** * Resolves one leaf url into the source page it references, the ref it selects, and any leftovers. * @@ -247,10 +266,9 @@ export async function discoverCases(options: DiscoverCasesOptions): Promise Date: Thu, 10 Sep 2026 06:33:11 +0200 Subject: [PATCH 040/173] [benchmark] Fail on a pnpm-workspace.yaml that cannot be read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catch was for a repository that has no such file, but it also absorbed a YAML syntax error and a permission error. The overrides decide a ref tree's dependency graph, so swallowing those resolves the un-overridden graph and benchmarks a dependency set the repository never ships — the exact failure the comment above the function warns about, reached through the one path it did not guard. --- packages/benchmark/src/tachometer/buildPages.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts index 21357e52f1..0be40d41b0 100644 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -97,12 +97,19 @@ async function rewriteWorkspaceDeps( * satisfy (a blocked resolution, a security pin) would fail here too. */ async function readOverrides(repoRoot: string): Promise> { + let raw: string; try { - const root = parse(await readFile(path.join(repoRoot, 'pnpm-workspace.yaml'), 'utf8')); - return root?.overrides ?? {}; - } catch { - return {}; + raw = await readFile(path.join(repoRoot, 'pnpm-workspace.yaml'), 'utf8'); + } catch (error) { + // A repository without the file simply has no overrides. Anything else — a malformed file, a + // permission error — would otherwise resolve the un-overridden graph and silently benchmark a + // dependency set the repository never ships, which is the failure this function exists to avoid. + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return {}; + } + throw error; } + return parse(raw)?.overrides ?? {}; } /** From a18bac1b0f42541ccc91ccb6462afd088d0ac019 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 06:33:26 +0200 Subject: [PATCH 041/173] [benchmark] Mark the build's output directory, and yield the index to a case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `tachometer.json` directly in `src/` is a supported layout, and such a case can own `index.html`. Rollup then drops the plugin's emit of the case list with no error and no warning, so the list just disappears. Skip it deliberately and say why — in that layout the case's own page is the landing page anyway. `prepareOutputDir` writes the `.gitignore` that makes `.tachometer/` ignore itself, but only `tacho run` called it. A harness that runs a plain `vite build` first — and does not list the directory in its own ignore file — was offered its built pages for commit, which is what the marker exists to prevent. --- packages/benchmark/src/tachometer/plugin.ts | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/benchmark/src/tachometer/plugin.ts b/packages/benchmark/src/tachometer/plugin.ts index a90bf00eca..52e1796f0c 100644 --- a/packages/benchmark/src/tachometer/plugin.ts +++ b/packages/benchmark/src/tachometer/plugin.ts @@ -3,7 +3,7 @@ import { readFile, realpath } from 'node:fs/promises'; import type { Plugin } from 'vite'; import { discoverCases, pagesOf } from './discoverCases'; import type { BenchmarkCase } from './discoverCases'; -import { buildsDirOf } from './outputDir'; +import { buildsDirOf, prepareOutputDir } from './outputDir'; /** * Vite plugin for a tachometer benchmark harness. @@ -124,7 +124,14 @@ export function tachometer(options: TachometerPluginOptions = {}): Plugin { // Only fill it in when the caller said nothing: a plugin's returned config is merged *over* // the inline config, so unconditionally setting it here would silently override // `vite build --outDir`, which is exactly how `tacho run` directs each ref's build. + const usingDefaultOutDir = userConfig.build?.outDir === undefined; const outDir = userConfig.build?.outDir ?? buildsDirOf(harnessDir, 'manual'); + // Mark the directory ignored from the inside before anything writes into it. `tacho run` does + // this itself and passes its own `outDir`, but a plain `vite build` would otherwise leave a + // harness that does not list `.tachometer` offering built pages for commit. + if (usingDefaultOutDir) { + await prepareOutputDir(harnessDir); + } if (env.command !== 'build') { return { root: srcDir, appType: 'mpa', build: { outDir } }; @@ -161,6 +168,17 @@ export function tachometer(options: TachometerPluginOptions = {}): Plugin { // case is only reachable through this list, so a build without it can only be entered by typing // urls from memory. generateBundle() { + // A `tachometer.json` directly in `src/` is a supported layout, and such a case can own + // `index.html` — which rollup emits under that very name. Rollup then drops this emit with no + // error and no warning, so the list simply vanishes; say so instead. The case's page is the + // landing page in that layout anyway. + if (pagesOf(buildCases).includes('index.html')) { + this.warn( + 'A case owns index.html, so the case list was not emitted. Move the case into a subfolder of src/ to get it back.', + ); + return; + } + this.emitFile({ type: 'asset', fileName: 'index.html', From 6a7292c2a461cfe286ae0c0efeda421e475a8a0a Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 06:33:26 +0200 Subject: [PATCH 042/173] [dashboard] Store the tachometer report, not an envelope around it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `tacho run` writes the report; S3 stored the upload envelope wrapping it. Both carry `reportType: 'tachometer'` at the top level, so `tacho report`'s guard passed for either and then read `cases` off an object that has none — the CI artifact its own help text tells you to download crashed it. Nothing needed the envelope here. The report already carries `version`, `reportType` and `head.sha`/`head.branch`, the repo is in the S3 key, and the generator takes its metadata from its own route params. Storing only the report leaves one shape, so there is nothing to tell apart. The benchmark axis keeps its envelope. Also pass `prNumber` to the details link, which the view already reads to title the page and link back to the PR, and drop the SHA the view repeated after an id that already contains it. --- .../app/api/ci-reports/upload/route.ts | 10 +++++++--- .../src/lib/ciReports/tachometerReport.ts | 12 +++++++----- .../src/lib/tachometer/types.ts | 12 ------------ .../src/utils/fetchCiReport.ts | 4 ++-- .../src/views/TachometerDetails.tsx | 14 ++++---------- packages/benchmark/src/cli/tachoReport.ts | 5 +++-- 6 files changed, 23 insertions(+), 34 deletions(-) diff --git a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts index 6b1fa79906..6a313a4502 100644 --- a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts +++ b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts @@ -8,9 +8,13 @@ import { findAssociatedPr } from '@/lib/ciReports/findAssociatedPr'; const VALID_REPORT_TYPES = new Set(['size-snapshot', 'benchmark', 'tachometer']); // Report types stored as the full upload envelope (version, timestamp, commitSha, repo, -// branch, prNumber, reportType, report). Older types keep their historic -// "just the inner report" storage. -const ENVELOPE_REPORT_TYPES = new Set(['benchmark', 'tachometer']); +// branch, prNumber, reportType, report). Every other type stores just the inner report. +// +// A tachometer report is not one of them: it already carries its own `version`, `reportType` and +// `head.sha`/`head.branch`, the repo is in the S3 key, and the generator reads the envelope's +// metadata from its own route params rather than from the artifact. Storing the envelope only +// created a second shape, which `tacho report` then could not tell apart from a local report. +const ENVELOPE_REPORT_TYPES = new Set(['benchmark']); const uploadSchema = z.object({ version: z.number(), diff --git a/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts b/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts index 587039389f..d5c84f67d9 100644 --- a/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts +++ b/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts @@ -5,7 +5,7 @@ import { } from '@/lib/tachometer/buildMarkdownReport'; import { DASHBOARD_ORIGIN } from '@/constants'; -import type { TachometerUpload } from '@/lib/tachometer/types'; +import type { TachometerReport } from '@/lib/tachometer/types'; import type { ReportOptions, ReportResult } from './types'; export { TACHOMETER_SECTION_TITLE }; @@ -25,17 +25,19 @@ export { TACHOMETER_SECTION_TITLE }; export async function generateTachometerReport( options: ReportOptions, ): Promise { - const { repo, commitSha } = options; + const { repo, prNumber, commitSha } = options; - const upload: TachometerUpload | null = await fetchCiReport(repo, commitSha, 'tachometer.json'); - if (!upload?.report) { + const report: TachometerReport | null = await fetchCiReport(repo, commitSha, 'tachometer.json'); + if (!report?.cases) { return null; } const detailsUrl = new URL(`${DASHBOARD_ORIGIN}/tachometer-details/${repo}`); detailsUrl.searchParams.set('sha', commitSha); + // The details view reads this to title the page and link back to the PR overview. + detailsUrl.searchParams.set('prNumber', String(prNumber)); return { - content: buildTachometerMarkdownReport(upload.report, { detailsUrl: detailsUrl.href }), + content: buildTachometerMarkdownReport(report, { detailsUrl: detailsUrl.href }), }; } diff --git a/apps/code-infra-dashboard/src/lib/tachometer/types.ts b/apps/code-infra-dashboard/src/lib/tachometer/types.ts index c49e4eadc2..d7fa2b1654 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/types.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/types.ts @@ -59,15 +59,3 @@ export interface TachometerReport { cases: TachometerCaseResult[]; raw?: Record; } - -/** The stored upload envelope, as the upload route writes it. */ -export interface TachometerUpload { - version: number; - timestamp: number; - commitSha: string; - repo: string; - reportType: 'tachometer'; - prNumber?: number; - branch: string; - report: TachometerReport; -} diff --git a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts index d3837fae7d..ce1d5a65d9 100644 --- a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts +++ b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts @@ -1,13 +1,13 @@ import type { SizeSnapshotWithMetadata } from '@/lib/bundleSize/types'; import type { BenchmarkReport, BenchmarkUpload } from '@/lib/benchmark/types'; -import type { TachometerUpload } from '@/lib/tachometer/types'; +import type { TachometerReport } from '@/lib/tachometer/types'; import { migrateBenchmarkReport } from '@/lib/benchmark/migrateBenchmarkReport'; export interface CiReportTypes { 'benchmark.json': BenchmarkUpload; 'size-snapshot.json': SizeSnapshotWithMetadata; // Always stored as the full envelope, so there is no legacy flat shape to normalize. - 'tachometer.json': TachometerUpload; + 'tachometer.json': TachometerReport; } export type CiReportName = keyof CiReportTypes; diff --git a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx index b73ce3f351..f38b7d5ef2 100644 --- a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx +++ b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx @@ -15,7 +15,7 @@ import TableHead from '@mui/material/TableHead'; import TableRow from '@mui/material/TableRow'; import Typography from '@mui/material/Typography'; import { fetchCiReport } from '@/utils/fetchCiReport'; -import type { ConfidenceInterval, TachometerUpload, Verdict } from '@/lib/tachometer/types'; +import type { ConfidenceInterval, TachometerReport, Verdict } from '@/lib/tachometer/types'; import { bytesPerVariant, groupCasesByVariantSet, @@ -266,10 +266,10 @@ export default function TachometerDetails() { const prNumber = searchParams.get('prNumber'); const { - data: upload, + data: report, isLoading, error, - } = useQuery({ + } = useQuery({ queryKey: ['tachometer-report', repo, sha], queryFn: () => fetchCiReport(repo, sha!, 'tachometer.json'), retry: 1, @@ -287,7 +287,6 @@ export default function TachometerDetails() { ); } - const report = upload?.report; const summarized = (report?.cases ?? []).filter(isSummarized); const failed = (report?.cases ?? []).filter((entry) => !isSummarized(entry)); const groups = groupCasesByVariantSet(summarized); @@ -366,12 +365,7 @@ export default function TachometerDetails() {     ))} - Builds:{' '} - {report.refs - .map( - (ref) => `${ref.id} = ${ref.label}${ref.sha ? ` (${ref.sha.slice(0, 9)})` : ''}`, - ) - .join(' · ')} + Builds: {report.refs.map((ref) => `${ref.id} = ${ref.label}`).join(' · ')} head: {report.head.sha.slice(0, 9)} ({report.head.branch || '?'}) · measured on the diff --git a/packages/benchmark/src/cli/tachoReport.ts b/packages/benchmark/src/cli/tachoReport.ts index 2f2486d9dc..e8f7d084b7 100644 --- a/packages/benchmark/src/cli/tachoReport.ts +++ b/packages/benchmark/src/cli/tachoReport.ts @@ -38,8 +38,9 @@ const command: CommandModule<{}, Args> = { }); const report = JSON.parse(raw); - // Reports from the other benchmark axis have the same envelope but different statistics, and - // rendering one here would silently read its numbers as confidence intervals on a difference. + // The report a run writes and the artifact the dashboard stores are the same shape, so this + // reads either. The check is for the other benchmark axis, whose reports carry different + // statistics that rendering here would silently read as confidence intervals on a difference. if (report?.reportType !== 'tachometer') { throw new Error( `${reportPath} is not a tachometer report (its "reportType" is ${JSON.stringify(report?.reportType)}).`, From ad6f4f3d79cc7eed38be723c3ae99849a47d9db4 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 06:33:26 +0200 Subject: [PATCH 043/173] [test] Describe how the harness page resolves its import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Under `tacho run` every ref resolves the workspace package from a packed tarball, the working tree included — that symmetry is the point, and the comment claimed the opposite. The workspace link applies to a plain `vite dev` or `vite build`. --- test/tachometer/src/workload/index.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/tachometer/src/workload/index.ts b/test/tachometer/src/workload/index.ts index 76cbad5521..e20ed2abfc 100644 --- a/test/tachometer/src/workload/index.ts +++ b/test/tachometer/src/workload/index.ts @@ -9,9 +9,10 @@ import { measureMount, numberParam } from '../_shared/measure'; * `benchmark tacho run` something real to drive end to end. * * It does import a workspace package (`@mui/internal-test-utils`) and put its value on screen. That - * is the part that matters: the working tree resolves it through the workspace link to the - * package's build output, while every other ref resolves it from a packed tarball in an isolated - * install. If either path breaks, this page fails to build rather than silently measuring nothing. + * is the part that matters: under `tacho run` every ref — the working tree included — resolves it + * from a packed tarball in an isolated install, so both sides of a comparison resolve identically. + * A plain `vite dev` or `vite build` resolves it through the workspace link instead. If either path + * breaks, this page fails to build rather than silently measuring nothing. */ /** A small linear congruential generator, so every sample does identical work. */ From 84bf0c383b927e53654fe51a08b8859d0c4d22cd Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 06:38:44 +0200 Subject: [PATCH 044/173] [benchmark] Cover the ref resolution this branch changed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `refs.test.ts` exercised only the pure token parser, so everything that decides what a run is compared against — tag peeling, base-branch matching, how a ref is named — had no test, and the three fixes in it rested on reasoning plus a CI run. Over a real repository rather than a stand-in for git: each case asks what git reports, and faking that would only restate the answer the code already assumes. All four fail against the previous behaviour. --- .../benchmark/src/tachometer/refs.test.ts | 86 ++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) diff --git a/packages/benchmark/src/tachometer/refs.test.ts b/packages/benchmark/src/tachometer/refs.test.ts index 428cc82bc3..820a72169c 100644 --- a/packages/benchmark/src/tachometer/refs.test.ts +++ b/packages/benchmark/src/tachometer/refs.test.ts @@ -1,5 +1,9 @@ +import * as path from 'node:path'; +import { writeFile } from 'node:fs/promises'; import { describe, expect, it } from 'vitest'; -import { parseRefToken } from './refs'; +import { execa } from 'execa'; +import { makeTempDir } from '../utils/testUtils'; +import { createRefResolver, parseRefToken } from './refs'; describe('parseRefToken', () => { it('treats an absent ref as the working tree', () => { @@ -46,3 +50,83 @@ describe('parseRefToken', () => { }); }); }); + +/** + * A repository with three commits on `main`, the first of them carrying an annotated tag. + * + * A real repository rather than a stand-in for git: every behaviour here is a question about what + * git reports — what a tag resolves to, which refnames exist — and a fake would only restate the + * answer the code already assumes. + */ +async function makeRepo(): Promise<{ repoRoot: string; shas: string[] }> { + const repoRoot = await makeTempDir(); + const git = (...args: string[]) => execa('git', args, { cwd: repoRoot }); + + await git('init', '--initial-branch=main'); + await git('config', 'user.email', 'fixture@example.com'); + await git('config', 'user.name', 'Fixture'); + + const shas: string[] = []; + for (const message of ['first', 'second', 'third']) { + // eslint-disable-next-line no-await-in-loop + await writeFile(path.join(repoRoot, `${message}.txt`), `${message}\n`); + // eslint-disable-next-line no-await-in-loop + await git('add', '.'); + // eslint-disable-next-line no-await-in-loop + await git('commit', '-m', message); + // eslint-disable-next-line no-await-in-loop + const { stdout } = await git('rev-parse', 'HEAD'); + shas.push(stdout.trim()); + } + + // Annotated (`-m`), so the tag is its own object with its own SHA. + await git('tag', '-a', 'v1.0.0', '-m', 'release', shas[0]); + + return { repoRoot, shas }; +} + +describe('createRefResolver', () => { + it('resolves an annotated tag to the commit it points at', async () => { + const { repoRoot, shas } = await makeRepo(); + + const ref = createRefResolver({ repoRoot }).parse('git:v1.0.0'); + + // Without `^{commit}` this is the tag object's own SHA, which is not a commit at all. + expect(ref.sha).toBe(shas[0]); + }); + + it('gives a tag and its commit one identity, so the commit is built once', async () => { + const { repoRoot, shas } = await makeRepo(); + const resolver = createRefResolver({ repoRoot }); + + const viaTag = resolver.parse('git:v1.0.0'); + const viaSha = resolver.parse(`git:${shas[0]}`); + + // `id` is the build directory and the dedupe key: two ids means packing and building twice. + expect(viaTag.id).toBe(viaSha.id); + }); + + it('labels a git ref with the revision it was given', async () => { + const { repoRoot } = await makeRepo(); + + const ref = createRefResolver({ repoRoot }).parse('git:v1.0.0'); + + // Not `v1.0.0 ()`: `id` already carries the short SHA, and the renderers print both. + expect(ref.label).toBe('v1.0.0'); + }); + + it('does not treat a branch matching the base branch as a pattern', async () => { + const { repoRoot, shas } = await makeRepo(); + const git = (...args: string[]) => execa('git', args, { cwd: repoRoot }); + // `v6-x`, not `v6.x` — only a regex reading `.` as a wildcard would accept it. + await git('update-ref', 'refs/remotes/origin/v6-x', shas[0]); + await git('checkout', '-b', 'feature'); + + const ref = createRefResolver({ repoRoot, baseBranch: 'v6.x' }).parse('baseline'); + + // No base branch exists, so the baseline falls back to HEAD~1. Matching `origin/v6-x` would + // instead have picked its merge base — the first commit — and benchmarked against the wrong one. + expect(ref.sha).toBe(shas[1]); + expect(ref.sha).not.toBe(shas[0]); + }); +}); From cc65a1e9d099b05c187ced948e5512ae22513719 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 07:09:03 +0200 Subject: [PATCH 045/173] [benchmark] Tidy up after the review fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `detectBaseBranch` and the new `isRemoteBase` each derived "the branch name is everything after the remote", which is the one subtle part — split on the *first* slash, so `release/7.x` keeps its own. One `branchNameOf` now holds it. The output directory's self-ignoring marker was written from the `config` hook whenever the caller named no `outDir`, so `vite dev` and `vite preview` — neither of which writes anything — created `.tachometer/` as a side effect of resolving configuration. It now runs only for a build, and keys on whether the output actually lands under that directory rather than on who chose it. Build and the dev server disagreed about who owns `index.html` when a case sits directly in `src/`: the build yielded to the case, the dev middleware kept intercepting `/`. They now ask one predicate. Also: the fixture repo spawned thirteen git processes per test where six do, and handed back no `git` helper so one test rebuilt it; a comment in `fetchCiReport` still claimed the tachometer artifact is an envelope, which the previous commit made false; and an assertion that could not fail, a single-use constant, and a verbatim-duplicated note are gone. --- .../src/utils/fetchCiReport.ts | 3 +- .../benchmark/src/tachometer/buildPages.ts | 5 +-- .../benchmark/src/tachometer/discoverCases.ts | 5 +-- packages/benchmark/src/tachometer/plugin.ts | 42 ++++++++++++------ .../benchmark/src/tachometer/refs.test.ts | 43 ++++++++++--------- packages/benchmark/src/tachometer/refs.ts | 28 ++++++++---- 6 files changed, 76 insertions(+), 50 deletions(-) diff --git a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts index ce1d5a65d9..93283b9f71 100644 --- a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts +++ b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts @@ -6,7 +6,8 @@ import { migrateBenchmarkReport } from '@/lib/benchmark/migrateBenchmarkReport'; export interface CiReportTypes { 'benchmark.json': BenchmarkUpload; 'size-snapshot.json': SizeSnapshotWithMetadata; - // Always stored as the full envelope, so there is no legacy flat shape to normalize. + // Stored as the report itself, the same shape `tacho run` writes: no envelope to unwrap, and + // no legacy flat shape to normalize. 'tachometer.json': TachometerReport; } diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts index 0be40d41b0..36abed3013 100644 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -7,9 +7,8 @@ import { pathToFileURL } from 'node:url'; import chalk from 'chalk'; import type * as Vite from 'vite'; import { parse, stringify } from 'yaml'; -// Self-referencing rather than `../../package.json`: this file is one directory deeper in the -// repository than in the published package, which is built from `build/`, so a relative path -// cannot reach the manifest from both — and the repository's workspace symlink hides the break. +// Self-referencing rather than `../../package.json`, which resolves only in the repository — see +// the note in ../cli/index.ts. import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; import { run } from '../utils/exec'; import { tarballFor } from '../utils/packWorkspace'; diff --git a/packages/benchmark/src/tachometer/discoverCases.ts b/packages/benchmark/src/tachometer/discoverCases.ts index 1d075d92dd..56808f8ced 100644 --- a/packages/benchmark/src/tachometer/discoverCases.ts +++ b/packages/benchmark/src/tachometer/discoverCases.ts @@ -118,9 +118,6 @@ function flattenExpansions(benchmark: any): any[] { return expand.flatMap((child) => flattenExpansions(child).map((leaf) => ({ ...rest, ...leaf }))); } -/** Base for parsing a benchmark url's query and fragment. Its path is never read. */ -const URL_PARTS_BASE = 'http://case.invalid/'; - /** * The same url with `?ref=` set. * @@ -131,7 +128,7 @@ const URL_PARTS_BASE = 'http://case.invalid/'; * fragment for a url that had one, where nothing reads it. */ function withRef(url: string, ref: string): string { - const parsed = new URL(url, URL_PARTS_BASE); + const parsed = new URL(url, 'http://case.invalid/'); parsed.searchParams.set('ref', ref); const [pathPart] = url.split(/[?#]/); return `${pathPart}${parsed.search}${parsed.hash}`; diff --git a/packages/benchmark/src/tachometer/plugin.ts b/packages/benchmark/src/tachometer/plugin.ts index 52e1796f0c..cb51b6b372 100644 --- a/packages/benchmark/src/tachometer/plugin.ts +++ b/packages/benchmark/src/tachometer/plugin.ts @@ -3,7 +3,7 @@ import { readFile, realpath } from 'node:fs/promises'; import type { Plugin } from 'vite'; import { discoverCases, pagesOf } from './discoverCases'; import type { BenchmarkCase } from './discoverCases'; -import { buildsDirOf, prepareOutputDir } from './outputDir'; +import { buildsDirOf, prepareOutputDir, OUTPUT_DIR } from './outputDir'; /** * Vite plugin for a tachometer benchmark harness. @@ -98,6 +98,17 @@ ${items} `; } +/** + * Whether a case's own page occupies the `index.html` the case list would be served at. + * + * A `tachometer.json` directly in `src/` is a supported layout, and such a case may own the root + * page. Asked in one place because the build and the dev server both have to give the same answer: + * otherwise `/` shows the case under one command and the list under the other. + */ +function aCaseOwnsIndex(cases: BenchmarkCase[]): boolean { + return pagesOf(cases).includes('index.html'); +} + /** Creates the tachometer harness plugin. */ export function tachometer(options: TachometerPluginOptions = {}): Plugin { const harnessDir = options.harnessDir ?? process.cwd(); @@ -124,19 +135,21 @@ export function tachometer(options: TachometerPluginOptions = {}): Plugin { // Only fill it in when the caller said nothing: a plugin's returned config is merged *over* // the inline config, so unconditionally setting it here would silently override // `vite build --outDir`, which is exactly how `tacho run` directs each ref's build. - const usingDefaultOutDir = userConfig.build?.outDir === undefined; const outDir = userConfig.build?.outDir ?? buildsDirOf(harnessDir, 'manual'); - // Mark the directory ignored from the inside before anything writes into it. `tacho run` does - // this itself and passes its own `outDir`, but a plain `vite build` would otherwise leave a - // harness that does not list `.tachometer` offering built pages for commit. - if (usingDefaultOutDir) { - await prepareOutputDir(harnessDir); - } if (env.command !== 'build') { return { root: srcDir, appType: 'mpa', build: { outDir } }; } + // Mark the output directory ignored from the inside before the build writes into it, so a + // harness that does not list `.tachometer` is not offered its built pages for commit. Keyed on + // where the output actually lands rather than on who chose it: `tacho run` passes its own + // `outDir` under the same directory and prepares it itself, and any other caller that writes + // there deserves the marker too. Vite resolves a relative `outDir` against `root`. + if (path.resolve(srcDir, outDir).startsWith(path.join(harnessDir, OUTPUT_DIR))) { + await prepareOutputDir(harnessDir); + } + const cases = await discoverCases({ harnessDir }); buildCases = cases; // Key the input map by the page path without its extension. Vite emits each page at its path @@ -168,11 +181,9 @@ export function tachometer(options: TachometerPluginOptions = {}): Plugin { // case is only reachable through this list, so a build without it can only be entered by typing // urls from memory. generateBundle() { - // A `tachometer.json` directly in `src/` is a supported layout, and such a case can own - // `index.html` — which rollup emits under that very name. Rollup then drops this emit with no - // error and no warning, so the list simply vanishes; say so instead. The case's page is the - // landing page in that layout anyway. - if (pagesOf(buildCases).includes('index.html')) { + // Rollup drops an emit whose name a bundle output already claims, with no error and no + // warning, so the list would simply vanish. Say so instead. + if (aCaseOwnsIndex(buildCases)) { this.warn( 'A case owns index.html, so the case list was not emitted. Move the case into a subfolder of src/ to get it back.', ); @@ -198,6 +209,11 @@ export function tachometer(options: TachometerPluginOptions = {}): Plugin { } try { const cases = await discoverCases({ harnessDir }); + if (aCaseOwnsIndex(cases)) { + // Let vite serve the case's own page, the way the build does. + next(); + return; + } res.setHeader('Content-Type', 'text/html; charset=utf-8'); res.end( renderIndex( diff --git a/packages/benchmark/src/tachometer/refs.test.ts b/packages/benchmark/src/tachometer/refs.test.ts index 820a72169c..a0bc6e0020 100644 --- a/packages/benchmark/src/tachometer/refs.test.ts +++ b/packages/benchmark/src/tachometer/refs.test.ts @@ -1,5 +1,3 @@ -import * as path from 'node:path'; -import { writeFile } from 'node:fs/promises'; import { describe, expect, it } from 'vitest'; import { execa } from 'execa'; import { makeTempDir } from '../utils/testUtils'; @@ -58,31 +56,38 @@ describe('parseRefToken', () => { * git reports — what a tag resolves to, which refnames exist — and a fake would only restate the * answer the code already assumes. */ -async function makeRepo(): Promise<{ repoRoot: string; shas: string[] }> { +async function makeRepo(): Promise<{ + repoRoot: string; + shas: string[]; + git: (...args: string[]) => Promise; +}> { const repoRoot = await makeTempDir(); - const git = (...args: string[]) => execa('git', args, { cwd: repoRoot }); + // Identity through the environment rather than `git config`, so the repo needs no extra processes + // and nothing depends on the machine's own git configuration. + const git = (...args: string[]) => + execa('git', args, { + cwd: repoRoot, + env: { + GIT_AUTHOR_NAME: 'Fixture', + GIT_AUTHOR_EMAIL: 'fixture@example.com', + GIT_COMMITTER_NAME: 'Fixture', + GIT_COMMITTER_EMAIL: 'fixture@example.com', + }, + }); await git('init', '--initial-branch=main'); - await git('config', 'user.email', 'fixture@example.com'); - await git('config', 'user.name', 'Fixture'); - - const shas: string[] = []; + // Empty commits: nothing here reads a file, only the commits' identities matter. for (const message of ['first', 'second', 'third']) { // eslint-disable-next-line no-await-in-loop - await writeFile(path.join(repoRoot, `${message}.txt`), `${message}\n`); - // eslint-disable-next-line no-await-in-loop - await git('add', '.'); - // eslint-disable-next-line no-await-in-loop - await git('commit', '-m', message); - // eslint-disable-next-line no-await-in-loop - const { stdout } = await git('rev-parse', 'HEAD'); - shas.push(stdout.trim()); + await git('commit', '--allow-empty', '-m', message); } + const { stdout } = await git('log', '--format=%H', '--reverse'); + const shas = stdout.trim().split('\n'); // Annotated (`-m`), so the tag is its own object with its own SHA. await git('tag', '-a', 'v1.0.0', '-m', 'release', shas[0]); - return { repoRoot, shas }; + return { repoRoot, shas, git }; } describe('createRefResolver', () => { @@ -116,8 +121,7 @@ describe('createRefResolver', () => { }); it('does not treat a branch matching the base branch as a pattern', async () => { - const { repoRoot, shas } = await makeRepo(); - const git = (...args: string[]) => execa('git', args, { cwd: repoRoot }); + const { repoRoot, shas, git } = await makeRepo(); // `v6-x`, not `v6.x` — only a regex reading `.` as a wildcard would accept it. await git('update-ref', 'refs/remotes/origin/v6-x', shas[0]); await git('checkout', '-b', 'feature'); @@ -127,6 +131,5 @@ describe('createRefResolver', () => { // No base branch exists, so the baseline falls back to HEAD~1. Matching `origin/v6-x` would // instead have picked its merge base — the first commit — and benchmarked against the wrong one. expect(ref.sha).toBe(shas[1]); - expect(ref.sha).not.toBe(shas[0]); }); }); diff --git a/packages/benchmark/src/tachometer/refs.ts b/packages/benchmark/src/tachometer/refs.ts index 3c1c316396..95722e8b16 100644 --- a/packages/benchmark/src/tachometer/refs.ts +++ b/packages/benchmark/src/tachometer/refs.ts @@ -86,6 +86,17 @@ export function parseRefToken(token?: string): RefDescriptor { /** Refname prefix every remote-tracking branch carries. */ const REMOTES_PREFIX = 'refs/remotes/'; +/** + * `origin/release/7.x` → `release/7.x`. + * + * Only the remote name comes off, so a branch whose own name contains a slash keeps it. Splitting on + * the last slash instead would turn `release/7.x` into `7.x` and match nothing. + */ +function branchNameOf(shortRef: string): string { + const slash = shortRef.indexOf('/'); + return slash === -1 ? shortRef : shortRef.slice(slash + 1); +} + /** Runs a git command, throwing on a non-zero exit. */ function gitCapture(args: string[], cwd: string): string { const result = execaSync('git', args, { cwd, reject: false }); @@ -104,11 +115,7 @@ function detectBaseBranch(cwd: string): string { if (result.exitCode !== 0) { return 'master'; } - // `origin/main` → `main`. Everything after the remote name is the branch name, so a default - // branch like `release/7.x` keeps its slash. - const shortRef = result.stdout.trim(); - const slash = shortRef.indexOf('/'); - return slash === -1 ? shortRef : shortRef.slice(slash + 1); + return branchNameOf(result.stdout.trim()); } /** @@ -134,13 +141,16 @@ function closestBaseBranch( // segment before it) or the local ``. Filtering short names on `/` would also catch a // local branch literally named e.g. `wip/master`. // - // Compared segment by segment rather than by a regex built from `baseBranch`: the branch name is + // Compared as a string rather than through a regex built from `baseBranch`: the branch name is // arbitrary, and a real one like `v6.x` would make `.` match any character — quietly admitting // `origin/v6-x` as a baseline candidate. const isRemoteBase = (refname: string): boolean => { - const tail = refname.startsWith(REMOTES_PREFIX) ? refname.slice(REMOTES_PREFIX.length) : ''; - const slash = tail.indexOf('/'); - return slash !== -1 && tail.slice(slash + 1) === baseBranch; + if (!refname.startsWith(REMOTES_PREFIX)) { + return false; + } + const withoutPrefix = refname.slice(REMOTES_PREFIX.length); + // A remote-tracking refname always has a remote to strip; `refs/remotes/foo` alone is not one. + return withoutPrefix.includes('/') && branchNameOf(withoutPrefix) === baseBranch; }; const candidates = gitCapture( ['for-each-ref', '--format=%(refname)', 'refs/remotes', 'refs/heads'], From fed52d5fc1e945f2238f0501f7dc5795f23169f6 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 07:46:34 +0200 Subject: [PATCH 046/173] [benchmark] Stop reporting transferred bytes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It answered nothing we act on. The `cold-start` measurement already charges a page for fetching, parsing and evaluating its bundle, in milliseconds — the unit the decision is made in — so the byte count was an input to a number already measured. Bundle weight as a thing to watch belongs to bundle-size-checker, which both repos run: per package, gzipped, against a baseline, with thresholds. What tachometer reports is neither of those: one page's uncompressed transfer over a local static server, including React and the scenario code. Sitting next to bundle-size-checker's figures it invites a comparison that does not hold. Empirically it was also constant where it was most prominent — the note fired for the two-variant regression cases, and all seven case-pairs across the last two CI runs reported the same size on both sides, because both build the same app from near-identical library commits. The raw tachometer JSON is still stored under `raw`, so the number stays recoverable for anyone who wants it. --- .../tachometer/buildMarkdownReport.test.ts | 1 - .../src/lib/tachometer/buildMarkdownReport.ts | 4 +-- .../src/lib/tachometer/formatInterval.ts | 5 --- .../src/lib/tachometer/groupCases.test.ts | 16 ++------- .../src/lib/tachometer/groupCases.ts | 13 ------- .../src/lib/tachometer/types.ts | 1 - .../src/views/TachometerDetails.tsx | 29 ++-------------- .../benchmark/src/tachometer/ciReport.test.ts | 1 - packages/benchmark/src/tachometer/ciReport.ts | 1 - .../benchmark/src/tachometer/renderReport.ts | 34 +------------------ .../src/tachometer/summarizeCase.test.ts | 1 - .../benchmark/src/tachometer/summarizeCase.ts | 3 -- 12 files changed, 6 insertions(+), 103 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts index 2035a49909..fa346a206f 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts @@ -16,7 +16,6 @@ function variant(name: string, refId: string | null) { refId, meanMs: { low: 20, high: 22 }, samples: 120, - bytesSent: 2969, }; } diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts index 47f6315aed..f875c37485 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts @@ -1,6 +1,6 @@ import { formatMarkdownTable } from '@/utils/formatters'; import type { ConfidenceInterval, TachometerReport } from './types'; -import { formatBytes, formatMean, formatPercent, formatSignedMs } from './formatInterval'; +import { formatMean, formatPercent, formatSignedMs } from './formatInterval'; import { isSummarized, shortNameOf } from './groupCases'; import type { SummarizedCase } from './groupCases'; @@ -88,7 +88,6 @@ function renderCaseTable(entry: SummarizedCase): string { mean: formatMean(variant.meanMs), versus, samples: variant.samples, - transferred: formatBytes(variant.bytesSent), }; }); }); @@ -100,7 +99,6 @@ function renderCaseTable(entry: SummarizedCase): string { { field: 'mean', header: 'Mean (95% CI)', align: 'right' }, { field: 'versus', header: 'vs reference', align: 'left' }, { field: 'samples', header: 'Samples', align: 'right' }, - { field: 'transferred', header: 'Transferred', align: 'right' }, ], rows, )}`; diff --git a/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts b/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts index 1366bf4474..adda3efe19 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts @@ -25,8 +25,3 @@ export function formatPercent(interval: ConfidenceInterval): string { export function formatSignedMs(interval: ConfidenceInterval): string { return signedInterval(interval, ' ms', 2); } - -/** Kibibytes, matching how tachometer's own table reports `bytesSent`. */ -export function formatBytes(bytes: number): string { - return `${(bytes / 1024).toFixed(1)} KiB`; -} diff --git a/apps/code-infra-dashboard/src/lib/tachometer/groupCases.test.ts b/apps/code-infra-dashboard/src/lib/tachometer/groupCases.test.ts index 8dcbfe7642..9f9d1bd115 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/groupCases.test.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/groupCases.test.ts @@ -1,14 +1,13 @@ import { describe, expect, it } from 'vitest'; -import { bytesPerVariant, groupCasesByVariantSet, shortNameOf, variantsOf } from './groupCases'; +import { groupCasesByVariantSet, shortNameOf, variantsOf } from './groupCases'; import type { SummarizedCase } from './groupCases'; -function variant(name: string, bytesSent = 2969) { +function variant(name: string) { return { variant: name, refId: 'current', meanMs: { low: 20, high: 22 }, samples: 120, - bytesSent, }; } @@ -99,14 +98,3 @@ describe('groupCasesByVariantSet', () => { ]); }); }); - -describe('bytesPerVariant', () => { - it('reports one figure per variant, not per measurement', () => { - const entry = caseWith('workload', ['[current]', '[baseline]'], ['mount', 'cold-start']); - - expect(bytesPerVariant(entry)).toEqual([ - ['[current]', 2969], - ['[baseline]', 2969], - ]); - }); -}); diff --git a/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts b/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts index 7825edb87f..cf9adf09df 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts @@ -61,16 +61,3 @@ export function groupCasesByVariantSet(cases: SummarizedCase[]): VariantGroup[] return [...groups.values()]; } - -/** What each of a case's variants transferred, for the cases whose table has no such column. */ -export function bytesPerVariant(entry: SummarizedCase): Array<[string, number]> { - const bytes = new Map(); - - for (const measurement of entry.measurements) { - for (const variant of measurement.variants) { - bytes.set(shortNameOf(entry.name, variant.variant), variant.bytesSent); - } - } - - return [...bytes]; -} diff --git a/apps/code-infra-dashboard/src/lib/tachometer/types.ts b/apps/code-infra-dashboard/src/lib/tachometer/types.ts index d7fa2b1654..07153db484 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/types.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/types.ts @@ -18,7 +18,6 @@ export interface TachometerVariantResult { refId: string | null; meanMs: ConfidenceInterval; samples: number; - bytesSent: number; } export interface TachometerComparison { diff --git a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx index f38b7d5ef2..d756ab910b 100644 --- a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx +++ b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx @@ -16,14 +16,9 @@ import TableRow from '@mui/material/TableRow'; import Typography from '@mui/material/Typography'; import { fetchCiReport } from '@/utils/fetchCiReport'; import type { ConfidenceInterval, TachometerReport, Verdict } from '@/lib/tachometer/types'; -import { - bytesPerVariant, - groupCasesByVariantSet, - isSummarized, - shortNameOf, -} from '@/lib/tachometer/groupCases'; +import { groupCasesByVariantSet, isSummarized, shortNameOf } from '@/lib/tachometer/groupCases'; import type { SummarizedCase } from '@/lib/tachometer/groupCases'; -import { formatBytes, formatMean, formatPercent } from '@/lib/tachometer/formatInterval'; +import { formatMean, formatPercent } from '@/lib/tachometer/formatInterval'; import Heading from '../components/Heading'; import ReportHeader from '../components/ReportHeader'; import ErrorDisplay from '../components/ErrorDisplay'; @@ -84,9 +79,6 @@ function Muted({ children }: { children: React.ReactNode }) { */ function VariantTable({ entry, variants }: { entry: SummarizedCase; variants: string[] }) { const [reference] = variants; - // Both are properties of the variant's page rather than of any one measurement, so they are read - // once per row instead of being accumulated while the measurement cells render. - const bytes = new Map(bytesPerVariant(entry)); const samplesOf = (variantName: string) => entry.measurements.flatMap((measurement) => measurement.variants @@ -106,14 +98,11 @@ function VariantTable({ entry, variants }: { entry: SummarizedCase; variants: st vs {reference} ))} - Transferred Samples     {variants.map((variantName) => { - const bytesSent = bytes.get(variantName); - return ( {variantName} @@ -148,9 +137,6 @@ function VariantTable({ entry, variants }: { entry: SummarizedCase; variants: st ); })} - - {bytesSent === undefined ? — : formatBytes(bytesSent)} - {formatSamples(samplesOf(variantName))} ); @@ -293,9 +279,6 @@ export default function TachometerDetails() { // Bundle weight is a property of the variant's page, not of a measurement, so it goes in a note // rather than down every row — except where a variant table already gives it a column. - const needBytesNote = groups - .filter((group) => group.variants.length <= 2) - .flatMap((group) => group.cases); // A tachometer report carries its own comparison, so the baseline is one of its own refs rather // than a separately fetched report. @@ -356,14 +339,6 @@ export default function TachometerDetails() { one page load. "unsure" means the interval still straddles zero: the difference did not resolve within the case's sampling budget. - {needBytesNote.map((entry) => ( - - {entry.name} transferred:{' '} - {bytesPerVariant(entry) - .map(([variantName, bytes]) => `${variantName} ${formatBytes(bytes)}`) - .join(' · ')} - - ))} Builds: {report.refs.map((ref) => `${ref.id} = ${ref.label}`).join(' · ')} diff --git a/packages/benchmark/src/tachometer/ciReport.test.ts b/packages/benchmark/src/tachometer/ciReport.test.ts index 96e99fac7e..bf4f887d49 100644 --- a/packages/benchmark/src/tachometer/ciReport.test.ts +++ b/packages/benchmark/src/tachometer/ciReport.test.ts @@ -31,7 +31,6 @@ function report(overrides = {}) { refId: 'current', meanMs: { low: 20.5, high: 22.7 }, samples: 120, - bytesSent: 2969, }, ], comparisons: [ diff --git a/packages/benchmark/src/tachometer/ciReport.ts b/packages/benchmark/src/tachometer/ciReport.ts index 4b779598e5..922cfba68e 100644 --- a/packages/benchmark/src/tachometer/ciReport.ts +++ b/packages/benchmark/src/tachometer/ciReport.ts @@ -25,7 +25,6 @@ const variantResultSchema = z.object({ refId: z.string().nullable(), meanMs: confidenceIntervalSchema, samples: z.number(), - bytesSent: z.number(), }); const comparisonSchema = differenceSchema.extend({ diff --git a/packages/benchmark/src/tachometer/renderReport.ts b/packages/benchmark/src/tachometer/renderReport.ts index 901ba8ca8c..ba619c7da8 100644 --- a/packages/benchmark/src/tachometer/renderReport.ts +++ b/packages/benchmark/src/tachometer/renderReport.ts @@ -75,11 +75,6 @@ function colorVerdict(verdict: Verdict, text: string): string { return chalk.dim(text); } -/** Kibibytes, matching how tachometer's own table reports `bytesSent`. */ -function formatBytes(bytes: number): string { - return `${(bytes / 1024).toFixed(1)} KiB`; -} - /** * Tachometer names each variant ` []`, and the case is already its own column, so * the prefix is dropped for display. Dropping it is also what makes columns line up across cases: @@ -122,7 +117,6 @@ function printVariantTable(entry: SummarizedCase, variants: string[]): void { const headers = [ entry.name, ...entry.measurements.flatMap((measurement) => [measurement.name, `vs ${reference}`]), - 'transferred', 'Samples', ]; @@ -137,8 +131,6 @@ function printVariantTable(entry: SummarizedCase, variants: string[]): void { (candidate) => shortNameOf(entry.name, candidate.variant) === variant, ), })); - // Transfer size is a property of the variant's page, so every measurement reports the same one. - const bytesSent = perMeasurement.find(({ found }) => found)?.found?.bytesSent; const sampleCounts = new Set( perMeasurement.flatMap(({ found }) => (found ? [found.samples] : [])), ); @@ -154,12 +146,7 @@ function printVariantTable(entry: SummarizedCase, variants: string[]): void { return [found ? formatInterval(found.meanMs) : chalk.dim('—'), delta]; }); - return [ - variant, - ...cells, - bytesSent === undefined ? chalk.dim('—') : formatBytes(bytesSent), - [...sampleCounts].join('/'), - ]; + return [variant, ...cells, [...sampleCounts].join('/')]; }); printTable(headers, rows, 1); @@ -296,25 +283,6 @@ export function renderTachometerReport(report: TachometerReport): void { '"unsure" means the interval still straddles zero: the difference did not resolve within the case\'s sampling budget.', ]; - // Bundle weight is a property of the variant's page, not of a measurement, so it belongs in a - // note rather than repeated down every row. It comes free with the run and is the number that - // explains a cold-start difference that mount time alone does not. A variant table already gives - // it a column, so only the cases in the smaller groups need the note. - for (const entry of [...groups.values()] - .filter((group) => group.variants.length <= 2) - .flatMap((group) => group.cases)) { - const bytes = new Map(); - for (const measurement of entry.measurements) { - for (const variant of measurement.variants) { - bytes.set(shortNameOf(entry.name, variant.variant), variant.bytesSent); - } - } - const perVariant = [...bytes].map(([variant, value]) => `${variant} ${formatBytes(value)}`); - if (perVariant.length > 0) { - notes.push(`${entry.name} transferred: ${perVariant.join(' · ')}`); - } - } - // No short SHA appended: `id` is already `git-`, and the label is the name the run was // given (a committish, or how a symbolic baseline resolved). const refLabels = report.refs.map((ref) => `${ref.id} = ${ref.label}`); diff --git a/packages/benchmark/src/tachometer/summarizeCase.test.ts b/packages/benchmark/src/tachometer/summarizeCase.test.ts index 31b471f0cd..2130e0ab95 100644 --- a/packages/benchmark/src/tachometer/summarizeCase.test.ts +++ b/packages/benchmark/src/tachometer/summarizeCase.test.ts @@ -17,7 +17,6 @@ function benchmark({ }): any { return { name, - bytesSent: 100, mean: { low: 1, high: 2 }, samples: [1, 2, 3], measurement: measurement === undefined ? undefined : { name: measurement }, diff --git a/packages/benchmark/src/tachometer/summarizeCase.ts b/packages/benchmark/src/tachometer/summarizeCase.ts index 75046083cb..f18b16b7b4 100644 --- a/packages/benchmark/src/tachometer/summarizeCase.ts +++ b/packages/benchmark/src/tachometer/summarizeCase.ts @@ -18,7 +18,6 @@ export interface TachometerDifference { export interface TachometerBenchmark { /** Variant name, suffixed with ` []` when a page has several. */ name: string; - bytesSent: number; /** Mean duration, as a 95% confidence interval. */ mean: ConfidenceInterval; samples: number[]; @@ -40,7 +39,6 @@ export interface VariantSummary { refId: string | null; meanMs: ConfidenceInterval; samples: number; - bytesSent: number; } export interface ComparisonSummary { @@ -156,7 +154,6 @@ export function summarizeCase(entry: BenchmarkCase, json: TachometerJson): CaseS refId: variant.refId, meanMs: found.benchmark.mean, samples: found.benchmark.samples.length, - bytesSent: found.benchmark.bytesSent, }); if (variant.name === reference.name) { continue; From bf0296e90b78e2b4c53c690126ad881af3ea6d00 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 07:54:43 +0200 Subject: [PATCH 047/173] [benchmark] Build both sides with one command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--working-tree-build-cmd` let the working tree be built by a different command than a ref's checkout, and nothing checked that the two produce the same artifacts. The design otherwise goes out of its way to keep the sides symmetric — even the working tree is packed into a tarball and installed, rather than linked, so that both resolve the library identically — and this reopened that hole one layer up, where a wrong value biases every number silently. The one real use was caching, not a different build. mosaic passed `pnpm build` against a default of `pnpm release:build`, and those differ only in `--concurrency` and `--skip-nx-cache`: same target, same packages, same output. Skipping the cache is already a no-op in a ref's checkout, which is a fresh directory with an empty one — so a single `--build-cmd 'pnpm build'` gets the thorough build where the cache is cold and the cached build where it is warm. If "this is already built" ever needs saying, it wants a symmetric flag rather than a second per-side command. --- packages/benchmark/src/cli/tachoRun.ts | 7 +------ packages/benchmark/src/tachometer/runTachometer.ts | 13 ++++--------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/packages/benchmark/src/cli/tachoRun.ts b/packages/benchmark/src/cli/tachoRun.ts index 6cf5e22a94..0ab2bd8d5f 100644 --- a/packages/benchmark/src/cli/tachoRun.ts +++ b/packages/benchmark/src/cli/tachoRun.ts @@ -28,12 +28,8 @@ const command: CommandModule<{}, Args> = { .option('build-cmd', { type: 'string', default: 'pnpm release:build', - describe: "Command that builds the publishable workspace packages of a ref's checkout", - }) - .option('working-tree-build-cmd', { - type: 'string', describe: - 'Command that builds the working tree. Defaults to --build-cmd; point it at a cached build when the two differ only in caching', + "Command that builds the publishable workspace packages, in each ref's checkout and in the working tree", }) .option('install', { type: 'boolean', @@ -62,7 +58,6 @@ const command: CommandModule<{}, Args> = { baseline: argv.baseline, baseBranch: argv.baseBranch, buildCmd: argv.buildCmd, - workingTreeBuildCmd: argv.workingTreeBuildCmd, install: argv.install, out: argv.out, upload: argv.upload, diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts index 7e30f0f928..e6a38f55b5 100644 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -43,12 +43,11 @@ export interface RunTachometerOptions { /** Branch PRs fork from. Defaults to detection via `origin/HEAD`. */ baseBranch?: string; /** - * Command that builds the publishable packages of a checked-out ref. Defaults to - * `pnpm release:build`. + * Command that builds the publishable workspace packages. Run for every ref — in each ref's own + * checkout, and in the working tree — so both sides of a comparison are built the same way. + * Defaults to `pnpm release:build`. */ buildCmd?: string; - /** Command that builds the working tree. Defaults to `buildCmd`. */ - workingTreeBuildCmd?: string; /** Whether to install inside a ref's checkout. Defaults to true. */ install?: boolean; /** Where to write the combined JSON report. Defaults to `.tachometer/results/report.json`. */ @@ -73,10 +72,6 @@ export async function runTachometer(options: RunTachometerOptions): Promise Date: Thu, 10 Sep 2026 09:12:31 +0200 Subject: [PATCH 048/173] [code-infra] Add a baseline command, and stop deriving one per tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every job that compares a branch to its base worked out the comparison point in shell, and the copies had drifted: mosaic's benchmark job skips the comparison on master while its tachometer job falls back to HEAD~1, and both hardcode `origin`. Bundle size does not derive one at all — the dashboard guesses with a candidate list because CI never told it. `code-infra baseline` prints that commit: the fork point from the closest base branch, or the previous commit when HEAD is on the base branch or already contained in it. Closest across every remote rather than `origin`'s, so a stale fork does not win over an up-to-date upstream — the one case none of the shell copies handle. Only the SHA goes to stdout, so it substitutes into a command. The benchmark package loses its own copy: `tacho run` no longer detects a base branch, `--base-branch` is gone with it, and an unbound baseline is HEAD~1, which needs no policy. Scripts pass the answer in, so nobody types it by hand. Adoption in mosaic waits for a code-infra release: pinning a preview there would bring back the subdependency overrides that pinning one cost last time. --- packages/benchmark/src/cli/tachoRun.ts | 8 +- .../benchmark/src/tachometer/refs.test.ts | 24 ++- packages/benchmark/src/tachometer/refs.ts | 126 ++----------- .../benchmark/src/tachometer/runTachometer.ts | 5 +- packages/code-infra/src/cli/cmdBaseline.mjs | 45 +++++ packages/code-infra/src/cli/index.mjs | 2 + packages/code-infra/src/utils/baseline.mjs | 167 ++++++++++++++++++ .../code-infra/src/utils/baseline.test.mjs | 92 ++++++++++ test/tachometer/README.md | 17 +- 9 files changed, 343 insertions(+), 143 deletions(-) create mode 100644 packages/code-infra/src/cli/cmdBaseline.mjs create mode 100644 packages/code-infra/src/utils/baseline.mjs create mode 100644 packages/code-infra/src/utils/baseline.test.mjs diff --git a/packages/benchmark/src/cli/tachoRun.ts b/packages/benchmark/src/cli/tachoRun.ts index 0ab2bd8d5f..297b863f8b 100644 --- a/packages/benchmark/src/cli/tachoRun.ts +++ b/packages/benchmark/src/cli/tachoRun.ts @@ -18,12 +18,7 @@ const command: CommandModule<{}, Args> = { .option('baseline', { type: 'string', describe: - 'Bind the "baseline" symbol, in the ref grammar (e.g. git:abc1234). Default: on the base branch HEAD~1, otherwise the fork point from the base branch', - }) - .option('base-branch', { - type: 'string', - describe: - 'Branch PRs fork from. Default: detected from origin/HEAD, falling back to master', + 'Bind the "baseline" symbol, in the ref grammar (e.g. git:abc1234). Defaults to HEAD~1; `code-infra baseline` resolves a fork point to pass here', }) .option('build-cmd', { type: 'string', @@ -56,7 +51,6 @@ const command: CommandModule<{}, Args> = { harnessDir: process.cwd(), filters: argv.filters ?? [], baseline: argv.baseline, - baseBranch: argv.baseBranch, buildCmd: argv.buildCmd, install: argv.install, out: argv.out, diff --git a/packages/benchmark/src/tachometer/refs.test.ts b/packages/benchmark/src/tachometer/refs.test.ts index a0bc6e0020..9210543c39 100644 --- a/packages/benchmark/src/tachometer/refs.test.ts +++ b/packages/benchmark/src/tachometer/refs.test.ts @@ -111,6 +111,17 @@ describe('createRefResolver', () => { expect(viaTag.id).toBe(viaSha.id); }); + it('falls back to the previous commit when nothing binds the baseline', async () => { + // Which commit a branch should be compared against is `code-infra baseline`'s question, and the + // harness scripts pass the answer in. Unbound, the previous commit is the only choice here that + // needs no policy of its own. + const { repoRoot, shas } = await makeRepo(); + + const ref = createRefResolver({ repoRoot }).parse('baseline'); + + expect(ref.sha).toBe(shas[1]); + }); + it('labels a git ref with the revision it was given', async () => { const { repoRoot } = await makeRepo(); @@ -119,17 +130,4 @@ describe('createRefResolver', () => { // Not `v1.0.0 ()`: `id` already carries the short SHA, and the renderers print both. expect(ref.label).toBe('v1.0.0'); }); - - it('does not treat a branch matching the base branch as a pattern', async () => { - const { repoRoot, shas, git } = await makeRepo(); - // `v6-x`, not `v6.x` — only a regex reading `.` as a wildcard would accept it. - await git('update-ref', 'refs/remotes/origin/v6-x', shas[0]); - await git('checkout', '-b', 'feature'); - - const ref = createRefResolver({ repoRoot, baseBranch: 'v6.x' }).parse('baseline'); - - // No base branch exists, so the baseline falls back to HEAD~1. Matching `origin/v6-x` would - // instead have picked its merge base — the first commit — and benchmarked against the wrong one. - expect(ref.sha).toBe(shas[1]); - }); }); diff --git a/packages/benchmark/src/tachometer/refs.ts b/packages/benchmark/src/tachometer/refs.ts index 95722e8b16..e9847cfdee 100644 --- a/packages/benchmark/src/tachometer/refs.ts +++ b/packages/benchmark/src/tachometer/refs.ts @@ -1,4 +1,3 @@ -import chalk from 'chalk'; import { execaSync } from 'execa'; import { revParseCommitArgs } from '../utils/git'; @@ -35,8 +34,6 @@ export type ResolvedRef = export interface RefResolverOptions { /** Repository to resolve revisions in. */ repoRoot: string; - /** Branch PRs fork from. Defaults to detection via `origin/HEAD`, then `master`. */ - baseBranch?: string; /** Binds the `baseline` symbol, in the ref grammar (e.g. `git:abc1234`). */ baselineOverride?: string; } @@ -83,20 +80,6 @@ export function parseRefToken(token?: string): RefDescriptor { ); } -/** Refname prefix every remote-tracking branch carries. */ -const REMOTES_PREFIX = 'refs/remotes/'; - -/** - * `origin/release/7.x` → `release/7.x`. - * - * Only the remote name comes off, so a branch whose own name contains a slash keeps it. Splitting on - * the last slash instead would turn `release/7.x` into `7.x` and match nothing. - */ -function branchNameOf(shortRef: string): string { - const slash = shortRef.indexOf('/'); - return slash === -1 ? shortRef : shortRef.slice(slash + 1); -} - /** Runs a git command, throwing on a non-zero exit. */ function gitCapture(args: string[], cwd: string): string { const result = execaSync('git', args, { cwd, reject: false }); @@ -106,77 +89,6 @@ function gitCapture(args: string[], cwd: string): string { return result.stdout.trim(); } -/** Detects the branch PRs fork from, from `origin`'s default branch, falling back to `master`. */ -function detectBaseBranch(cwd: string): string { - const result = execaSync('git', ['symbolic-ref', '--short', 'refs/remotes/origin/HEAD'], { - cwd, - reject: false, - }); - if (result.exitCode !== 0) { - return 'master'; - } - return branchNameOf(result.stdout.trim()); -} - -/** - * Among the available base branches — every remote's `/` plus a local `` — - * returns the one whose merge base with HEAD is the most recent commit (the closest fork point), or - * undefined if none exist. - * - * Picking by most-recent merge base prefers an up-to-date upstream over a stale fork's base branch - * without hardcoding which remote is authoritative: locally `origin` may be a months-behind fork - * while `upstream` tracks the real repo, yet in CI `origin` *is* that repo. - */ -function closestBaseBranch( - repoRoot: string, - baseBranch: string, -): { ref: string; mergeBase: string } | undefined { - // On ties (same merge base), prefer upstream's, then origin's, then a local base branch. - const preference = [`upstream/${baseBranch}`, `origin/${baseBranch}`, baseBranch]; - const priority = (ref: string): number => { - const index = preference.indexOf(ref); - return index === -1 ? preference.length : index; - }; - // Match only real base branches from full refnames: a remote's `/` (exactly one - // segment before it) or the local ``. Filtering short names on `/` would also catch a - // local branch literally named e.g. `wip/master`. - // - // Compared as a string rather than through a regex built from `baseBranch`: the branch name is - // arbitrary, and a real one like `v6.x` would make `.` match any character — quietly admitting - // `origin/v6-x` as a baseline candidate. - const isRemoteBase = (refname: string): boolean => { - if (!refname.startsWith(REMOTES_PREFIX)) { - return false; - } - const withoutPrefix = refname.slice(REMOTES_PREFIX.length); - // A remote-tracking refname always has a remote to strip; `refs/remotes/foo` alone is not one. - return withoutPrefix.includes('/') && branchNameOf(withoutPrefix) === baseBranch; - }; - const candidates = gitCapture( - ['for-each-ref', '--format=%(refname)', 'refs/remotes', 'refs/heads'], - repoRoot, - ) - .split('\n') - .filter((ref) => isRemoteBase(ref) || ref === `refs/heads/${baseBranch}`) - .map((ref) => ref.replace(/^refs\/(remotes|heads)\//, '')) - .sort((a, b) => priority(a) - priority(b)); - - let best: { ref: string; mergeBase: string; when: number } | undefined; - for (const ref of candidates) { - const result = execaSync('git', ['merge-base', 'HEAD', ref], { cwd: repoRoot, reject: false }); - if (result.exitCode !== 0) { - continue; - } - const mergeBase = result.stdout.trim(); - const when = Number(gitCapture(['show', '-s', '--format=%ct', mergeBase], repoRoot)); - // Strictly greater, so on ties the higher-priority (earlier-sorted) ref keeps a clean label. - if (!best || when > best.when) { - best = { ref, mergeBase, when }; - } - } - return best && { ref: best.ref, mergeBase: best.mergeBase }; -} - /** * Creates a resolver that turns {@link RefDescriptor}s into {@link ResolvedRef}s, memoizing the * `baseline` symbol so it is computed at most once per run. @@ -185,9 +97,6 @@ export function createRefResolver(options: RefResolverOptions): { parse: (token?: string) => ResolvedRef; } { const { repoRoot, baselineOverride } = options; - // Detected on demand: only the `baseline` symbol needs it, so a run whose cases all pin `git:` - // refs — or none at all — never shells out for it. - let baseBranch = options.baseBranch; let baselineRef: ResolvedRef | undefined; // Several cases commonly pin the same commit, and resolving one spawns a git process. @@ -212,35 +121,22 @@ export function createRefResolver(options: RefResolverOptions): { } /** - * Computes the `baseline` symbol: the override when given, otherwise the sensible default — on - * the base branch the previous commit, on any other branch the fork point from the closest base - * branch, which isolates what this branch changed. + * Computes the `baseline` symbol from whatever bound it. + * + * Which commit that should be is not decided here: `code-infra baseline` answers that for every + * job that compares against a base — bundle size as well as benchmarks — so the harness scripts + * pass the answer in rather than each tool having its own opinion of a fork point. Without a + * binding the previous commit is the only choice that needs no policy. */ function computeBaselineRef(): ResolvedRef { - if (baselineOverride !== undefined) { - const descriptor = parseRefToken(baselineOverride); - if (descriptor.kind === 'baseline') { - throw new Error('The baseline cannot itself be "baseline" — that is the symbol it binds.'); - } - return resolve(descriptor); - } - - baseBranch ??= detectBaseBranch(repoRoot); - - if (gitCapture(['rev-parse', '--abbrev-ref', 'HEAD'], repoRoot) === baseBranch) { + if (baselineOverride === undefined) { return gitRef('HEAD~1'); } - - const base = closestBaseBranch(repoRoot, baseBranch); - if (!base) { - console.warn(chalk.yellow(`No ${baseBranch} branch found; using HEAD~1 as the baseline.`)); - return gitRef('HEAD~1'); - } - // Fork point is HEAD itself (HEAD already contained in the base branch) — nothing to diff. - if (base.mergeBase === gitCapture(['rev-parse', 'HEAD'], repoRoot)) { - return gitRef('HEAD~1'); + const descriptor = parseRefToken(baselineOverride); + if (descriptor.kind === 'baseline') { + throw new Error('The baseline cannot itself be "baseline" — that is the symbol it binds.'); } - return gitRef(base.mergeBase, `merge-base with ${base.ref}`); + return resolve(descriptor); } function resolve(descriptor: RefDescriptor): ResolvedRef { diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts index e6a38f55b5..657b4520bf 100644 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -40,8 +40,6 @@ export interface RunTachometerOptions { filters?: string[]; /** Binds the `baseline` symbol, in the ref grammar (e.g. `git:abc1234`). */ baseline?: string; - /** Branch PRs fork from. Defaults to detection via `origin/HEAD`. */ - baseBranch?: string; /** * Command that builds the publishable workspace packages. Run for every ref — in each ref's own * checkout, and in the working tree — so both sides of a comparison are built the same way. @@ -70,7 +68,6 @@ export async function runTachometer(options: RunTachometerOptions): Promise(); diff --git a/packages/code-infra/src/cli/cmdBaseline.mjs b/packages/code-infra/src/cli/cmdBaseline.mjs new file mode 100644 index 0000000000..b987b7c284 --- /dev/null +++ b/packages/code-infra/src/cli/cmdBaseline.mjs @@ -0,0 +1,45 @@ +#!/usr/bin/env node + +/* eslint-disable no-console */ + +import { resolveBaseline } from '../utils/baseline.mjs'; + +/** + * @typedef {Object} Args + * @property {string} [baseBranch] - Branch pull requests fork from + * @property {boolean} [verbose] - Explain the choice on stderr + */ + +export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ + command: 'baseline', + describe: 'Print the commit HEAD should be compared against', + builder: (yargs) => { + return yargs + .option('base-branch', { + type: 'string', + description: + 'Branch pull requests fork from. Default: detected from origin/HEAD, falling back to master', + }) + .option('verbose', { + type: 'boolean', + default: false, + description: 'Explain the choice on stderr', + }) + .example( + '$0 baseline', + 'Print the fork point from the base branch, or the previous commit when on it', + ) + .example( + 'BASE=$($0 baseline)', + 'Capture it — only the SHA goes to stdout, so this stays substitutable', + ); + }, + handler: async (argv) => { + const { sha, reason } = await resolveBaseline({ baseBranch: argv.baseBranch }); + if (argv.verbose) { + // stderr, so command substitution captures the SHA alone. + console.error(`Baseline: ${sha} (${reason})`); + } + console.log(sha); + }, +}); diff --git a/packages/code-infra/src/cli/index.mjs b/packages/code-infra/src/cli/index.mjs index 58982e6ccf..93a70f7dad 100644 --- a/packages/code-infra/src/cli/index.mjs +++ b/packages/code-infra/src/cli/index.mjs @@ -4,6 +4,7 @@ import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; import cmdArgosPush from './cmdArgosPush.mjs'; +import cmdBaseline from './cmdBaseline.mjs'; import cmdBuild from './cmdBuild.mjs'; import cmdCopyFiles from './cmdCopyFiles.mjs'; import cmdExtractErrorCodes from './cmdExtractErrorCodes.mjs'; @@ -38,6 +39,7 @@ await yargs(hideBin(process.argv)) globalArgv = argv; }, true) .command(cmdArgosPush) + .command(cmdBaseline) .command(cmdBuild) .command(cmdCopyFiles) .command(cmdExtractErrorCodes) diff --git a/packages/code-infra/src/utils/baseline.mjs b/packages/code-infra/src/utils/baseline.mjs new file mode 100644 index 0000000000..c25f183ad3 --- /dev/null +++ b/packages/code-infra/src/utils/baseline.mjs @@ -0,0 +1,167 @@ +import { execa } from 'execa'; + +/** + * Resolving "which commit should this branch be compared against". + * + * CI jobs that compare a pull request against its base — bundle size, benchmarks — each used to + * spell this out in shell, and the copies drifted: some fell back to `HEAD~1` on the base branch, + * some skipped the comparison entirely, and all of them hardcoded `origin`. + */ + +const REMOTES_PREFIX = 'refs/remotes/'; + +/** + * `origin/release/7.x` → `release/7.x`. + * + * Only the remote name comes off, so a branch whose own name contains a slash keeps it. Splitting on + * the last slash instead would turn `release/7.x` into `7.x` and match nothing. + * @param {string} shortRef + * @returns {string} + */ +function branchNameOf(shortRef) { + const slash = shortRef.indexOf('/'); + return slash === -1 ? shortRef : shortRef.slice(slash + 1); +} + +/** + * Runs a git command, throwing on a non-zero exit. + * @param {string[]} args + * @param {string} cwd + * @returns {Promise} + */ +async function gitCapture(args, cwd) { + const result = await execa('git', args, { cwd, reject: false }); + if (result.exitCode !== 0) { + throw new Error(`git ${args.join(' ')} failed: ${String(result.stderr).trim()}`); + } + return result.stdout.trim(); +} + +/** + * The branch pull requests fork from, read from `origin`'s default branch and falling back to + * `master` when the repository has no `origin/HEAD` — which is common in a CI clone. + * @param {string} cwd + * @returns {Promise} + */ +export async function detectBaseBranch(cwd) { + const result = await execa('git', ['symbolic-ref', '--short', 'refs/remotes/origin/HEAD'], { + cwd, + reject: false, + }); + if (result.exitCode !== 0) { + return 'master'; + } + return branchNameOf(result.stdout.trim()); +} + +/** + * Among the available base branches — every remote's `/` plus a local `` — the + * one whose merge base with HEAD is the most recent commit (the closest fork point), or undefined + * if none exist. + * + * Picking by most-recent merge base prefers an up-to-date upstream over a stale fork's base branch + * without hardcoding which remote is authoritative: locally `origin` may be a months-behind fork + * while `upstream` tracks the real repo, yet in CI `origin` *is* that repo. + * @param {string} repoRoot + * @param {string} baseBranch + * @returns {Promise<{ ref: string, mergeBase: string } | undefined>} + */ +export async function closestBaseBranch(repoRoot, baseBranch) { + // On ties (same merge base), prefer upstream's, then origin's, then a local base branch. + const preference = [`upstream/${baseBranch}`, `origin/${baseBranch}`, baseBranch]; + /** @param {string} ref */ + const priority = (ref) => { + const index = preference.indexOf(ref); + return index === -1 ? preference.length : index; + }; + // Match only real base branches from full refnames: a remote's `/` (exactly one + // segment before it) or the local ``. Filtering short names on `/` would also catch a + // local branch literally named e.g. `wip/master`. + // + // Compared as a string rather than through a regex built from `baseBranch`: the branch name is + // arbitrary, and a real one like `v6.x` would make `.` match any character — quietly admitting + // `origin/v6-x` as a baseline candidate. + /** @param {string} refname */ + const isRemoteBase = (refname) => { + if (!refname.startsWith(REMOTES_PREFIX)) { + return false; + } + const withoutPrefix = refname.slice(REMOTES_PREFIX.length); + // A remote-tracking refname always has a remote to strip; `refs/remotes/foo` alone is not one. + return withoutPrefix.includes('/') && branchNameOf(withoutPrefix) === baseBranch; + }; + + const candidates = ( + await gitCapture( + ['for-each-ref', '--format=%(refname)', 'refs/remotes', 'refs/heads'], + repoRoot, + ) + ) + .split('\n') + .filter((ref) => isRemoteBase(ref) || ref === `refs/heads/${baseBranch}`) + .map((ref) => ref.replace(/^refs\/(remotes|heads)\//, '')) + .sort((a, b) => priority(a) - priority(b)); + + /** @type {{ ref: string, mergeBase: string, when: number } | undefined} */ + let best; + for (const ref of candidates) { + // Sequential on purpose: each iteration is a git process, and the list is a handful of refs. + // eslint-disable-next-line no-await-in-loop + const result = await execa('git', ['merge-base', 'HEAD', ref], { + cwd: repoRoot, + reject: false, + }); + if (result.exitCode !== 0) { + continue; + } + const mergeBase = result.stdout.trim(); + // eslint-disable-next-line no-await-in-loop + const when = Number(await gitCapture(['show', '-s', '--format=%ct', mergeBase], repoRoot)); + // Strictly greater, so on ties the higher-priority (earlier-sorted) ref wins. + if (!best || when > best.when) { + best = { ref, mergeBase, when }; + } + } + return best && { ref: best.ref, mergeBase: best.mergeBase }; +} + +/** + * @typedef {Object} Baseline + * @property {string} sha - The commit to compare against. + * @property {string} reason - How it was chosen, for a log line. + */ + +/** + * The commit HEAD should be compared against: on a feature branch the fork point from the closest + * base branch, so a comparison reflects only what the branch changed; on the base branch itself the + * previous commit, since there is no meaningful fork point there. + * @param {Object} [options] + * @param {string} [options.cwd] + * @param {string} [options.baseBranch] - Overrides detection from `origin/HEAD`. + * @returns {Promise} + */ +export async function resolveBaseline(options = {}) { + const cwd = options.cwd ?? process.cwd(); + const repoRoot = await gitCapture(['rev-parse', '--show-toplevel'], cwd); + const baseBranch = options.baseBranch ?? (await detectBaseBranch(repoRoot)); + + const previous = async () => ({ + sha: await gitCapture(['rev-parse', 'HEAD~1'], repoRoot), + reason: 'previous commit', + }); + + if ((await gitCapture(['rev-parse', '--abbrev-ref', 'HEAD'], repoRoot)) === baseBranch) { + return previous(); + } + + const base = await closestBaseBranch(repoRoot, baseBranch); + if (!base) { + return previous(); + } + // The fork point is HEAD itself — HEAD is already contained in the base branch, so there is + // nothing this branch changed to compare. + if (base.mergeBase === (await gitCapture(['rev-parse', 'HEAD'], repoRoot))) { + return previous(); + } + return { sha: base.mergeBase, reason: `merge base with ${base.ref}` }; +} diff --git a/packages/code-infra/src/utils/baseline.test.mjs b/packages/code-infra/src/utils/baseline.test.mjs new file mode 100644 index 0000000000..f739c76b0e --- /dev/null +++ b/packages/code-infra/src/utils/baseline.test.mjs @@ -0,0 +1,92 @@ +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { describe, expect, it, onTestFinished } from 'vitest'; +import { execa } from 'execa'; +import { resolveBaseline } from './baseline.mjs'; + +/** + * A repository with three commits on `main`. + * + * A real repository rather than a stand-in for git: every question here is about what git reports — + * which refnames exist, where two branches diverge — and faking that would only restate the answer + * the code already assumes. + * @returns {Promise<{ repoRoot: string, shas: string[], git: (...args: string[]) => Promise }>} + */ +async function makeRepo() { + const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'baseline-test-')); + onTestFinished(async () => { + await fs.rm(repoRoot, { recursive: true, force: true }); + }); + + // Identity through the environment rather than `git config`, so the repo needs no extra processes + // and nothing depends on the machine's own git configuration. + const git = (/** @type {string[]} */ ...args) => + execa('git', args, { + cwd: repoRoot, + env: { + GIT_AUTHOR_NAME: 'Fixture', + GIT_AUTHOR_EMAIL: 'fixture@example.com', + GIT_COMMITTER_NAME: 'Fixture', + GIT_COMMITTER_EMAIL: 'fixture@example.com', + }, + }); + + await git('init', '--initial-branch=main'); + for (const message of ['first', 'second', 'third']) { + // Empty commits: nothing here reads a file, only the commits' identities matter. + // eslint-disable-next-line no-await-in-loop + await git('commit', '--allow-empty', '-m', message); + } + const { stdout } = await git('log', '--format=%H', '--reverse'); + return { repoRoot, shas: stdout.trim().split('\n'), git }; +} + +describe('resolveBaseline', () => { + it('uses the previous commit when HEAD is on the base branch', async () => { + const { repoRoot, shas } = await makeRepo(); + + const baseline = await resolveBaseline({ cwd: repoRoot, baseBranch: 'main' }); + + expect(baseline.sha).toBe(shas[1]); + }); + + it('uses the fork point when HEAD is on a branch', async () => { + const { repoRoot, shas, git } = await makeRepo(); + // Branch off the first commit, so the fork point is not simply HEAD~1. + await git('checkout', '-b', 'feature', shas[0]); + await git('commit', '--allow-empty', '-m', 'work'); + + const baseline = await resolveBaseline({ cwd: repoRoot, baseBranch: 'main' }); + + expect(baseline.sha).toBe(shas[0]); + expect(baseline.reason).toContain('main'); + }); + + it('prefers the remote whose fork point is most recent', async () => { + const { repoRoot, shas, git } = await makeRepo(); + // A stale fork and an up-to-date upstream, the case that makes hardcoding `origin` wrong. + await git('update-ref', 'refs/remotes/origin/main', shas[0]); + await git('update-ref', 'refs/remotes/upstream/main', shas[1]); + await git('checkout', '-b', 'feature'); + await git('commit', '--allow-empty', '-m', 'work'); + + const baseline = await resolveBaseline({ cwd: repoRoot, baseBranch: 'main' }); + + expect(baseline.sha).toBe(shas[1]); + expect(baseline.reason).toContain('upstream/main'); + }); + + it('does not treat a branch matching the base branch as a pattern', async () => { + const { repoRoot, shas, git } = await makeRepo(); + // `v6-x`, not `v6.x` — only a regex reading `.` as a wildcard would accept it. + await git('update-ref', 'refs/remotes/origin/v6-x', shas[0]); + await git('checkout', '-b', 'feature'); + + const baseline = await resolveBaseline({ cwd: repoRoot, baseBranch: 'v6.x' }); + + // No base branch exists, so it falls back to the previous commit. Matching `origin/v6-x` would + // instead have picked its merge base — the first commit — and compared against the wrong one. + expect(baseline.sha).toBe(shas[1]); + }); +}); diff --git a/test/tachometer/README.md b/test/tachometer/README.md index b819696bc2..6d12d722d7 100644 --- a/test/tachometer/README.md +++ b/test/tachometer/README.md @@ -87,10 +87,19 @@ copy wins over the root pin. If the check reports a driver you did not install, ### A note on the baseline -The default baseline is the fork point from the base branch, so a ref older than the introduction of -`benchmark tacho` will fail to build its pages: this harness's vite config imports a plugin that -did not exist at that commit. Pass `--baseline git:HEAD` to compare the working tree against the -current commit while that is still true. +`tacho run` does not work out which commit to compare against — `code-infra baseline` does, for +every job that compares a branch to its base, so bundle size and the benchmarks agree on the answer +instead of each deriving it. Pass it in: + +```bash +pnpm test:tacho --baseline "git:$(pnpm code-infra baseline)" +``` + +Without `--baseline` the previous commit is used, which needs no policy to decide. + +A ref older than the introduction of `benchmark tacho` will fail to build its pages: this harness's +vite config imports a plugin that did not exist at that commit. `--baseline git:HEAD` compares the +working tree against the current commit while that is still true. ## Working on a case by hand From a9cb4f19a41e62ce4e89a482e04de99ca49113aa Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:36:02 +0200 Subject: [PATCH 049/173] [code-infra] Keep the baseline helpers with the rest of git MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They were git plumbing in a module of their own — `rev-parse`, `symbolic-ref`, `for-each-ref`, `merge-base`, `show`, and nothing else — sitting beside a `git.mjs` that already shells out to git and already had `rev-parse HEAD` as `getCurrentGitSha`. So the new module both hid the functions from anyone looking where they belong, and grew a second way to run a git command. Folded in, using the same `$` form as its neighbours, and `getCurrentGitSha` takes a `cwd` rather than the baseline code carrying its own copy of it. All three existing callers pass nothing, so the parameter is additive. --- packages/code-infra/src/cli/cmdBaseline.mjs | 2 +- packages/code-infra/src/utils/baseline.mjs | 167 ------------------ packages/code-infra/src/utils/git.mjs | 146 ++++++++++++++- .../utils/{baseline.test.mjs => git.test.mjs} | 2 +- 4 files changed, 146 insertions(+), 171 deletions(-) delete mode 100644 packages/code-infra/src/utils/baseline.mjs rename packages/code-infra/src/utils/{baseline.test.mjs => git.test.mjs} (98%) diff --git a/packages/code-infra/src/cli/cmdBaseline.mjs b/packages/code-infra/src/cli/cmdBaseline.mjs index b987b7c284..a1c779ab92 100644 --- a/packages/code-infra/src/cli/cmdBaseline.mjs +++ b/packages/code-infra/src/cli/cmdBaseline.mjs @@ -2,7 +2,7 @@ /* eslint-disable no-console */ -import { resolveBaseline } from '../utils/baseline.mjs'; +import { resolveBaseline } from '../utils/git.mjs'; /** * @typedef {Object} Args diff --git a/packages/code-infra/src/utils/baseline.mjs b/packages/code-infra/src/utils/baseline.mjs deleted file mode 100644 index c25f183ad3..0000000000 --- a/packages/code-infra/src/utils/baseline.mjs +++ /dev/null @@ -1,167 +0,0 @@ -import { execa } from 'execa'; - -/** - * Resolving "which commit should this branch be compared against". - * - * CI jobs that compare a pull request against its base — bundle size, benchmarks — each used to - * spell this out in shell, and the copies drifted: some fell back to `HEAD~1` on the base branch, - * some skipped the comparison entirely, and all of them hardcoded `origin`. - */ - -const REMOTES_PREFIX = 'refs/remotes/'; - -/** - * `origin/release/7.x` → `release/7.x`. - * - * Only the remote name comes off, so a branch whose own name contains a slash keeps it. Splitting on - * the last slash instead would turn `release/7.x` into `7.x` and match nothing. - * @param {string} shortRef - * @returns {string} - */ -function branchNameOf(shortRef) { - const slash = shortRef.indexOf('/'); - return slash === -1 ? shortRef : shortRef.slice(slash + 1); -} - -/** - * Runs a git command, throwing on a non-zero exit. - * @param {string[]} args - * @param {string} cwd - * @returns {Promise} - */ -async function gitCapture(args, cwd) { - const result = await execa('git', args, { cwd, reject: false }); - if (result.exitCode !== 0) { - throw new Error(`git ${args.join(' ')} failed: ${String(result.stderr).trim()}`); - } - return result.stdout.trim(); -} - -/** - * The branch pull requests fork from, read from `origin`'s default branch and falling back to - * `master` when the repository has no `origin/HEAD` — which is common in a CI clone. - * @param {string} cwd - * @returns {Promise} - */ -export async function detectBaseBranch(cwd) { - const result = await execa('git', ['symbolic-ref', '--short', 'refs/remotes/origin/HEAD'], { - cwd, - reject: false, - }); - if (result.exitCode !== 0) { - return 'master'; - } - return branchNameOf(result.stdout.trim()); -} - -/** - * Among the available base branches — every remote's `/` plus a local `` — the - * one whose merge base with HEAD is the most recent commit (the closest fork point), or undefined - * if none exist. - * - * Picking by most-recent merge base prefers an up-to-date upstream over a stale fork's base branch - * without hardcoding which remote is authoritative: locally `origin` may be a months-behind fork - * while `upstream` tracks the real repo, yet in CI `origin` *is* that repo. - * @param {string} repoRoot - * @param {string} baseBranch - * @returns {Promise<{ ref: string, mergeBase: string } | undefined>} - */ -export async function closestBaseBranch(repoRoot, baseBranch) { - // On ties (same merge base), prefer upstream's, then origin's, then a local base branch. - const preference = [`upstream/${baseBranch}`, `origin/${baseBranch}`, baseBranch]; - /** @param {string} ref */ - const priority = (ref) => { - const index = preference.indexOf(ref); - return index === -1 ? preference.length : index; - }; - // Match only real base branches from full refnames: a remote's `/` (exactly one - // segment before it) or the local ``. Filtering short names on `/` would also catch a - // local branch literally named e.g. `wip/master`. - // - // Compared as a string rather than through a regex built from `baseBranch`: the branch name is - // arbitrary, and a real one like `v6.x` would make `.` match any character — quietly admitting - // `origin/v6-x` as a baseline candidate. - /** @param {string} refname */ - const isRemoteBase = (refname) => { - if (!refname.startsWith(REMOTES_PREFIX)) { - return false; - } - const withoutPrefix = refname.slice(REMOTES_PREFIX.length); - // A remote-tracking refname always has a remote to strip; `refs/remotes/foo` alone is not one. - return withoutPrefix.includes('/') && branchNameOf(withoutPrefix) === baseBranch; - }; - - const candidates = ( - await gitCapture( - ['for-each-ref', '--format=%(refname)', 'refs/remotes', 'refs/heads'], - repoRoot, - ) - ) - .split('\n') - .filter((ref) => isRemoteBase(ref) || ref === `refs/heads/${baseBranch}`) - .map((ref) => ref.replace(/^refs\/(remotes|heads)\//, '')) - .sort((a, b) => priority(a) - priority(b)); - - /** @type {{ ref: string, mergeBase: string, when: number } | undefined} */ - let best; - for (const ref of candidates) { - // Sequential on purpose: each iteration is a git process, and the list is a handful of refs. - // eslint-disable-next-line no-await-in-loop - const result = await execa('git', ['merge-base', 'HEAD', ref], { - cwd: repoRoot, - reject: false, - }); - if (result.exitCode !== 0) { - continue; - } - const mergeBase = result.stdout.trim(); - // eslint-disable-next-line no-await-in-loop - const when = Number(await gitCapture(['show', '-s', '--format=%ct', mergeBase], repoRoot)); - // Strictly greater, so on ties the higher-priority (earlier-sorted) ref wins. - if (!best || when > best.when) { - best = { ref, mergeBase, when }; - } - } - return best && { ref: best.ref, mergeBase: best.mergeBase }; -} - -/** - * @typedef {Object} Baseline - * @property {string} sha - The commit to compare against. - * @property {string} reason - How it was chosen, for a log line. - */ - -/** - * The commit HEAD should be compared against: on a feature branch the fork point from the closest - * base branch, so a comparison reflects only what the branch changed; on the base branch itself the - * previous commit, since there is no meaningful fork point there. - * @param {Object} [options] - * @param {string} [options.cwd] - * @param {string} [options.baseBranch] - Overrides detection from `origin/HEAD`. - * @returns {Promise} - */ -export async function resolveBaseline(options = {}) { - const cwd = options.cwd ?? process.cwd(); - const repoRoot = await gitCapture(['rev-parse', '--show-toplevel'], cwd); - const baseBranch = options.baseBranch ?? (await detectBaseBranch(repoRoot)); - - const previous = async () => ({ - sha: await gitCapture(['rev-parse', 'HEAD~1'], repoRoot), - reason: 'previous commit', - }); - - if ((await gitCapture(['rev-parse', '--abbrev-ref', 'HEAD'], repoRoot)) === baseBranch) { - return previous(); - } - - const base = await closestBaseBranch(repoRoot, baseBranch); - if (!base) { - return previous(); - } - // The fork point is HEAD itself — HEAD is already contained in the base branch, so there is - // nothing this branch changed to compare. - if (base.mergeBase === (await gitCapture(['rev-parse', 'HEAD'], repoRoot))) { - return previous(); - } - return { sha: base.mergeBase, reason: `merge base with ${base.ref}` }; -} diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index f3135769d4..ea36e61e1e 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -67,10 +67,11 @@ export async function getRepositoryInfo(cwd = process.cwd()) { /** * Get current git SHA + * @param {string} [cwd=process.cwd()] * @returns {Promise} Current git commit SHA */ -export async function getCurrentGitSha() { - const result = await $`git rev-parse HEAD`; +export async function getCurrentGitSha(cwd = process.cwd()) { + const result = await $({ cwd })`git rev-parse HEAD`; return result.stdout.trim(); } @@ -84,3 +85,144 @@ export async function remoteGitTagExists(tagName, cwd = process.cwd()) { const { stdout } = await $({ cwd })`git ls-remote --tags origin refs/tags/${tagName}`; return stdout.trim().length > 0; } + +const REMOTES_PREFIX = 'refs/remotes/'; + +/** + * `origin/release/7.x` → `release/7.x`. + * + * Only the remote name comes off, so a branch whose own name contains a slash keeps it. Splitting on + * the last slash instead would turn `release/7.x` into `7.x` and match nothing. + * @param {string} shortRef + * @returns {string} + */ +function branchNameOf(shortRef) { + const slash = shortRef.indexOf('/'); + return slash === -1 ? shortRef : shortRef.slice(slash + 1); +} + +/** + * The branch pull requests fork from, read from `origin`'s default branch and falling back to + * `master` when the repository has no `origin/HEAD` — which is common in a CI clone. + * @param {string} [cwd=process.cwd()] + * @returns {Promise} + */ +export async function detectBaseBranch(cwd = process.cwd()) { + const result = await $({ cwd, reject: false })`git symbolic-ref --short refs/remotes/origin/HEAD`; + if (result.exitCode !== 0) { + return 'master'; + } + return branchNameOf(result.stdout.trim()); +} + +/** + * Among the available base branches — every remote's `/` plus a local `` — the + * one whose merge base with HEAD is the most recent commit (the closest fork point), or undefined + * if none exist. + * + * Picking by most-recent merge base prefers an up-to-date upstream over a stale fork's base branch + * without hardcoding which remote is authoritative: locally `origin` may be a months-behind fork + * while `upstream` tracks the real repo, yet in CI `origin` *is* that repo. + * @param {string} baseBranch + * @param {string} [cwd=process.cwd()] + * @returns {Promise<{ ref: string, mergeBase: string } | undefined>} + */ +export async function closestBaseBranch(baseBranch, cwd = process.cwd()) { + // On ties (same merge base), prefer upstream's, then origin's, then a local base branch. + const preference = [`upstream/${baseBranch}`, `origin/${baseBranch}`, baseBranch]; + /** @param {string} ref */ + const priority = (ref) => { + const index = preference.indexOf(ref); + return index === -1 ? preference.length : index; + }; + // Match only real base branches from full refnames: a remote's `/` (exactly one + // segment before it) or the local ``. Filtering short names on `/` would also catch a + // local branch literally named e.g. `wip/master`. + // + // Compared as a string rather than through a regex built from `baseBranch`: the branch name is + // arbitrary, and a real one like `v6.x` would make `.` match any character — quietly admitting + // `origin/v6-x` as a baseline candidate. + /** @param {string} refname */ + const isRemoteBase = (refname) => { + if (!refname.startsWith(REMOTES_PREFIX)) { + return false; + } + const withoutPrefix = refname.slice(REMOTES_PREFIX.length); + // A remote-tracking refname always has a remote to strip; `refs/remotes/foo` alone is not one. + return withoutPrefix.includes('/') && branchNameOf(withoutPrefix) === baseBranch; + }; + + const { stdout } = await $({ + cwd, + })`git for-each-ref --format=%(refname) refs/remotes refs/heads`; + const candidates = stdout + .trim() + .split('\n') + .filter((ref) => isRemoteBase(ref) || ref === `refs/heads/${baseBranch}`) + .map((ref) => ref.replace(/^refs\/(remotes|heads)\//, '')) + .sort((a, b) => priority(a) - priority(b)); + + /** @type {{ ref: string, mergeBase: string, when: number } | undefined} */ + let best; + for (const ref of candidates) { + // Sequential on purpose: each iteration is a git process, and the list is a handful of refs. + // eslint-disable-next-line no-await-in-loop + const result = await $({ cwd, reject: false })`git merge-base HEAD ${ref}`; + if (result.exitCode !== 0) { + continue; + } + const mergeBase = result.stdout.trim(); + // eslint-disable-next-line no-await-in-loop + const shown = await $({ cwd })`git show -s --format=%ct ${mergeBase}`; + const when = Number(shown.stdout.trim()); + // Strictly greater, so on ties the higher-priority (earlier-sorted) ref wins. + if (!best || when > best.when) { + best = { ref, mergeBase, when }; + } + } + return best && { ref: best.ref, mergeBase: best.mergeBase }; +} + +/** + * @typedef {Object} Baseline + * @property {string} sha - The commit to compare against. + * @property {string} reason - How it was chosen, for a log line. + */ + +/** + * The commit HEAD should be compared against: on a feature branch the fork point from the closest + * base branch, so a comparison reflects only what the branch changed; on the base branch itself the + * previous commit, since there is no meaningful fork point there. + * + * One answer for every job that compares a branch to its base — bundle size, benchmarks — which + * each used to derive it in shell, and had drifted apart doing so. + * @param {Object} [options] + * @param {string} [options.cwd] + * @param {string} [options.baseBranch] - Overrides detection from `origin/HEAD`. + * @returns {Promise} + */ +export async function resolveBaseline(options = {}) { + const cwd = options.cwd ?? process.cwd(); + const baseBranch = options.baseBranch ?? (await detectBaseBranch(cwd)); + + const previous = async () => ({ + sha: (await $({ cwd })`git rev-parse HEAD~1`).stdout.trim(), + reason: 'previous commit', + }); + + const branch = (await $({ cwd })`git rev-parse --abbrev-ref HEAD`).stdout.trim(); + if (branch === baseBranch) { + return previous(); + } + + const base = await closestBaseBranch(baseBranch, cwd); + if (!base) { + return previous(); + } + // The fork point is HEAD itself — HEAD is already contained in the base branch, so there is + // nothing this branch changed to compare. + if (base.mergeBase === (await getCurrentGitSha(cwd))) { + return previous(); + } + return { sha: base.mergeBase, reason: `merge base with ${base.ref}` }; +} diff --git a/packages/code-infra/src/utils/baseline.test.mjs b/packages/code-infra/src/utils/git.test.mjs similarity index 98% rename from packages/code-infra/src/utils/baseline.test.mjs rename to packages/code-infra/src/utils/git.test.mjs index f739c76b0e..95e7f83c74 100644 --- a/packages/code-infra/src/utils/baseline.test.mjs +++ b/packages/code-infra/src/utils/git.test.mjs @@ -3,7 +3,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { describe, expect, it, onTestFinished } from 'vitest'; import { execa } from 'execa'; -import { resolveBaseline } from './baseline.mjs'; +import { resolveBaseline } from './git.mjs'; /** * A repository with three commits on `main`. From 519a0aa7d3e5ec97a5367c31f8f660b8387f04e8 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:14:40 +0200 Subject: [PATCH 050/173] [benchmark] Document the tachometer axis, and compare against the fork point MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ref grammar, how a baseline is chosen and how packed builds are cached were written down only in a consuming repository's harness README, where they described the tool rather than that repository's benchmarks. They belong here. The intro said the package was a Vitest tool; it has been two things for a while. The smoke test now compares against the fork point from master rather than HEAD, the same comparison a consuming repository makes. The verdict still is not the point — this repository ships tooling, and the measured page is a synthetic workload — but packing and building a genuinely different commit is the path a real run takes, and the previous shape never exercised it. `code-infra baseline` decides which commit that is, which also gives it its first caller. It covers being on master, and CircleCI's detached HEAD, that a test on the branch name does not. --- .circleci/config.yml | 32 +++++-- packages/benchmark/README.md | 91 ++++++++++++++++++- .../src/changelog/fetchChangelogs.mjs | 21 ----- packages/code-infra/src/changelog/index.mjs | 3 + packages/code-infra/src/utils/git.mjs | 23 +++++ packages/code-infra/src/utils/git.test.mjs | 44 +++++---- 6 files changed, 170 insertions(+), 44 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e19ecf0e29..35329cb78f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -117,6 +117,24 @@ jobs: # runs is itself a workspace package that publishes from `build/`. name: Build packages command: pnpm release:build + - run: + name: Resolve the baseline commit + # The fork point from master, the same comparison a consuming repository makes. No render + # time here is worth tracking — this repository ships tooling, and the measured page is a + # synthetic workload — so the verdict is not the point; packing and building a genuinely + # different commit is, because that is the path a real run takes. + # + # `code-infra baseline` decides, rather than a `merge-base` spelled out here: it also + # covers being on master, and CircleCI's detached HEAD, which a branch-name test does not. + command: | + set -e + git fetch --no-tags origin master:refs/remotes/origin/master + BASE_SHA=$(pnpm code-infra baseline --verbose) + echo "$BASE_SHA" > .tachometer-baseline-sha + - restore_cache: + name: Restore the packed-baseline cache + keys: + - tachometer-cache-v1-{{ checksum ".tachometer-baseline-sha" }} - run: name: Run the tachometer smoke test # Packing a ref installs and builds it in a quiet worktree; sampling then prints per case. @@ -127,12 +145,14 @@ jobs: # request lands, so it would reject the upload. Points at this pull request's own # dashboard preview until then — remove the line before merging. CI_REPORT_API_URL: https://code-infra-dashboard-pr-1829.onrender.com - # This repository ships tooling rather than a browser library, so no render time here is - # worth tracking against the base branch. The working tree is measured against HEAD — - # identical code on both sides, so the verdict is always "unsure" — because the point is - # to exercise the pipeline: packing a ref, installing it in isolation, building the pages, - # sampling under Chrome, and reporting. - command: pnpm test:tacho --baseline git:HEAD + command: pnpm test:tacho --baseline "git:$(cat .tachometer-baseline-sha)" + - save_cache: + name: Save the packed-baseline cache + key: tachometer-cache-v1-{{ checksum ".tachometer-baseline-sha" }} + paths: + # Tarballs only; the isolated installs beside them are cheap to recreate and their store + # links would not survive the container boundary anyway. + - test/tachometer/.tachometer/packed test_browser: <<: *default-job diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index bdbb0f34dd..92e7083f92 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -1,6 +1,15 @@ # Benchmark -A React component render benchmarking tool built on Vitest and Playwright. Runs benchmarks in a real browser using React's profiling build to capture accurate render durations. +Two ways to measure, for two different questions. + +**Render durations** — a React component benchmarking tool built on Vitest and Playwright, using +React's profiling build to capture accurate render durations against your source. Everything up to +[Tachometer](#tachometer) covers it. + +**Wall-clock time** — [Tachometer](#tachometer) below: Google's +[tachometer](https://github.com/google/tachometer) driving a real browser against your **built** +package, comparing two commits of it with interleaved sampling. No profiler and no render counts, +only elapsed milliseconds on what a consumer installs. ## Features @@ -276,6 +285,86 @@ BENCHMARK_BASELINE_PATH=/tmp/base-bench.json pnpm test:bench # head run, inlin The feature is opt-in — without `BENCHMARK_BASELINE_PATH` (or the `baselinePath` config option), the dashboard falls back to fetching the base from S3 by merge-base SHA as before. +## Tachometer + +`benchmark tacho run` measures a harness package in a real browser, building the workspace once per +commit under comparison and serving the pages from those builds. Run it from the harness. + +A harness is a package holding a `src/` tree of **case folders**, each with a page and a plain, +unmodified [tachometer config](https://github.com/google/tachometer#config-file). The configs are the +source of truth for which pages exist: the runner reads the selected ones, works out which pages they +reference, builds one variant per distinct commit, and rewrites each url to the built page. Filtering +to one case therefore narrows the build too. + +Sampling — `sampleSize`, `autoSampleConditions`, `timeout` — is set per case in its own config, +because tachometer rejects those as flags once a config file is in play. A case samples until its +difference resolves or the timeout is hit; two builds of equal speed cannot be separated, so such a +case reports `unsure`, which is the expected "nothing changed" outcome rather than a failure. + +### Refs: which build a page loads + +A benchmark url may carry a `?ref=` selecting **which build of the workspace** to load. A ref always +names a commit and covers every package in the workspace, and is resolved to an immutable SHA before +anything is cached. + +| `?ref=` | Meaning | +| :---------------------------- | :------------------------------------------------------ | +| _(absent)_ | the working tree | +| `baseline` | symbolic — whatever `--baseline` bound | +| `git:` | this repository at `` (SHA, tag, branch, `HEAD~1`) | +| `github:/#` | another remote _(not implemented)_ | +| `preview:` | pkg.pr.new artifacts _(not implemented)_ | + +Anything else is an error. A bare value is never assumed to be a revision, since an absent ref +already means the working tree. + +A case whose benchmark declares no `expand` is the ordinary regression shape, and is expanded for you +into `[current]` (the working tree) against `[baseline]`: + +```jsonc +{ "benchmarks": [{ "name": "init", "url": "./index.html" }] } +``` + +A case that declares its own `expand` owns that axis, and only the refs its leaves mention get built. +Because `expand` nests, this is how a ref axis combines with anything else — browsers, or entirely +different libraries: + +```jsonc +"expand": [ + { "name": "[mosaic]", "url": "./index.html" }, + { "name": "[tanstack]", "url": "./tanstack.html" } +] +``` + +Leaving `?ref` off every variant, as above, compares implementations at a single commit rather than +two builds of one library — so no regression can be reported, by construction. + +### Baselines + +`--baseline` binds the `baseline` symbol, taking a ref in the grammar above. It defaults to `HEAD~1`, +which needs no policy to decide. Which commit a branch should actually be compared against is +`code-infra baseline`'s question — it answers the same way for every job that compares a branch to +its base — so pass it in: + +```bash +benchmark tacho run --baseline "git:$(code-infra baseline)" +``` + +### Builds and caching + +Both sides are built the same way, from packed tarballs installed into an isolated tree, so neither +resolves the library through a workspace link. The benchmark pages themselves stay on the current +branch and only the built library changes between refs — so a commit whose public API differs from +today's pages will fail that ref's build, with the error surfaced. + +Packed builds are cached by commit SHA under `.tachometer/packed/`, so repeating a comparison against +the same commit skips the rebuild; the working tree is never cached. In CI, cache that directory +keyed on the baseline SHA, and check out with full history so a fork point can be resolved. A run +never modifies the checkout it was started from. + +`--no-install` skips a ref's install. `benchmark tacho run --help` lists the rest, and +`benchmark tacho report` prints the table again from a saved JSON report. + ## API - `benchmark` — define a benchmark test case diff --git a/packages/code-infra/src/changelog/fetchChangelogs.mjs b/packages/code-infra/src/changelog/fetchChangelogs.mjs index 03822409dd..084284794e 100644 --- a/packages/code-infra/src/changelog/fetchChangelogs.mjs +++ b/packages/code-infra/src/changelog/fetchChangelogs.mjs @@ -1,7 +1,5 @@ import { Octokit } from '@octokit/rest'; -import { $ } from 'execa'; -import { getRepositoryInfo } from '../utils/git.mjs'; import { persistentAuthStrategy } from '../utils/github.mjs'; /** @@ -20,25 +18,6 @@ import { persistentAuthStrategy } from '../utils/github.mjs'; * @property {string} [org='mui'] - GitHub organization name, defaults to 'mui' */ -/** - * @param {Object} opts - * @param {string} opts.cwd - * @param {boolean} [opts.fetchAll=true] Whether to fetch all tags from all remotes before finding the latest tag. - * @returns {Promise} - */ -export async function findLatestTaggedVersion(opts) { - const $$ = $({ cwd: opts.cwd }); - const fetchAll = opts.fetchAll ?? true; - if (fetchAll) { - const { remoteName } = await getRepositoryInfo(); - // Fetch all tags from the mui remote to ensure we have the latest tags. - // --force to update any existing tags that may have changed to avoid the clobering error. - await $$`git fetch --tags --force ${remoteName}`; - } - const { stdout } = await $$`git describe --tags --abbrev=0 --match ${'v*'}`; // only include "version-tags" - return stdout.trim(); -} - /** * Fetches commits between two refs (lastRelease..release) including PR details. * Automatically handles GitHub OAuth authentication if none provided. diff --git a/packages/code-infra/src/changelog/index.mjs b/packages/code-infra/src/changelog/index.mjs index a886ed164e..2135e2e382 100644 --- a/packages/code-infra/src/changelog/index.mjs +++ b/packages/code-infra/src/changelog/index.mjs @@ -5,3 +5,6 @@ export { generateChangelog } from './generateChangelog.mjs'; export * from './loadChangelogConfig.mjs'; export * from './fetchChangelogs.mjs'; +// Moved to the git utilities, where the rest of the local git plumbing lives. Re-exported because +// `@mui/internal-code-infra/changelog` is a published entry point and this was part of it. +export { findLatestTaggedVersion } from '../utils/git.mjs'; diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index ea36e61e1e..9784e7584c 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -226,3 +226,26 @@ export async function resolveBaseline(options = {}) { } return { sha: base.mergeBase, reason: `merge base with ${base.ref}` }; } + +/** + * The most recent version tag reachable from HEAD — where a changelog's range starts. + * + * Not a baseline: {@link resolveBaseline} answers where a *branch* diverged, which on a feature + * branch is a different commit entirely, and the two are not interchangeable. + * @param {Object} opts + * @param {string} opts.cwd + * @param {boolean} [opts.fetchAll=true] Whether to fetch all tags from all remotes before finding the latest tag. + * @returns {Promise} + */ +export async function findLatestTaggedVersion(opts) { + const $$ = $({ cwd: opts.cwd }); + const fetchAll = opts.fetchAll ?? true; + if (fetchAll) { + const { remoteName } = await getRepositoryInfo(); + // Fetch all tags from the mui remote to ensure we have the latest tags. + // --force to update any existing tags that may have changed to avoid the clobering error. + await $$`git fetch --tags --force ${remoteName}`; + } + const { stdout } = await $$`git describe --tags --abbrev=0 --match ${'v*'}`; // only include "version-tags" + return stdout.trim(); +} diff --git a/packages/code-infra/src/utils/git.test.mjs b/packages/code-infra/src/utils/git.test.mjs index 95e7f83c74..c263b41c08 100644 --- a/packages/code-infra/src/utils/git.test.mjs +++ b/packages/code-infra/src/utils/git.test.mjs @@ -21,22 +21,29 @@ async function makeRepo() { // Identity through the environment rather than `git config`, so the repo needs no extra processes // and nothing depends on the machine's own git configuration. - const git = (/** @type {string[]} */ ...args) => - execa('git', args, { - cwd: repoRoot, - env: { - GIT_AUTHOR_NAME: 'Fixture', - GIT_AUTHOR_EMAIL: 'fixture@example.com', - GIT_COMMITTER_NAME: 'Fixture', - GIT_COMMITTER_EMAIL: 'fixture@example.com', - }, - }); + /** @param {string} [date] Commit date, so ranking by recency is not left to the clock. */ + const gitWith = + (date) => + (/** @type {string[]} */ ...args) => + execa('git', args, { + cwd: repoRoot, + env: { + GIT_AUTHOR_NAME: 'Fixture', + GIT_AUTHOR_EMAIL: 'fixture@example.com', + GIT_COMMITTER_NAME: 'Fixture', + GIT_COMMITTER_EMAIL: 'fixture@example.com', + ...(date ? { GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date } : {}), + }, + }); + const git = gitWith(undefined); await git('init', '--initial-branch=main'); - for (const message of ['first', 'second', 'third']) { + // A day apart each: `closestBaseBranch` picks the candidate whose merge base is most recent, and + // three commits made in a loop otherwise share a second and leave that comparison to a tie-break. + for (const [index, message] of ['first', 'second', 'third'].entries()) { // Empty commits: nothing here reads a file, only the commits' identities matter. // eslint-disable-next-line no-await-in-loop - await git('commit', '--allow-empty', '-m', message); + await gitWith(`2020-01-0${index + 1}T00:00:00Z`)('commit', '--allow-empty', '-m', message); } const { stdout } = await git('log', '--format=%H', '--reverse'); return { repoRoot, shas: stdout.trim().split('\n'), git }; @@ -65,16 +72,21 @@ describe('resolveBaseline', () => { it('prefers the remote whose fork point is most recent', async () => { const { repoRoot, shas, git } = await makeRepo(); - // A stale fork and an up-to-date upstream, the case that makes hardcoding `origin` wrong. - await git('update-ref', 'refs/remotes/origin/main', shas[0]); - await git('update-ref', 'refs/remotes/upstream/main', shas[1]); + // `upstream` is the preferred remote on a tie, so it is deliberately the *stale* one here: + // picking by recency has to override that preference, and a "first candidate wins" reading would + // answer `upstream` instead. + await git('update-ref', 'refs/remotes/upstream/main', shas[0]); + await git('update-ref', 'refs/remotes/origin/main', shas[1]); await git('checkout', '-b', 'feature'); await git('commit', '--allow-empty', '-m', 'work'); + // Drop the local `main`, which is a candidate in its own right and sits ahead of both remotes — + // it would win on recency and say nothing about which remote was chosen. + await git('branch', '-D', 'main'); const baseline = await resolveBaseline({ cwd: repoRoot, baseBranch: 'main' }); expect(baseline.sha).toBe(shas[1]); - expect(baseline.reason).toContain('upstream/main'); + expect(baseline.reason).toContain('origin/main'); }); it('does not treat a branch matching the base branch as a pattern', async () => { From c205334d3b0e7224fe24bb1ef5f34727021fb220 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 10:20:01 +0200 Subject: [PATCH 051/173] [code-infra] Print the baseline, nothing else MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `--verbose` explained which ref the commit came from, and `resolveBaseline` carried a `reason` string for it to print. One flag and one field for a log line the caller can write itself — the CI step now echoes the commit it captured. The tests asserted on that string alongside the SHA; the SHA was always the assertion doing the work, and still fails on its own if the ranking breaks. --- .circleci/config.yml | 3 ++- packages/code-infra/src/cli/cmdBaseline.mjs | 19 ++----------------- packages/code-infra/src/utils/git.mjs | 21 ++++++--------------- packages/code-infra/src/utils/git.test.mjs | 18 ++++++++---------- 4 files changed, 18 insertions(+), 43 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 35329cb78f..df16204b53 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,7 +129,8 @@ jobs: command: | set -e git fetch --no-tags origin master:refs/remotes/origin/master - BASE_SHA=$(pnpm code-infra baseline --verbose) + BASE_SHA=$(pnpm code-infra baseline) + echo "Baseline commit: $BASE_SHA" echo "$BASE_SHA" > .tachometer-baseline-sha - restore_cache: name: Restore the packed-baseline cache diff --git a/packages/code-infra/src/cli/cmdBaseline.mjs b/packages/code-infra/src/cli/cmdBaseline.mjs index a1c779ab92..4ffd1fda45 100644 --- a/packages/code-infra/src/cli/cmdBaseline.mjs +++ b/packages/code-infra/src/cli/cmdBaseline.mjs @@ -7,7 +7,6 @@ import { resolveBaseline } from '../utils/git.mjs'; /** * @typedef {Object} Args * @property {string} [baseBranch] - Branch pull requests fork from - * @property {boolean} [verbose] - Explain the choice on stderr */ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ @@ -20,26 +19,12 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ description: 'Branch pull requests fork from. Default: detected from origin/HEAD, falling back to master', }) - .option('verbose', { - type: 'boolean', - default: false, - description: 'Explain the choice on stderr', - }) - .example( - '$0 baseline', - 'Print the fork point from the base branch, or the previous commit when on it', - ) .example( 'BASE=$($0 baseline)', - 'Capture it — only the SHA goes to stdout, so this stays substitutable', + 'Nothing but the commit is printed, so this stays substitutable', ); }, handler: async (argv) => { - const { sha, reason } = await resolveBaseline({ baseBranch: argv.baseBranch }); - if (argv.verbose) { - // stderr, so command substitution captures the SHA alone. - console.error(`Baseline: ${sha} (${reason})`); - } - console.log(sha); + console.log(await resolveBaseline({ baseBranch: argv.baseBranch })); }, }); diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index 9784e7584c..1c4c6c0093 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -183,12 +183,6 @@ export async function closestBaseBranch(baseBranch, cwd = process.cwd()) { return best && { ref: best.ref, mergeBase: best.mergeBase }; } -/** - * @typedef {Object} Baseline - * @property {string} sha - The commit to compare against. - * @property {string} reason - How it was chosen, for a log line. - */ - /** * The commit HEAD should be compared against: on a feature branch the fork point from the closest * base branch, so a comparison reflects only what the branch changed; on the base branch itself the @@ -199,32 +193,29 @@ export async function closestBaseBranch(baseBranch, cwd = process.cwd()) { * @param {Object} [options] * @param {string} [options.cwd] * @param {string} [options.baseBranch] - Overrides detection from `origin/HEAD`. - * @returns {Promise} + * @returns {Promise} */ export async function resolveBaseline(options = {}) { const cwd = options.cwd ?? process.cwd(); const baseBranch = options.baseBranch ?? (await detectBaseBranch(cwd)); - const previous = async () => ({ - sha: (await $({ cwd })`git rev-parse HEAD~1`).stdout.trim(), - reason: 'previous commit', - }); + const previousCommit = async () => (await $({ cwd })`git rev-parse HEAD~1`).stdout.trim(); const branch = (await $({ cwd })`git rev-parse --abbrev-ref HEAD`).stdout.trim(); if (branch === baseBranch) { - return previous(); + return previousCommit(); } const base = await closestBaseBranch(baseBranch, cwd); if (!base) { - return previous(); + return previousCommit(); } // The fork point is HEAD itself — HEAD is already contained in the base branch, so there is // nothing this branch changed to compare. if (base.mergeBase === (await getCurrentGitSha(cwd))) { - return previous(); + return previousCommit(); } - return { sha: base.mergeBase, reason: `merge base with ${base.ref}` }; + return base.mergeBase; } /** diff --git a/packages/code-infra/src/utils/git.test.mjs b/packages/code-infra/src/utils/git.test.mjs index c263b41c08..7acf9a8a5f 100644 --- a/packages/code-infra/src/utils/git.test.mjs +++ b/packages/code-infra/src/utils/git.test.mjs @@ -53,9 +53,9 @@ describe('resolveBaseline', () => { it('uses the previous commit when HEAD is on the base branch', async () => { const { repoRoot, shas } = await makeRepo(); - const baseline = await resolveBaseline({ cwd: repoRoot, baseBranch: 'main' }); + const sha = await resolveBaseline({ cwd: repoRoot, baseBranch: 'main' }); - expect(baseline.sha).toBe(shas[1]); + expect(sha).toBe(shas[1]); }); it('uses the fork point when HEAD is on a branch', async () => { @@ -64,10 +64,9 @@ describe('resolveBaseline', () => { await git('checkout', '-b', 'feature', shas[0]); await git('commit', '--allow-empty', '-m', 'work'); - const baseline = await resolveBaseline({ cwd: repoRoot, baseBranch: 'main' }); + const sha = await resolveBaseline({ cwd: repoRoot, baseBranch: 'main' }); - expect(baseline.sha).toBe(shas[0]); - expect(baseline.reason).toContain('main'); + expect(sha).toBe(shas[0]); }); it('prefers the remote whose fork point is most recent', async () => { @@ -83,10 +82,9 @@ describe('resolveBaseline', () => { // it would win on recency and say nothing about which remote was chosen. await git('branch', '-D', 'main'); - const baseline = await resolveBaseline({ cwd: repoRoot, baseBranch: 'main' }); + const sha = await resolveBaseline({ cwd: repoRoot, baseBranch: 'main' }); - expect(baseline.sha).toBe(shas[1]); - expect(baseline.reason).toContain('origin/main'); + expect(sha).toBe(shas[1]); }); it('does not treat a branch matching the base branch as a pattern', async () => { @@ -95,10 +93,10 @@ describe('resolveBaseline', () => { await git('update-ref', 'refs/remotes/origin/v6-x', shas[0]); await git('checkout', '-b', 'feature'); - const baseline = await resolveBaseline({ cwd: repoRoot, baseBranch: 'v6.x' }); + const sha = await resolveBaseline({ cwd: repoRoot, baseBranch: 'v6.x' }); // No base branch exists, so it falls back to the previous commit. Matching `origin/v6-x` would // instead have picked its merge base — the first commit — and compared against the wrong one. - expect(baseline.sha).toBe(shas[1]); + expect(sha).toBe(shas[1]); }); }); From a1ae0911ca39dd0a250a6095829c208fdf1925f9 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:03:17 +0200 Subject: [PATCH 052/173] [benchmark] One definition of the report, and of how it reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dashboard kept its own copy of the report's types and of every formatter, on the grounds that stored artifacts may predate today's shape. The report carries a version for exactly that, so the shape is now a union keyed on it: a new shape joins with its own literal version and a reader narrows before touching anything. Mirroring bought nothing the discriminant does not, and the copies had already drifted — the dashboard had `raw` optional where the schema requires it. So the types and the table formatting come from the package that defines them. That also means `refs` can say what a ref is rather than how it reads: the revision asked for, not a rendered label, with one formatter for the three places that print it. `committish` went with it, having been a second copy of `sha`. `tacho report` now parses against the schema instead of checking one field, so an artifact this version cannot read says which field is wrong rather than printing `undefined`. Also: `@playwright/test` through `import()` — node exposes its named exports, so the comment claiming otherwise was wrong; `semver` for the browser version, with a regex still picking the version out of the surrounding words, since `coerce` alone reads "2 browsers found: Chrome 151.0.7922.34" as version 2; and pnpm asked for the harness's resolved versions in one call rather than reading `node_modules` manifests one at a time. Carries the comment trims from the working tree: describe the code, not the decision behind it. --- .circleci/config.yml | 12 -- .../app/api/ci-reports/upload/route.ts | 14 +-- .../src/lib/ciReports/tachometerReport.ts | 11 +- .../tachometer/buildMarkdownReport.test.ts | 21 ++-- .../src/lib/tachometer/buildMarkdownReport.ts | 22 ++-- .../src/lib/tachometer/formatInterval.ts | 27 ----- .../src/lib/tachometer/groupCases.ts | 63 ----------- .../src/lib/tachometer/types.ts | 60 ---------- .../src/utils/fetchCiReport.ts | 4 +- .../src/views/TachometerDetails.tsx | 39 +++---- packages/benchmark/package.json | 3 + packages/benchmark/src/cli/cmdTacho.ts | 4 - packages/benchmark/src/cli/tachoReport.ts | 36 +++--- packages/benchmark/src/tachometer/browser.ts | 56 ++++------ .../benchmark/src/tachometer/buildPages.ts | 83 +++++--------- .../benchmark/src/tachometer/ciReport.test.ts | 4 +- packages/benchmark/src/tachometer/ciReport.ts | 42 +++++-- .../benchmark/src/tachometer/format.test.ts | 4 +- packages/benchmark/src/tachometer/format.ts | 93 ++++++++++++++++ .../benchmark/src/tachometer/refs.test.ts | 6 +- packages/benchmark/src/tachometer/refs.ts | 25 ++--- .../benchmark/src/tachometer/renderReport.ts | 104 ++++-------------- .../benchmark/src/tachometer/runTachometer.ts | 18 +-- packages/benchmark/src/utils/pnpm.ts | 31 ++++++ pnpm-lock.yaml | 6 + 25 files changed, 315 insertions(+), 473 deletions(-) delete mode 100644 apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts delete mode 100644 apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts delete mode 100644 apps/code-infra-dashboard/src/lib/tachometer/types.ts rename apps/code-infra-dashboard/src/lib/tachometer/groupCases.test.ts => packages/benchmark/src/tachometer/format.test.ts (97%) create mode 100644 packages/benchmark/src/tachometer/format.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index df16204b53..15d77a722d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -113,19 +113,10 @@ jobs: - checkout - code-infra/install-deps - run: - # The harness resolves the library under test through its build output, and the CLI it - # runs is itself a workspace package that publishes from `build/`. name: Build packages command: pnpm release:build - run: name: Resolve the baseline commit - # The fork point from master, the same comparison a consuming repository makes. No render - # time here is worth tracking — this repository ships tooling, and the measured page is a - # synthetic workload — so the verdict is not the point; packing and building a genuinely - # different commit is, because that is the path a real run takes. - # - # `code-infra baseline` decides, rather than a `merge-base` spelled out here: it also - # covers being on master, and CircleCI's detached HEAD, which a branch-name test does not. command: | set -e git fetch --no-tags origin master:refs/remotes/origin/master @@ -138,7 +129,6 @@ jobs: - tachometer-cache-v1-{{ checksum ".tachometer-baseline-sha" }} - run: name: Run the tachometer smoke test - # Packing a ref installs and builds it in a quiet worktree; sampling then prints per case. no_output_timeout: 30m environment: TACHO_UPLOAD: 'true' @@ -151,8 +141,6 @@ jobs: name: Save the packed-baseline cache key: tachometer-cache-v1-{{ checksum ".tachometer-baseline-sha" }} paths: - # Tarballs only; the isolated installs beside them are cheap to recreate and their store - # links would not survive the container boundary anyway. - test/tachometer/.tachometer/packed test_browser: diff --git a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts index 6a313a4502..1129bf6587 100644 --- a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts +++ b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts @@ -7,15 +7,6 @@ import { findAssociatedPr } from '@/lib/ciReports/findAssociatedPr'; const VALID_REPORT_TYPES = new Set(['size-snapshot', 'benchmark', 'tachometer']); -// Report types stored as the full upload envelope (version, timestamp, commitSha, repo, -// branch, prNumber, reportType, report). Every other type stores just the inner report. -// -// A tachometer report is not one of them: it already carries its own `version`, `reportType` and -// `head.sha`/`head.branch`, the repo is in the S3 key, and the generator reads the envelope's -// metadata from its own route params rather than from the artifact. Storing the envelope only -// created a second shape, which `tacho report` then could not tell apart from a local report. -const ENVELOPE_REPORT_TYPES = new Set(['benchmark']); - const uploadSchema = z.object({ version: z.number(), timestamp: z.number(), @@ -64,9 +55,8 @@ export async function POST(request: NextRequest) { const { commitSha, repo, reportType, branch, report } = parsed.data; - const storedBody = ENVELOPE_REPORT_TYPES.has(reportType) - ? JSON.stringify(parsed.data) - : JSON.stringify(report); + const storedBody = + reportType === 'benchmark' ? JSON.stringify(parsed.data) : JSON.stringify(report); if (!VALID_REPORT_TYPES.has(reportType)) { return NextResponse.json( diff --git a/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts b/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts index d5c84f67d9..7224b5c0ee 100644 --- a/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts +++ b/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts @@ -5,22 +5,13 @@ import { } from '@/lib/tachometer/buildMarkdownReport'; import { DASHBOARD_ORIGIN } from '@/constants'; -import type { TachometerReport } from '@/lib/tachometer/types'; +import type { TachometerReport } from '@mui/internal-benchmark/tachometerReport'; import type { ReportOptions, ReportResult } from './types'; export { TACHOMETER_SECTION_TITLE }; /** * Generates the tachometer section of the pull request comment. - * - * Unlike the benchmark section, this fetches exactly one artifact and does no comparison. A - * tachometer report already contains both sides: `[current]` and `[baseline]` are sampled - * round-robin in a single browser session, and the report carries tachometer's own confidence - * interval on the difference. There is no base report to look up, and so no chance of pairing - * against a mismatched one. - * - * Returns null when the head commit has no report, which is how a repository that did not run the - * job produces no section at all. */ export async function generateTachometerReport( options: ReportOptions, diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts index fa346a206f..e97b4d9bae 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts @@ -1,11 +1,11 @@ import { describe, expect, it } from 'vitest'; -import { buildTachometerMarkdownReport, findRegressions } from './buildMarkdownReport'; import type { - TachometerCaseResult, - TachometerMeasurementResult, + CaseResult, + MeasurementResult, TachometerReport, Verdict, -} from './types'; +} from '@mui/internal-benchmark/tachometerReport'; +import { buildTachometerMarkdownReport, findRegressions } from './buildMarkdownReport'; /** * A variant on a given build. @@ -50,8 +50,8 @@ function comparison(name: string, verdict: Verdict) { /** * A case comparing `[current]` against `[baseline]` — two different builds. */ -function regressionCase(name: string, verdicts: Record): TachometerCaseResult { - const measurements: TachometerMeasurementResult[] = Object.entries(verdicts).map( +function regressionCase(name: string, verdicts: Record): CaseResult { + const measurements: MeasurementResult[] = Object.entries(verdicts).map( ([measurement, verdict]) => ({ name: measurement, variants: [ @@ -67,7 +67,7 @@ function regressionCase(name: string, verdicts: Record): Tachom /** * A case comparing libraries with each other, so every variant comes from the same build. */ -function libraryCase(name: string, verdict: Verdict): TachometerCaseResult { +function libraryCase(name: string, verdict: Verdict): CaseResult { return { name, reference: `${name} [mosaic]`, @@ -81,17 +81,18 @@ function libraryCase(name: string, verdict: Verdict): TachometerCaseResult { }; } -function report(cases: TachometerCaseResult[]): TachometerReport { +function report(cases: CaseResult[]): TachometerReport { return { version: 1, reportType: 'tachometer', generatedAt: '2026-09-02T00:00:00.000Z', head: { ref: 'HEAD', sha: 'a'.repeat(40), branch: 'feature' }, refs: [ - { id: 'current', kind: 'worktree', label: 'working tree' }, - { id: 'git-abc123456', kind: 'git', label: 'merge-base', sha: 'b'.repeat(40) }, + { id: 'current', kind: 'worktree' }, + { id: 'git-abc123456', kind: 'git', requested: 'HEAD~1', sha: 'b'.repeat(40) }, ], cases, + raw: {}, }; } diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts index f875c37485..32df13b4c3 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts @@ -1,8 +1,16 @@ import { formatMarkdownTable } from '@/utils/formatters'; -import type { ConfidenceInterval, TachometerReport } from './types'; -import { formatMean, formatPercent, formatSignedMs } from './formatInterval'; -import { isSummarized, shortNameOf } from './groupCases'; -import type { SummarizedCase } from './groupCases'; +import type { + ConfidenceInterval, + TachometerReport, +} from '@mui/internal-benchmark/tachometerReport'; +import { + formatMean, + formatPercent, + formatSignedMs, + isSummarized, + shortNameOf, +} from '@mui/internal-benchmark/tachometerFormat'; +import type { SummarizedCase } from '@mui/internal-benchmark/tachometerFormat'; export const TACHOMETER_SECTION_TITLE = 'Tachometer'; @@ -21,9 +29,6 @@ interface BuildOptions { /** * Collects the comparisons that mean "this pull request made something slower". - * - * `comparisons` express the reference relative to each other variant, so a `slower` verdict means - * the reference — `[current]` for an auto-expanded case — is the slow side. */ function comparisonsAcrossRefs(entry: SummarizedCase) { const found = []; @@ -72,9 +77,6 @@ function renderCaseTable(entry: SummarizedCase): string { measurement.comparisons.map((comparison) => [comparison.variant, comparison]), ); return measurement.variants.map((variant) => { - // Each row reads as "this variant, compared to the reference", so it needs the direction - // whose subject is the variant. The comparison itself holds the opposite one — the reference - // relative to the variant — which is what a regression is stated in, above the fold. const againstReference = byVariant.get(variant.variant)?.versusReference; let versus = '—'; if (variant.variant === entry.reference) { diff --git a/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts b/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts deleted file mode 100644 index adda3efe19..0000000000 --- a/apps/code-infra-dashboard/src/lib/tachometer/formatInterval.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { ConfidenceInterval } from './types'; - -/** - * How a tachometer report's numbers are written, shared by the pull request comment and the details - * page so the two never disagree about precision. - */ - -/** `20.50 – 22.67 ms`, with the unit written once. */ -export function formatMean(interval: ConfidenceInterval): string { - return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; -} - -/** Both bounds signed, so the direction of a difference reads without the verdict beside it. */ -function signedInterval(interval: ConfidenceInterval, unit: string, digits: number): string { - const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(digits)}${unit}`; - return `${signed(interval.low)} – ${signed(interval.high)}`; -} - -/** `+2.7% – +6.8%`. */ -export function formatPercent(interval: ConfidenceInterval): string { - return signedInterval(interval, '%', 1); -} - -/** `+6.10 ms – +15.20 ms`. Only a regression line states an absolute difference. */ -export function formatSignedMs(interval: ConfidenceInterval): string { - return signedInterval(interval, ' ms', 2); -} diff --git a/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts b/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts deleted file mode 100644 index cf9adf09df..0000000000 --- a/apps/code-infra-dashboard/src/lib/tachometer/groupCases.ts +++ /dev/null @@ -1,63 +0,0 @@ -import type { TachometerCaseResult, TachometerMeasurementResult } from './types'; - -export type SummarizedCase = TachometerCaseResult & { - measurements: TachometerMeasurementResult[]; -}; - -export interface VariantGroup { - /** The variant set the group's cases share, reference first. */ - variants: string[]; - cases: SummarizedCase[]; -} - -/** - * Tachometer names each variant ` []`, and the case is named by the table itself, so - * the prefix is dropped for display. Dropping it is also what lets cases share a table: every - * auto-expanded case then contributes the same `[current]`/`[baseline]` pair rather than a pair of - * its own. - */ -export function shortNameOf(caseName: string, variant: string): string { - return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; -} - -/** A case that produced results, as opposed to one whose run failed and carries only an error. */ -export function isSummarized(entry: TachometerCaseResult): entry is SummarizedCase { - return entry.measurements !== undefined && entry.measurements.length > 0; -} - -/** The distinct variants a case reports, in order of first appearance. */ -export function variantsOf(entry: SummarizedCase): string[] { - const variants: string[] = []; - - for (const measurement of entry.measurements) { - for (const variant of measurement.variants) { - const short = shortNameOf(entry.name, variant.variant); - if (!variants.includes(short)) { - variants.push(short); - } - } - } - - return variants; -} - -/** - * Splits cases into the groups that each get their own table, the way the terminal report does. - * - * A regression case carries `[current]`/`[baseline]`, a cross-library case `[ours]`/`[theirs]`, and - * each set has its own reference — the first variant. One table over the union of the columns would - * leave a blank cell in every row of every case, and could name only one reference for all of them. - */ -export function groupCasesByVariantSet(cases: SummarizedCase[]): VariantGroup[] { - const groups = new Map(); - - for (const entry of cases) { - const variants = variantsOf(entry); - const key = JSON.stringify(variants); - const group = groups.get(key) ?? { variants, cases: [] }; - group.cases.push(entry); - groups.set(key, group); - } - - return [...groups.values()]; -} diff --git a/apps/code-infra-dashboard/src/lib/tachometer/types.ts b/apps/code-infra-dashboard/src/lib/tachometer/types.ts deleted file mode 100644 index 07153db484..0000000000 --- a/apps/code-infra-dashboard/src/lib/tachometer/types.ts +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Types mirroring @mui/internal-benchmark/tachometerReport. - * - * Mirrored rather than imported on purpose: these describe artifacts already sitting in S3, written - * by whatever version of the tooling a repository had at the time. Importing the current definitions - * would make the dashboard assume every stored report matches today's shape. - */ - -export interface ConfidenceInterval { - low: number; - high: number; -} - -export type Verdict = 'faster' | 'slower' | 'unsure'; - -export interface TachometerVariantResult { - variant: string; - refId: string | null; - meanMs: ConfidenceInterval; - samples: number; -} - -export interface TachometerComparison { - variant: string; - verdict: Verdict; - /** The reference relative to `variant`: positive means the reference is slower. */ - absoluteMs: ConfidenceInterval; - percentChange: ConfidenceInterval; - /** The same pair the other way round. Each direction has its own denominator. */ - versusReference?: { - verdict: Verdict; - absoluteMs: ConfidenceInterval; - percentChange: ConfidenceInterval; - }; -} - -export interface TachometerMeasurementResult { - name: string; - variants: TachometerVariantResult[]; - comparisons: TachometerComparison[]; -} - -export interface TachometerCaseResult { - name: string; - /** The variant every comparison is expressed against. Absent when the case failed to summarize. */ - reference?: string; - measurements?: TachometerMeasurementResult[]; - error?: string; -} - -export interface TachometerReport { - version: number; - reportType: 'tachometer'; - generatedAt: string; - head: { ref: string; sha: string; branch?: string }; - browser?: string; - refs: Array<{ id: string; kind: string; label: string; sha?: string }>; - cases: TachometerCaseResult[]; - raw?: Record; -} diff --git a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts index 93283b9f71..075735d832 100644 --- a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts +++ b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts @@ -1,13 +1,11 @@ import type { SizeSnapshotWithMetadata } from '@/lib/bundleSize/types'; import type { BenchmarkReport, BenchmarkUpload } from '@/lib/benchmark/types'; -import type { TachometerReport } from '@/lib/tachometer/types'; +import type { TachometerReport } from '@mui/internal-benchmark/tachometerReport'; import { migrateBenchmarkReport } from '@/lib/benchmark/migrateBenchmarkReport'; export interface CiReportTypes { 'benchmark.json': BenchmarkUpload; 'size-snapshot.json': SizeSnapshotWithMetadata; - // Stored as the report itself, the same shape `tacho run` writes: no envelope to unwrap, and - // no legacy flat shape to normalize. 'tachometer.json': TachometerReport; } diff --git a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx index d756ab910b..d2dbdc7e06 100644 --- a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx +++ b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx @@ -15,10 +15,20 @@ import TableHead from '@mui/material/TableHead'; import TableRow from '@mui/material/TableRow'; import Typography from '@mui/material/Typography'; import { fetchCiReport } from '@/utils/fetchCiReport'; -import type { ConfidenceInterval, TachometerReport, Verdict } from '@/lib/tachometer/types'; -import { groupCasesByVariantSet, isSummarized, shortNameOf } from '@/lib/tachometer/groupCases'; -import type { SummarizedCase } from '@/lib/tachometer/groupCases'; -import { formatMean, formatPercent } from '@/lib/tachometer/formatInterval'; +import type { + ConfidenceInterval, + TachometerReport, + Verdict, +} from '@mui/internal-benchmark/tachometerReport'; +import { + formatMean, + formatPercent, + refLabel, + groupCasesByVariantSet, + isSummarized, + shortNameOf, +} from '@mui/internal-benchmark/tachometerFormat'; +import type { SummarizedCase } from '@mui/internal-benchmark/tachometerFormat'; import Heading from '../components/Heading'; import ReportHeader from '../components/ReportHeader'; import ErrorDisplay from '../components/ErrorDisplay'; @@ -26,10 +36,6 @@ import ErrorDisplay from '../components/ErrorDisplay'; /** * The tables the run prints in CI, as HTML: cases grouped by their variant set, a group of two * variants read down its measurements, a group of more read across its variants. - * - * Deliberately plain — no charts or bars. A confidence interval is two numbers and a verdict, and - * drawing it as a length invites reading the picture as significance when the interval is what - * actually carries that. */ /** Only a resolved difference is coloured; `unsure` is the expected result and stays neutral. */ @@ -72,10 +78,6 @@ function Muted({ children }: { children: React.ReactNode }) { /** * One case as a table of variants, for a comparison with more than two of them. - * - * Rows are variants, not measurements: with several libraries, a column per variant plus a Δ column - * per pair would run off the page. Each measurement then shows the variant's interval next to its - * difference relative to the reference — the direction a row about that library reads in. */ function VariantTable({ entry, variants }: { entry: SummarizedCase; variants: string[] }) { const [reference] = variants; @@ -149,9 +151,6 @@ function VariantTable({ entry, variants }: { entry: SummarizedCase; variants: st /** * A group of cases sharing a variant set, one row per case and measurement. - * - * Each variant gets a column of its own, followed by the difference of the reference against it — - * which is the direction a regression is stated in, and what the Δ heading names. */ function CaseGroupTable({ cases, variants }: { cases: SummarizedCase[]; variants: string[] }) { const [reference, ...others] = variants; @@ -277,9 +276,6 @@ export default function TachometerDetails() { const failed = (report?.cases ?? []).filter((entry) => !isSummarized(entry)); const groups = groupCasesByVariantSet(summarized); - // Bundle weight is a property of the variant's page, not of a measurement, so it goes in a note - // rather than down every row — except where a variant table already gives it a column. - // A tachometer report carries its own comparison, so the baseline is one of its own refs rather // than a separately fetched report. const baselineRef = report?.refs.find((ref) => ref.kind !== 'worktree'); @@ -293,9 +289,8 @@ export default function TachometerDetails() { sha={sha} baseSha={baselineRef?.sha ?? null} prNumber={prNumber ? Number(prNumber) : undefined} - // Labels `baseSha`, so it has to describe the baseline — the run's own branch would read - // "comparing against ()". - baseRef={baselineRef?.label} + // Labels `baseSha`, so it has to describe the baseline. + baseRef={baselineRef ? refLabel(baselineRef) : undefined} /> @@ -340,7 +335,7 @@ export default function TachometerDetails() { difference did not resolve within the case's sampling budget. - Builds: {report.refs.map((ref) => `${ref.id} = ${ref.label}`).join(' · ')} + Builds: {report.refs.map((ref) => `${ref.id} = ${refLabel(ref)}`).join(' · ')} head: {report.head.sha.slice(0, 9)} ({report.head.branch || '?'}) · measured on the diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index 7b57964cce..099206e390 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -16,6 +16,7 @@ "./vitest": "./src/vitest.ts", "./tachometerPlugin": "./src/tachometer/plugin.ts", "./tachometerReport": "./src/tachometer/renderReport.ts", + "./tachometerFormat": "./src/tachometer/format.ts", "./cli": "./src/cli/index.ts" }, "scripts": { @@ -30,6 +31,7 @@ "env-ci": "^11.2.0", "execa": "^10.0.1", "globby": "^16.2.4", + "semver": "^7.8.5", "yaml": "^2.9.0", "yargs": "^18.1.0", "zod": "^4.5.4" @@ -47,6 +49,7 @@ "@types/env-ci": "3.1.4", "@types/react": "19.2.18", "@types/react-dom": "19.2.7", + "@types/semver": "^7.8.0", "@types/yargs": "17.0.35", "vite": "^8.2.2", "vitest": "4.1.11" diff --git a/packages/benchmark/src/cli/cmdTacho.ts b/packages/benchmark/src/cli/cmdTacho.ts index 1933bbd8e3..050b2f0065 100644 --- a/packages/benchmark/src/cli/cmdTacho.ts +++ b/packages/benchmark/src/cli/cmdTacho.ts @@ -4,10 +4,6 @@ import tachoRun from './tachoRun'; /** * The `tacho` command group. - * - * Its subcommands are registered here rather than as separate top-level commands: yargs keys a - * command on its first word, so `tacho run …` and `tacho report …` registered side by side would - * both claim `tacho` and the later one would answer for both. */ const command: CommandModule<{}, {}> = { command: 'tacho ', diff --git a/packages/benchmark/src/cli/tachoReport.ts b/packages/benchmark/src/cli/tachoReport.ts index e8f7d084b7..49b59569ec 100644 --- a/packages/benchmark/src/cli/tachoReport.ts +++ b/packages/benchmark/src/cli/tachoReport.ts @@ -1,5 +1,6 @@ import * as path from 'node:path'; import { readFile } from 'node:fs/promises'; +import { z } from 'zod/v4'; import type { CommandModule } from 'yargs'; import { OUTPUT_DIR } from '../tachometer/outputDir'; @@ -15,19 +16,15 @@ const command: CommandModule<{}, Args> = { describe: 'Print the table for a JSON report written by `tacho run`. Reads the last run of the harness in the current directory when no path is given.', builder: (yargs) => { - return ( - yargs - // Not named `report`: yargs binds a positional whose name repeats its own subcommand word to - // the literal token, so `tacho report` would resolve the path "report". - .positional('file', { - type: 'string', - default: DEFAULT_REPORT, - describe: 'Path to the JSON report', - }) - .epilogue( - 'The same table `tacho run` prints when it finishes, so a report kept from an earlier run — or downloaded from CI — can be read without sampling again.', - ) - ); + return yargs + .positional('file', { + type: 'string', + default: DEFAULT_REPORT, + describe: 'Path to the JSON report', + }) + .epilogue( + 'The same table `tacho run` prints when it finishes, so a report kept from an earlier run — or downloaded from CI — can be read without sampling again.', + ); }, handler: async (argv) => { const reportPath = path.resolve(argv.file); @@ -37,18 +34,17 @@ const command: CommandModule<{}, Args> = { ); }); - const report = JSON.parse(raw); - // The report a run writes and the artifact the dashboard stores are the same shape, so this - // reads either. The check is for the other benchmark axis, whose reports carry different - // statistics that rendering here would silently read as confidence intervals on a difference. - if (report?.reportType !== 'tachometer') { + const { tachometerReportSchema } = await import('../tachometer/ciReport'); + const parsed = tachometerReportSchema.safeParse(JSON.parse(raw)); + if (!parsed.success) { throw new Error( - `${reportPath} is not a tachometer report (its "reportType" is ${JSON.stringify(report?.reportType)}).`, + `${reportPath} is not a tachometer report this version can read.\n` + + `${z.prettifyError(parsed.error)}`, ); } const { renderTachometerReport } = await import('../tachometer/renderReport'); - renderTachometerReport(report); + renderTachometerReport(parsed.data); }, }; diff --git a/packages/benchmark/src/tachometer/browser.ts b/packages/benchmark/src/tachometer/browser.ts index 1364da74d2..bc2c334741 100644 --- a/packages/benchmark/src/tachometer/browser.ts +++ b/packages/benchmark/src/tachometer/browser.ts @@ -1,17 +1,12 @@ import * as path from 'node:path'; import { createRequire } from 'node:module'; +import { pathToFileURL } from 'node:url'; import { execFileSync } from 'node:child_process'; import chalk from 'chalk'; +import * as semver from 'semver'; import type * as PlaywrightTest from '@playwright/test'; import { pathExists } from '../utils/path'; -/** - * Which browser the benchmarks run on, and whether the driver can actually drive it. - * - * Both checks happen before anything is built: a driver that cannot open the browser fails the run - * either way, and finding out now costs seconds instead of minutes of packing and installing. - */ - /** * Resolves the browser every case runs on: Playwright's pinned Chrome for Testing, rather than * whatever Chrome the machine has auto-updated to. @@ -19,12 +14,11 @@ import { pathExists } from '../utils/path'; * The path is machine-specific, so callers inject it into each config instead of committing it. */ export async function resolveBrowserBinary(harnessDir: string): Promise { + // Resolved from the harness, so the browser is the one its own `@playwright/test` pins. const require = createRequire(path.join(harnessDir, 'package.json')); - // `@playwright/test` is CommonJS, so `require` hands back its exports directly; a dynamic - // `import()` would bury them under `default`. let playwright: typeof PlaywrightTest; try { - playwright = require('@playwright/test'); + playwright = await import(pathToFileURL(require.resolve('@playwright/test')).href); } catch { throw new Error( `Could not resolve "@playwright/test" from ${harnessDir}. Add it as a devDependency — ` + @@ -42,18 +36,21 @@ export async function resolveBrowserBinary(harnessDir: string): Promise return binary; } -/** Extracts the major version out of output like `Google Chrome for Testing 151.0.7922.34`. */ +/** + * The major version in output like `Google Chrome for Testing 151.0.7922.34`, or undefined when + * there is no version in it. + * + * The version is picked out of the surrounding words first, requiring at least three components so + * a stray count is not read as one. Chrome's fourth component makes it invalid semver, so what is + * left is coerced rather than parsed. + */ export function majorVersionOf(versionOutput: string): number | undefined { - const match = /(\d+)\.\d+\.\d+/.exec(versionOutput); - return match ? Number(match[1]) : undefined; + const version = /\d+(?:\.\d+){2,}/.exec(versionOutput)?.[0]; + return version ? (semver.coerce(version)?.major ?? undefined) : undefined; } /** * Throws when the chromedriver tachometer will use cannot drive `binary`. - * - * A chromedriver only drives its own Chrome major. The driver is resolved the way tachometer - * resolves it — from tachometer's own package root — so this checks the pair that will actually be - * used rather than whatever else is installed. */ export function assertDriverMatchesBrowser(harnessDir: string, binary: string): void { const browserMajor = majorVersionOf( @@ -111,19 +108,9 @@ const DEFAULT_BROWSER = 'chrome'; const HEADLESS_SUFFIX = '-headless'; /** - * Normalises whatever a case put in `browser` into the object form, the way tachometer does. - * - * Tachometer accepts three things there: nothing at all, the string shorthand - * `"[-headless][@]"`, and an object. Only the object form can carry the binary and - * the arguments a run has to add — and tachometer validates the whole config against its own schema - * before reading any of it, where every browser variant requires `name` and forbids unknown keys. - * So merging into the other two forms without converting first yields a config tachometer rejects - * outright: `{ binary }` has no `name`, and spreading a string produces one key per character. - * - * Only `name` is filled in for the other two forms: tachometer's own `parseBrowserObject` supplies - * `headless` and `windowSize` from the defaults it would otherwise have applied, so naming the - * browser is all it takes to reproduce what an absent `browser` did. The string shorthand is the - * exception — it encodes `headless` itself, so that is read out of it. + * Whatever a case put in `browser` as the object form tachometer's schema requires: `name` is + * mandatory there and unknown keys are rejected, so the string shorthand + * `"[-headless][@]"` and an absent `browser` are both expanded here. */ function asBrowserObject(browser: string | BrowserConfig | undefined): BrowserConfig { if (browser === undefined) { @@ -144,13 +131,8 @@ function asBrowserObject(browser: string | BrowserConfig | undefined): BrowserCo } /** - * Merges what the run decides into a case's own `browser` config. - * - * Neither of these belongs in a committed `tachometer.json`. The binary is machine-specific. And - * whether the sandbox has to come off depends on the user the run happens under: as root Chrome - * exits during launch without explaining itself, leaving the driver to report nothing more than - * `session not created: Chrome instance exited`. Containerised CI runs as root, so the flag goes - * on there and nowhere else — a local run keeps the sandbox. + * Merges what the run decides into a case's own `browser` config: the binary, which is + * machine-specific, and the sandbox flag, which depends on the user the run happens under. */ export function withBrowserDefaults( browser: string | BrowserConfig | undefined, diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts index 36abed3013..8dab241fff 100644 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -13,7 +13,8 @@ import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' } import { run } from '../utils/exec'; import { tarballFor } from '../utils/packWorkspace'; import type { PackedPackage } from '../utils/packWorkspace'; -import { readPackageJson, writePackageJson } from '../utils/pnpm'; +import { installedVersions, readPackageJson, writePackageJson } from '../utils/pnpm'; +import { refLabel } from './format'; import type { ResolvedRef } from './refs'; /** @@ -25,65 +26,34 @@ import type { ResolvedRef } from './refs'; * to install: tachometer pulls a chromedriver, `@playwright/test` a browser, and this package a * dependency tree many times the size of a harness. */ -const RUNNER_ONLY_DEPS = [ - 'tachometer', - 'chromedriver', - '@playwright/test', - // Read rather than spelled out: this is the one entry naming *this* package, and a rename that - // left a literal behind would quietly restore a multi-minute install per ref. - pkgJson.name, -]; +const RUNNER_ONLY_DEPS = ['tachometer', 'chromedriver', '@playwright/test', pkgJson.name]; /** - * The version of `name` the harness actually has installed, or `undefined` when it has none. + * A dependency map for a ref's tree: every `workspace:` entry points at that ref's packed tarball, + * and everything else is pinned to the version the harness resolved rather than the range it + * declares. * - * Read from the harness's own `node_modules` rather than a lockfile: pnpm links every direct - * dependency there, and these maps only ever hold direct dependencies. + * The tree installs without the repository's lockfile, so a range would float to whatever is newest + * at run time — a different library than the repository builds against, measured as if it were + * ours. */ -async function installedVersionOf(harnessDir: string, name: string): Promise { - try { - const manifest = path.join(harnessDir, 'node_modules', ...name.split('/'), 'package.json'); - return JSON.parse(await readFile(manifest, 'utf8')).version; - } catch { - return undefined; - } -} - -/** - * Rewrites a dependency map, pointing every `workspace:`-protocol entry at its packed tarball and - * copying everything else verbatim. - * - * This is what lets the tree be derived rather than configured: the harness already declares the - * library under test as `workspace:*`, so there is no library name to hardcode and no - * hand-maintained list of versions to pin. Third-party dependencies — including the competitor - * libraries a cross-library case compares against — come along, so every page builds against every - * ref. - * - * Everything else is pinned to the version the harness *resolved*, not the range it declares. The - * tree is its own workspace root and installs without the repository's lockfile, so a range would - * float to whatever is newest at run time: a different library than the repository builds against, - * quietly measured as if it were ours. A release that also drops an export fails the build outright, - * which is the visible half of the same problem. - */ -async function rewriteWorkspaceDeps( - harnessDir: string, +function rewriteWorkspaceDeps( + installed: Map, deps: Partial> | undefined, packages: PackedPackage[], omit: string[] = [], -): Promise> { +): Record { const entries = Object.entries(deps ?? {}).filter( ([name, version]) => version !== undefined && !omit.includes(name), ) as Array<[string, string]>; - const resolved = await Promise.all( - entries.map(async ([name, version]): Promise<[string, string]> => { - if (version.startsWith('workspace:')) { - return [name, `file:${tarballFor(packages, name)}`]; - } - const installed = await installedVersionOf(harnessDir, name); - return [name, installed ?? version]; - }), + + return Object.fromEntries( + entries.map(([name, version]) => + version.startsWith('workspace:') + ? [name, `file:${tarballFor(packages, name)}`] + : [name, installed.get(name) ?? version], + ), ); - return Object.fromEntries(resolved); } /** @@ -154,16 +124,19 @@ async function installRefTree({ /** Where to install. */ treeDir: string; }): Promise { - const harnessPkg = await readPackageJson(harnessDir); - await mkdir(treeDir, { recursive: true }); + const [harnessPkg, installed] = await Promise.all([ + readPackageJson(harnessDir), + installedVersions(harnessDir), + mkdir(treeDir, { recursive: true }), + ]); await writePackageJson(treeDir, { name: 'tacho-resolve-tree', private: true, version: '0.0.0', - dependencies: await rewriteWorkspaceDeps(harnessDir, harnessPkg.dependencies, packages), - devDependencies: await rewriteWorkspaceDeps( - harnessDir, + dependencies: rewriteWorkspaceDeps(installed, harnessPkg.dependencies, packages), + devDependencies: rewriteWorkspaceDeps( + installed, harnessPkg.devDependencies, packages, RUNNER_ONLY_DEPS, @@ -224,7 +197,7 @@ export async function buildRefPages(options: { outDir: string; }): Promise { const { harnessDir, repoRoot, ref, packages, treeDir, outDir } = options; - console.log(chalk.cyan(`\nBuilding benchmark pages for "${ref.label}"…`)); + console.log(chalk.cyan(`\nBuilding benchmark pages for "${refLabel(ref)}"…`)); await installRefTree({ harnessDir, repoRoot, packages, treeDir }); diff --git a/packages/benchmark/src/tachometer/ciReport.test.ts b/packages/benchmark/src/tachometer/ciReport.test.ts index bf4f887d49..93349354d5 100644 --- a/packages/benchmark/src/tachometer/ciReport.test.ts +++ b/packages/benchmark/src/tachometer/ciReport.test.ts @@ -15,8 +15,8 @@ function report(overrides = {}) { head: { ref: 'HEAD', sha: 'a'.repeat(40), branch: 'feature' }, browser: '/path/to/Chrome for Testing', refs: [ - { id: 'current', kind: 'worktree', label: 'working tree' }, - { id: 'git-abc123456', kind: 'git', label: 'merge-base', sha: 'b'.repeat(40) }, + { id: 'current', kind: 'worktree' }, + { id: 'git-abc123456', kind: 'git', requested: 'HEAD~1', sha: 'b'.repeat(40) }, ], cases: [ { diff --git a/packages/benchmark/src/tachometer/ciReport.ts b/packages/benchmark/src/tachometer/ciReport.ts index 922cfba68e..45ccb0f5b8 100644 --- a/packages/benchmark/src/tachometer/ciReport.ts +++ b/packages/benchmark/src/tachometer/ciReport.ts @@ -47,26 +47,45 @@ const caseResultSchema = z.object({ error: z.string().optional(), }); -const tachometerReportSchema = z.object({ - version: z.number(), +/** + * A build the pages were loaded from. The working tree has no commit behind it and a `git:` ref + * always does, so the two carry different fields. + * + * `requested` is the revision as the run was given it — `HEAD~1`, a tag, a SHA — kept instead of a + * ready-made display string so a renderer decides how it reads. + */ +const refSchema = z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('worktree'), id: z.string() }), + z.object({ + kind: z.literal('git'), + id: z.string(), + sha: z.string(), + requested: z.string(), + }), +]); + +const tachometerReportV1Schema = z.object({ + version: z.literal(1), reportType: z.literal('tachometer'), generatedAt: z.string(), head: z.object({ ref: z.string(), sha: z.string(), branch: z.string().optional() }), browser: z.string().optional(), - refs: z.array( - z.object({ - id: z.string(), - kind: z.string(), - label: z.string(), - sha: z.string().optional(), - }), - ), + refs: z.array(refSchema), cases: z.array(caseResultSchema), // Every sample tachometer produced. Not read by the comment, but kept so a richer view can be // built later without changing what CI uploads. raw: z.record(z.string(), z.unknown()), }); +/** + * Every report shape the store may hold, keyed on `version`. + * + * Reports written months ago by an older version of this tooling are still in S3 and still get + * rendered. A new shape joins this union with its own literal version rather than replacing the old + * one, so a reader narrows on `version` and the compiler makes it handle each. + */ +export const tachometerReportSchema = z.discriminatedUnion('version', [tachometerReportV1Schema]); + /** * The upload envelope. Built from the same factory the Vitest axis uses, so both report types reach * the dashboard's upload route in the shape it expects. @@ -75,7 +94,10 @@ export const tachometerUploadSchema = ciReportUploadSchema('tachometer', 1, tach export type TachometerUpload = z.infer; export type TachometerReport = z.infer; +export type ReportRef = z.infer; export type CaseResult = z.infer; +export type ConfidenceInterval = z.infer; +export type Verdict = z.infer['verdict']; export type MeasurementResult = z.infer; export type VariantResult = z.infer; export type Comparison = z.infer; diff --git a/apps/code-infra-dashboard/src/lib/tachometer/groupCases.test.ts b/packages/benchmark/src/tachometer/format.test.ts similarity index 97% rename from apps/code-infra-dashboard/src/lib/tachometer/groupCases.test.ts rename to packages/benchmark/src/tachometer/format.test.ts index 9f9d1bd115..d698108c71 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/groupCases.test.ts +++ b/packages/benchmark/src/tachometer/format.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { groupCasesByVariantSet, shortNameOf, variantsOf } from './groupCases'; -import type { SummarizedCase } from './groupCases'; +import { groupCasesByVariantSet, shortNameOf, variantsOf } from './format'; +import type { SummarizedCase } from './format'; function variant(name: string) { return { diff --git a/packages/benchmark/src/tachometer/format.ts b/packages/benchmark/src/tachometer/format.ts new file mode 100644 index 0000000000..6bdd4f56b8 --- /dev/null +++ b/packages/benchmark/src/tachometer/format.ts @@ -0,0 +1,93 @@ +import type { CaseResult, MeasurementResult, ReportRef, ConfidenceInterval } from './ciReport'; + +/** + * How a report's numbers and names are written, for every renderer of one. + * + * The terminal table and the pull request comment show the same run, so they format it here rather + * than each their own way. Pure, and free of anything node-only, so a browser can import it. + */ + +/** A case that produced results, as opposed to one carrying only the error that stopped it. */ +export type SummarizedCase = CaseResult & { measurements: MeasurementResult[] }; + +export interface VariantGroup { + /** The variant set the group's cases share, reference first. */ + variants: string[]; + cases: SummarizedCase[]; +} + +/** `20.50 – 22.67 ms`, with the unit written once. */ +export function formatMean(interval: ConfidenceInterval): string { + return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; +} + +/** Both bounds signed, so the direction of a difference reads without the verdict beside it. */ +function signedInterval(interval: ConfidenceInterval, unit: string, digits: number): string { + const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(digits)}${unit}`; + return `${signed(interval.low)} – ${signed(interval.high)}`; +} + +/** `+2.7% – +6.8%`. */ +export function formatPercent(interval: ConfidenceInterval): string { + return signedInterval(interval, '%', 1); +} + +/** `+6.10 ms – +15.20 ms`. */ +export function formatSignedMs(interval: ConfidenceInterval): string { + return signedInterval(interval, ' ms', 2); +} + +/** What a ref is called in a report: the working tree, or the revision the run was given. */ +export function refLabel(ref: ReportRef): string { + return ref.kind === 'worktree' ? 'working tree' : ref.requested; +} + +/** + * Tachometer names each variant ` []`, and the case is named by the table itself, so + * the prefix is dropped for display. Dropping it is also what lets cases share a table: every + * auto-expanded case then contributes the same `[current]`/`[baseline]` pair rather than a pair of + * its own. + */ +export function shortNameOf(caseName: string, variant: string): string { + return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; +} + +/** Whether a case produced results, as opposed to carrying only the error that stopped it. */ +export function isSummarized(entry: CaseResult): entry is SummarizedCase { + return entry.measurements !== undefined && entry.measurements.length > 0; +} + +/** The distinct variants a case reports, in order of first appearance. */ +export function variantsOf(entry: SummarizedCase): string[] { + const variants: string[] = []; + for (const measurement of entry.measurements) { + for (const variant of measurement.variants) { + const short = shortNameOf(entry.name, variant.variant); + if (!variants.includes(short)) { + variants.push(short); + } + } + } + return variants; +} + +/** + * Splits cases into the groups that each get their own table. + * + * A regression case carries `[current]`/`[baseline]`, a cross-library case `[ours]`/`[theirs]`, and + * each set has its own reference, the first variant. One table over the union of the columns would + * leave a blank cell in every row, and could name only one reference for all of them. + */ +export function groupCasesByVariantSet(cases: SummarizedCase[]): VariantGroup[] { + const groups = new Map(); + + for (const entry of cases) { + const variants = variantsOf(entry); + const key = JSON.stringify(variants); + const group = groups.get(key) ?? { variants, cases: [] }; + group.cases.push(entry); + groups.set(key, group); + } + + return [...groups.values()]; +} diff --git a/packages/benchmark/src/tachometer/refs.test.ts b/packages/benchmark/src/tachometer/refs.test.ts index 9210543c39..940788d8cf 100644 --- a/packages/benchmark/src/tachometer/refs.test.ts +++ b/packages/benchmark/src/tachometer/refs.test.ts @@ -122,12 +122,12 @@ describe('createRefResolver', () => { expect(ref.sha).toBe(shas[1]); }); - it('labels a git ref with the revision it was given', async () => { + it('keeps the revision it was given', async () => { const { repoRoot } = await makeRepo(); const ref = createRefResolver({ repoRoot }).parse('git:v1.0.0'); - // Not `v1.0.0 ()`: `id` already carries the short SHA, and the renderers print both. - expect(ref.label).toBe('v1.0.0'); + // The report shows this, so a tag stays a tag rather than becoming the SHA it points at. + expect(ref.requested).toBe('v1.0.0'); }); }); diff --git a/packages/benchmark/src/tachometer/refs.ts b/packages/benchmark/src/tachometer/refs.ts index e9847cfdee..5a05dd7040 100644 --- a/packages/benchmark/src/tachometer/refs.ts +++ b/packages/benchmark/src/tachometer/refs.ts @@ -15,21 +15,17 @@ import { revParseCommitArgs } from '../utils/git'; export type RefDescriptor = { kind: 'worktree' } | { kind: 'baseline' } | { kind: 'git'; committish: string }; -interface RefIdentity { - /** Build directory name, e.g. `current` or `git-230342ee2`. Doubles as the dedupe identity. */ - id: string; - /** Human-readable label for logs and the report. */ - label: string; -} - /** * A ref with its build identity. The working tree has no commit behind it and a `git:` ref always * does, so the two carry different fields rather than sharing optional ones a reader has to pair * with the right `kind` by hand. + * + * `id` is the build directory name — `current`, or `git-230342ee2` — and doubles as the dedupe + * identity. `requested` is the revision as it was given, which is what a report shows. */ export type ResolvedRef = - | (RefIdentity & { kind: 'worktree'; sha?: undefined; committish?: undefined }) - | (RefIdentity & { kind: 'git'; sha: string; committish: string }); + | { kind: 'worktree'; id: string; sha?: undefined; requested?: undefined } + | { kind: 'git'; id: string; sha: string; requested: string }; export interface RefResolverOptions { /** Repository to resolve revisions in. */ @@ -39,11 +35,7 @@ export interface RefResolverOptions { } /** The working tree — never cached, since it has no immutable identity. */ -const WORKTREE_REF: ResolvedRef = { - kind: 'worktree', - id: 'current', - label: 'working tree', -}; +const WORKTREE_REF: ResolvedRef = { kind: 'worktree', id: 'current' }; /** * Parses a `?ref=` token into a descriptor, validating the grammar. Makes no git calls. @@ -103,7 +95,7 @@ export function createRefResolver(options: RefResolverOptions): { const gitRefs = new Map(); /** Resolves a git committish to a `ResolvedRef` keyed by its immutable SHA. */ - function gitRef(committish: string, label?: string): ResolvedRef { + function gitRef(committish: string): ResolvedRef { const cached = gitRefs.get(committish); if (cached) { return cached; @@ -112,9 +104,8 @@ export function createRefResolver(options: RefResolverOptions): { const ref: ResolvedRef = { kind: 'git', id: `git-${sha.slice(0, 9)}`, - label: label ?? committish, sha, - committish: sha, + requested: committish, }; gitRefs.set(committish, ref); return ref; diff --git a/packages/benchmark/src/tachometer/renderReport.ts b/packages/benchmark/src/tachometer/renderReport.ts index ba619c7da8..1ad46cd643 100644 --- a/packages/benchmark/src/tachometer/renderReport.ts +++ b/packages/benchmark/src/tachometer/renderReport.ts @@ -3,33 +3,29 @@ // Column widths have to measure visible characters, so chalk's escapes come off first. import { stripVTControlCharacters as stripAnsi } from 'node:util'; import chalk from 'chalk'; -import type { ConfidenceInterval, Verdict } from './summarizeCase'; -import type { CaseResult, MeasurementResult, TachometerReport } from './ciReport'; - -/** - * Renders the report `tacho run` produces. - * - * Lives beside the runner rather than in the consuming repository because the runner defines the - * report's shape: keeping the producer and its renderer apart is how a change to one silently - * breaks the other. - */ - -/** - * The wire format is defined once, by the schema CI validates against and S3 stores. Re-declaring - * it here is how the exported type and the stored one drift apart — the render side would keep - * compiling while the report grew a field it never learned about. - */ +import { + formatMean, + formatPercent, + groupCasesByVariantSet, + isSummarized, + refLabel, + shortNameOf, +} from './format'; +import type { SummarizedCase } from './format'; +import type { TachometerReport, Verdict } from './ciReport'; + +/** The report's types, from the schema that defines it. */ export type { TachometerReport, CaseResult, MeasurementResult, VariantResult, Comparison, + ReportRef, + ConfidenceInterval, + Verdict, } from './ciReport'; -/** A case that produced results, as opposed to one carrying only the error that stopped it. */ -type SummarizedCase = CaseResult & { measurements: MeasurementResult[] }; - /** Prints a table, padding each column to its widest visible cell. */ function printTable(headers: string[], rows: string[][], leftColumns: number): void { const widths = headers.map((header, index) => @@ -52,19 +48,6 @@ function printTable(headers: string[], rows: string[][], leftColumns: number): v } } -/** `low – high`, with the unit written once. */ -function formatInterval(interval: ConfidenceInterval): string { - // Fixed to two places so a column lines up on the decimal point, and written the same way the - // dashboard writes it, so the terminal and the pull request comment agree digit for digit. - return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; -} - -/** Both bounds as signed percentages. */ -function formatPercentInterval(interval: ConfidenceInterval): string { - const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(1)}%`; - return `${signed(interval.low)} – ${signed(interval.high)}`; -} - function colorVerdict(verdict: Verdict, text: string): string { if (verdict === 'faster') { return chalk.green(text); @@ -75,35 +58,6 @@ function colorVerdict(verdict: Verdict, text: string): string { return chalk.dim(text); } -/** - * Tachometer names each variant ` []`, and the case is already its own column, so - * the prefix is dropped for display. Dropping it is also what makes columns line up across cases: - * every auto-expanded case then contributes the same `[current]`/`[baseline]` pair rather than a - * column of its own. - */ -function shortNameOf(caseName: string, variant: string): string { - return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; -} - -/** Whether a case produced results, as opposed to carrying only the error that stopped it. */ -function isSummarized(entry: CaseResult): entry is SummarizedCase { - return entry.measurements !== undefined && entry.measurements.length > 0; -} - -/** The distinct variants a case reports, in order of first appearance. */ -function variantsOf(entry: SummarizedCase): string[] { - const variants: string[] = []; - for (const measurement of entry.measurements) { - for (const variant of measurement.variants) { - const short = shortNameOf(entry.name, variant.variant); - if (!variants.includes(short)) { - variants.push(short); - } - } - } - return variants; -} - /** * Renders one case as a table of variants, for a comparison with more than two of them. * @@ -141,9 +95,9 @@ function printVariantTable(entry: SummarizedCase, variants: string[]): void { delta = chalk.dim('reference'); } else if (comparison?.versusReference) { const { verdict, percentChange } = comparison.versusReference; - delta = colorVerdict(verdict, `${verdict} ${formatPercentInterval(percentChange)}`); + delta = colorVerdict(verdict, `${verdict} ${formatPercent(percentChange)}`); } - return [found ? formatInterval(found.meanMs) : chalk.dim('—'), delta]; + return [found ? formatMean(found.meanMs) : chalk.dim('—'), delta]; }); return [variant, ...cells, [...sampleCounts].join('/')]; @@ -194,7 +148,7 @@ function printCaseTable(cases: SummarizedCase[], variants: string[]): void { measurement.name, ...variants.map((variant) => { const found = byVariant.get(variant); - return found ? formatInterval(found.meanMs) : chalk.dim('—'); + return found ? formatMean(found.meanMs) : chalk.dim('—'); }), ...others.map((variant) => { const comparison = byComparison.get(variant); @@ -203,7 +157,7 @@ function printCaseTable(cases: SummarizedCase[], variants: string[]): void { } return colorVerdict( comparison.verdict, - `${comparison.verdict} ${formatPercentInterval(comparison.percentChange)}`, + `${comparison.verdict} ${formatPercent(comparison.percentChange)}`, ); }), sampleCounts.join('/'), @@ -249,20 +203,7 @@ export function renderTachometerReport(report: TachometerReport): void { return; } - // One table per variant set, in order of first appearance: a regression case carries - // `[current]`/`[baseline]`, a cross-library case `[ours]`/`[theirs]`, and each set has its own - // reference (the first variant). A single table over the union of the columns would put a blank - // cell in every row of every case, and could name only one reference for all of them. - const groups = new Map(); - for (const entry of usable) { - const variants = variantsOf(entry); - const key = JSON.stringify(variants); - const group = groups.get(key) ?? { variants, cases: [] }; - group.cases.push(entry); - groups.set(key, group); - } - - for (const [groupIndex, group] of [...groups.values()].entries()) { + for (const [groupIndex, group] of groupCasesByVariantSet(usable).entries()) { if (groupIndex > 0) { console.log(''); } @@ -283,9 +224,8 @@ export function renderTachometerReport(report: TachometerReport): void { '"unsure" means the interval still straddles zero: the difference did not resolve within the case\'s sampling budget.', ]; - // No short SHA appended: `id` is already `git-`, and the label is the name the run was - // given (a committish, or how a symbolic baseline resolved). - const refLabels = report.refs.map((ref) => `${ref.id} = ${ref.label}`); + // No short SHA appended: `id` already ends in one. + const refLabels = report.refs.map((ref) => `${ref.id} = ${refLabel(ref)}`); if (refLabels.length > 0) { notes.push(`Builds: ${refLabels.join(' · ')}`); } diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts index 657b4520bf..f05ac72060 100644 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -12,6 +12,7 @@ import { createRefResolver } from './refs'; import type { ResolvedRef } from './refs'; import { discoverCases, pagesOf } from './discoverCases'; import type { BenchmarkCase } from './discoverCases'; +import { refLabel } from './format'; import { buildRefPages } from './buildPages'; import { assertDriverMatchesBrowser, resolveBrowserBinary, withBrowserDefaults } from './browser'; import { summarizeCase } from './summarizeCase'; @@ -102,11 +103,9 @@ export async function runTachometer(options: RunTachometerOptions): Promise - ref.sha ? `${ref.label} (${ref.sha.slice(0, 9)})` : ref.label; + ref.sha ? `${refLabel(ref)} (${ref.sha.slice(0, 9)})` : refLabel(ref); // Before any build: a driver that cannot open the browser fails the run either way, and finding // out now costs seconds instead of minutes of packing and installing. @@ -141,7 +140,7 @@ export async function runTachometer(options: RunTachometerOptions): Promise ({ - id: ref.id, - kind: ref.kind, - label: ref.label, - sha: ref.sha, - })), + refs: [...refs.values()], // Summarising is best-effort per case: a case that produced no usable benchmarks must not // cost the whole run its report, since `raw` below is the only surviving copy of every other // case's samples once the temp dir is cleaned up. diff --git a/packages/benchmark/src/utils/pnpm.ts b/packages/benchmark/src/utils/pnpm.ts index 0d8d0f7154..8e260e35d9 100644 --- a/packages/benchmark/src/utils/pnpm.ts +++ b/packages/benchmark/src/utils/pnpm.ts @@ -50,3 +50,34 @@ export async function writePackageJson(packagePath: string, packageJson: object) const content = `${JSON.stringify(packageJson, null, 2)}\n`; await fs.writeFile(path.join(packagePath, 'package.json'), content); } + +/** One project, as `pnpm ls` reports it. */ +interface ListedProject { + dependencies?: Record; + devDependencies?: Record; +} + +/** + * The version the project in `cwd` resolved for each of its direct dependencies. + * + * Asked of pnpm rather than read out of `node_modules`, so the answer follows however pnpm chose to + * lay that directory out. `--filter .` keeps it to the one project; without it pnpm reports every + * workspace member. + */ +export async function installedVersions(cwd: string): Promise> { + const result = await $({ cwd })`pnpm ls --json --depth 0 --filter .`; + const [project]: ListedProject[] = JSON.parse(result.stdout); + const versions = new Map(); + + for (const section of [project?.dependencies, project?.devDependencies]) { + for (const [name, entry] of Object.entries(section ?? {})) { + // A workspace link reports `link:../…` here rather than a version. Callers want a version to + // pin, so leave those out and let them fall back to whatever the manifest declared. + if (entry.version && !entry.version.startsWith('link:')) { + versions.set(name, entry.version); + } + } + } + + return versions; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3eb3108d15..38c5d6c2bd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -514,6 +514,9 @@ importers: react-dom: specifier: ^18.0.0 || ^19.0.0 version: 19.2.8(react@19.2.8) + semver: + specifier: ^7.8.5 + version: 7.8.5 yaml: specifier: ^2.9.0 version: 2.9.0 @@ -536,6 +539,9 @@ importers: '@types/react-dom': specifier: 19.2.7 version: 19.2.7(@types/react@19.2.18) + '@types/semver': + specifier: ^7.8.0 + version: 7.8.0 '@types/yargs': specifier: 17.0.35 version: 17.0.35 From 7e6d0a06ff39007ae620aa88e1fed5b92e92f2b1 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:08:24 +0200 Subject: [PATCH 053/173] [benchmark] Import playwright by name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolving it from the harness restated what a peer dependency already decides: pnpm resolves the peer against the dependent that pulled this package in, which is the harness, so both land on the same install — checked, byte for byte. The catch stays: the peer is optional, so it installs cleanly when absent and Node's own error names a path inside this package, which reads like a bug here rather than a missing dependency there. --- packages/benchmark/src/tachometer/browser.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/benchmark/src/tachometer/browser.ts b/packages/benchmark/src/tachometer/browser.ts index bc2c334741..eeed935d1d 100644 --- a/packages/benchmark/src/tachometer/browser.ts +++ b/packages/benchmark/src/tachometer/browser.ts @@ -1,6 +1,5 @@ import * as path from 'node:path'; import { createRequire } from 'node:module'; -import { pathToFileURL } from 'node:url'; import { execFileSync } from 'node:child_process'; import chalk from 'chalk'; import * as semver from 'semver'; @@ -14,16 +13,13 @@ import { pathExists } from '../utils/path'; * The path is machine-specific, so callers inject it into each config instead of committing it. */ export async function resolveBrowserBinary(harnessDir: string): Promise { - // Resolved from the harness, so the browser is the one its own `@playwright/test` pins. - const require = createRequire(path.join(harnessDir, 'package.json')); let playwright: typeof PlaywrightTest; try { - playwright = await import(pathToFileURL(require.resolve('@playwright/test')).href); + playwright = await import('@playwright/test'); } catch { + // An optional peer, so it installs cleanly when absent and only fails here. throw new Error( - `Could not resolve "@playwright/test" from ${harnessDir}. Add it as a devDependency — ` + - `the benchmarks run on its pinned Chrome for Testing so results do not move when the ` + - `machine's own Chrome updates.`, + `"@playwright/test" is not installed. Add it as a devDependency of ${harnessDir}.`, ); } From 14dec34b69ded9d57ca0c09c0e8946a74b021167 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:25:45 +0200 Subject: [PATCH 054/173] [benchmark] Fail when no chromedriver is reachable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tachometer uses whichever chromedriver resolves next to itself, and downloads the newest when none does. Playwright's Chrome is pinned behind the current release, so that download is normally a driver the browser cannot work with — and it surfaces as a WebDriver error minutes later, once every ref has been packed and built. The browser's major is already known at that point, so say which one to pin instead of warning and carrying on into the failure. The check had no tests. Three now cover it, over real directories and real module resolution, since what is being checked is which chromedriver node finds from tachometer. --- packages/benchmark/package.json | 2 - .../benchmark/src/tachometer/browser.test.ts | 67 ++++++++++++++++++- packages/benchmark/src/tachometer/browser.ts | 38 +++++++---- pnpm-lock.yaml | 6 -- 4 files changed, 91 insertions(+), 22 deletions(-) diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index 099206e390..507ffe2d96 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -31,7 +31,6 @@ "env-ci": "^11.2.0", "execa": "^10.0.1", "globby": "^16.2.4", - "semver": "^7.8.5", "yaml": "^2.9.0", "yargs": "^18.1.0", "zod": "^4.5.4" @@ -49,7 +48,6 @@ "@types/env-ci": "3.1.4", "@types/react": "19.2.18", "@types/react-dom": "19.2.7", - "@types/semver": "^7.8.0", "@types/yargs": "17.0.35", "vite": "^8.2.2", "vitest": "4.1.11" diff --git a/packages/benchmark/src/tachometer/browser.test.ts b/packages/benchmark/src/tachometer/browser.test.ts index 8fc1d592f6..29f970ea1d 100644 --- a/packages/benchmark/src/tachometer/browser.test.ts +++ b/packages/benchmark/src/tachometer/browser.test.ts @@ -1,5 +1,8 @@ +import * as path from 'node:path'; +import { chmod, mkdir, writeFile } from 'node:fs/promises'; import { describe, expect, it } from 'vitest'; -import { majorVersionOf, withBrowserDefaults } from './browser'; +import { assertDriverMatchesBrowser, majorVersionOf, withBrowserDefaults } from './browser'; +import { makeTempDir } from '../utils/testUtils'; describe('majorVersionOf', () => { it('reads the major from Chrome for Testing output', () => { @@ -105,3 +108,65 @@ describe('withBrowserDefaults', () => { expect(withBrowserDefaults({ headless: true }, BINARY, false).name).toBe('chrome'); }); }); + +describe('assertDriverMatchesBrowser', () => { + /** + * A harness with tachometer installed, and optionally a chromedriver where tachometer can reach + * it — one directory up from its own, the way a workspace root is reachable in a real install. + * + * Real directories and real module resolution rather than a stand-in: what is being checked is + * which chromedriver node finds from tachometer, and faking that would assume the answer. + */ + async function makeHarness(driverVersion?: string): Promise { + const harnessDir = await makeTempDir(); + const modules = path.join(harnessDir, 'node_modules'); + + await writeFile(path.join(harnessDir, 'package.json'), JSON.stringify({ name: 'harness' })); + await mkdir(path.join(modules, 'tachometer'), { recursive: true }); + await writeFile( + path.join(modules, 'tachometer', 'package.json'), + JSON.stringify({ name: 'tachometer', version: '0.7.2' }), + ); + + if (driverVersion) { + await mkdir(path.join(modules, 'chromedriver'), { recursive: true }); + await writeFile( + path.join(modules, 'chromedriver', 'package.json'), + JSON.stringify({ name: 'chromedriver', version: driverVersion }), + ); + } + + return harnessDir; + } + + /** An executable standing in for the browser, reporting `version` to `--version`. */ + async function makeBrowser(version: string): Promise { + const dir = await makeTempDir(); + const binary = path.join(dir, 'chrome'); + await writeFile(binary, `#!/bin/sh\necho "Google Chrome for Testing ${version}"\n`); + await chmod(binary, 0o755); + return binary; + } + + it('passes when the driver and the browser share a major', async () => { + const harnessDir = await makeHarness('151.0.5'); + const binary = await makeBrowser('151.0.7922.34'); + + expect(() => assertDriverMatchesBrowser(harnessDir, binary)).not.toThrow(); + }); + + it('fails when the driver is on a different major', async () => { + const harnessDir = await makeHarness('150.0.3'); + const binary = await makeBrowser('151.0.7922.34'); + + expect(() => assertDriverMatchesBrowser(harnessDir, binary)).toThrow(/cannot drive Chrome/); + }); + + it('fails when no driver is reachable, naming the major to pin', async () => { + // Tachometer would install the newest chromedriver here, which does not drive a pinned Chrome. + const harnessDir = await makeHarness(); + const binary = await makeBrowser('151.0.7922.34'); + + expect(() => assertDriverMatchesBrowser(harnessDir, binary)).toThrow(/"chromedriver": "\^151"/); + }); +}); diff --git a/packages/benchmark/src/tachometer/browser.ts b/packages/benchmark/src/tachometer/browser.ts index eeed935d1d..befb21b5e4 100644 --- a/packages/benchmark/src/tachometer/browser.ts +++ b/packages/benchmark/src/tachometer/browser.ts @@ -2,10 +2,24 @@ import * as path from 'node:path'; import { createRequire } from 'node:module'; import { execFileSync } from 'node:child_process'; import chalk from 'chalk'; -import * as semver from 'semver'; import type * as PlaywrightTest from '@playwright/test'; import { pathExists } from '../utils/path'; +/** + * Picking the browser to benchmark in, and checking it can actually be driven. + * + * Two separate programs are involved. The browser itself comes from Playwright, which downloads a + * fixed Chrome build — fixed on purpose, because the Chrome on a machine updates itself and timings + * would move with it. Driving that browser needs a second program, `chromedriver`, and tachometer + * supplies that one: it uses whichever it can find next to itself, and downloads the newest if it + * finds none. + * + * A chromedriver only drives the Chrome version it was built for. Since Playwright's Chrome is + * deliberately a little behind the latest, letting tachometer fetch the newest driver usually gets a + * pair that cannot work together — and the failure arrives late, after everything has been built. + * So the pair is checked here first, before any of that work. + */ + /** * Resolves the browser every case runs on: Playwright's pinned Chrome for Testing, rather than * whatever Chrome the machine has auto-updated to. @@ -36,13 +50,11 @@ export async function resolveBrowserBinary(harnessDir: string): Promise * The major version in output like `Google Chrome for Testing 151.0.7922.34`, or undefined when * there is no version in it. * - * The version is picked out of the surrounding words first, requiring at least three components so - * a stray count is not read as one. Chrome's fourth component makes it invalid semver, so what is - * left is coerced rather than parsed. + * At least three components are required, so a stray count in the same line is not read as one. */ export function majorVersionOf(versionOutput: string): number | undefined { - const version = /\d+(?:\.\d+){2,}/.exec(versionOutput)?.[0]; - return version ? (semver.coerce(version)?.major ?? undefined) : undefined; + const match = /(\d+)\.\d+\.\d+/.exec(versionOutput); + return match ? Number(match[1]) : undefined; } /** @@ -53,20 +65,20 @@ export function assertDriverMatchesBrowser(harnessDir: string, binary: string): execFileSync(binary, ['--version'], { encoding: 'utf8' }).trim(), ); + // The same lookup tachometer does before launching: it uses whichever chromedriver resolves from + // its own package root, and installs the newest one when none does. let driverVersion: string | undefined; try { const fromHarness = createRequire(path.join(harnessDir, 'package.json')); const fromTachometer = createRequire(fromHarness.resolve('tachometer/package.json')); driverVersion = fromTachometer('chromedriver/package.json').version; } catch { - console.warn( - chalk.yellow( - 'Could not resolve chromedriver from tachometer; it will install the latest major itself, ' + - 'which fails against any browser not on that major yet. Pin a "chromedriver" ' + - 'devDependency to keep the two aligned.', - ), + throw new Error( + `No chromedriver is reachable from tachometer, so it will install the newest one — which ` + + `will not drive Chrome for Testing ${browserMajor}, since Playwright pins a Chrome behind ` + + `the current release. Add "chromedriver": "^${browserMajor}" to the workspace root, where ` + + `tachometer can resolve it.`, ); - return; } const driverMajor = majorVersionOf(driverVersion ?? ''); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 38c5d6c2bd..3eb3108d15 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -514,9 +514,6 @@ importers: react-dom: specifier: ^18.0.0 || ^19.0.0 version: 19.2.8(react@19.2.8) - semver: - specifier: ^7.8.5 - version: 7.8.5 yaml: specifier: ^2.9.0 version: 2.9.0 @@ -539,9 +536,6 @@ importers: '@types/react-dom': specifier: 19.2.7 version: 19.2.7(@types/react@19.2.18) - '@types/semver': - specifier: ^7.8.0 - version: 7.8.0 '@types/yargs': specifier: 17.0.35 version: 17.0.35 From 21104b8e210683e8bc8413fa630b1cd2c60dfd80 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:35:56 +0200 Subject: [PATCH 055/173] [benchmark] Trim the browser check's header comment --- packages/benchmark/src/tachometer/browser.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/benchmark/src/tachometer/browser.ts b/packages/benchmark/src/tachometer/browser.ts index befb21b5e4..7241fd7e96 100644 --- a/packages/benchmark/src/tachometer/browser.ts +++ b/packages/benchmark/src/tachometer/browser.ts @@ -9,15 +9,13 @@ import { pathExists } from '../utils/path'; * Picking the browser to benchmark in, and checking it can actually be driven. * * Two separate programs are involved. The browser itself comes from Playwright, which downloads a - * fixed Chrome build — fixed on purpose, because the Chrome on a machine updates itself and timings - * would move with it. Driving that browser needs a second program, `chromedriver`, and tachometer + * fixed Chrome build. Driving that browser needs a second program, `chromedriver`, and tachometer * supplies that one: it uses whichever it can find next to itself, and downloads the newest if it * finds none. * * A chromedriver only drives the Chrome version it was built for. Since Playwright's Chrome is * deliberately a little behind the latest, letting tachometer fetch the newest driver usually gets a - * pair that cannot work together — and the failure arrives late, after everything has been built. - * So the pair is checked here first, before any of that work. + * pair that cannot work together. */ /** From ab456d923a89ab3f998e895d0b9199e5ac272ceb Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:53:11 +0200 Subject: [PATCH 056/173] [benchmark] Make the baseline the only ref, and `expand` what picks it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A case is one of two things: the working tree against the baseline, or the pages it names against each other. `expand` says which, so nothing has to carry a ref through a url to find out. The `?ref=` grammar goes with that — no config used one, and the auto-expanded baseline was writing a token into a url only to parse it straight back out. Discovery now never touches git or resolves a ref, the runner builds at most two trees, and `--baseline` keeps the grammar as a flag: a bare revision, or `git:`, with `github:`/`preview:` still reserved. Each variant's `refId` leaves the report; the case's `comparison` is what tells a regression from a library comparison, which is the only thing it was for. --- .circleci/config.yml | 2 +- .../tachometer/buildMarkdownReport.test.ts | 26 +-- .../src/lib/tachometer/buildMarkdownReport.ts | 32 ++-- packages/benchmark/README.md | 48 +++-- packages/benchmark/src/cli/tachoRun.ts | 2 +- .../benchmark/src/tachometer/ciReport.test.ts | 6 +- packages/benchmark/src/tachometer/ciReport.ts | 7 +- .../src/tachometer/discoverCases.test.ts | 99 ++++------- .../benchmark/src/tachometer/discoverCases.ts | 142 +++++++-------- .../benchmark/src/tachometer/format.test.ts | 2 +- .../benchmark/src/tachometer/refs.test.ts | 119 ++++++------- packages/benchmark/src/tachometer/refs.ts | 164 +++++------------- .../benchmark/src/tachometer/runTachometer.ts | 58 ++++--- .../src/tachometer/summarizeCase.test.ts | 3 +- .../benchmark/src/tachometer/summarizeCase.ts | 24 ++- test/tachometer/README.md | 20 +-- 16 files changed, 314 insertions(+), 440 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 15d77a722d..9905c72681 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -136,7 +136,7 @@ jobs: # request lands, so it would reject the upload. Points at this pull request's own # dashboard preview until then — remove the line before merging. CI_REPORT_API_URL: https://code-infra-dashboard-pr-1829.onrender.com - command: pnpm test:tacho --baseline "git:$(cat .tachometer-baseline-sha)" + command: pnpm test:tacho --baseline "$(cat .tachometer-baseline-sha)" - save_cache: name: Save the packed-baseline cache key: tachometer-cache-v1-{{ checksum ".tachometer-baseline-sha" }} diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts index e97b4d9bae..a086e22459 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts @@ -7,13 +7,9 @@ import type { } from '@mui/internal-benchmark/tachometerReport'; import { buildTachometerMarkdownReport, findRegressions } from './buildMarkdownReport'; -/** - * A variant on a given build. - */ -function variant(name: string, refId: string | null) { +function variant(name: string) { return { variant: name, - refId, meanMs: { low: 20, high: 22 }, samples: 120, }; @@ -47,34 +43,28 @@ function comparison(name: string, verdict: Verdict) { }; } -/** - * A case comparing `[current]` against `[baseline]` — two different builds. - */ +/** A case measuring the working tree against the baseline, where a difference is a regression. */ function regressionCase(name: string, verdicts: Record): CaseResult { const measurements: MeasurementResult[] = Object.entries(verdicts).map( ([measurement, verdict]) => ({ name: measurement, - variants: [ - variant(`${name} [current]`, 'current'), - variant(`${name} [baseline]`, 'git-abc123456'), - ], + variants: [variant(`${name} [current]`), variant(`${name} [baseline]`)], comparisons: [comparison(`${name} [baseline]`, verdict)], }), ); - return { name, reference: `${name} [current]`, measurements }; + return { name, comparison: 'baseline', reference: `${name} [current]`, measurements }; } -/** - * A case comparing libraries with each other, so every variant comes from the same build. - */ +/** A case comparing libraries with each other, every variant built from the working tree. */ function libraryCase(name: string, verdict: Verdict): CaseResult { return { name, + comparison: 'variants', reference: `${name} [mosaic]`, measurements: [ { name: 'mount', - variants: [variant(`${name} [mosaic]`, 'current'), variant(`${name} [ag-grid]`, 'current')], + variants: [variant(`${name} [mosaic]`), variant(`${name} [ag-grid]`)], comparisons: [comparison(`${name} [ag-grid]`, verdict)], }, ], @@ -210,7 +200,7 @@ describe('buildTachometerMarkdownReport', () => { it('reports a case that produced no result', () => { const markdown = buildTachometerMarkdownReport( - report([{ name: 'broken', error: 'produced no benchmarks' }]), + report([{ name: 'broken', comparison: 'baseline', error: 'produced no benchmarks' }]), ); expect(visiblePart(markdown)).toContain('broken'); diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts index 32df13b4c3..cd3ff28bc5 100644 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts @@ -28,23 +28,19 @@ interface BuildOptions { } /** - * Collects the comparisons that mean "this pull request made something slower". + * The comparisons that say something about this pull request. + * + * Only a case measuring the working tree against the baseline does. One comparing pages built from + * the same tree — this library against another — is a standing measurement, and a difference there + * is the point rather than a regression. */ -function comparisonsAcrossRefs(entry: SummarizedCase) { - const found = []; - - for (const measurement of entry.measurements) { - const refIds = new Map(measurement.variants.map((variant) => [variant.variant, variant.refId])); - const referenceRefId = refIds.get(entry.reference ?? ''); - - for (const comparison of measurement.comparisons) { - if (refIds.get(comparison.variant) !== referenceRefId) { - found.push({ measurement, comparison }); - } - } +function baselineComparisons(entry: SummarizedCase) { + if (entry.comparison !== 'baseline') { + return []; } - - return found; + return entry.measurements.flatMap((measurement) => + measurement.comparisons.map((comparison) => ({ measurement, comparison })), + ); } export function findRegressions(report: TachometerReport): Regression[] { @@ -54,7 +50,7 @@ export function findRegressions(report: TachometerReport): Regression[] { if (!isSummarized(entry)) { continue; } - for (const { measurement, comparison } of comparisonsAcrossRefs(entry)) { + for (const { measurement, comparison } of baselineComparisons(entry)) { if (comparison.verdict === 'slower') { regressions.push({ caseName: entry.name, @@ -137,13 +133,13 @@ export function buildTachometerMarkdownReport( // Everything that did not regress collapses to a single line, so the regressions are what the eye // lands on. `unsure` is the expected result for two equivalent builds, not a warning. Counted in - // cases, like the total beside them, and over the same cross-ref comparisons a regression is read + // cases, like the total beside them, and over the same baseline comparisons a regression is read // from — a case that beats a competing library has not got faster. const regressedCases = new Set(regressions.map((regression) => regression.caseName)); const faster = summarized.filter( (entry) => !regressedCases.has(entry.name) && - comparisonsAcrossRefs(entry).some(({ comparison }) => comparison.verdict === 'faster'), + baselineComparisons(entry).some(({ comparison }) => comparison.verdict === 'faster'), ).length; const unchanged = summarized.length - regressedCases.size - faster; const summary = [ diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 92e7083f92..89fffa0daa 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -301,33 +301,19 @@ because tachometer rejects those as flags once a config file is in play. A case difference resolves or the timeout is hit; two builds of equal speed cannot be separated, so such a case reports `unsure`, which is the expected "nothing changed" outcome rather than a failure. -### Refs: which build a page loads +### What a case compares -A benchmark url may carry a `?ref=` selecting **which build of the workspace** to load. A ref always -names a commit and covers every package in the workspace, and is resolved to an immutable SHA before -anything is cached. +A case is one of two things, and its `expand` decides which. -| `?ref=` | Meaning | -| :---------------------------- | :------------------------------------------------------ | -| _(absent)_ | the working tree | -| `baseline` | symbolic — whatever `--baseline` bound | -| `git:` | this repository at `` (SHA, tag, branch, `HEAD~1`) | -| `github:/#` | another remote _(not implemented)_ | -| `preview:` | pkg.pr.new artifacts _(not implemented)_ | - -Anything else is an error. A bare value is never assumed to be a revision, since an absent ref -already means the working tree. - -A case whose benchmark declares no `expand` is the ordinary regression shape, and is expanded for you -into `[current]` (the working tree) against `[baseline]`: +**No `expand`** is the ordinary regression shape. It is expanded for you into `[current]` — the +working tree — against `[baseline]`, both loading the same page: ```jsonc { "benchmarks": [{ "name": "init", "url": "./index.html" }] } ``` -A case that declares its own `expand` owns that axis, and only the refs its leaves mention get built. -Because `expand` nests, this is how a ref axis combines with anything else — browsers, or entirely -different libraries: +**Its own `expand`** compares the pages it names with each other, every one of them built from the +working tree. That is how one implementation is measured against another: ```jsonc "expand": [ @@ -336,20 +322,28 @@ different libraries: ] ``` -Leaving `?ref` off every variant, as above, compares implementations at a single commit rather than -two builds of one library — so no regression can be reported, by construction. +Such a case reports no regression, by construction: there is no baseline in it to regress against. +Every run says which shape each case has before it starts, so this is visible rather than inferred +from an empty result. To sweep a parameter and still measure it against the baseline, give each +value its own case folder pointing at the shared page — `expand` would turn the case into the other +shape. ### Baselines -`--baseline` binds the `baseline` symbol, taking a ref in the grammar above. It defaults to `HEAD~1`, -which needs no policy to decide. Which commit a branch should actually be compared against is -`code-infra baseline`'s question — it answers the same way for every job that compares a branch to -its base — so pass it in: +`--baseline` names the build every `[baseline]` variant loads: a revision — a SHA, a tag, a branch, +`HEAD~1` — on its own or behind `git:`. It is resolved to an immutable SHA before anything is +cached, and defaults to `HEAD~1`, which needs no policy to decide. + +Which commit a branch should actually be compared against is `code-infra baseline`'s question — it +answers the same way for every job that compares a branch to its base — so pass it in: ```bash -benchmark tacho run --baseline "git:$(code-infra baseline)" +benchmark tacho run --baseline "$(code-infra baseline)" ``` +`github:/#` and `preview:` are reserved for a baseline that is not a commit in +this repository. Neither is implemented, and both are rejected by name rather than handed to git. + ### Builds and caching Both sides are built the same way, from packed tarballs installed into an isolated tree, so neither diff --git a/packages/benchmark/src/cli/tachoRun.ts b/packages/benchmark/src/cli/tachoRun.ts index 297b863f8b..895412675f 100644 --- a/packages/benchmark/src/cli/tachoRun.ts +++ b/packages/benchmark/src/cli/tachoRun.ts @@ -18,7 +18,7 @@ const command: CommandModule<{}, Args> = { .option('baseline', { type: 'string', describe: - 'Bind the "baseline" symbol, in the ref grammar (e.g. git:abc1234). Defaults to HEAD~1; `code-infra baseline` resolves a fork point to pass here', + 'The build to compare against: a revision, on its own or as git:. Defaults to HEAD~1; `code-infra baseline` resolves a fork point to pass here', }) .option('build-cmd', { type: 'string', diff --git a/packages/benchmark/src/tachometer/ciReport.test.ts b/packages/benchmark/src/tachometer/ciReport.test.ts index 93349354d5..0e10d03b22 100644 --- a/packages/benchmark/src/tachometer/ciReport.test.ts +++ b/packages/benchmark/src/tachometer/ciReport.test.ts @@ -21,6 +21,7 @@ function report(overrides = {}) { cases: [ { name: 'workload', + comparison: 'baseline', reference: 'workload [current]', measurements: [ { @@ -28,7 +29,6 @@ function report(overrides = {}) { variants: [ { variant: 'workload [current]', - refId: 'current', meanMs: { low: 20.5, high: 22.7 }, samples: 120, }, @@ -99,7 +99,9 @@ describe('tachometerUploadSchema', () => { it('accepts a case that failed to summarize', () => { // Kept in the report rather than dropped, so the comment can say the case produced nothing. const withFailure = envelope({ - report: report({ cases: [{ name: 'broken', error: 'produced no benchmarks' }] }), + report: report({ + cases: [{ name: 'broken', comparison: 'baseline', error: 'produced no benchmarks' }], + }), }); expect(tachometerUploadSchema.safeParse(withFailure).success).toBe(true); diff --git a/packages/benchmark/src/tachometer/ciReport.ts b/packages/benchmark/src/tachometer/ciReport.ts index 45ccb0f5b8..8fe6cb09d4 100644 --- a/packages/benchmark/src/tachometer/ciReport.ts +++ b/packages/benchmark/src/tachometer/ciReport.ts @@ -22,7 +22,6 @@ const differenceSchema = z.object({ const variantResultSchema = z.object({ variant: z.string(), - refId: z.string().nullable(), meanMs: confidenceIntervalSchema, samples: z.number(), }); @@ -42,13 +41,17 @@ const measurementResultSchema = z.object({ // rather than dropped, so the comment can say a case produced nothing instead of quietly omitting it. const caseResultSchema = z.object({ name: z.string(), + // `baseline` compares the working tree against the baseline build, so a difference there is a + // regression; `variants` compares pages built from the same tree, where a difference is the + // point. Carried per case because that is where the two are told apart. + comparison: z.enum(['baseline', 'variants']), reference: z.string().optional(), measurements: z.array(measurementResultSchema).optional(), error: z.string().optional(), }); /** - * A build the pages were loaded from. The working tree has no commit behind it and a `git:` ref + * A build the pages were loaded from. The working tree has no commit behind it and the baseline * always does, so the two carry different fields. * * `requested` is the revision as the run was given it — `HEAD~1`, a tag, a SHA — kept instead of a diff --git a/packages/benchmark/src/tachometer/discoverCases.test.ts b/packages/benchmark/src/tachometer/discoverCases.test.ts index f6846f4f73..9edafdb53d 100644 --- a/packages/benchmark/src/tachometer/discoverCases.test.ts +++ b/packages/benchmark/src/tachometer/discoverCases.test.ts @@ -144,58 +144,46 @@ describe('discoverCases', () => { expect(entry.config.benchmarks).toEqual([ { name: 'alpha [current]', url: './index.html' }, - { name: 'alpha [baseline]', url: './index.html?ref=baseline' }, + { name: 'alpha [baseline]', url: './index.html' }, ]); }); - it('hands tachometer benchmarks it cannot expand again', async () => { - // The config is parsed afresh, so a benchmark that still carried `expand` would be expanded a - // second time — and an un-rewritten parent url would point at a page that was never built. + it('points the two variants at the two builds', async () => { const harnessDir = await makeHarness({ alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, }); const [entry] = await discoverCases({ harnessDir }); - for (const benchmark of entry.config.benchmarks) { - expect(benchmark).not.toHaveProperty('expand'); - } + expect(entry.comparison).toBe('baseline'); + expect(entry.leaves.map((leaf) => leaf.ref)).toEqual(['current', 'baseline']); }); - it('joins the ref with an ampersand when the url already has a query', async () => { + it('hands tachometer benchmarks it cannot expand again', async () => { + // The config is parsed afresh, so a benchmark that still carried `expand` would be expanded a + // second time — and an un-rewritten parent url would point at a page that was never built. const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html?rows=100'), pages: ['index.html'] }, + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, }); const [entry] = await discoverCases({ harnessDir }); - expect(entry.config.benchmarks[1].url).toBe('./index.html?rows=100&ref=baseline'); + for (const benchmark of entry.config.benchmarks) { + expect(benchmark).not.toHaveProperty('expand'); + } }); - it('keeps the ref in the query when the url has a fragment', async () => { - // Appending to the url text put `ref` after the `#`, where it is part of the fragment and - // `searchParams` never sees it — so the baseline variant silently loaded the working tree and - // the case could never report a regression. + it('gives both variants the same page, query and fragment', async () => { + // Everything after the page path belongs to the page. A parameterised case that lost it on one + // side would compare the parameters against the defaults and call the difference a regression. const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html#state'), pages: ['index.html'] }, + alpha: { config: config('alpha', './index.html?rows=100#state'), pages: ['index.html'] }, }); const [entry] = await discoverCases({ harnessDir }); + const target = `${path.join('alpha', 'index.html')}?rows=100#state`; - expect(entry.config.benchmarks[1].url).toBe('./index.html?ref=baseline#state'); - }); - - it('leaves a traversing path untouched', async () => { - // The url is relative to its own config's directory and is allowed to leave it; resolving it - // against a base to edit the query would collapse the `..` segments. - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - beta: { config: config('beta', '../alpha/index.html?rows=100') }, - }); - - const [entry] = await discoverCases({ harnessDir, filters: ['beta'] }); - - expect(entry.config.benchmarks[1].url).toBe('../alpha/index.html?rows=100&ref=baseline'); + expect(entry.leaves.map((leaf) => `${leaf.page}${leaf.suffix}`)).toEqual([target, target]); }); it('merges an expansion over its parent, nearest value winning', async () => { @@ -254,6 +242,9 @@ describe('discoverCases', () => { const [entry] = await discoverCases({ harnessDir }); expect(entry.leaves.map((leaf) => leaf.page)).toEqual(['libs/ours.html', 'libs/theirs.html']); + // Its own variants, so they are compared with each other and all load the working tree. + expect(entry.comparison).toBe('variants'); + expect(entry.leaves.map((leaf) => leaf.ref)).toEqual(['current', 'current']); }); }); @@ -285,15 +276,12 @@ describe('discoverCases', () => { ]); }); - it('consumes the ref and carries every other query parameter through', async () => { + it('carries the query and the fragment through as written', async () => { const harnessDir = await makeHarness({ alpha: { config: { benchmarks: [ - { - name: 'alpha', - expand: [{ name: 'a', url: './index.html?rows=100&ref=baseline#x' }], - }, + { name: 'alpha', expand: [{ name: 'a', url: './index.html?rows=100&cols=8#x' }] }, ], }, pages: ['index.html'], @@ -302,7 +290,7 @@ describe('discoverCases', () => { const [entry] = await discoverCases({ harnessDir }); - expect(entry.leaves[0].suffix).toBe('?rows=100#x'); + expect(entry.leaves[0].suffix).toBe('?rows=100&cols=8#x'); }); it('resolves a case that owns no page but parameterises a sibling', async () => { @@ -348,34 +336,20 @@ describe('discoverCases', () => { await expect(discoverCases({ harnessDir })).rejects.toThrow(/missing page/); }); - - it('rejects an unknown ref scheme even when refs are not being resolved', async () => { - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [ - { name: 'alpha', expand: [{ name: 'a', url: './index.html?ref=nonsense' }] }, - ], - }, - pages: ['index.html'], - }, - }); - - await expect(discoverCases({ harnessDir })).rejects.toThrow(/Unknown ref "nonsense"/); - }); }); - describe('without a ref resolver', () => { - it('leaves refs unresolved and makes no git calls', async () => { - // The temp harness is not a git repository at all, so anything reaching for git would fail. + describe('outside a repository', () => { + it('names both builds without asking git anything', async () => { + // The temp harness is no repository at all, so anything reaching for git would fail here. + // Which build a variant loads follows from the case's shape, which is what lets a plain + // `vite build` run in a checkout with no history. const harnessDir = await makeHarness({ alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, }); const [entry] = await discoverCases({ harnessDir }); - expect(entry.leaves).toHaveLength(2); - expect(entry.leaves.every((leaf) => leaf.ref === null)).toBe(true); + expect(entry.leaves.map((leaf) => leaf.ref)).toEqual(['current', 'baseline']); }); it('still yields the full page list', async () => { @@ -474,10 +448,7 @@ describe('case variants and measurements', () => { const [entry] = await discoverCases({ harnessDir }); - expect(entry.variants.map((variant) => variant.name)).toEqual([ - 'alpha [current]', - 'alpha [baseline]', - ]); + expect(entry.variants).toEqual(['alpha [current]', 'alpha [baseline]']); expect(entry.measurements).toEqual(['mount']); }); @@ -493,10 +464,7 @@ describe('case variants and measurements', () => { const [entry] = await discoverCases({ harnessDir }); - expect(entry.variants.map((variant) => variant.name)).toEqual([ - 'alpha [current]', - 'alpha [baseline]', - ]); + expect(entry.variants).toEqual(['alpha [current]', 'alpha [baseline]']); }); it('takes a variant name from the nearest node that sets one', async () => { @@ -521,10 +489,7 @@ describe('case variants and measurements', () => { const [entry] = await discoverCases({ harnessDir }); - expect(entry.variants.map((variant) => variant.name)).toEqual([ - 'alpha [ours]', - 'alpha [theirs]', - ]); + expect(entry.variants).toEqual(['alpha [ours]', 'alpha [theirs]']); }); it('collects every measurement a benchmark declares', async () => { diff --git a/packages/benchmark/src/tachometer/discoverCases.ts b/packages/benchmark/src/tachometer/discoverCases.ts index 56808f8ced..f8d3d6f7e1 100644 --- a/packages/benchmark/src/tachometer/discoverCases.ts +++ b/packages/benchmark/src/tachometer/discoverCases.ts @@ -1,38 +1,40 @@ import * as path from 'node:path'; import { readFile } from 'node:fs/promises'; -import { pathToFileURL } from 'node:url'; import { globby } from 'globby'; import { pathExists } from '../utils/path'; -import { parseRefToken } from './refs'; -import type { ResolvedRef } from './refs'; /** * Case discovery: the `tachometer.json` files under a harness's `src/` are the source of truth for * which benchmark cases exist and which pages they reference. * * Both the runner and the vite plugin read them through here, so there is exactly one notion of - * "what pages exist". The plugin passes no `resolveRef`, which keeps discovery free of git — a - * plain `vite build` must not require a checkout with full history. + * "what pages exist". Discovery reads only the filesystem — which builds a case loads follows from + * its shape rather than from git, so a plain `vite build` needs no checkout with history. */ +/** Which of a run's two builds a variant loads. */ +export type BuildRef = 'current' | 'baseline'; + +/** + * What a case compares. + * + * `baseline` is the working tree against the baseline build: the regression case, and what a case + * gets when it declares no variants of its own. `variants` is the pages the case names compared + * against each other, every one of them built from the working tree. + */ +export type CaseComparison = 'baseline' | 'variants'; + export interface Leaf { /** The config node whose `url` this is; rewritten in place once builds exist. */ node: { url?: string }; /** Page path relative to `src`, e.g. `data-grid-init/index.html`. */ page: string; - /** The resolved ref, or null when discovery ran without `resolveRef`. */ - ref: ResolvedRef | null; - /** Everything after the page path once `ref` was consumed: remaining query plus any fragment. */ + /** The build this variant loads. */ + ref: BuildRef; + /** Everything after the page path: query and fragment, as written. */ suffix: string; } -export interface CaseVariant { - /** The variant's name, as tachometer will report it. */ - name: string; - /** The ref this variant loads, or null when refs were not resolved. */ - refId: string | null; -} - export interface BenchmarkCase { /** The case name, as its benchmark declares it. */ name: string; @@ -40,10 +42,12 @@ export interface BenchmarkCase { configPath: string; /** The parsed config, mutated in place as urls are rewritten. */ config: any; + /** What the case compares. */ + comparison: CaseComparison; /** Every node that selects a page. */ leaves: Leaf[]; - /** Declared variants in order; the first is every comparison's reference. */ - variants: CaseVariant[]; + /** Variant names in order, as tachometer will report them; the first is the reference. */ + variants: string[]; /** Measurement names, as tachometer will name them in its output. */ measurements: string[]; } @@ -56,8 +60,6 @@ export interface DiscoverCasesOptions { * case-insensitively. */ filters?: string[]; - /** Resolves a ref token. Omit to skip resolution entirely (no git). */ - resolveRef?: (token?: string) => ResolvedRef; } /** The file that marks a directory under `src/` as a benchmark case. */ @@ -119,44 +121,22 @@ function flattenExpansions(benchmark: any): any[] { } /** - * The same url with `?ref=` set. + * Resolves one variant's url into the source page it references and whatever follows it. * - * The path is carried through as written, not resolved: these urls are relative to their own - * config's directory and traverse out of it (`../row-updates/index.html?throttle=16`), and `URL` - * would clamp that traversal against whatever base it was given. Only the query and fragment go - * through `URL`, which is where the syntax rules are — appending text instead put `ref` *inside* a - * fragment for a url that had one, where nothing reads it. - */ -function withRef(url: string, ref: string): string { - const parsed = new URL(url, 'http://case.invalid/'); - parsed.searchParams.set('ref', ref); - const [pathPart] = url.split(/[?#]/); - return `${pathPart}${parsed.search}${parsed.hash}`; -} - -/** - * Resolves one leaf url into the source page it references, the ref it selects, and any leftovers. - * - * `ref` selects the build and is consumed here; everything else belongs to the page and has to - * survive the rewrite, or a parameterised benchmark would silently run its defaults. + * The path is carried through as written rather than through `URL`: these urls are relative to their + * own config's directory and traverse out of it (`../row-updates/index.html?throttle=16`), and `URL` + * would clamp that traversal against whatever base it was given. Everything after the path belongs + * to the page and survives the rewrite verbatim, or a parameterised benchmark would silently run its + * defaults. */ async function parseLeafUrl( url: string, configDir: string, srcDir: string, - resolveRef: ((token?: string) => ResolvedRef) | undefined, -): Promise<{ page: string; ref: ResolvedRef | null; suffix: string }> { - const parsed = new URL(url, pathToFileURL(path.join(configDir, path.sep))); - const token = parsed.searchParams.get('ref') ?? undefined; - // Validate the grammar even when not resolving, so a typo fails the build rather than silently - // building a page the runner would later reject. - parseRefToken(token); - const ref = resolveRef ? resolveRef(token) : null; - const absolute = decodeURIComponent(parsed.pathname); - - parsed.searchParams.delete('ref'); - const search = parsed.searchParams.toString(); - const suffix = `${search ? `?${search}` : ''}${parsed.hash}`; +): Promise<{ page: string; suffix: string }> { + const [pathPart] = url.split(/[?#]/); + const suffix = url.slice(pathPart.length); + const absolute = path.resolve(configDir, decodeURIComponent(pathPart)); const page = path.relative(srcDir, absolute); if (page.startsWith('..') || path.isAbsolute(page)) { @@ -165,7 +145,7 @@ async function parseLeafUrl( if (!(await pathExists(absolute))) { throw new Error(`Benchmark url "${url}" points at a missing page (${absolute}).`); } - return { page, ref, suffix }; + return { page, suffix }; } /** @@ -188,15 +168,16 @@ async function findCaseLocations(srcDir: string): Promise { } /** - * Reads every `tachometer.json` under `src/` (optionally filtered by location), expands the sugar - * for cases that declare no variants of their own, and resolves each leaf's page and ref. + * Reads every `tachometer.json` under `src/` (optionally filtered by location) and resolves the page + * each of its variants loads. * - * A benchmark with no `expand` is the common regression case: it is expanded into `[current]` (the - * working tree) versus `[baseline]`. A benchmark that declares its own `expand` owns its variant - * axis — only the refs its leaves reference are resolved. + * `expand` decides what a case compares. A benchmark that declares one owns its variant axis, and + * every variant loads the working tree — a comparison between pages, such as one library against + * another. A benchmark without one is expanded into `[current]` and `[baseline]` over the same page, + * which is the regression case. */ export async function discoverCases(options: DiscoverCasesOptions): Promise { - const { harnessDir, filters = [], resolveRef } = options; + const { harnessDir, filters = [] } = options; const srcDir = path.join(harnessDir, 'src'); const located = await findCaseLocations(srcDir); @@ -247,7 +228,7 @@ export async function discoverCases(options: DiscoverCasesOptions): Promise(); - const benchmarks: any[] = []; + const nodes: Array<{ node: any; ref: BuildRef }> = []; for (const benchmark of config.benchmarks ?? []) { // A benchmark need not name itself, and the case name is the same fallback its own name gets // — without it the auto-expanded variants would read "undefined [current]". Set before @@ -258,48 +239,51 @@ export async function discoverCases(options: DiscoverCasesOptions): Promise 0; + const expanded = flattenExpansions(benchmark); + if (declaresVariants) { + for (const node of expanded) { + nodes.push({ node, ref: 'current' }); + } + } else { + // Without `expand` exactly one benchmark comes back, and the same page is measured twice. + const [base] = expanded; + if (base.url === undefined) { throw new Error(`Benchmark "${benchmark.name}" in ${configPath} has no "url".`); } - benchmark.expand = [ - { name: `${benchmark.name} [current]`, url: base }, - { name: `${benchmark.name} [baseline]`, url: withRef(base, 'baseline') }, - ]; + nodes.push( + { node: { ...base, name: `${benchmark.name} [current]` }, ref: 'current' }, + { node: { ...base, name: `${benchmark.name} [baseline]` }, ref: 'baseline' }, + ); } - benchmarks.push(...flattenExpansions(benchmark)); } // A declared `expand` tree may bottom out in a variant that names no url and inherits none. - for (const benchmark of benchmarks) { - if (benchmark.url === undefined) { + for (const { node } of nodes) { + if (node.url === undefined) { throw new Error( - `Benchmark "${benchmark.name}" in ${configPath} expands to a variant with no "url".`, + `Benchmark "${node.name}" in ${configPath} expands to a variant with no "url".`, ); } } // The flattened list is what tachometer is handed: it expands to exactly this, and every later // step — the url rewrite, the browser defaults — then addresses one whole benchmark at a time. - config.benchmarks = benchmarks; + config.benchmarks = nodes.map((entry) => entry.node); // eslint-disable-next-line no-await-in-loop const resolved = await Promise.all( - benchmarks.map((benchmark) => parseLeafUrl(benchmark.url, configDir, srcDir, resolveRef)), + nodes.map((entry) => parseLeafUrl(entry.node.url, configDir, srcDir)), ); - const leaves = benchmarks.map((node, index) => ({ node, ...resolved[index] })); - const variants = benchmarks.map((node, index) => ({ - name: node.name as string, - refId: resolved[index].ref?.id ?? null, - })); cases.push({ name, configPath, config, - leaves, - variants, + comparison: nodes.some((entry) => entry.ref === 'baseline') ? 'baseline' : 'variants', + leaves: nodes.map((entry, index) => ({ ...entry, ...resolved[index] })), + variants: nodes.map((entry) => entry.node.name as string), measurements: [...measurements], }); } diff --git a/packages/benchmark/src/tachometer/format.test.ts b/packages/benchmark/src/tachometer/format.test.ts index d698108c71..19cca7d4a0 100644 --- a/packages/benchmark/src/tachometer/format.test.ts +++ b/packages/benchmark/src/tachometer/format.test.ts @@ -5,7 +5,6 @@ import type { SummarizedCase } from './format'; function variant(name: string) { return { variant: name, - refId: 'current', meanMs: { low: 20, high: 22 }, samples: 120, }; @@ -14,6 +13,7 @@ function variant(name: string) { function caseWith(name: string, variantNames: string[], measurements = ['mount']): SummarizedCase { return { name, + comparison: 'baseline', reference: `${name} ${variantNames[0]}`, measurements: measurements.map((measurement) => ({ name: measurement, diff --git a/packages/benchmark/src/tachometer/refs.test.ts b/packages/benchmark/src/tachometer/refs.test.ts index 940788d8cf..04a74648ad 100644 --- a/packages/benchmark/src/tachometer/refs.test.ts +++ b/packages/benchmark/src/tachometer/refs.test.ts @@ -1,53 +1,7 @@ import { describe, expect, it } from 'vitest'; import { execa } from 'execa'; import { makeTempDir } from '../utils/testUtils'; -import { createRefResolver, parseRefToken } from './refs'; - -describe('parseRefToken', () => { - it('treats an absent ref as the working tree', () => { - expect(parseRefToken(undefined)).toEqual({ kind: 'worktree' }); - }); - - it('treats an empty ref as the working tree', () => { - expect(parseRefToken('')).toEqual({ kind: 'worktree' }); - }); - - it('recognises the baseline symbol', () => { - expect(parseRefToken('baseline')).toEqual({ kind: 'baseline' }); - }); - - it('parses a git revision', () => { - expect(parseRefToken('git:abc1234')).toEqual({ kind: 'git', committish: 'abc1234' }); - }); - - it('keeps the whole revision, including characters that look like a scheme', () => { - expect(parseRefToken('git:origin/master')).toEqual({ - kind: 'git', - committish: 'origin/master', - }); - }); - - describe('reserved schemes', () => { - it.each(['github:owner/repo#abc1234', 'preview:abc1234'])( - 'reports %s as recognised but not implemented', - (token) => { - expect(() => parseRefToken(token)).toThrow(/recognised but not implemented/); - }, - ); - }); - - describe('unknown tokens', () => { - it('suggests the git scheme for a bare revision-shaped value', () => { - expect(() => parseRefToken('HEAD~1')).toThrow(/did you mean "git:HEAD~1"\?/); - }); - - it('does not suggest the git scheme for a value that cannot be a revision', () => { - // A bare value is never auto-prefixed, since an absent ref already means "working tree". - expect(() => parseRefToken('what is this')).toThrow(/Unknown ref "what is this"/); - expect(() => parseRefToken('what is this')).not.toThrow(/did you mean/); - }); - }); -}); +import { resolveBaseline } from './refs'; /** * A repository with three commits on `main`, the first of them carrying an annotated tag. @@ -90,44 +44,83 @@ async function makeRepo(): Promise<{ return { repoRoot, shas, git }; } -describe('createRefResolver', () => { - it('resolves an annotated tag to the commit it points at', async () => { +describe('resolveBaseline', () => { + it('takes a bare value as a revision', async () => { const { repoRoot, shas } = await makeRepo(); - const ref = createRefResolver({ repoRoot }).parse('git:v1.0.0'); + const ref = await resolveBaseline(shas[0], repoRoot); - // Without `^{commit}` this is the tag object's own SHA, which is not a commit at all. expect(ref.sha).toBe(shas[0]); }); - it('gives a tag and its commit one identity, so the commit is built once', async () => { + it('takes the same revision behind the git scheme', async () => { const { repoRoot, shas } = await makeRepo(); - const resolver = createRefResolver({ repoRoot }); - const viaTag = resolver.parse('git:v1.0.0'); - const viaSha = resolver.parse(`git:${shas[0]}`); + const ref = await resolveBaseline(`git:${shas[0]}`, repoRoot); - // `id` is the build directory and the dedupe key: two ids means packing and building twice. - expect(viaTag.id).toBe(viaSha.id); + expect(ref.sha).toBe(shas[0]); }); - it('falls back to the previous commit when nothing binds the baseline', async () => { + it('resolves an annotated tag to the commit it points at', async () => { + const { repoRoot, shas } = await makeRepo(); + + const ref = await resolveBaseline('v1.0.0', repoRoot); + + // Without `^{commit}` this is the tag object's own SHA, which is not a commit at all. + expect(ref.sha).toBe(shas[0]); + }); + + it('falls back to the previous commit when nothing names one', async () => { // Which commit a branch should be compared against is `code-infra baseline`'s question, and the // harness scripts pass the answer in. Unbound, the previous commit is the only choice here that // needs no policy of its own. const { repoRoot, shas } = await makeRepo(); - const ref = createRefResolver({ repoRoot }).parse('baseline'); + const ref = await resolveBaseline(undefined, repoRoot); expect(ref.sha).toBe(shas[1]); }); - it('keeps the revision it was given', async () => { + it('keeps the revision it was given, without the scheme', async () => { const { repoRoot } = await makeRepo(); - const ref = createRefResolver({ repoRoot }).parse('git:v1.0.0'); - // The report shows this, so a tag stays a tag rather than becoming the SHA it points at. - expect(ref.requested).toBe('v1.0.0'); + expect((await resolveBaseline('v1.0.0', repoRoot)).requested).toBe('v1.0.0'); + expect((await resolveBaseline('git:v1.0.0', repoRoot)).requested).toBe('v1.0.0'); + }); + + it('names a revision and a tag the same build, so the commit is built once', async () => { + const { repoRoot, shas } = await makeRepo(); + + const viaTag = await resolveBaseline('v1.0.0', repoRoot); + const viaSha = await resolveBaseline(shas[0], repoRoot); + + // `id` is the build directory: two ids for one commit means packing and building it twice. + expect(viaTag.id).toBe(viaSha.id); + }); + + it.each(['github:owner/repo#abc1234', 'preview:abc1234'])( + 'reports %s as recognised but not implemented', + async (token) => { + const { repoRoot } = await makeRepo(); + + await expect(() => resolveBaseline(token, repoRoot)).rejects.toThrow( + /recognised but not implemented/, + ); + }, + ); + + it('rejects the git scheme with no revision after it', async () => { + const { repoRoot } = await makeRepo(); + + await expect(() => resolveBaseline('git:', repoRoot)).rejects.toThrow(/names no revision/); + }); + + it('reports what git said about a revision it cannot resolve', async () => { + const { repoRoot } = await makeRepo(); + + // Git is the authority on what a revision is, so an unknown one surfaces as its own failure + // rather than as a grammar this code would have to keep in step with git's. + await expect(() => resolveBaseline('no-such-thing', repoRoot)).rejects.toThrow(/rev-parse/); }); }); diff --git a/packages/benchmark/src/tachometer/refs.ts b/packages/benchmark/src/tachometer/refs.ts index 5a05dd7040..9716b51770 100644 --- a/packages/benchmark/src/tachometer/refs.ts +++ b/packages/benchmark/src/tachometer/refs.ts @@ -1,80 +1,35 @@ -import { execaSync } from 'execa'; +import { execa } from 'execa'; import { revParseCommitArgs } from '../utils/git'; /** - * The `?ref=` grammar: which build of the workspace a benchmark page should load. + * The builds a run measures: the working tree, and the baseline it is compared against. * - * A ref always denotes a commit and covers every package in the workspace. Parsing is split in two - * so that consumers who only need to know *which pages exist* never shell out to git: - * - * - {@link parseRefToken} validates the grammar and returns a descriptor. Pure. - * - {@link createRefResolver} turns a descriptor into a {@link ResolvedRef} with an immutable SHA. + * Those are the only two. A case either compares the working tree against the baseline or compares + * the pages it names against each other, all built from the working tree — so the only build anyone + * chooses is the baseline, and `--baseline` is where it is named. */ -/** Only a `git:` ref carries a revision, which the union states rather than leaving to a comment. */ -export type RefDescriptor = - { kind: 'worktree' } | { kind: 'baseline' } | { kind: 'git'; committish: string }; - /** - * A ref with its build identity. The working tree has no commit behind it and a `git:` ref always + * A build with its identity. The working tree has no commit behind it and a git revision always * does, so the two carry different fields rather than sharing optional ones a reader has to pair * with the right `kind` by hand. * - * `id` is the build directory name — `current`, or `git-230342ee2` — and doubles as the dedupe - * identity. `requested` is the revision as it was given, which is what a report shows. + * `id` is the build directory name — `current`, or `git-230342ee2`. `requested` is the revision as + * it was given, which is what a report shows. */ export type ResolvedRef = | { kind: 'worktree'; id: string; sha?: undefined; requested?: undefined } | { kind: 'git'; id: string; sha: string; requested: string }; -export interface RefResolverOptions { - /** Repository to resolve revisions in. */ - repoRoot: string; - /** Binds the `baseline` symbol, in the ref grammar (e.g. `git:abc1234`). */ - baselineOverride?: string; -} - -/** The working tree — never cached, since it has no immutable identity. */ -const WORKTREE_REF: ResolvedRef = { kind: 'worktree', id: 'current' }; - -/** - * Parses a `?ref=` token into a descriptor, validating the grammar. Makes no git calls. - * - * Bare values are never auto-prefixed: an absent ref already means "working tree", so letting a - * bare value mean a git revision would give one token two meanings depending on where it appears. - */ -export function parseRefToken(token?: string): RefDescriptor { - if (token === undefined || token === '') { - return { kind: 'worktree' }; - } - - if (token === 'baseline') { - return { kind: 'baseline' }; - } - - if (token.startsWith('git:') && token.length > 'git:'.length) { - return { kind: 'git', committish: token.slice('git:'.length) }; - } - - if (token.startsWith('github:') || token.startsWith('preview:')) { - const scheme = token.split(':', 1)[0]; - throw new Error( - `Ref scheme "${scheme}:" is recognised but not implemented yet (in "${token}"). ` + - `Supported today: an absent ref (working tree), "baseline", and "git:".`, - ); - } +/** The working tree, which every run builds. Never cached, having no immutable identity. */ +export const WORKTREE_REF: ResolvedRef = { kind: 'worktree', id: 'current' }; - // Only advertise the schemes that actually work here; the reserved `github:`/`preview:` schemes - // are handled above with their own message, so listing them as valid would mislead. - throw new Error( - `Unknown ref "${token}". Expected an absent ref (working tree), "baseline", or "git:"` + - `${/^[0-9a-zA-Z._/~^-]+$/.test(token) ? ` — did you mean "git:${token}"?` : '.'}`, - ); -} +/** Schemes a baseline may eventually name, each rejected until it does something. */ +const RESERVED_SCHEMES = ['github', 'preview']; /** Runs a git command, throwing on a non-zero exit. */ -function gitCapture(args: string[], cwd: string): string { - const result = execaSync('git', args, { cwd, reject: false }); +async function gitCapture(args: string[], cwd: string): Promise { + const result = await execa('git', args, { cwd, reject: false }); if (result.exitCode !== 0) { throw new Error(`git ${args.join(' ')} failed: ${String(result.stderr).trim()}`); } @@ -82,67 +37,44 @@ function gitCapture(args: string[], cwd: string): string { } /** - * Creates a resolver that turns {@link RefDescriptor}s into {@link ResolvedRef}s, memoizing the - * `baseline` symbol so it is computed at most once per run. + * The revision a baseline token names. + * + * A bare value is a revision and `git:` says so explicitly, which leaves room for a baseline that is + * not a commit in this repository — a published preview, another repository — to arrive as its own + * scheme without the flag changing shape. Anything else goes to git, which is the authority on what + * a revision is. */ -export function createRefResolver(options: RefResolverOptions): { - parse: (token?: string) => ResolvedRef; -} { - const { repoRoot, baselineOverride } = options; - - let baselineRef: ResolvedRef | undefined; - // Several cases commonly pin the same commit, and resolving one spawns a git process. - const gitRefs = new Map(); - - /** Resolves a git committish to a `ResolvedRef` keyed by its immutable SHA. */ - function gitRef(committish: string): ResolvedRef { - const cached = gitRefs.get(committish); - if (cached) { - return cached; - } - const sha = gitCapture(revParseCommitArgs(committish), repoRoot); - const ref: ResolvedRef = { - kind: 'git', - id: `git-${sha.slice(0, 9)}`, - sha, - requested: committish, - }; - gitRefs.set(committish, ref); - return ref; +function committishOf(token: string | undefined): string { + if (!token) { + // Which commit a pull request should be measured against is not decided here: `code-infra + // baseline` answers that for every job that compares against a base — bundle size as well as + // benchmarks — and the harness scripts pass the answer in. Without one, the previous commit is + // the only choice that needs no policy. + return 'HEAD~1'; } - - /** - * Computes the `baseline` symbol from whatever bound it. - * - * Which commit that should be is not decided here: `code-infra baseline` answers that for every - * job that compares against a base — bundle size as well as benchmarks — so the harness scripts - * pass the answer in rather than each tool having its own opinion of a fork point. Without a - * binding the previous commit is the only choice that needs no policy. - */ - function computeBaselineRef(): ResolvedRef { - if (baselineOverride === undefined) { - return gitRef('HEAD~1'); + if (token.startsWith('git:')) { + const revision = token.slice('git:'.length); + if (!revision) { + throw new Error('The baseline "git:" names no revision.'); } - const descriptor = parseRefToken(baselineOverride); - if (descriptor.kind === 'baseline') { - throw new Error('The baseline cannot itself be "baseline" — that is the symbol it binds.'); - } - return resolve(descriptor); + return revision; } - - function resolve(descriptor: RefDescriptor): ResolvedRef { - if (descriptor.kind === 'worktree') { - return WORKTREE_REF; - } - if (descriptor.kind === 'baseline') { - baselineRef ??= computeBaselineRef(); - return baselineRef; - } - // The union has no fourth member, so this is the `git:` case and carries a revision. - return gitRef(descriptor.committish); + const reserved = RESERVED_SCHEMES.find((scheme) => token.startsWith(`${scheme}:`)); + if (reserved) { + throw new Error( + `Baseline scheme "${reserved}:" is recognised but not implemented yet (in "${token}"). ` + + `Pass a git revision, on its own or as "git:".`, + ); } + return token; +} - return { - parse: (token?: string) => resolve(parseRefToken(token)), - }; +/** Resolves the baseline to the commit behind it, so a run stays interpretable after the fact. */ +export async function resolveBaseline( + token: string | undefined, + repoRoot: string, +): Promise { + const committish = committishOf(token); + const sha = await gitCapture(revParseCommitArgs(committish), repoRoot); + return { kind: 'git', id: `git-${sha.slice(0, 9)}`, sha, requested: committish }; } diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts index f05ac72060..8e14f3ea83 100644 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -8,10 +8,10 @@ import { execaSync } from 'execa'; import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; import { packRef, packWorkingTree } from '../utils/packWorkspace'; import type { PackedPackage } from '../utils/packWorkspace'; -import { createRefResolver } from './refs'; +import { resolveBaseline, WORKTREE_REF } from './refs'; import type { ResolvedRef } from './refs'; import { discoverCases, pagesOf } from './discoverCases'; -import type { BenchmarkCase } from './discoverCases'; +import type { BenchmarkCase, Leaf } from './discoverCases'; import { refLabel } from './format'; import { buildRefPages } from './buildPages'; import { assertDriverMatchesBrowser, resolveBrowserBinary, withBrowserDefaults } from './browser'; @@ -39,7 +39,10 @@ export interface RunTachometerOptions { * Only run cases whose path under `src` contains one of these substrings, case-insensitively. */ filters?: string[]; - /** Binds the `baseline` symbol, in the ref grammar (e.g. `git:abc1234`). */ + /** + * The build every `[baseline]` variant loads: a revision, on its own or as `git:`. Defaults + * to `HEAD~1`. + */ baseline?: string; /** * Command that builds the publishable workspace packages. Run for every ref — in each ref's own @@ -91,22 +94,30 @@ export async function runTachometer(options: RunTachometerOptions): Promise(); - for (const entry of cases) { - for (const leaf of entry.leaves) { - if (leaf.ref) { - refs.set(leaf.ref.id, leaf.ref); - } - } - } + // Resolved only when something compares against it: a harness that only puts pages side by side + // needs no second commit, and asking for one would fail in a clone too shallow to hold it. + const baselineRef = cases.some((entry) => entry.comparison === 'baseline') + ? await resolveBaseline(baseline, repoRoot) + : undefined; + const refs = baselineRef ? [WORKTREE_REF, baselineRef] : [WORKTREE_REF]; // The commit follows the name, so a revision like `HEAD~1` still says which commit it landed on. const describeRef = (ref: ResolvedRef) => ref.sha ? `${refLabel(ref)} (${ref.sha.slice(0, 9)})` : refLabel(ref); + /** The build directory a variant's pages were written to. */ + const buildIdOf = (leaf: Leaf) => { + if (leaf.ref === 'current') { + return WORKTREE_REF.id; + } + if (!baselineRef) { + throw new Error(`"${leaf.page}" loads the baseline, which this run did not resolve.`); + } + return baselineRef.id; + }; + // Before any build: a driver that cannot open the browser fails the run either way, and finding // out now costs seconds instead of minutes of packing and installing. const browserBinary = await resolveBrowserBinary(harnessDir); @@ -114,9 +125,16 @@ export async function runTachometer(options: RunTachometerOptions): Promise entry.name).join(', ')}`)); + // What each case compares, so one that carries no baseline says so up front rather than through + // an empty regression list at the end. + const describeCase = (entry: BenchmarkCase) => + entry.comparison === 'baseline' + ? `${entry.name} (vs baseline)` + : `${entry.name} (${entry.variants.length} variants)`; + + console.log(chalk.cyan(`Cases: ${cases.map(describeCase).join(', ')}`)); console.log(chalk.cyan(`Pages: ${pagesOf(cases).join(', ')}`)); - console.log(chalk.cyan(`Refs: ${[...refs.values()].map(describeRef).join(', ')}`)); + console.log(chalk.cyan(`Refs: ${refs.map(describeRef).join(', ')}`)); console.log( chalk.cyan(`Browser: ${browserBinary}${asRoot ? ' (as root, so without its sandbox)' : ''}`), ); @@ -126,7 +144,7 @@ export async function runTachometer(options: RunTachometerOptions): Promise = []; for (const entry of cases) { for (const leaf of entry.leaves) { - const refId = leaf.ref ? leaf.ref.id : 'current'; - leaf.node.url = `${path.join(buildsDir, refId, leaf.page)}${leaf.suffix}`; + leaf.node.url = `${path.join(buildsDir, buildIdOf(leaf), leaf.page)}${leaf.suffix}`; } // Discovery flattened `expand` away, so every benchmark already carries its own effective // browser and there is no inheritance left to walk. @@ -214,8 +231,7 @@ export async function runTachometer(options: RunTachometerOptions): Promise [entry.name, json])), diff --git a/packages/benchmark/src/tachometer/summarizeCase.test.ts b/packages/benchmark/src/tachometer/summarizeCase.test.ts index 2130e0ab95..11678777dc 100644 --- a/packages/benchmark/src/tachometer/summarizeCase.test.ts +++ b/packages/benchmark/src/tachometer/summarizeCase.test.ts @@ -40,8 +40,9 @@ function discovered(variants: string[], measurements: string[]): any { name: 'example', configPath: '/tmp/tachometer.json', config: {}, + comparison: 'baseline', leaves: [], - variants: variants.map((name) => ({ name, refId: 'current' })), + variants, measurements, }; } diff --git a/packages/benchmark/src/tachometer/summarizeCase.ts b/packages/benchmark/src/tachometer/summarizeCase.ts index f18b16b7b4..19ab0c9da6 100644 --- a/packages/benchmark/src/tachometer/summarizeCase.ts +++ b/packages/benchmark/src/tachometer/summarizeCase.ts @@ -1,5 +1,5 @@ import chalk from 'chalk'; -import type { BenchmarkCase } from './discoverCases'; +import type { BenchmarkCase, CaseComparison } from './discoverCases'; /** Reduces tachometer's raw JSON output to a flat, interpretable summary. */ @@ -36,7 +36,6 @@ export type Verdict = 'faster' | 'slower' | 'unsure'; export interface VariantSummary { variant: string; - refId: string | null; meanMs: ConfidenceInterval; samples: number; } @@ -61,6 +60,8 @@ export interface MeasurementSummary { export interface CaseSummary { name: string; + /** What the case compares, which is what says whether a difference is a regression. */ + comparison: CaseComparison; reference: string; measurements: MeasurementSummary[]; } @@ -130,32 +131,29 @@ export function summarizeCase(entry: BenchmarkCase, json: TachometerJson): CaseS const byVariant = byMeasurement.get(measurement); const lookup = (variant: string) => byVariant?.get(variant); - const referenceResult = lookup(reference.name); + const referenceResult = lookup(reference); if (!referenceResult) { throw new Error( - `Case "${entry.name}": no "${measurement}" result for the reference "${reference.name}".`, + `Case "${entry.name}": no "${measurement}" result for the reference "${reference}".`, ); } const variants: VariantSummary[] = []; const comparisons: ComparisonSummary[] = []; for (const variant of entry.variants) { - const found = lookup(variant.name); + const found = lookup(variant); if (!found) { console.warn( - chalk.yellow( - ` ${entry.name}: no "${measurement}" result for "${variant.name}"; skipping.`, - ), + chalk.yellow(` ${entry.name}: no "${measurement}" result for "${variant}"; skipping.`), ); continue; } variants.push({ - variant: variant.name, - refId: variant.refId, + variant, meanMs: found.benchmark.mean, samples: found.benchmark.samples.length, }); - if (variant.name === reference.name) { + if (variant === reference) { continue; } const difference = referenceResult.benchmark.differences[found.index]; @@ -166,7 +164,7 @@ export function summarizeCase(entry: BenchmarkCase, json: TachometerJson): CaseS // denominator, so it is not the negation of `difference`. const reverse = found.benchmark.differences[referenceResult.index]; comparisons.push({ - variant: variant.name, + variant, verdict: verdictOf(difference), absoluteMs: difference.absolute, percentChange: difference.percentChange, @@ -182,5 +180,5 @@ export function summarizeCase(entry: BenchmarkCase, json: TachometerJson): CaseS return { name: measurement, variants, comparisons }; }); - return { name: entry.name, reference: reference.name, measurements }; + return { name: entry.name, comparison: entry.comparison, reference, measurements }; } diff --git a/test/tachometer/README.md b/test/tachometer/README.md index 6d12d722d7..f17882ea83 100644 --- a/test/tachometer/README.md +++ b/test/tachometer/README.md @@ -5,8 +5,8 @@ A working harness for `benchmark tacho run`, used to exercise that tooling end t This is **not** a meaningful benchmark, and it is not trying to be. This repository ships build and test tooling, not a browser library, so there is nothing here whose render time is worth tracking. The cases run a deterministic CPU workload purely so the pipeline has something real to drive: -discovering cases, resolving refs, packing a ref's workspace to tarballs, building each variant, and -running Chrome under tachometer. +discovering cases, resolving the baseline, packing a commit's workspace to tarballs, building each +variant, and running Chrome under tachometer. For the real thing, see the consumers this tooling was extracted from — `base-ui-mosaic` and `base-ui-charts` benchmark actual data grids and charts. @@ -19,10 +19,10 @@ The cases are chosen to walk the parts of the pipeline most likely to break: | :--------------- | :------------------------------------------------------------------------------- | | `workload` | The ordinary regression shape: one page, auto-expanded into current vs baseline | | `workload-large` | A case owning **no page of its own**, parameterising its sibling's with `?size=` | -| `libs-sort` | Three variants from **one build**: the cross-library shape, with no ref in sight | +| `libs-sort` | Three variants from **one build**: the cross-library shape, with no baseline | -`workload-large` exercises that a case's query parameters survive the url rewrite, that `?ref=` is -appended with `&` when a url already has a query, and that two cases referencing one page build it +`workload-large` exercises that a case's query parameters survive the url rewrite, that a url +traversing out of its own case folder resolves, and that two cases referencing one page build it only once. `libs-sort` covers the other axis, where nothing is compared across commits. Its three variants are @@ -49,7 +49,7 @@ Or from here, with the usual filters and flags: ```bash pnpm -F ./test/tachometer test:tacho large -pnpm -F ./test/tachometer test:tacho --baseline git:HEAD +pnpm -F ./test/tachometer test:tacho --baseline HEAD ``` A filter is a case-insensitive substring of a case's path under `src/`, the way vitest matches test @@ -92,14 +92,14 @@ every job that compares a branch to its base, so bundle size and the benchmarks instead of each deriving it. Pass it in: ```bash -pnpm test:tacho --baseline "git:$(pnpm code-infra baseline)" +pnpm test:tacho --baseline "$(pnpm code-infra baseline)" ``` Without `--baseline` the previous commit is used, which needs no policy to decide. -A ref older than the introduction of `benchmark tacho` will fail to build its pages: this harness's -vite config imports a plugin that did not exist at that commit. `--baseline git:HEAD` compares the -working tree against the current commit while that is still true. +A baseline older than the introduction of `benchmark tacho` will fail to build its pages: this +harness's vite config imports a plugin that did not exist at that commit. `--baseline HEAD` compares +the working tree against the current commit while that is still true. ## Working on a case by hand From aa12530bef3e66fd8e3d4f1ae403d7ca05ebd81f Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:01:20 +0200 Subject: [PATCH 057/173] [core] Build packages before the static and unit checks The dashboard imports `@mui/internal-benchmark`, which publishes from `build/` and is therefore linked there rather than at its source. Without that directory the subpath exports resolve to nothing: every type from the package degrades to `any` and the value import fails outright. Both jobs passed locally only because a developer's tree already has a build. --- .circleci/config.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9905c72681..96c6d38fc1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -66,6 +66,11 @@ jobs: # make sure the netlify ignore command has the correct dependencies name: '`pnpm update-netlify-ignore` changes committed?' command: pnpm code-infra netlify-ignore @apps/code-infra-dashboard --check + - run: + # A package that publishes from `build/` is linked there, not at its source, so a consumer + # of one sees no types at all until it is built. + name: Build packages + command: pnpm release:build - run: name: Tests TypeScript command: pnpm typescript @@ -74,6 +79,11 @@ jobs: steps: - checkout - code-infra/install-deps + - run: + # A package that publishes from `build/` is linked there, not at its source, so an import + # from one fails to resolve until it is built. + name: Build packages + command: pnpm release:build - run: name: 'Run tests' command: pnpm test From 392914244d4ab315d75c75043cf8b0c63337fdbc Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 17:25:55 +0200 Subject: [PATCH 058/173] [benchmark] Fold the review's cleanups into the baseline change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One resolver for a committish, in the utils module that already held the half of it both callers shared. Leaves are resolved into their own shape rather than zipped back onto the node list by index, and the runner asks one lookup which build a variant loads — its two branches are total, so there is no longer an unreachable error to explain. The degrade path is typed against the report schema, so a new case field is a compile error there rather than an upload failure. --- .../src/tachometer/discoverCases.test.ts | 78 +++++++++---------- .../benchmark/src/tachometer/discoverCases.ts | 12 ++- .../benchmark/src/tachometer/refs.test.ts | 12 ++- packages/benchmark/src/tachometer/refs.ts | 14 +--- .../benchmark/src/tachometer/runTachometer.ts | 48 +++++++----- packages/benchmark/src/utils/git.ts | 19 +++-- packages/benchmark/src/utils/packWorkspace.ts | 14 +--- 7 files changed, 96 insertions(+), 101 deletions(-) diff --git a/packages/benchmark/src/tachometer/discoverCases.test.ts b/packages/benchmark/src/tachometer/discoverCases.test.ts index 9edafdb53d..95f3aa0c98 100644 --- a/packages/benchmark/src/tachometer/discoverCases.test.ts +++ b/packages/benchmark/src/tachometer/discoverCases.test.ts @@ -149,6 +149,9 @@ describe('discoverCases', () => { }); it('points the two variants at the two builds', async () => { + // The temp harness is no repository at all, so anything reaching for git would fail here. + // Which build a variant loads follows from the case's shape, which is what lets a plain + // `vite build` run in a checkout with no history. const harnessDir = await makeHarness({ alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, }); @@ -329,53 +332,24 @@ describe('discoverCases', () => { await expect(discoverCases({ harnessDir })).rejects.toThrow(/resolves outside/); }); - it('rejects a url pointing at a missing page', async () => { + it('rejects a benchmark that names no url', async () => { + // Named for the benchmark the author wrote rather than for a variant they did not, which is + // what the auto-expanded pair would otherwise be reported as. const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './missing.html') }, + alpha: { config: { benchmarks: [{ name: 'alpha' }] }, pages: ['index.html'] }, }); - await expect(discoverCases({ harnessDir })).rejects.toThrow(/missing page/); - }); - }); - - describe('outside a repository', () => { - it('names both builds without asking git anything', async () => { - // The temp harness is no repository at all, so anything reaching for git would fail here. - // Which build a variant loads follows from the case's shape, which is what lets a plain - // `vite build` run in a checkout with no history. - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.leaves.map((leaf) => leaf.ref)).toEqual(['current', 'baseline']); + await expect(discoverCases({ harnessDir })).rejects.toThrow( + /Benchmark "alpha" in .* has no "url"/, + ); }); - it('still yields the full page list', async () => { + it('rejects a url pointing at a missing page', async () => { const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - libs: { - config: { - benchmarks: [ - { - name: 'libs', - expand: [ - { name: 'libs [ours]', url: './ours.html' }, - { name: 'libs [theirs]', url: './theirs.html' }, - ], - }, - ], - }, - pages: ['ours.html', 'theirs.html'], - }, + alpha: { config: config('alpha', './missing.html') }, }); - expect(pagesOf(await discoverCases({ harnessDir }))).toEqual([ - path.join('alpha', 'index.html'), - path.join('libs', 'ours.html'), - path.join('libs', 'theirs.html'), - ]); + await expect(discoverCases({ harnessDir })).rejects.toThrow(/missing page/); }); }); }); @@ -391,6 +365,32 @@ describe('pagesOf', () => { path.join('alpha', 'index.html'), ]); }); + + it('unions the pages of every case, sorted', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + libs: { + config: { + benchmarks: [ + { + name: 'libs', + expand: [ + { name: 'libs [ours]', url: './ours.html' }, + { name: 'libs [theirs]', url: './theirs.html' }, + ], + }, + ], + }, + pages: ['ours.html', 'theirs.html'], + }, + }); + + expect(pagesOf(await discoverCases({ harnessDir }))).toEqual([ + path.join('alpha', 'index.html'), + path.join('libs', 'ours.html'), + path.join('libs', 'theirs.html'), + ]); + }); }); describe('measurementNameOf', () => { diff --git a/packages/benchmark/src/tachometer/discoverCases.ts b/packages/benchmark/src/tachometer/discoverCases.ts index f8d3d6f7e1..e19d59753b 100644 --- a/packages/benchmark/src/tachometer/discoverCases.ts +++ b/packages/benchmark/src/tachometer/discoverCases.ts @@ -273,16 +273,20 @@ export async function discoverCases(options: DiscoverCasesOptions): Promise entry.node); // eslint-disable-next-line no-await-in-loop - const resolved = await Promise.all( - nodes.map((entry) => parseLeafUrl(entry.node.url, configDir, srcDir)), + const leaves: Leaf[] = await Promise.all( + nodes.map(async ({ node, ref }) => ({ + node, + ref, + ...(await parseLeafUrl(node.url, configDir, srcDir)), + })), ); cases.push({ name, configPath, config, - comparison: nodes.some((entry) => entry.ref === 'baseline') ? 'baseline' : 'variants', - leaves: nodes.map((entry, index) => ({ ...entry, ...resolved[index] })), + comparison: leaves.some((leaf) => leaf.ref === 'baseline') ? 'baseline' : 'variants', + leaves, variants: nodes.map((entry) => entry.node.name as string), measurements: [...measurements], }); diff --git a/packages/benchmark/src/tachometer/refs.test.ts b/packages/benchmark/src/tachometer/refs.test.ts index 04a74648ad..53d184d40f 100644 --- a/packages/benchmark/src/tachometer/refs.test.ts +++ b/packages/benchmark/src/tachometer/refs.test.ts @@ -10,11 +10,7 @@ import { resolveBaseline } from './refs'; * git reports — what a tag resolves to, which refnames exist — and a fake would only restate the * answer the code already assumes. */ -async function makeRepo(): Promise<{ - repoRoot: string; - shas: string[]; - git: (...args: string[]) => Promise; -}> { +async function makeRepo(): Promise<{ repoRoot: string; shas: string[] }> { const repoRoot = await makeTempDir(); // Identity through the environment rather than `git config`, so the repo needs no extra processes // and nothing depends on the machine's own git configuration. @@ -41,7 +37,7 @@ async function makeRepo(): Promise<{ // Annotated (`-m`), so the tag is its own object with its own SHA. await git('tag', '-a', 'v1.0.0', '-m', 'release', shas[0]); - return { repoRoot, shas, git }; + return { repoRoot, shas }; } describe('resolveBaseline', () => { @@ -121,6 +117,8 @@ describe('resolveBaseline', () => { // Git is the authority on what a revision is, so an unknown one surfaces as its own failure // rather than as a grammar this code would have to keep in step with git's. - await expect(() => resolveBaseline('no-such-thing', repoRoot)).rejects.toThrow(/rev-parse/); + await expect(() => resolveBaseline('no-such-thing', repoRoot)).rejects.toThrow( + /Could not resolve git ref "no-such-thing"/, + ); }); }); diff --git a/packages/benchmark/src/tachometer/refs.ts b/packages/benchmark/src/tachometer/refs.ts index 9716b51770..e29eb86d99 100644 --- a/packages/benchmark/src/tachometer/refs.ts +++ b/packages/benchmark/src/tachometer/refs.ts @@ -1,5 +1,4 @@ -import { execa } from 'execa'; -import { revParseCommitArgs } from '../utils/git'; +import { resolveCommit } from '../utils/git'; /** * The builds a run measures: the working tree, and the baseline it is compared against. @@ -27,15 +26,6 @@ export const WORKTREE_REF: ResolvedRef = { kind: 'worktree', id: 'current' }; /** Schemes a baseline may eventually name, each rejected until it does something. */ const RESERVED_SCHEMES = ['github', 'preview']; -/** Runs a git command, throwing on a non-zero exit. */ -async function gitCapture(args: string[], cwd: string): Promise { - const result = await execa('git', args, { cwd, reject: false }); - if (result.exitCode !== 0) { - throw new Error(`git ${args.join(' ')} failed: ${String(result.stderr).trim()}`); - } - return result.stdout.trim(); -} - /** * The revision a baseline token names. * @@ -75,6 +65,6 @@ export async function resolveBaseline( repoRoot: string, ): Promise { const committish = committishOf(token); - const sha = await gitCapture(revParseCommitArgs(committish), repoRoot); + const sha = await resolveCommit(repoRoot, committish); return { kind: 'git', id: `git-${sha.slice(0, 9)}`, sha, requested: committish }; } diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts index 8e14f3ea83..9fde3118b2 100644 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -20,7 +20,7 @@ import { renderTachometerReport } from './renderReport'; import { getCiMetadata } from '../ciReport'; import { syncPrComment } from '../syncPrComment'; import { uploadCiReport } from './ciReport'; -import type { TachometerReport } from './ciReport'; +import type { CaseResult, TachometerReport } from './ciReport'; import { buildsDirOf, prepareOutputDir } from './outputDir'; import { run } from '../utils/exec'; @@ -32,6 +32,29 @@ function fileSlugOf(name: string): string { return name.replace(/[^a-zA-Z0-9._-]+/g, '-'); } +/** + * The builds a run measures, and which of them a given variant loads. + * + * The baseline is resolved only when a case compares against it: a harness that only puts pages + * side by side needs no second commit, and asking for one would fail in a clone too shallow to hold + * it. The lookup comes back with the list so that "there is no baseline" is settled once, here, + * rather than being a case every later step has to allow for. + */ +async function resolveBuilds( + cases: BenchmarkCase[], + baseline: string | undefined, + repoRoot: string, +): Promise<{ refs: ResolvedRef[]; buildFor: (leaf: Leaf) => ResolvedRef }> { + if (!cases.some((entry) => entry.comparison === 'baseline')) { + return { refs: [WORKTREE_REF], buildFor: () => WORKTREE_REF }; + } + const baselineRef = await resolveBaseline(baseline, repoRoot); + return { + refs: [WORKTREE_REF, baselineRef], + buildFor: (leaf) => (leaf.ref === 'baseline' ? baselineRef : WORKTREE_REF), + }; +} + export interface RunTachometerOptions { /** The harness package directory (where the command was run). */ harnessDir: string; @@ -95,29 +118,12 @@ export async function runTachometer(options: RunTachometerOptions): Promise entry.comparison === 'baseline') - ? await resolveBaseline(baseline, repoRoot) - : undefined; - const refs = baselineRef ? [WORKTREE_REF, baselineRef] : [WORKTREE_REF]; + const { refs, buildFor } = await resolveBuilds(cases, baseline, repoRoot); // The commit follows the name, so a revision like `HEAD~1` still says which commit it landed on. const describeRef = (ref: ResolvedRef) => ref.sha ? `${refLabel(ref)} (${ref.sha.slice(0, 9)})` : refLabel(ref); - /** The build directory a variant's pages were written to. */ - const buildIdOf = (leaf: Leaf) => { - if (leaf.ref === 'current') { - return WORKTREE_REF.id; - } - if (!baselineRef) { - throw new Error(`"${leaf.page}" loads the baseline, which this run did not resolve.`); - } - return baselineRef.id; - }; - // Before any build: a driver that cannot open the browser fails the run either way, and finding // out now costs seconds instead of minutes of packing and installing. const browserBinary = await resolveBrowserBinary(harnessDir); @@ -184,7 +190,7 @@ export async function runTachometer(options: RunTachometerOptions): Promise = []; for (const entry of cases) { for (const leaf of entry.leaves) { - leaf.node.url = `${path.join(buildsDir, buildIdOf(leaf), leaf.page)}${leaf.suffix}`; + leaf.node.url = `${path.join(buildsDir, buildFor(leaf).id, leaf.page)}${leaf.suffix}`; } // Discovery flattened `expand` away, so every benchmark already carries its own effective // browser and there is no inheritance left to walk. @@ -235,7 +241,7 @@ export async function runTachometer(options: RunTachometerOptions): Promise { + cases: results.map(({ entry, json }): CaseResult => { try { return summarizeCase(entry, json); } catch (error) { diff --git a/packages/benchmark/src/utils/git.ts b/packages/benchmark/src/utils/git.ts index 12e192d0f2..4072b54810 100644 --- a/packages/benchmark/src/utils/git.ts +++ b/packages/benchmark/src/utils/git.ts @@ -1,14 +1,23 @@ +import { execa } from 'execa'; + /** - * Args that resolve a committish to the commit it names. + * Resolves a committish to the commit it names. * * `^{commit}` peels an annotated tag to its commit: `git rev-parse v1.2.3` returns the tag *object*, * whose SHA is not the commit's, so the same commit reached by a tag and by its SHA would look like * two different builds. `--verify` turns an unknown ref into a non-zero exit rather than an echoed * argument. * - * Shared because two callers resolve a ref — the ref resolver, and `packRef` to key its cache — and - * they disagreed about this. + * Shared because two callers resolve a ref — the baseline, and `packRef` to key its cache — and a + * commit they disagreed about would be built twice. */ -export function revParseCommitArgs(ref: string): string[] { - return ['rev-parse', '--verify', `${ref}^{commit}`]; +export async function resolveCommit(repoRoot: string, ref: string): Promise { + const result = await execa('git', ['rev-parse', '--verify', `${ref}^{commit}`], { + cwd: repoRoot, + reject: false, + }); + if (result.exitCode !== 0) { + throw new Error(`Could not resolve git ref "${ref}": ${String(result.stderr).trim()}`); + } + return result.stdout.trim(); } diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index 8ff13cc6f7..68f7361fb7 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -11,7 +11,7 @@ import { execa, parseCommandString } from 'execa'; import { mapConcurrently } from './build'; import { run } from './exec'; import { pathExists } from './path'; -import { revParseCommitArgs } from './git'; +import { resolveCommit } from './git'; import { listPublishablePackages } from './pnpm'; /** @@ -78,18 +78,6 @@ export interface PackRefOptions { /** Name of the file written into each packed folder describing its contents. */ const MANIFEST = 'manifest.json'; -/** Resolves a git ref to its commit SHA. */ -async function resolveCommit(repoRoot: string, ref: string): Promise { - const result = await execa('git', revParseCommitArgs(ref), { - cwd: repoRoot, - reject: false, - }); - if (result.exitCode !== 0) { - throw new Error(`Could not resolve git ref "${ref}": ${String(result.stderr).trim()}`); - } - return result.stdout.trim(); -} - /** Turns a package name into a filesystem-safe tarball basename (`@scope/pkg` → `scope-pkg.tgz`). */ export function tarballName(pkgName: string): string { return `${pkgName.replace(/^@/, '').replace(/\//g, '-')}.tgz`; From 8a6cf63d7734dd04686055703d4e00efa68e3ea5 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:27:30 +0200 Subject: [PATCH 059/173] [benchmark] Act on the review: measurements, variant names, Windows paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measurements are read off the flattened nodes, so an expansion that declares its own is not reported as the parent's default — which tachometer would then have no matching result for. Two variants that end up sharing a name now fail discovery. An expansion that names itself nothing inherits its parent's, so one silently shadowed the other in the results. A page path is posix, because it is a url as often as it is a path: the case index links to it and rollup names a chunk after it, both of which a native separator breaks on Windows. The index escapes what it interpolates. The list of dependencies a ref's tree does without applies to `dependencies` too; neither reason for the list turns on which section a package sits in. --- .../benchmark/src/tachometer/buildPages.ts | 10 ++- .../src/tachometer/discoverCases.test.ts | 64 ++++++++++++++++--- .../benchmark/src/tachometer/discoverCases.ts | 39 ++++++++--- packages/benchmark/src/tachometer/plugin.ts | 16 ++++- packages/code-infra/src/utils/git.mjs | 54 +++++++++++++--- packages/code-infra/src/utils/git.test.mjs | 59 ++++++++++++++++- 6 files changed, 210 insertions(+), 32 deletions(-) diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts index 8dab241fff..d3a1a24c7d 100644 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -18,7 +18,8 @@ import { refLabel } from './format'; import type { ResolvedRef } from './refs'; /** - * Dependencies the run needs but a page never imports, so a ref's tree does without them. + * Dependencies the run needs but a page never imports, so a ref's tree does without them — + * whichever section of the harness's manifest they sit in, since neither reason below turns on that. * * Every one is resolved from the harness rather than from here: the browser and its driver by the * runner, tachometer by the command that samples, and this package by the harness's own vite config @@ -134,7 +135,12 @@ async function installRefTree({ name: 'tacho-resolve-tree', private: true, version: '0.0.0', - dependencies: rewriteWorkspaceDeps(installed, harnessPkg.dependencies, packages), + dependencies: rewriteWorkspaceDeps( + installed, + harnessPkg.dependencies, + packages, + RUNNER_ONLY_DEPS, + ), devDependencies: rewriteWorkspaceDeps( installed, harnessPkg.devDependencies, diff --git a/packages/benchmark/src/tachometer/discoverCases.test.ts b/packages/benchmark/src/tachometer/discoverCases.test.ts index 95f3aa0c98..08d3285563 100644 --- a/packages/benchmark/src/tachometer/discoverCases.test.ts +++ b/packages/benchmark/src/tachometer/discoverCases.test.ts @@ -184,7 +184,7 @@ describe('discoverCases', () => { }); const [entry] = await discoverCases({ harnessDir }); - const target = `${path.join('alpha', 'index.html')}?rows=100#state`; + const target = 'alpha/index.html?rows=100#state'; expect(entry.leaves.map((leaf) => `${leaf.page}${leaf.suffix}`)).toEqual([target, target]); }); @@ -315,7 +315,7 @@ describe('discoverCases', () => { const throttled = cases.find((entry) => entry.name === 'updates-throttled'); expect(throttled?.leaves[0]).toMatchObject({ - page: path.join('updates', 'index.html'), + page: 'updates/index.html', suffix: '?throttle=16', }); }); @@ -361,9 +361,7 @@ describe('pagesOf', () => { alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, }); - expect(pagesOf(await discoverCases({ harnessDir }))).toEqual([ - path.join('alpha', 'index.html'), - ]); + expect(pagesOf(await discoverCases({ harnessDir }))).toEqual(['alpha/index.html']); }); it('unions the pages of every case, sorted', async () => { @@ -386,9 +384,9 @@ describe('pagesOf', () => { }); expect(pagesOf(await discoverCases({ harnessDir }))).toEqual([ - path.join('alpha', 'index.html'), - path.join('libs', 'ours.html'), - path.join('libs', 'theirs.html'), + 'alpha/index.html', + 'libs/ours.html', + 'libs/theirs.html', ]); }); }); @@ -517,6 +515,56 @@ describe('case variants and measurements', () => { expect(entry.measurements).toEqual(['mount', 'scroll']); }); + it('collects the measurements an expansion declares for itself', async () => { + // Read from the parent, a case whose children each measure something different reports the + // parent's default while tachometer reports theirs, and pairing the two then finds nothing. + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [ + { + name: 'alpha', + expand: [ + { + name: 'alpha [one]', + url: './index.html', + measurement: { mode: 'performance', entryName: 'mount' }, + }, + { + name: 'alpha [two]', + url: './other.html', + measurement: { mode: 'performance', entryName: 'paint' }, + }, + ], + }, + ], + }, + pages: ['index.html', 'other.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.measurements).toEqual(['mount', 'paint']); + }); + + it('refuses two variants that end up with the same name', async () => { + // An expansion that names itself nothing inherits its parent's name, so this is reachable from + // a config that looks fine. Results are paired back by name, so one would shadow the other. + const harnessDir = await makeHarness({ + libs: { + config: { + benchmarks: [ + { name: 'libs', expand: [{ url: './ours.html' }, { url: './theirs.html' }] }, + ], + }, + pages: ['ours.html', 'theirs.html'], + }, + }); + + await expect(discoverCases({ harnessDir })).rejects.toThrow(/has two variants named "libs"/); + }); + it('defaults to the callback measurement when none is declared', async () => { const harnessDir = await makeHarness({ alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, diff --git a/packages/benchmark/src/tachometer/discoverCases.ts b/packages/benchmark/src/tachometer/discoverCases.ts index e19d59753b..dbf1b1ce47 100644 --- a/packages/benchmark/src/tachometer/discoverCases.ts +++ b/packages/benchmark/src/tachometer/discoverCases.ts @@ -27,7 +27,7 @@ export type CaseComparison = 'baseline' | 'variants'; export interface Leaf { /** The config node whose `url` this is; rewritten in place once builds exist. */ node: { url?: string }; - /** Page path relative to `src`, e.g. `data-grid-init/index.html`. */ + /** Page path relative to `src`, posix, e.g. `data-grid-init/index.html`. */ page: string; /** The build this variant loads. */ ref: BuildRef; @@ -138,14 +138,16 @@ async function parseLeafUrl( const suffix = url.slice(pathPart.length); const absolute = path.resolve(configDir, decodeURIComponent(pathPart)); - const page = path.relative(srcDir, absolute); - if (page.startsWith('..') || path.isAbsolute(page)) { + const relative = path.relative(srcDir, absolute); + if (relative.startsWith('..') || path.isAbsolute(relative)) { throw new Error(`Benchmark url "${url}" resolves outside ${srcDir}.`); } if (!(await pathExists(absolute))) { throw new Error(`Benchmark url "${url}" points at a missing page (${absolute}).`); } - return { page, suffix }; + // Posix, because a page path is a url as often as it is a path — the case index links to it and + // rollup names a chunk after it — and node takes forward slashes on Windows for the rest. + return { page: relative.split(path.sep).join('/'), suffix }; } /** @@ -234,14 +236,19 @@ export async function discoverCases(options: DiscoverCasesOptions): Promise 0; const expanded = flattenExpansions(benchmark); + + // Read from the flattened nodes, not the parent: an expansion may set its own `measurement`, + // and a parent that names none would otherwise contribute `callback` while tachometer reports + // what the child asked for. `measurement` may also be a single entry or a list, and a page + // with several is exactly the case whose results are paired by name rather than by position. + for (const node of expanded) { + for (const measurement of [node.measurement ?? 'callback'].flat()) { + measurements.add(measurementNameOf(measurement, node.measurementExpression)); + } + } if (declaresVariants) { for (const node of expanded) { nodes.push({ node, ref: 'current' }); @@ -268,6 +275,20 @@ export async function discoverCases(options: DiscoverCasesOptions): Promise(); + for (const { node } of nodes) { + if (named.has(node.name)) { + throw new Error( + `Benchmark case "${name}" has two variants named "${node.name}" (${configPath}). ` + + `Names identify a variant in the report and in its results, so they have to be unique.`, + ); + } + named.add(node.name); + } + // The flattened list is what tachometer is handed: it expands to exactly this, and every later // step — the url rewrite, the browser defaults — then addresses one whole benchmark at a time. config.benchmarks = nodes.map((entry) => entry.node); diff --git a/packages/benchmark/src/tachometer/plugin.ts b/packages/benchmark/src/tachometer/plugin.ts index cb51b6b372..0d2a7b72e6 100644 --- a/packages/benchmark/src/tachometer/plugin.ts +++ b/packages/benchmark/src/tachometer/plugin.ts @@ -57,6 +57,15 @@ async function assertWorkspaceDepsBuilt( ); } +/** A case name and a query both come from a config file, so neither is trusted as markup. */ +function escapeHtml(text: string): string { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + /** * Renders the case index: one link per case, carrying whatever query the case parameterises its * page with. @@ -68,9 +77,12 @@ function renderIndex(cases: BenchmarkCase[], note: string): string { // with different queries (a parameterised case), so dedupe on the pair. const targets = [...new Set(entry.leaves.map((leaf) => `${leaf.page}${leaf.suffix}`))].sort(); const links = targets - .map((target) => `
  • ${target}
    • `) + .map( + (target) => + `
  • ${escapeHtml(target)}
    • `, + ) .join('\n '); - return `
  • \n ${entry.name}\n
      \n ${links}\n
    \n
    • `; + return `
  • \n ${escapeHtml(entry.name)}\n
      \n ${links}\n
    \n
    • `; }) .join('\n'); diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index 1c4c6c0093..f8885c4c1a 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -102,17 +102,37 @@ function branchNameOf(shortRef) { } /** - * The branch pull requests fork from, read from `origin`'s default branch and falling back to - * `master` when the repository has no `origin/HEAD` — which is common in a CI clone. + * The branch pull requests fork from, read from `origin`'s default branch. + * + * A CI clone often has no `origin/HEAD` — it records only the refs it fetched — so the conventional + * names are tried against the refs that are actually there. Neither guessing between them nor + * assuming one is safe: the wrong guess resolves to no ref at all, and the baseline then silently + * degrades to the previous commit, which looks like an answer. * @param {string} [cwd=process.cwd()] * @returns {Promise} */ export async function detectBaseBranch(cwd = process.cwd()) { - const result = await $({ cwd, reject: false })`git symbolic-ref --short refs/remotes/origin/HEAD`; - if (result.exitCode !== 0) { - return 'master'; + const head = await $({ cwd, reject: false })`git symbolic-ref --short refs/remotes/origin/HEAD`; + if (head.exitCode === 0) { + return branchNameOf(head.stdout.trim()); } - return branchNameOf(result.stdout.trim()); + + for (const candidate of ['main', 'master']) { + // Sequential on purpose: two git processes at most, and the first hit wins. + // eslint-disable-next-line no-await-in-loop + const ref = await $({ + cwd, + reject: false, + })`git rev-parse --verify --quiet ${`refs/remotes/origin/${candidate}`}`; + if (ref.exitCode === 0) { + return candidate; + } + } + + throw new Error( + 'Could not tell which branch this one forks from: there is no "origin/HEAD", and neither ' + + '"origin/main" nor "origin/master" is present. Fetch the base branch, or name it explicitly.', + ); } /** @@ -172,9 +192,18 @@ export async function closestBaseBranch(baseBranch, cwd = process.cwd()) { continue; } const mergeBase = result.stdout.trim(); + // `--no-show-signature` because `log.showSignature` would otherwise prepend verification lines + // to the output, and the timestamp would read as NaN — which loses every later comparison and + // leaves whichever candidate came first standing. // eslint-disable-next-line no-await-in-loop - const shown = await $({ cwd })`git show -s --format=%ct ${mergeBase}`; + const shown = await $({ + cwd, + reject: false, + })`git show -s --no-show-signature --format=%ct ${mergeBase}`; const when = Number(shown.stdout.trim()); + if (shown.exitCode !== 0 || !Number.isFinite(when)) { + continue; + } // Strictly greater, so on ties the higher-priority (earlier-sorted) ref wins. if (!best || when > best.when) { best = { ref, mergeBase, when }; @@ -199,7 +228,16 @@ export async function resolveBaseline(options = {}) { const cwd = options.cwd ?? process.cwd(); const baseBranch = options.baseBranch ?? (await detectBaseBranch(cwd)); - const previousCommit = async () => (await $({ cwd })`git rev-parse HEAD~1`).stdout.trim(); + const previousCommit = async () => { + const result = await $({ cwd, reject: false })`git rev-parse --verify HEAD~1`; + if (result.exitCode !== 0) { + throw new Error( + 'HEAD has no parent commit, so there is nothing here to compare against. A shallow clone ' + + 'is the usual cause — fetch more history, or name the base branch explicitly.', + ); + } + return result.stdout.trim(); + }; const branch = (await $({ cwd })`git rev-parse --abbrev-ref HEAD`).stdout.trim(); if (branch === baseBranch) { diff --git a/packages/code-infra/src/utils/git.test.mjs b/packages/code-infra/src/utils/git.test.mjs index 7acf9a8a5f..b0c71334d8 100644 --- a/packages/code-infra/src/utils/git.test.mjs +++ b/packages/code-infra/src/utils/git.test.mjs @@ -3,7 +3,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { describe, expect, it, onTestFinished } from 'vitest'; import { execa } from 'execa'; -import { resolveBaseline } from './git.mjs'; +import { detectBaseBranch, resolveBaseline } from './git.mjs'; /** * A repository with three commits on `main`. @@ -19,8 +19,10 @@ async function makeRepo() { await fs.rm(repoRoot, { recursive: true, force: true }); }); - // Identity through the environment rather than `git config`, so the repo needs no extra processes - // and nothing depends on the machine's own git configuration. + // Everything through the environment rather than `git config`, so the repo needs no extra + // processes. The two `GIT_CONFIG_*` variables are what make the second half true: without them a + // contributor's own `commit.gpgsign` or `core.hooksPath` still applies, and these commits fail + // for a reason that has nothing to do with the code under test. /** @param {string} [date] Commit date, so ranking by recency is not left to the clock. */ const gitWith = (date) => @@ -32,6 +34,8 @@ async function makeRepo() { GIT_AUTHOR_EMAIL: 'fixture@example.com', GIT_COMMITTER_NAME: 'Fixture', GIT_COMMITTER_EMAIL: 'fixture@example.com', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_CONFIG_SYSTEM: '/dev/null', ...(date ? { GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date } : {}), }, }); @@ -100,3 +104,52 @@ describe('resolveBaseline', () => { expect(sha).toBe(shas[1]); }); }); + +describe('detectBaseBranch', () => { + it('reads the default branch from origin/HEAD', async () => { + const { repoRoot, git } = await makeRepo(); + await git('update-ref', 'refs/remotes/origin/trunk', 'HEAD'); + await git('symbolic-ref', 'refs/remotes/origin/HEAD', 'refs/remotes/origin/trunk'); + + expect(await detectBaseBranch(repoRoot)).toBe('trunk'); + }); + + it('falls back to whichever conventional branch is actually there', async () => { + // A CI clone records only the refs it fetched, so `origin/HEAD` is usually absent. Assuming a + // name resolves to no ref at all, and the baseline then quietly becomes the previous commit. + const { repoRoot, git } = await makeRepo(); + await git('update-ref', 'refs/remotes/origin/main', 'HEAD'); + + expect(await detectBaseBranch(repoRoot)).toBe('main'); + }); + + it('fails when nothing says which branch is the base', async () => { + const { repoRoot } = await makeRepo(); + + await expect(detectBaseBranch(repoRoot)).rejects.toThrow(/Could not tell which branch/); + }); +}); + +describe('resolveBaseline without history', () => { + it('explains that a root commit has nothing behind it', async () => { + const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'baseline-root-')); + onTestFinished(async () => { + await fs.rm(repoRoot, { recursive: true, force: true }); + }); + const env = { + GIT_AUTHOR_NAME: 'Fixture', + GIT_AUTHOR_EMAIL: 'fixture@example.com', + GIT_COMMITTER_NAME: 'Fixture', + GIT_COMMITTER_EMAIL: 'fixture@example.com', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_CONFIG_SYSTEM: '/dev/null', + }; + await execa('git', ['init', '--initial-branch=main'], { cwd: repoRoot, env }); + await execa('git', ['commit', '--allow-empty', '-m', 'first'], { cwd: repoRoot, env }); + + // A shallow clone reaches the same place, which is the common one: the parent was never fetched. + await expect(resolveBaseline({ cwd: repoRoot, baseBranch: 'main' })).rejects.toThrow( + /HEAD has no parent commit/, + ); + }); +}); From bc8b33393e04f77bbeb9c54b969e413bae113529 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:34:28 +0200 Subject: [PATCH 060/173] [code-infra] Read the base branch from a remote's HEAD, never from a name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `/HEAD` is git's own record of a remote's default branch, so it is right for a repository whose default is neither `main` nor `master`. Probing those names is a guess that can be confidently wrong, and a wrong one resolves to no ref at all — leaving the baseline as the previous commit. It is often absent: only `git clone` writes one, only for `origin`. A remote added by hand has none, and neither does a CI checkout — so the tachometer job names the branch it already fetches a line above. --- .circleci/config.yml | 3 +- packages/code-infra/src/cli/cmdBaseline.mjs | 2 +- packages/code-infra/src/utils/git.mjs | 32 ++++++++++----------- packages/code-infra/src/utils/git.test.mjs | 21 ++++++++++---- 4 files changed, 34 insertions(+), 24 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 96c6d38fc1..162abbe248 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -130,7 +130,8 @@ jobs: command: | set -e git fetch --no-tags origin master:refs/remotes/origin/master - BASE_SHA=$(pnpm code-infra baseline) + # Named, because a CI checkout records no `origin/HEAD` for it to be detected from. + BASE_SHA=$(pnpm code-infra baseline --base-branch master) echo "Baseline commit: $BASE_SHA" echo "$BASE_SHA" > .tachometer-baseline-sha - restore_cache: diff --git a/packages/code-infra/src/cli/cmdBaseline.mjs b/packages/code-infra/src/cli/cmdBaseline.mjs index 4ffd1fda45..9dcb58f5d5 100644 --- a/packages/code-infra/src/cli/cmdBaseline.mjs +++ b/packages/code-infra/src/cli/cmdBaseline.mjs @@ -17,7 +17,7 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ .option('base-branch', { type: 'string', description: - 'Branch pull requests fork from. Default: detected from origin/HEAD, falling back to master', + "Branch pull requests fork from. Default: read from a remote's HEAD, which a CI checkout does not record", }) .example( 'BASE=$($0 baseline)', diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index f8885c4c1a..90451a8c3d 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -102,36 +102,34 @@ function branchNameOf(shortRef) { } /** - * The branch pull requests fork from, read from `origin`'s default branch. + * The branch pull requests fork from, from `/HEAD` — the only local record of a remote's + * default branch, and often absent: only `git clone` writes one, and only for `origin`. * - * A CI clone often has no `origin/HEAD` — it records only the refs it fetched — so the conventional - * names are tried against the refs that are actually there. Neither guessing between them nor - * assuming one is safe: the wrong guess resolves to no ref at all, and the baseline then silently - * degrades to the previous commit, which looks like an answer. + * Throws rather than guessing a conventional name, which resolves to no ref in + * {@link closestBaseBranch} and leaves the baseline as the previous commit. * @param {string} [cwd=process.cwd()] * @returns {Promise} */ export async function detectBaseBranch(cwd = process.cwd()) { - const head = await $({ cwd, reject: false })`git symbolic-ref --short refs/remotes/origin/HEAD`; - if (head.exitCode === 0) { - return branchNameOf(head.stdout.trim()); - } + const listed = (await $({ cwd })`git remote`).stdout.trim().split('\n').filter(Boolean); + // `origin` first: it is the one a clone records a HEAD for. + const remotes = [...new Set(['origin', ...listed])].filter((remote) => listed.includes(remote)); - for (const candidate of ['main', 'master']) { - // Sequential on purpose: two git processes at most, and the first hit wins. + for (const remote of remotes) { + // Sequential: a handful of remotes, and the first hit wins. // eslint-disable-next-line no-await-in-loop - const ref = await $({ + const head = await $({ cwd, reject: false, - })`git rev-parse --verify --quiet ${`refs/remotes/origin/${candidate}`}`; - if (ref.exitCode === 0) { - return candidate; + })`git symbolic-ref --short ${`refs/remotes/${remote}/HEAD`}`; + if (head.exitCode === 0) { + return branchNameOf(head.stdout.trim()); } } throw new Error( - 'Could not tell which branch this one forks from: there is no "origin/HEAD", and neither ' + - '"origin/main" nor "origin/master" is present. Fetch the base branch, or name it explicitly.', + 'Could not tell which branch this one forks from: no remote records a default branch. ' + + 'Run `git remote set-head origin -a` to write one, or name the base branch explicitly.', ); } diff --git a/packages/code-infra/src/utils/git.test.mjs b/packages/code-infra/src/utils/git.test.mjs index b0c71334d8..131784b42f 100644 --- a/packages/code-infra/src/utils/git.test.mjs +++ b/packages/code-infra/src/utils/git.test.mjs @@ -108,22 +108,33 @@ describe('resolveBaseline', () => { describe('detectBaseBranch', () => { it('reads the default branch from origin/HEAD', async () => { const { repoRoot, git } = await makeRepo(); + await git('remote', 'add', 'origin', repoRoot); await git('update-ref', 'refs/remotes/origin/trunk', 'HEAD'); await git('symbolic-ref', 'refs/remotes/origin/HEAD', 'refs/remotes/origin/trunk'); expect(await detectBaseBranch(repoRoot)).toBe('trunk'); }); - it('falls back to whichever conventional branch is actually there', async () => { - // A CI clone records only the refs it fetched, so `origin/HEAD` is usually absent. Assuming a - // name resolves to no ref at all, and the baseline then quietly becomes the previous commit. + it('reads it from another remote when origin records none', async () => { + // A remote added by hand has no HEAD unless someone wrote one. If they did, it still counts. const { repoRoot, git } = await makeRepo(); + await git('remote', 'add', 'upstream', repoRoot); + await git('update-ref', 'refs/remotes/upstream/release', 'HEAD'); + await git('symbolic-ref', 'refs/remotes/upstream/HEAD', 'refs/remotes/upstream/release'); + + expect(await detectBaseBranch(repoRoot)).toBe('release'); + }); + + it('does not take a conventional name as evidence', async () => { + // The branch exists, but nothing says it is the default. + const { repoRoot, git } = await makeRepo(); + await git('remote', 'add', 'origin', repoRoot); await git('update-ref', 'refs/remotes/origin/main', 'HEAD'); - expect(await detectBaseBranch(repoRoot)).toBe('main'); + await expect(detectBaseBranch(repoRoot)).rejects.toThrow(/no remote records a default branch/); }); - it('fails when nothing says which branch is the base', async () => { + it('fails when there is no remote at all', async () => { const { repoRoot } = await makeRepo(); await expect(detectBaseBranch(repoRoot)).rejects.toThrow(/Could not tell which branch/); From 0c83353ed14a13e0bb86508d281c09de21a37333 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 10 Sep 2026 19:48:12 +0200 Subject: [PATCH 061/173] [benchmark] Finish the review's small items A run where every case failed now exits non-zero, after writing and uploading the report so the errors are still readable. It measured nothing, which a green job does not say. Packing refuses two packages whose tarballs would share a name, rather than letting the last writer win and installing one package's build as the other. `resolveBaselineRef` distinguishes this from code-infra's `resolveBaseline`, which answers a different question. The harness is typechecked: it had a tsconfig that nothing ran. Also restores the upload route's account of which report types are stored as the whole envelope, and threads `cwd` into the remote lookup that ignored it. --- .../app/api/ci-reports/upload/route.ts | 4 +++ .../benchmark/src/tachometer/refs.test.ts | 26 +++++++++---------- packages/benchmark/src/tachometer/refs.ts | 2 +- .../benchmark/src/tachometer/runTachometer.ts | 12 ++++++--- packages/benchmark/src/utils/packWorkspace.ts | 12 +++++++++ packages/code-infra/src/utils/git.mjs | 2 +- test/tachometer/package.json | 1 + tsconfig.json | 3 ++- 8 files changed, 43 insertions(+), 19 deletions(-) diff --git a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts index 1129bf6587..e672f85a19 100644 --- a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts +++ b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts @@ -55,6 +55,10 @@ export async function POST(request: NextRequest) { const { commitSha, repo, reportType, branch, report } = parsed.data; + // A benchmark report is stored as the whole envelope, every other type as just the inner report. + // Historic, and kept because artifacts written that way are still read back. A tachometer report + // needs none of it: it carries its own `version`, `reportType` and `head`, the repo is in the S3 + // key, and the generator reads the rest from its own route params. const storedBody = reportType === 'benchmark' ? JSON.stringify(parsed.data) : JSON.stringify(report); diff --git a/packages/benchmark/src/tachometer/refs.test.ts b/packages/benchmark/src/tachometer/refs.test.ts index 53d184d40f..4d55ec205c 100644 --- a/packages/benchmark/src/tachometer/refs.test.ts +++ b/packages/benchmark/src/tachometer/refs.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import { execa } from 'execa'; import { makeTempDir } from '../utils/testUtils'; -import { resolveBaseline } from './refs'; +import { resolveBaselineRef } from './refs'; /** * A repository with three commits on `main`, the first of them carrying an annotated tag. @@ -40,11 +40,11 @@ async function makeRepo(): Promise<{ repoRoot: string; shas: string[] }> { return { repoRoot, shas }; } -describe('resolveBaseline', () => { +describe('resolveBaselineRef', () => { it('takes a bare value as a revision', async () => { const { repoRoot, shas } = await makeRepo(); - const ref = await resolveBaseline(shas[0], repoRoot); + const ref = await resolveBaselineRef(shas[0], repoRoot); expect(ref.sha).toBe(shas[0]); }); @@ -52,7 +52,7 @@ describe('resolveBaseline', () => { it('takes the same revision behind the git scheme', async () => { const { repoRoot, shas } = await makeRepo(); - const ref = await resolveBaseline(`git:${shas[0]}`, repoRoot); + const ref = await resolveBaselineRef(`git:${shas[0]}`, repoRoot); expect(ref.sha).toBe(shas[0]); }); @@ -60,7 +60,7 @@ describe('resolveBaseline', () => { it('resolves an annotated tag to the commit it points at', async () => { const { repoRoot, shas } = await makeRepo(); - const ref = await resolveBaseline('v1.0.0', repoRoot); + const ref = await resolveBaselineRef('v1.0.0', repoRoot); // Without `^{commit}` this is the tag object's own SHA, which is not a commit at all. expect(ref.sha).toBe(shas[0]); @@ -72,7 +72,7 @@ describe('resolveBaseline', () => { // needs no policy of its own. const { repoRoot, shas } = await makeRepo(); - const ref = await resolveBaseline(undefined, repoRoot); + const ref = await resolveBaselineRef(undefined, repoRoot); expect(ref.sha).toBe(shas[1]); }); @@ -81,15 +81,15 @@ describe('resolveBaseline', () => { const { repoRoot } = await makeRepo(); // The report shows this, so a tag stays a tag rather than becoming the SHA it points at. - expect((await resolveBaseline('v1.0.0', repoRoot)).requested).toBe('v1.0.0'); - expect((await resolveBaseline('git:v1.0.0', repoRoot)).requested).toBe('v1.0.0'); + expect((await resolveBaselineRef('v1.0.0', repoRoot)).requested).toBe('v1.0.0'); + expect((await resolveBaselineRef('git:v1.0.0', repoRoot)).requested).toBe('v1.0.0'); }); it('names a revision and a tag the same build, so the commit is built once', async () => { const { repoRoot, shas } = await makeRepo(); - const viaTag = await resolveBaseline('v1.0.0', repoRoot); - const viaSha = await resolveBaseline(shas[0], repoRoot); + const viaTag = await resolveBaselineRef('v1.0.0', repoRoot); + const viaSha = await resolveBaselineRef(shas[0], repoRoot); // `id` is the build directory: two ids for one commit means packing and building it twice. expect(viaTag.id).toBe(viaSha.id); @@ -100,7 +100,7 @@ describe('resolveBaseline', () => { async (token) => { const { repoRoot } = await makeRepo(); - await expect(() => resolveBaseline(token, repoRoot)).rejects.toThrow( + await expect(() => resolveBaselineRef(token, repoRoot)).rejects.toThrow( /recognised but not implemented/, ); }, @@ -109,7 +109,7 @@ describe('resolveBaseline', () => { it('rejects the git scheme with no revision after it', async () => { const { repoRoot } = await makeRepo(); - await expect(() => resolveBaseline('git:', repoRoot)).rejects.toThrow(/names no revision/); + await expect(() => resolveBaselineRef('git:', repoRoot)).rejects.toThrow(/names no revision/); }); it('reports what git said about a revision it cannot resolve', async () => { @@ -117,7 +117,7 @@ describe('resolveBaseline', () => { // Git is the authority on what a revision is, so an unknown one surfaces as its own failure // rather than as a grammar this code would have to keep in step with git's. - await expect(() => resolveBaseline('no-such-thing', repoRoot)).rejects.toThrow( + await expect(() => resolveBaselineRef('no-such-thing', repoRoot)).rejects.toThrow( /Could not resolve git ref "no-such-thing"/, ); }); diff --git a/packages/benchmark/src/tachometer/refs.ts b/packages/benchmark/src/tachometer/refs.ts index e29eb86d99..ac94c1c6c8 100644 --- a/packages/benchmark/src/tachometer/refs.ts +++ b/packages/benchmark/src/tachometer/refs.ts @@ -60,7 +60,7 @@ function committishOf(token: string | undefined): string { } /** Resolves the baseline to the commit behind it, so a run stays interpretable after the fact. */ -export async function resolveBaseline( +export async function resolveBaselineRef( token: string | undefined, repoRoot: string, ): Promise { diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts index 9fde3118b2..dba24e1741 100644 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -8,11 +8,11 @@ import { execaSync } from 'execa'; import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; import { packRef, packWorkingTree } from '../utils/packWorkspace'; import type { PackedPackage } from '../utils/packWorkspace'; -import { resolveBaseline, WORKTREE_REF } from './refs'; +import { resolveBaselineRef, WORKTREE_REF } from './refs'; import type { ResolvedRef } from './refs'; import { discoverCases, pagesOf } from './discoverCases'; import type { BenchmarkCase, Leaf } from './discoverCases'; -import { refLabel } from './format'; +import { isSummarized, refLabel } from './format'; import { buildRefPages } from './buildPages'; import { assertDriverMatchesBrowser, resolveBrowserBinary, withBrowserDefaults } from './browser'; import { summarizeCase } from './summarizeCase'; @@ -48,7 +48,7 @@ async function resolveBuilds( if (!cases.some((entry) => entry.comparison === 'baseline')) { return { refs: [WORKTREE_REF], buildFor: () => WORKTREE_REF }; } - const baselineRef = await resolveBaseline(baseline, repoRoot); + const baselineRef = await resolveBaselineRef(baseline, repoRoot); return { refs: [WORKTREE_REF, baselineRef], buildFor: (leaf) => (leaf.ref === 'baseline' ? baselineRef : WORKTREE_REF), @@ -261,6 +261,12 @@ export async function runTachometer(options: RunTachometerOptions): Promise !isSummarized(entry))) { + throw new Error('No case produced any measurement.'); + } } finally { await rm(tmpBase, { recursive: true, force: true }); } diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index 68f7361fb7..f5e9792894 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -186,6 +186,18 @@ export async function packBuiltPackages( if (packages.length === 0) { throw new Error(`No public workspace packages found in ${checkoutDir}.`); } + // `@a/b-c` and `@a-b/c` both name `a-b-c.tgz`, and these pack concurrently into one directory — + // so without this the last writer wins and a ref's tree installs one package's build under the + // other's name. + const byTarball = new Map(); + for (const { name } of packages) { + const tarball = tarballName(name); + const clash = byTarball.get(tarball); + if (clash) { + throw new Error(`"${clash}" and "${name}" would both pack to ${tarball}.`); + } + byTarball.set(tarball, name); + } // Each `pnpm pack` is its own node process, so a repository with a dozen public packages would // otherwise start a dozen at once. return mapConcurrently( diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index 90451a8c3d..58cb3724f0 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -268,7 +268,7 @@ export async function findLatestTaggedVersion(opts) { const $$ = $({ cwd: opts.cwd }); const fetchAll = opts.fetchAll ?? true; if (fetchAll) { - const { remoteName } = await getRepositoryInfo(); + const { remoteName } = await getRepositoryInfo(opts.cwd); // Fetch all tags from the mui remote to ensure we have the latest tags. // --force to update any existing tags that may have changed to avoid the clobering error. await $$`git fetch --tags --force ${remoteName}`; diff --git a/test/tachometer/package.json b/test/tachometer/package.json index 9dbe39fa24..cca8712442 100644 --- a/test/tachometer/package.json +++ b/test/tachometer/package.json @@ -5,6 +5,7 @@ "dev": "vite", "build": "vite build", "preview": "vite preview", + "typescript": "tsc -noEmit", "test:tacho": "node ./scripts/runBenchmark.mjs tacho run" }, "dependencies": { diff --git a/tsconfig.json b/tsconfig.json index 39611bdd1c..97d14d0db3 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,7 +25,8 @@ { "path": "tsconfig.node.json" }, { "path": "scripts" }, { "path": "docs" }, - { "path": "test/performance" } + { "path": "test/performance" }, + { "path": "test/tachometer" } ], "files": [] } From 735838da36211cc48732ac506e7a3288538fdf2b Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Fri, 11 Sep 2026 10:40:52 +0200 Subject: [PATCH 062/173] [benchmark] Fold the tacho command group into the CLI entry Six lines of grouping with one consumer. Also names tachometer as what needs chromedriver, rather than leaving the sentence without an actor. --- packages/benchmark/src/cli/cmdTacho.ts | 16 ---------------- packages/benchmark/src/cli/index.ts | 12 ++++++++++-- packages/benchmark/src/tachometer/browser.ts | 5 ++--- 3 files changed, 12 insertions(+), 21 deletions(-) delete mode 100644 packages/benchmark/src/cli/cmdTacho.ts diff --git a/packages/benchmark/src/cli/cmdTacho.ts b/packages/benchmark/src/cli/cmdTacho.ts deleted file mode 100644 index 050b2f0065..0000000000 --- a/packages/benchmark/src/cli/cmdTacho.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { CommandModule } from 'yargs'; -import tachoReport from './tachoReport'; -import tachoRun from './tachoRun'; - -/** - * The `tacho` command group. - */ -const command: CommandModule<{}, {}> = { - command: 'tacho ', - describe: 'Benchmark a tachometer harness, and read the reports it writes.', - builder: (yargs) => - yargs.command(tachoRun).command(tachoReport).demandCommand(1, 'Specify a tacho subcommand.'), - handler: () => {}, -}; - -export default command; diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index e0c07d0dec..a67b7bf25d 100644 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -8,7 +8,8 @@ import { hideBin } from 'yargs/helpers'; // repository than in the published package, which is built from `build/`, so a relative path // cannot reach the manifest from both — and the repository's workspace symlink hides the break. import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; -import cmdTacho from './cmdTacho'; +import tachoReport from './tachoReport'; +import tachoRun from './tachoRun'; let globalArgv: { verbose?: boolean } = {}; @@ -25,7 +26,14 @@ await yargs(hideBin(process.argv)) .middleware((argv) => { globalArgv = argv; }, true) - .command(cmdTacho) + .command({ + command: 'tacho ', + describe: 'Benchmark a tachometer harness, and read the reports it writes.', + builder: (group) => + group.command(tachoRun).command(tachoReport).demandCommand(1, 'Specify a tacho subcommand.'), + // The group itself does nothing; `demandCommand` above means a subcommand always runs. + handler: () => {}, + }) .fail((msg, err, yargsInstance) => { if (msg) { yargsInstance.showHelp(); diff --git a/packages/benchmark/src/tachometer/browser.ts b/packages/benchmark/src/tachometer/browser.ts index 7241fd7e96..efb7bec783 100644 --- a/packages/benchmark/src/tachometer/browser.ts +++ b/packages/benchmark/src/tachometer/browser.ts @@ -9,9 +9,8 @@ import { pathExists } from '../utils/path'; * Picking the browser to benchmark in, and checking it can actually be driven. * * Two separate programs are involved. The browser itself comes from Playwright, which downloads a - * fixed Chrome build. Driving that browser needs a second program, `chromedriver`, and tachometer - * supplies that one: it uses whichever it can find next to itself, and downloads the newest if it - * finds none. + * fixed Chrome build. Tachometer needs `chromedriver` to drive that browser, and supplies it itself: + * it uses whichever one it can find next to itself, and downloads the newest if it finds none. * * A chromedriver only drives the Chrome version it was built for. Since Playwright's Chrome is * deliberately a little behind the latest, letting tachometer fetch the newest driver usually gets a From d8cd60d011b51129db2d97a2dae4d0849c95263f Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Fri, 11 Sep 2026 10:45:24 +0200 Subject: [PATCH 063/173] [benchmark] Say who supplies chromedriver, and why it sits at the root --- packages/benchmark/src/tachometer/browser.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/benchmark/src/tachometer/browser.ts b/packages/benchmark/src/tachometer/browser.ts index efb7bec783..95e539eebc 100644 --- a/packages/benchmark/src/tachometer/browser.ts +++ b/packages/benchmark/src/tachometer/browser.ts @@ -15,6 +15,12 @@ import { pathExists } from '../utils/path'; * A chromedriver only drives the Chrome version it was built for. Since Playwright's Chrome is * deliberately a little behind the latest, letting tachometer fetch the newest driver usually gets a * pair that cannot work together. + * + * So the consumer supplies it instead: a `chromedriver` devDependency at the workspace root, which + * is the only place tachometer resolves one from. Declared beside `@playwright/test`, so one place + * governs both versions and Renovate moves them together. All this module does is compare the + * version tachometer would resolve against the browser's own — which version that is stays the + * consumer's to choose. */ /** From 7ac72a686ea9c6a3b946c4aac7c52799c0f06ad3 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Fri, 11 Sep 2026 12:37:51 +0200 Subject: [PATCH 064/173] [benchmark] Add an in-place resolve mode behind a flag `--resolve-mode in-place` pins the packed build in the repository's own pnpm-workspace.yaml and installs it there, so the pages resolve the library ordinarily and the isolated tree and its resolver plugin are not used. The manifest is copied aside, rewritten from that copy for each ref, and restored with a reinstall when the run ends, including after a failure. Default stays `isolated`, so CI measures what it measured before. The pins are global to the workspace for the length of the run: pnpm scopes an override by parent package name, and a harness has none. --- packages/benchmark/README.md | 6 ++ packages/benchmark/src/cli/tachoRun.ts | 8 ++ .../benchmark/src/tachometer/buildPages.ts | 94 ++++++++++++++++++- .../benchmark/src/tachometer/runTachometer.ts | 13 ++- 4 files changed, 115 insertions(+), 6 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 89fffa0daa..c295f4dbe8 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -356,6 +356,12 @@ the same commit skips the rebuild; the working tree is never cached. In CI, cach keyed on the baseline SHA, and check out with full history so a fork point can be resolved. A run never modifies the checkout it was started from. +`--resolve-mode in-place` is an experiment in doing without the isolated tree: it pins the packed +build in the repository's own `pnpm-workspace.yaml` instead, installs it there, and resolution is +then ordinary. The repository is put back, and reinstalled, when the run ends — including when it +fails. Until the run ends, though, a tracked file names tarballs under `.tachometer/`, so do not +commit while one is in flight. The default, `isolated`, leaves the repository alone. + `--no-install` skips a ref's install. `benchmark tacho run --help` lists the rest, and `benchmark tacho report` prints the table again from a saved JSON report. diff --git a/packages/benchmark/src/cli/tachoRun.ts b/packages/benchmark/src/cli/tachoRun.ts index 895412675f..00bf41f03c 100644 --- a/packages/benchmark/src/cli/tachoRun.ts +++ b/packages/benchmark/src/cli/tachoRun.ts @@ -20,6 +20,13 @@ const command: CommandModule<{}, Args> = { describe: 'The build to compare against: a revision, on its own or as git:. Defaults to HEAD~1; `code-infra baseline` resolves a fork point to pass here', }) + .option('resolve-mode', { + type: 'string', + choices: ['isolated', 'in-place'] as const, + default: 'isolated' as const, + describe: + "How pages find the library: 'isolated' installs each ref beside the repository, 'in-place' pins it in the repository's own pnpm-workspace.yaml for the length of the run", + }) .option('build-cmd', { type: 'string', default: 'pnpm release:build', @@ -51,6 +58,7 @@ const command: CommandModule<{}, Args> = { harnessDir: process.cwd(), filters: argv.filters ?? [], baseline: argv.baseline, + resolveMode: argv.resolveMode, buildCmd: argv.buildCmd, install: argv.install, out: argv.out, diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts index d3a1a24c7d..093c98de12 100644 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -2,7 +2,7 @@ import * as path from 'node:path'; import { createRequire } from 'node:module'; -import { readFile, mkdir, writeFile } from 'node:fs/promises'; +import { readFile, mkdir, rm, writeFile } from 'node:fs/promises'; import { pathToFileURL } from 'node:url'; import chalk from 'chalk'; import type * as Vite from 'vite'; @@ -16,6 +16,17 @@ import type { PackedPackage } from '../utils/packWorkspace'; import { installedVersions, readPackageJson, writePackageJson } from '../utils/pnpm'; import { refLabel } from './format'; import type { ResolvedRef } from './refs'; +import { pathExists } from '../utils/path'; + +/** + * How a ref's pages find the library under test. + * + * `isolated` installs that ref's packed build into a directory of its own and points resolution + * there, leaving the repository untouched. `in-place` pins the packed build in the repository's own + * `pnpm-workspace.yaml` and installs it, so resolution is ordinary — at the cost of mutating a + * tracked file for as long as the run lasts. + */ +export type ResolveMode = 'isolated' | 'in-place'; /** * Dependencies the run needs but a page never imports, so a ref's tree does without them — @@ -109,6 +120,60 @@ function resolveFromTree(treeDir: string): Vite.Plugin { }; } +/** The packed pins an in-place run adds to the repository's overrides. */ +function packedPins(packages: PackedPackage[]): Record { + return Object.fromEntries(packages.map((pkg) => [pkg.name, `file:${pkg.tarball}`])); +} + +/** Where the repository's own manifest is kept while an in-place run has it pinned. */ +function backupPathOf(outputDir: string): string { + return path.join(outputDir, 'pnpm-workspace.yaml.orig'); +} + +/** + * Points the repository's own install at this ref's packed build, through the same overrides the + * isolated mode applies to a workspace of its own. + * + * The manifest is copied aside first and rewritten from that copy each time, so a second ref pins + * over the original rather than over the first ref's pins. A run that dies leaves both the pins and + * the copy: the manifest then names tarballs under the output directory, which fails the next + * install rather than resolving something else and saying nothing. + */ +async function pinPackedPackages( + repoRoot: string, + outputDir: string, + packages: PackedPackage[], +): Promise { + const manifestPath = path.join(repoRoot, 'pnpm-workspace.yaml'); + const backupPath = backupPathOf(outputDir); + if (!(await pathExists(backupPath))) { + await writeFile(backupPath, await readFile(manifestPath, 'utf8')); + } + + const config = parse(await readFile(backupPath, 'utf8')) ?? {}; + await writeFile( + manifestPath, + stringify({ ...config, overrides: { ...config.overrides, ...packedPins(packages) } }), + ); +} + +/** + * Puts the repository's manifest back, and installs from it. + * + * The install is the point: without it the tree on disk still resolves the library to a tarball + * under the output directory, and that surfaces at the next unrelated command instead of here. + */ +export async function restoreWorkspace(repoRoot: string, outputDir: string): Promise { + const backupPath = backupPathOf(outputDir); + if (!(await pathExists(backupPath))) { + return; + } + console.log(chalk.cyan('\nRestoring the repository install…')); + await writeFile(path.join(repoRoot, 'pnpm-workspace.yaml'), await readFile(backupPath, 'utf8')); + await rm(backupPath, { force: true }); + await run('pnpm', ['install', '--prefer-offline'], repoRoot); +} + /** Installs one ref's packed build, with the harness's own dependencies around it. */ async function installRefTree({ harnessDir, @@ -197,15 +262,33 @@ export async function buildRefPages(options: { ref: ResolvedRef; /** That ref's packed workspace packages. */ packages: PackedPackage[]; - /** Persistent directory holding this ref's installed tree. */ + /** Persistent directory holding this ref's installed tree. Unused when resolving in place. */ treeDir: string; + /** The run's output directory, where an in-place run keeps the manifest it replaced. */ + outputDir: string; /** Absolute output directory for the built pages. */ outDir: string; + /** How the pages find the library. Defaults to `isolated`. */ + resolveMode?: ResolveMode; }): Promise { - const { harnessDir, repoRoot, ref, packages, treeDir, outDir } = options; + const { + harnessDir, + repoRoot, + ref, + packages, + treeDir, + outputDir, + outDir, + resolveMode = 'isolated', + } = options; console.log(chalk.cyan(`\nBuilding benchmark pages for "${refLabel(ref)}"…`)); - await installRefTree({ harnessDir, repoRoot, packages, treeDir }); + if (resolveMode === 'in-place') { + await pinPackedPackages(repoRoot, outputDir, packages); + await run('pnpm', ['install', '--prefer-offline'], repoRoot); + } else { + await installRefTree({ harnessDir, repoRoot, packages, treeDir }); + } // vite is the harness's own, not this package's: the harness declares the version its pages are // built with, and the config being run is the harness's. @@ -217,7 +300,8 @@ export async function buildRefPages(options: { await vite.build({ root: harnessDir, logLevel: 'warn', - plugins: [resolveFromTree(treeDir)], + // Resolving in place needs no plugin: the repository's own install is the one that changed. + plugins: resolveMode === 'in-place' ? [] : [resolveFromTree(treeDir)], build: { outDir }, }); } diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts index dba24e1741..a124ae3d97 100644 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -13,7 +13,8 @@ import type { ResolvedRef } from './refs'; import { discoverCases, pagesOf } from './discoverCases'; import type { BenchmarkCase, Leaf } from './discoverCases'; import { isSummarized, refLabel } from './format'; -import { buildRefPages } from './buildPages'; +import { buildRefPages, restoreWorkspace } from './buildPages'; +import type { ResolveMode } from './buildPages'; import { assertDriverMatchesBrowser, resolveBrowserBinary, withBrowserDefaults } from './browser'; import { summarizeCase } from './summarizeCase'; import { renderTachometerReport } from './renderReport'; @@ -79,6 +80,8 @@ export interface RunTachometerOptions { out?: string; /** Upload the report and refresh the pull request comment. */ upload?: boolean; + /** How a ref's pages find the library under test. Defaults to `isolated`. */ + resolveMode?: ResolveMode; } /** @@ -99,6 +102,7 @@ export async function runTachometer(options: RunTachometerOptions): Promise Date: Fri, 11 Sep 2026 12:45:05 +0200 Subject: [PATCH 065/173] [benchmark] Resolve in place by default `--resolve-mode isolated` keeps the old behaviour. A run killed outright never reaches the restore, so the copy it leaves behind is now put back at the start of the next run, whatever mode that run is in. That is the case the default makes likely: a benchmark is slow enough to interrupt. --- packages/benchmark/README.md | 15 ++++++++++----- packages/benchmark/src/cli/index.ts | 0 packages/benchmark/src/cli/tachoRun.ts | 2 +- packages/benchmark/src/tachometer/buildPages.ts | 4 ++-- .../benchmark/src/tachometer/runTachometer.ts | 9 +++++++-- 5 files changed, 20 insertions(+), 10 deletions(-) mode change 100644 => 100755 packages/benchmark/src/cli/index.ts diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index c295f4dbe8..3fba5cd25b 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -356,11 +356,16 @@ the same commit skips the rebuild; the working tree is never cached. In CI, cach keyed on the baseline SHA, and check out with full history so a fork point can be resolved. A run never modifies the checkout it was started from. -`--resolve-mode in-place` is an experiment in doing without the isolated tree: it pins the packed -build in the repository's own `pnpm-workspace.yaml` instead, installs it there, and resolution is -then ordinary. The repository is put back, and reinstalled, when the run ends — including when it -fails. Until the run ends, though, a tracked file names tarballs under `.tachometer/`, so do not -commit while one is in flight. The default, `isolated`, leaves the repository alone. +By default a run resolves **in place**: it pins the packed build in the repository's own +`pnpm-workspace.yaml`, installs it there, and resolution is then ordinary. The repository is put +back, and reinstalled, when the run ends — including when it fails, and at the start of the next run +if one was killed outright. Two things follow from it. A tracked file names tarballs under +`.tachometer/` until the run ends, so do not commit while one is in flight. And the pins are global +to the workspace for that time, because pnpm scopes an override by parent package name and a harness +has none — so nothing else should build against the same checkout meanwhile. + +`--resolve-mode isolated` avoids both: it installs each ref into a directory of its own and points +resolution there, leaving the repository untouched, at the cost of a second install tree per ref. `--no-install` skips a ref's install. `benchmark tacho run --help` lists the rest, and `benchmark tacho report` prints the table again from a saved JSON report. diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts old mode 100644 new mode 100755 diff --git a/packages/benchmark/src/cli/tachoRun.ts b/packages/benchmark/src/cli/tachoRun.ts index 00bf41f03c..a6c6343e0d 100644 --- a/packages/benchmark/src/cli/tachoRun.ts +++ b/packages/benchmark/src/cli/tachoRun.ts @@ -23,7 +23,7 @@ const command: CommandModule<{}, Args> = { .option('resolve-mode', { type: 'string', choices: ['isolated', 'in-place'] as const, - default: 'isolated' as const, + default: 'in-place' as const, describe: "How pages find the library: 'isolated' installs each ref beside the repository, 'in-place' pins it in the repository's own pnpm-workspace.yaml for the length of the run", }) diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts index 093c98de12..b268de7553 100644 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -268,7 +268,7 @@ export async function buildRefPages(options: { outputDir: string; /** Absolute output directory for the built pages. */ outDir: string; - /** How the pages find the library. Defaults to `isolated`. */ + /** How the pages find the library. Defaults to `in-place`. */ resolveMode?: ResolveMode; }): Promise { const { @@ -279,7 +279,7 @@ export async function buildRefPages(options: { treeDir, outputDir, outDir, - resolveMode = 'isolated', + resolveMode = 'in-place', } = options; console.log(chalk.cyan(`\nBuilding benchmark pages for "${refLabel(ref)}"…`)); diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts index a124ae3d97..298e3df3e9 100644 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -80,7 +80,7 @@ export interface RunTachometerOptions { out?: string; /** Upload the report and refresh the pull request comment. */ upload?: boolean; - /** How a ref's pages find the library under test. Defaults to `isolated`. */ + /** How a ref's pages find the library under test. Defaults to `in-place`. */ resolveMode?: ResolveMode; } @@ -102,7 +102,7 @@ export async function runTachometer(options: RunTachometerOptions): Promise Date: Fri, 11 Sep 2026 13:17:24 +0200 Subject: [PATCH 066/173] [benchmark] Capture the output of commands worth nothing on success MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `pnpm pack` listed every file of every package, twice over, and an install printed its progress and a dependency diff. Neither says anything a successful run needs. Commands are captured by default now and `verbose` opts one back in: a ref's install and build, and the sampling run itself. Nothing is lost on failure — execa carries both streams on the error it throws. Vite reports what each ref built, which is the one part of a build step worth seeing when it works. --- .../benchmark/src/tachometer/buildPages.ts | 2 +- .../benchmark/src/tachometer/runTachometer.ts | 1 + packages/benchmark/src/utils/exec.ts | 20 ++++++++++++++++--- packages/benchmark/src/utils/packWorkspace.ts | 6 +++--- 4 files changed, 22 insertions(+), 7 deletions(-) diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts index b268de7553..35f6181ddf 100644 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -299,7 +299,7 @@ export async function buildRefPages(options: { // it to `src/`, the same as for a plain `vite build`. await vite.build({ root: harnessDir, - logLevel: 'warn', + logLevel: 'info', // Resolving in place needs no plugin: the repository's own install is the one that changed. plugins: resolveMode === 'in-place' ? [] : [resolveFromTree(treeDir)], build: { outDir }, diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts index 298e3df3e9..45ffd3ccc2 100644 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -226,6 +226,7 @@ export async function runTachometer(options: RunTachometerOptions): Promise { + const { env, verbose = false } = options; console.log(chalk.dim(`$ ${file} ${args.join(' ')} (in ${cwd})`)); - await execa(file, args, { cwd, stdio: 'inherit', env }); + await execa(file, args, { cwd, env, stdio: verbose ? 'inherit' : 'pipe' }); } diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index f5e9792894..05285f9d4c 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -256,14 +256,14 @@ export async function packRef(options: PackRefOptions): Promise if (installCmd) { console.log(chalk.cyan(`\nInstalling dependencies for ${sha.slice(0, 9)}…`)); const [installFile, ...installArgs] = parseCommandString(installCmd); - await run(installFile, installArgs, checkout); + await run(installFile, installArgs, checkout, { verbose: true }); } console.log(chalk.cyan(`\nBuilding packages for ${sha.slice(0, 9)}…`)); // Disable the nx daemon (lerna runs builds through nx): a lingering daemon keeps writing into // the checkout and makes removal fail with "Directory not empty". execa's `env` extends the // current environment, so only NX_DAEMON is overridden. const [buildFile, ...buildArgs] = parseCommandString(buildCmd); - await run(buildFile, buildArgs, checkout, { NX_DAEMON: 'false' }); + await run(buildFile, buildArgs, checkout, { env: { NX_DAEMON: 'false' }, verbose: true }); const packages = await packBuiltPackages(checkout, staging); // Store tarballs by basename so the folder is relocatable; record buildCmd so a later run can @@ -317,7 +317,7 @@ export async function packWorkingTree(options: { console.log(chalk.cyan(`\nBuilding workspace packages for "working tree" (${buildCmd})…`)); // Disable the nx daemon: it keeps writing into the workspace after the build returns. const [file, ...args] = parseCommandString(buildCmd); - await run(file, args, repoRoot, { NX_DAEMON: 'false' }); + await run(file, args, repoRoot, { env: { NX_DAEMON: 'false' }, verbose: true }); console.log(chalk.cyan('\nPacking the working tree…')); // Stage next to the destination so the renames cannot cross filesystems. From 59d8566c4b6cd6bfe577630d195a1a911f81520a Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:19:40 +0200 Subject: [PATCH 067/173] [benchmark] Let the in-place installs write a lockfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pnpm turns a frozen install on by itself when CI is set, and changing the overrides is exactly what these installs do — so CI refused the pin with ERR_PNPM_LOCKFILE_CONFIG_MISMATCH while every local run passed, frozen being off by default there. The restore needs it too: by then the lockfile carries the pins it is undoing. --- packages/benchmark/src/tachometer/buildPages.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts index 35f6181ddf..a892508246 100644 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -162,6 +162,9 @@ async function pinPackedPackages( * * The install is the point: without it the tree on disk still resolves the library to a tarball * under the output directory, and that surfaces at the next unrelated command instead of here. + * + * `--no-frozen-lockfile` because pnpm turns a frozen install on by itself in CI, and the lockfile + * here still carries the pins this is undoing. */ export async function restoreWorkspace(repoRoot: string, outputDir: string): Promise { const backupPath = backupPathOf(outputDir); @@ -171,7 +174,7 @@ export async function restoreWorkspace(repoRoot: string, outputDir: string): Pro console.log(chalk.cyan('\nRestoring the repository install…')); await writeFile(path.join(repoRoot, 'pnpm-workspace.yaml'), await readFile(backupPath, 'utf8')); await rm(backupPath, { force: true }); - await run('pnpm', ['install', '--prefer-offline'], repoRoot); + await run('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], repoRoot); } /** Installs one ref's packed build, with the harness's own dependencies around it. */ @@ -285,7 +288,9 @@ export async function buildRefPages(options: { if (resolveMode === 'in-place') { await pinPackedPackages(repoRoot, outputDir, packages); - await run('pnpm', ['install', '--prefer-offline'], repoRoot); + // `--no-frozen-lockfile` because changing the overrides is the point, and pnpm turns a frozen + // install on by itself in CI — where it would refuse the very change being made. + await run('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], repoRoot); } else { await installRefTree({ harnessDir, repoRoot, packages, treeDir }); } From db1a69960ddca2783a753f6d83fd92d6da0a9987 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Fri, 11 Sep 2026 15:37:10 +0200 Subject: [PATCH 068/173] [benchmark] Keep an in-place run from pinning its own tooling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pins covered every packed workspace package, this one included — so while building the baseline, the harness's vite config was handed that ref's copy of the tool doing the building. Against a commit whose benchmark package does not export the plugin yet, the config fails to load at all. The isolated tree already left these alone; the pins now use the same list. Local runs missed it by comparing against HEAD, where the ref's copy is this copy. `--baseline ` reproduces it. --- packages/benchmark/src/tachometer/buildPages.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts index a892508246..4b910cae47 100644 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -120,9 +120,19 @@ function resolveFromTree(treeDir: string): Vite.Plugin { }; } -/** The packed pins an in-place run adds to the repository's overrides. */ +/** + * The packed pins an in-place run adds to the repository's overrides. + * + * Everything the run itself resolves is left alone, this package above all: pinning it would hand + * the harness's vite config the ref's copy of the very tool doing the building, and a ref old enough + * not to export the plugin then fails to configure at all. + */ function packedPins(packages: PackedPackage[]): Record { - return Object.fromEntries(packages.map((pkg) => [pkg.name, `file:${pkg.tarball}`])); + return Object.fromEntries( + packages + .filter((pkg) => !RUNNER_ONLY_DEPS.includes(pkg.name)) + .map((pkg) => [pkg.name, `file:${pkg.tarball}`]), + ); } /** Where the repository's own manifest is kept while an in-place run has it pinned. */ From 513ba7838d0c7a7a9ae3fb7006c104022c968dcc Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Mon, 14 Sep 2026 11:22:27 +0200 Subject: [PATCH 069/173] [code-infra] Fetch the base branch, and refuse when there is none MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not finding the base branch used to return the previous commit, which is a plausible number for a question nobody asked: every comparison silently became "since the last commit" instead of "since the fork point". Being on the base branch, and having HEAD already contained in it, are both handled before that point — so reaching it means the branch named is one this repository cannot see. The command fetches it too. It knows which ref it needs and the caller cannot do better, so the CI step that used to fetch first is gone. --- .circleci/config.yml | 4 +- packages/code-infra/src/utils/git.mjs | 43 +++++++++++++++++++++- packages/code-infra/src/utils/git.test.mjs | 41 +++++++++++++++++++-- 3 files changed, 81 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 162abbe248..0ae37c8d95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,8 +129,8 @@ jobs: name: Resolve the baseline commit command: | set -e - git fetch --no-tags origin master:refs/remotes/origin/master - # Named, because a CI checkout records no `origin/HEAD` for it to be detected from. + # Named, because a CI checkout records no `origin/HEAD` to detect it from. Fetching it + # is the command's own job, so there is nothing to keep in step with it here. BASE_SHA=$(pnpm code-infra baseline --base-branch master) echo "Baseline commit: $BASE_SHA" echo "$BASE_SHA" > .tachometer-baseline-sha diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index 58cb3724f0..1f2bff16f1 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -210,6 +210,40 @@ export async function closestBaseBranch(baseBranch, cwd = process.cwd()) { return best && { ref: best.ref, mergeBase: best.mergeBase }; } +/** + * Makes the base branch reachable, fetching it when nothing local has it yet. + * + * A CI checkout fetches only the branch under test, so the ref a fork point is measured from is + * usually absent there. Fetching it here rather than leaving it to the caller keeps the command's + * one job — say which commit to compare against — from depending on a step somewhere else. + * @param {string} baseBranch + * @param {string} cwd + * @returns {Promise} + */ +async function fetchBaseBranch(baseBranch, cwd) { + const listed = (await $({ cwd })`git remote`).stdout.trim().split('\n').filter(Boolean); + for (const ref of [ + `refs/heads/${baseBranch}`, + ...listed.map((r) => `refs/remotes/${r}/${baseBranch}`), + ]) { + // Sequential on purpose: a handful of refs, and the first hit means there is nothing to fetch. + // eslint-disable-next-line no-await-in-loop + const found = await $({ cwd, reject: false })`git rev-parse --verify --quiet ${ref}`; + if (found.exitCode === 0) { + return; + } + } + if (!listed.includes('origin')) { + return; + } + // Best-effort: a failure here is not the error to report — the caller's own check says plainly + // that no ref for the base branch exists, which is what a reader needs either way. + await $({ + cwd, + reject: false, + })`git fetch --no-tags origin ${`${baseBranch}:refs/remotes/origin/${baseBranch}`}`; +} + /** * The commit HEAD should be compared against: on a feature branch the fork point from the closest * base branch, so a comparison reflects only what the branch changed; on the base branch itself the @@ -242,9 +276,16 @@ export async function resolveBaseline(options = {}) { return previousCommit(); } + await fetchBaseBranch(baseBranch, cwd); const base = await closestBaseBranch(baseBranch, cwd); if (!base) { - return previousCommit(); + // Not a fallback: being on the base branch and having HEAD already contained in it are both + // handled above, so reaching here means the branch named is one this repository cannot see. + // Answering with the previous commit would be a plausible number for a question left unasked. + throw new Error( + `No ref for base branch "${baseBranch}", so there is no fork point to measure from. ` + + `Name a branch this repository has, or make it fetchable.`, + ); } // The fork point is HEAD itself — HEAD is already contained in the base branch, so there is // nothing this branch changed to compare. diff --git a/packages/code-infra/src/utils/git.test.mjs b/packages/code-infra/src/utils/git.test.mjs index 131784b42f..3601563799 100644 --- a/packages/code-infra/src/utils/git.test.mjs +++ b/packages/code-infra/src/utils/git.test.mjs @@ -91,17 +91,50 @@ describe('resolveBaseline', () => { expect(sha).toBe(shas[1]); }); + it('fetches the base branch when the checkout never pulled it', async () => { + // What CI looks like: the branch under test is there, the branch it forked from is not. Without + // the fetch there is no fork point to find, and the answer silently became the previous commit. + const { repoRoot, shas, git } = await makeRepo(); + const clone = await fs.mkdtemp(path.join(os.tmpdir(), 'baseline-clone-')); + onTestFinished(async () => { + await fs.rm(clone, { recursive: true, force: true }); + }); + + await git('checkout', '-b', 'feature', shas[0]); + await git('commit', '--allow-empty', '-m', 'work'); + const branchTip = (await git('rev-parse', 'HEAD')).stdout.trim(); + + // A single-branch clone, so `origin/main` is absent exactly as it is after a CI checkout. + await execa('git', ['clone', '--single-branch', '--branch', 'feature', repoRoot, clone], { + env: { GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null' }, + }); + expect( + ( + await execa('git', ['rev-parse', '--verify', '--quiet', 'refs/remotes/origin/main'], { + cwd: clone, + reject: false, + }) + ).exitCode, + ).not.toBe(0); + + const sha = await resolveBaseline({ cwd: clone, baseBranch: 'main' }); + + // The fork point, not `branchTip~1` — which is what the previous-commit fallback would give. + expect(sha).toBe(shas[0]); + expect(sha).not.toBe(branchTip); + }); + it('does not treat a branch matching the base branch as a pattern', async () => { const { repoRoot, shas, git } = await makeRepo(); // `v6-x`, not `v6.x` — only a regex reading `.` as a wildcard would accept it. await git('update-ref', 'refs/remotes/origin/v6-x', shas[0]); await git('checkout', '-b', 'feature'); - const sha = await resolveBaseline({ cwd: repoRoot, baseBranch: 'v6.x' }); - - // No base branch exists, so it falls back to the previous commit. Matching `origin/v6-x` would + // No ref matches, which is the answer rather than a fallback. Matching `origin/v6-x` would // instead have picked its merge base — the first commit — and compared against the wrong one. - expect(sha).toBe(shas[1]); + await expect(resolveBaseline({ cwd: repoRoot, baseBranch: 'v6.x' })).rejects.toThrow( + /No ref for base branch "v6.x"/, + ); }); }); From 45df3703e5424fce98adfb7df6cf936e0cd99bfa Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:30:49 +0200 Subject: [PATCH 070/173] [benchmark] Export runCase() as a single-iteration primitive benchmark()'s it() loops over runCase(renderFn, interaction?, options?), which mounts, interacts with and unmounts a case exactly once; warmup: true skips bench:paint and gates custom metrics for that iteration. Based on the direction of #1878. --- packages/benchmark/README.md | 23 +++ packages/benchmark/src/index.tsx | 257 +++++++++++++++++++------------ 2 files changed, 185 insertions(+), 95 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 3fba5cd25b..0bec73a807 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -211,6 +211,28 @@ benchmark('name', renderFn, interaction, { }); ``` +### Custom drivers + +`benchmark()` loops over `runCase()`, which mounts, interacts with and unmounts a case exactly once. Call it directly to control the iterations yourself — e.g. to alternate two variants so machine drift affects both equally, warming each one up right before it is measured: + +```tsx +import { runCase } from '@mui/internal-benchmark'; + +it('A/B', async () => { + for (let i = 0; i < 20; i += 1) { + await runCase(renderA, interaction, { warmup: true }); + const a = await runCase(renderA, interaction); + + await runCase(renderB, interaction, { warmup: true }); + const b = await runCase(renderB, interaction); + + // compare a.renders with b.renders, check a.renderError, ... + } +}); +``` + +`runCase()` takes the same `renderFn`, interaction and `afterEach`/`reactRecordingPaused` options as `benchmark()`, plus `warmup`, which skips `bench:paint` and drops custom metrics for that iteration. It registers and asserts nothing. Measured iterations record metrics against the running Vitest test, so call it from inside one. + ### Running ```bash @@ -373,6 +395,7 @@ resolution there, leaving the repository untouched, at the cost of a second inst ## API - `benchmark` — define a benchmark test case +- `runCase` — run a single iteration of a case, for custom drivers - `ElementTiming` — invisible marker component for paint timing (renders a `` tracked by the Element Timing API) - `ScalarMetric` — record a continuous custom measurement (with a `console.time`-style timing helper) - `DiscreteMetric` — record a discrete custom count diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index 6ffed8c6e3..1c82c41d24 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -1,5 +1,6 @@ import * as React from 'react'; -import { expect, it } from 'vitest'; +import { expect, it, TestRunner } from 'vitest'; +import type { RunnerTestCase } from 'vitest'; import * as ReactDOMClient from 'react-dom/client'; // aliased to react-dom/profiling by Vite import * as ReactDOM from 'react-dom'; import type { RenderEvent, IterationData, InteractionContext, BenchmarkCaseRuntime } from './types'; @@ -231,9 +232,9 @@ function createCaseRuntime({ }; } -interface BenchmarkOptions { - runs?: number; - warmupRuns?: number; +export type BenchmarkInteraction = (ctx: InteractionContext) => Promise | void; + +interface CaseOptions { afterEach?: () => Promise | void; /** * Start each iteration with React render/paint recording paused. The interaction callback then @@ -243,10 +244,154 @@ interface BenchmarkOptions { reactRecordingPaused?: boolean; } +export interface BenchmarkOptions extends CaseOptions { + runs?: number; + warmupRuns?: number; +} + +export interface RunCaseOptions extends CaseOptions { + /** + * Run the iteration without recording anything: no `bench:paint`, and custom metrics recorded + * inside the case are dropped. Defaults to `false`. + */ + warmup?: boolean; +} + +export interface RunCaseResult { + /** Renders captured while React recording was active. */ + renders: RenderEvent[]; + /** The error a render threw, if one did. The iteration stops at mount when it does. */ + renderError: unknown; + /** Whether a recording window was active yet captured no renders. */ + hadEmptyActiveWindow: boolean; +} + +// Paint timings are recorded as one harness-owned `bench:paint` metric: the default sentinel +// is the base series (`bench:paint`) and named `elementtiming` markers are sub-series +// (`bench:paint#grid-header`, …), all sharing a single definition. Paint is informational (no +// alarm): it dominates each test's total duration, so a per-test paint alarm just duplicates the +// Duration regression signal and floods the report on any broadly-regressed run. +const paint = new ScalarMetric({ + name: 'bench:paint', + format: { style: 'unit', unit: 'millisecond', maximumFractionDigits: 2 }, +}); + +async function measureIteration( + renderFn: () => React.ReactElement, + interaction: BenchmarkInteraction | undefined, + options: RunCaseOptions | undefined, +): Promise { + // Per-iteration switch for the harness's React render/paint recording. Starts paused when + // `reactRecordingPaused` is set; the interaction callback drives it from there. + const recording = createReactRecordingControls(!(options?.reactRecordingPaused ?? false)); + + // Drain event loop from previous unmount, then double GC for thorough cleanup + await settle(); + forceGC(); + + const captures: RenderEvent[] = []; + const timing = createElementTimingWaiter(); + let renderError: unknown = null; + + const runtime = createCaseRuntime({ + // Wrap the case in BenchProfiler so its renders are captured; the runtime mounts whatever + // renderFn returns (profiling passes the case bare). + renderFn: () => ( + + {renderFn()} + + ), + interaction, + context: { + waitForElementTiming: timing.waitForElementTiming, + pauseReactRecording: recording.pauseReactRecording, + resumeReactRecording: recording.resumeReactRecording, + }, + onUncaughtError: (error) => { + renderError = error; + }, + }); + + const iterationStart = performance.now(); + + runtime.mount(); + + if (renderError) { + timing.disconnect(); + runtime.unmount(); + return { renders: captures, renderError, hadEmptyActiveWindow: false }; + } + + await runtime.interact?.(); + + // Wait for the bench sentinel paint entry (relies on test timeout) + await timing.waitForElementTiming('default', 0); + + // Close the final window so an active window that measured no renders is flagged. + recording.finalizeWindow(); + + timing.disconnect(); + + runtime.unmount(); + + if (!options?.warmup) { + for (const entry of timing.elementEntries) { + // Skip paints that happened while recording was paused. Attribute by the paint's + // `paintTime`, not by when the observer callback fired (which can lag the paint). + if (!recording.activeAt(entry.paintTime)) { + continue; + } + // The default sentinel is the base series; named markers become sub-series. + const id = entry.identifier === 'default' ? undefined : entry.identifier; + paint.record(entry.paintTime - iterationStart, id !== undefined ? { id } : undefined); + } + } + + if (options?.afterEach) { + await options.afterEach(); + } + + return { renders: captures, renderError, hadEmptyActiveWindow: recording.hadEmptyActiveWindow }; +} + +/** + * Mounts, interacts with and unmounts a case exactly once, recording its renders and paint. + * + * This is the primitive `benchmark()` loops over. How many iterations to run, in which order, and + * how to aggregate them is up to the caller, so a driver can run cases differently — e.g. an A/B + * runner alternating two builds iteration by iteration instead of in blocks. It registers and + * asserts nothing; the caller decides what a render error or empty recording window means. + * + * Metrics resolve the running Vitest test when they record, so measured (non-warmup) iterations + * must run inside one. + */ +export async function runCase( + renderFn: () => React.ReactElement, + interactionOrOptions?: BenchmarkInteraction | RunCaseOptions, + maybeOptions?: RunCaseOptions, +): Promise { + const interaction = typeof interactionOrOptions === 'function' ? interactionOrOptions : undefined; + const options = typeof interactionOrOptions === 'object' ? interactionOrOptions : maybeOptions; + + // Custom metrics recorded inside the case honor warmup exclusion through the gate, the same way + // renders and `bench:paint` are excluded during warmup. The gate is keyed on the running test, + // and is re-enabled afterwards so metrics the driver records between iterations are kept. + const test = TestRunner.getCurrentTest(); + if (!test) { + return measureIteration(renderFn, interaction, options); + } + metricsGate.setRecordingEnabled(test, !options?.warmup); + try { + return await measureIteration(renderFn, interaction, options); + } finally { + metricsGate.setRecordingEnabled(test, true); + } +} + export function benchmark( name: string, renderFn: () => React.ReactElement, - interactionOrOptions?: ((ctx: InteractionContext) => Promise | void) | BenchmarkOptions, + interactionOrOptions?: BenchmarkInteraction | BenchmarkOptions, maybeOptions?: BenchmarkOptions, ) { const interaction = typeof interactionOrOptions === 'function' ? interactionOrOptions : undefined; @@ -280,112 +425,34 @@ export function benchmark( const runs = options?.runs ?? 20; const warmupRuns = options?.warmupRuns ?? 10; - const totalRuns = warmupRuns + runs; - const iterations: IterationData[] = []; - - // Paint timings are recorded as one harness-owned `bench:paint` metric: the default sentinel - // is the base series (`bench:paint`) and named `elementtiming` markers are sub-series - // (`bench:paint#grid-header`, …), all sharing a single definition. Paint is informational (no - // alarm): it dominates each test's total duration, so a per-test paint alarm just duplicates the - // Duration regression signal and floods the report on any broadly-regressed run. - const paint = new ScalarMetric({ - name: 'bench:paint', - format: { style: 'unit', unit: 'millisecond', maximumFractionDigits: 2 }, - }); - if (typeof window.gc !== 'function') { console.warn( 'window.gc is not available. Run with --js-flags=--expose-gc for consistent GC between iterations.', ); } + const iterations: IterationData[] = []; let renderError: unknown = null; // Set if any iteration had a recording window that was active yet captured no renders. let sawEmptyActiveWindow = false; - for (let i = 0; i < totalRuns; i += 1) { - const isWarmup = i < warmupRuns; - - // Custom metrics recorded inside the benchmark honor warmup exclusion through the gate, the - // same way renders and `bench:paint` are excluded during warmup. - metricsGate.setRecordingEnabled(task, !isWarmup); - - // Per-iteration switch for the harness's React render/paint recording. Starts paused when - // `reactRecordingPaused` is set; the interaction callback drives it from there. - const recording = createReactRecordingControls(!(options?.reactRecordingPaused ?? false)); - - // Drain event loop from previous unmount, then double GC for thorough cleanup + for (let i = 0; i < warmupRuns + runs; i += 1) { + const warmup = i < warmupRuns; // eslint-disable-next-line no-await-in-loop - await settle(); - forceGC(); - - const captures: RenderEvent[] = []; - const timing = createElementTimingWaiter(); - - const runtime = createCaseRuntime({ - // Wrap the case in BenchProfiler so its renders are captured; the runtime mounts whatever - // renderFn returns (profiling passes the case bare). - renderFn: () => ( - - {renderFn()} - - ), - interaction, - context: { - waitForElementTiming: timing.waitForElementTiming, - pauseReactRecording: recording.pauseReactRecording, - resumeReactRecording: recording.resumeReactRecording, - }, - // eslint-disable-next-line @typescript-eslint/no-loop-func - onUncaughtError: (error) => { - renderError = error; - }, + const result = await runCase(renderFn, interaction, { + afterEach: options?.afterEach, + reactRecordingPaused: options?.reactRecordingPaused, + warmup, }); - - const iterationStart = performance.now(); - - runtime.mount(); - - if (renderError) { - timing.disconnect(); - runtime.unmount(); + if (result.renderError) { + renderError = result.renderError; break; } - - // eslint-disable-next-line no-await-in-loop - await runtime.interact?.(); - - // Wait for the bench sentinel paint entry (relies on test timeout) - // eslint-disable-next-line no-await-in-loop - await timing.waitForElementTiming('default', 0); - - // Close the final window and remember if any active window measured no renders. - recording.finalizeWindow(); - if (recording.hadEmptyActiveWindow) { + if (result.hadEmptyActiveWindow) { sawEmptyActiveWindow = true; } - - timing.disconnect(); - - runtime.unmount(); - - if (!isWarmup) { - for (const entry of timing.elementEntries) { - // Skip paints that happened while recording was paused. Attribute by the paint's - // `paintTime`, not by when the observer callback fired (which can lag the paint). - if (!recording.activeAt(entry.paintTime)) { - continue; - } - // The default sentinel is the base series; named markers become sub-series. - const id = entry.identifier === 'default' ? undefined : entry.identifier; - paint.record(entry.paintTime - iterationStart, id !== undefined ? { id } : undefined); - } - iterations.push({ renders: captures }); - } - - if (options?.afterEach) { - // eslint-disable-next-line no-await-in-loop - await options.afterEach(); + if (!warmup) { + iterations.push({ renders: result.renders }); } } From 457fa3c794321e4143b4e642da674d8b5fb0a356 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Fri, 25 Sep 2026 13:47:39 +0200 Subject: [PATCH 071/173] [benchmark] Prototype an interleaved engine for tacho run Adds `tacho run --engine interleaved`: instead of handing each case to tachometer, it drives Chromium through Playwright, samples every variant once per round in a shuffled order, and computes confidence intervals on per-round paired differences. The output is tachometer-shaped JSON, so summarizeCase, the report, the upload and the PR comment are unchanged. It runs tachometer.json cases (fresh page load per sample, performance entries) and *.bench.tsx files written with benchmark(). The plugin generates a page per bench file where @mui/internal-benchmark resolves to a page runtime; the page stays open so iterations are warm, and results leave the page as performance.measure entries. The harness core (mount, profiler, paint wait) moves to caseRuntime.tsx so both drivers share it without Vitest; Metric.record() goes through a pluggable recorder. Interactions get `input`: trusted CDP gestures (scroll, pinch, tap), via Vitest's cdp() or a binding the runner exposes. A/A runs showed a renderer process keeps a stable speed offset for its lifetime (several percent), which reads as a difference when a build is tied to one process. Page loads are therefore dealt over the contexts anew each round, and bench cases run in epochs that reopen their pages. --- packages/benchmark/README.md | 49 +++ packages/benchmark/package.json | 2 + packages/benchmark/src/DiscreteMetric.ts | 2 +- packages/benchmark/src/Metric.test.ts | 2 + packages/benchmark/src/Metric.ts | 110 ++---- packages/benchmark/src/ScalarMetric.ts | 2 +- packages/benchmark/src/caseRuntime.tsx | 331 ++++++++++++++++ packages/benchmark/src/cli/tachoRun.ts | 26 ++ packages/benchmark/src/index.tsx | 314 ++------------- packages/benchmark/src/input.ts | 71 ++++ packages/benchmark/src/launchArgs.ts | 13 + packages/benchmark/src/metricCore.ts | 58 +++ packages/benchmark/src/page/page.tsx | 178 +++++++++ packages/benchmark/src/page/vitestStub.ts | 26 ++ .../src/tachometer/benchFiles.test.ts | 68 ++++ .../benchmark/src/tachometer/benchFiles.ts | 96 +++++ .../benchmark/src/tachometer/discoverCases.ts | 24 +- .../src/tachometer/pairedStats.test.ts | 135 +++++++ .../benchmark/src/tachometer/pairedStats.ts | 142 +++++++ packages/benchmark/src/tachometer/plugin.ts | 58 ++- .../src/tachometer/runInterleaved.ts | 363 ++++++++++++++++++ .../benchmark/src/tachometer/runTachometer.ts | 86 ++++- .../src/tachometer/serveDirectory.ts | 64 +++ packages/benchmark/src/types.ts | 7 + packages/benchmark/src/vitest.ts | 17 +- pnpm-lock.yaml | 6 + test/tachometer/package.json | 4 +- test/tachometer/src/list/list.bench.tsx | 59 +++ test/tachometer/tsconfig.json | 1 + 29 files changed, 1923 insertions(+), 391 deletions(-) create mode 100644 packages/benchmark/src/caseRuntime.tsx create mode 100644 packages/benchmark/src/input.ts create mode 100644 packages/benchmark/src/launchArgs.ts create mode 100644 packages/benchmark/src/metricCore.ts create mode 100644 packages/benchmark/src/page/page.tsx create mode 100644 packages/benchmark/src/page/vitestStub.ts create mode 100644 packages/benchmark/src/tachometer/benchFiles.test.ts create mode 100644 packages/benchmark/src/tachometer/benchFiles.ts create mode 100644 packages/benchmark/src/tachometer/pairedStats.test.ts create mode 100644 packages/benchmark/src/tachometer/pairedStats.ts create mode 100644 packages/benchmark/src/tachometer/runInterleaved.ts create mode 100644 packages/benchmark/src/tachometer/serveDirectory.ts create mode 100644 test/tachometer/src/list/list.bench.tsx diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 0bec73a807..90cf295a48 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -70,6 +70,23 @@ benchmark( ); ``` +For input that should behave like a real mouse or trackpad, the interaction context has `input`: +trusted gestures the browser generates itself at frame cadence, one command per gesture. + +```tsx +benchmark( + 'Chart zoom', + () => , + async ({ input }) => { + await input.scroll({ x: 400, y: 300, deltaY: 600 }); + await input.pinch({ x: 400, y: 300, scaleFactor: 2 }); + }, +); +``` + +A gesture's event count follows what the browser coalesced, so its render count can vary from one +iteration to the next. + ### Scoping which renders are measured By default a benchmark records every React render and paint, from the mount through the whole interaction. To measure only part of an interaction — or to exclude the mount — pause and resume recording from the interaction callback: @@ -392,6 +409,38 @@ resolution there, leaving the repository untouched, at the cost of a second inst `--no-install` skips a ref's install. `benchmark tacho run --help` lists the rest, and `benchmark tacho report` prints the table again from a saved JSON report. +### Interleaved engine + +`--engine interleaved` measures the same builds without tachometer: it drives Chromium through +Playwright, samples every variant once per round in a shuffled order, and judges a difference on the +per-round differences rather than on two independent sets of samples. Whatever the machine was doing +during a round — thermal throttling, a background process — then affects both sides of it and +cancels out, instead of counting as noise or, worse, as a difference. The report, the upload and the +PR comment are the same as tachometer's. + +It runs two kinds of case side by side: + +- **`tachometer.json` cases**, unchanged. Every sample is a fresh page load, measured by the + `performance.measure` entry (or `fcp`) the case names. Other measurement modes are tachometer-only. +- **`*.bench.tsx` files** under `src/`, written with `benchmark()` exactly as for Vitest. The plugin + generates a page per file under `src/__bench__/`, where `@mui/internal-benchmark` resolves to a page + runtime instead of Vitest. The page stays open, so every sample is one warm iteration and + module-scope data is built once; renders, `bench:paint` and custom metrics are reported as + measurements. The iteration counts in `benchmark()`'s options are ignored — `--samples` and + `--warmup` decide. + +```bash +benchmark tacho run --engine interleaved --baseline "$(code-infra baseline)" --samples 30 +``` + +Every variant runs in a browser context of its own, but a context's renderer process keeps whatever +speed it started with — on a machine with performance and efficiency cores, which kind it landed on +— and two identical builds can differ by several percent for as long as their processes live. Page +loads are therefore dealt out over the contexts anew every round. A `*.bench.tsx` page has to stay +open to stay warm, so its rounds run in epochs instead (`--epoch-size`, 10 rounds by default): each +epoch reopens the pages in fresh processes and warms them up again, so the process a build lands on +is drawn once per epoch rather than once per case. + ## API - `benchmark` — define a benchmark test case diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index 507ffe2d96..c8317d56b7 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -14,6 +14,8 @@ "./ciReport": "./src/ciReport.ts", "./reporter": "./src/reporter.ts", "./vitest": "./src/vitest.ts", + "./page": "./src/page/page.tsx", + "./page/vitest": "./src/page/vitestStub.ts", "./tachometerPlugin": "./src/tachometer/plugin.ts", "./tachometerReport": "./src/tachometer/renderReport.ts", "./tachometerFormat": "./src/tachometer/format.ts", diff --git a/packages/benchmark/src/DiscreteMetric.ts b/packages/benchmark/src/DiscreteMetric.ts index 90cec67c5d..b1c27862f8 100644 --- a/packages/benchmark/src/DiscreteMetric.ts +++ b/packages/benchmark/src/DiscreteMetric.ts @@ -1,4 +1,4 @@ -import { Metric } from './Metric'; +import { Metric } from './metricCore'; import type { MetricConfig, MetricKind } from './types'; /** diff --git a/packages/benchmark/src/Metric.test.ts b/packages/benchmark/src/Metric.test.ts index fc061eb074..51a6682160 100644 --- a/packages/benchmark/src/Metric.test.ts +++ b/packages/benchmark/src/Metric.test.ts @@ -1,4 +1,6 @@ import { describe, it, expect } from 'vitest'; +// Installs the Vitest metric recorder. +import './Metric'; import { ScalarMetric } from './ScalarMetric'; import { DiscreteMetric } from './DiscreteMetric'; diff --git a/packages/benchmark/src/Metric.ts b/packages/benchmark/src/Metric.ts index 2cdc2cf492..c84a4087fc 100644 --- a/packages/benchmark/src/Metric.ts +++ b/packages/benchmark/src/Metric.ts @@ -3,9 +3,13 @@ import type { RunnerTestCase } from 'vitest'; import type { MetricConfig, MetricKind, MetricReport, MetricSampleStats } from './types'; import { aggregateSamples } from './stats'; import { metricsGate } from './metricsGate'; +import { setMetricRecorder } from './metricCore'; +import type { Metric } from './metricCore'; // Import for TaskMeta augmentation side effect import './taskMetaAugmentation'; +export { Metric, type MetricRecordOptions } from './metricCore'; + interface SeriesAccumulator { /** The instance that registered this name, used to reject two metrics sharing a name. */ owner: Metric; @@ -34,78 +38,46 @@ function flush(test: RunnerTestCase, accumulator: TestAccumulator): void { test.meta.benchmarkMetrics = store; } -export interface MetricRecordOptions { - /** Sub-series label. Recorded under `${name}#${id}` in the report; omit for the base series. */ - id?: string; -} - -/** - * Base class for custom benchmark metrics. Use `ScalarMetric` or `DiscreteMetric`. - * - * Records are tied to the test that is running when `record()` is called (resolved via - * `getCurrentTest()`), so a single instance can be declared at module scope and reused across - * tests and loop iterations, inside or outside React. - */ -export abstract class Metric { - abstract readonly kind: MetricKind; - - readonly name: string; - - protected readonly config: MetricConfig; - - constructor(config: MetricConfig | string) { - this.config = typeof config === 'string' ? { name: config } : config; - this.name = this.config.name; - if (this.name.includes('#')) { - throw new Error( - `Metric name "${this.name}" must not contain "#" — it is reserved as the sub-series separator.`, - ); - } +// Records are tied to the test that is running when `record()` is called (resolved via +// `getCurrentTest()`), and aggregated once when that test finishes. +setMetricRecorder((metric, value, options) => { + const test = TestRunner.getCurrentTest(); + if (!test) { + throw new Error( + `${metric.constructor.name}.record() must be called inside a running Vitest test.`, + ); } - /** - * Records a single measured value. Samples accumulate in browser memory and are aggregated - * once when the test finishes. Pass `options.id` to split into a labeled sub-series. - */ - record(value: number, options?: MetricRecordOptions): void { - const test = TestRunner.getCurrentTest(); - if (!test) { - throw new Error( - `${this.constructor.name}.record() must be called inside a running Vitest test.`, - ); - } - - // The harness disables the gate during warmup iterations so custom metrics recorded inside a - // benchmark honor the same warmup exclusion as renders and `bench:paint`. Standalone `it()` - // loops never touch the gate, so they keep recording every value. - if (!metricsGate.isRecordingEnabled(test)) { - return; - } + // The harness disables the gate during warmup iterations so custom metrics recorded inside a + // benchmark honor the same warmup exclusion as renders and `bench:paint`. Standalone `it()` + // loops never touch the gate, so they keep recording every value. + if (!metricsGate.isRecordingEnabled(test)) { + return; + } - let accumulator = accumulators.get(test); - if (!accumulator) { - const created: TestAccumulator = new Map(); - accumulator = created; - accumulators.set(test, created); - onTestFinished(() => flush(test, created)); - } + let accumulator = accumulators.get(test); + if (!accumulator) { + const created: TestAccumulator = new Map(); + accumulator = created; + accumulators.set(test, created); + onTestFinished(() => flush(test, created)); + } - let entry = accumulator.get(this.name); - if (!entry) { - entry = { owner: this, kind: this.kind, config: this.config, series: new Map() }; - accumulator.set(this.name, entry); - } else if (entry.owner !== this) { - throw new Error( - `Two metrics share the name "${this.name}". Metric names must be unique; reuse a single instance instead.`, - ); - } + let entry = accumulator.get(metric.name); + if (!entry) { + entry = { owner: metric, kind: metric.kind, config: metric.config, series: new Map() }; + accumulator.set(metric.name, entry); + } else if (entry.owner !== metric) { + throw new Error( + `Two metrics share the name "${metric.name}". Metric names must be unique; reuse a single instance instead.`, + ); + } - const seriesId = options?.id ?? ''; - let samples = entry.series.get(seriesId); - if (!samples) { - samples = []; - entry.series.set(seriesId, samples); - } - samples.push(value); + const seriesId = options?.id ?? ''; + let samples = entry.series.get(seriesId); + if (!samples) { + samples = []; + entry.series.set(seriesId, samples); } -} + samples.push(value); +}); diff --git a/packages/benchmark/src/ScalarMetric.ts b/packages/benchmark/src/ScalarMetric.ts index ce730327a4..58bbafff3c 100644 --- a/packages/benchmark/src/ScalarMetric.ts +++ b/packages/benchmark/src/ScalarMetric.ts @@ -1,4 +1,4 @@ -import { Metric } from './Metric'; +import { Metric } from './metricCore'; import type { MetricKind } from './types'; /** diff --git a/packages/benchmark/src/caseRuntime.tsx b/packages/benchmark/src/caseRuntime.tsx new file mode 100644 index 0000000000..cc472d6c43 --- /dev/null +++ b/packages/benchmark/src/caseRuntime.tsx @@ -0,0 +1,331 @@ +import * as React from 'react'; +import * as ReactDOMClient from 'react-dom/client'; // aliased to react-dom/profiling by Vite +import * as ReactDOM from 'react-dom'; +import type { RenderEvent, InteractionContext, BenchmarkCaseRuntime } from './types'; +import type { BenchmarkInput } from './input'; +import { ElementTiming } from './ElementTiming'; +import { createReactRecordingControls } from './reactRecording'; +import type { ReactRecordingControls } from './reactRecording'; + +// Everything here runs a case without knowing who drives it: the Vitest harness (`index.tsx`) and +// the A/B page (`page/page.tsx`) both build on it, so nothing in this module may import Vitest. + +interface PerformanceElementTiming extends PerformanceEntry { + readonly entryType: 'element'; + // When the browser started painting the element (i.e. the render phase ended). Preferred over + // `renderTime`, which reports when the pixels reached the screen: that includes the wait for the + // next display refresh, adding variance and time unrelated to the CPU-bound render work these + // benchmarks optimize. + readonly paintTime: DOMHighResTimeStamp; + readonly identifier: string; +} + +export type BenchmarkInteraction = (ctx: InteractionContext) => Promise | void; + +export interface CaseOptions { + afterEach?: () => Promise | void; + /** + * Start each iteration with React render/paint recording paused. The interaction callback then + * calls `resumeReactRecording()` at the point it cares about — useful to exclude the mount and + * measure only the renders/paint of a later interaction. Defaults to `false` (mount recorded). + */ + reactRecordingPaused?: boolean; +} + +function BenchProfiler({ + captures, + recording, + children, +}: { + captures: RenderEvent[]; + recording: ReactRecordingControls; + children: React.ReactNode; +}) { + const onRender = React.useCallback( + (id, phase, actualDuration, _baseDuration, startTime) => { + // Skip renders captured while React recording is paused (e.g. the mount when the benchmark + // starts paused, or a span the interaction explicitly excludes). + if (recording.active) { + captures.push({ id, phase, actualDuration, startTime }); + recording.markRendered(); + } + }, + [captures, recording], + ); + + return ( + + {children} + + ); +} + +// Double GC: the first pass collects garbage, the second catches weak refs +// and prevent leaking into the next iteration. +function forceGC() { + if (typeof window.gc === 'function') { + window.gc(); + window.gc(); + } +} + +declare global { + interface Window { + gc?: () => void; + } +} + +// Flush pending microtasks and React cleanup effects (e.g. from a previous unmount) +// so they don't interfere with the next iteration's timing. +function settle(): Promise { + return new Promise((resolve) => { + setTimeout(resolve, 0); + }); +} + +function supportsElementTiming(): boolean { + return PerformanceObserver.supportedEntryTypes.includes('element'); +} + +interface ElementTimingWaiter { + elementEntries: PerformanceElementTiming[]; + waitForElementTiming: (identifier: string, timeout?: number) => Promise; + disconnect: () => void; +} + +// Sets up a PerformanceObserver for the Element Timing API and exposes a promise-based +// `waitForElementTiming` helper. Used by the measurement loop (which also reads `elementEntries` +// to record paint metrics) and the interactive profiling session. +export function createElementTimingWaiter(): ElementTimingWaiter { + const hasElementTiming = supportsElementTiming(); + const elementEntries: PerformanceElementTiming[] = []; + const elementResolvers = new Map void>(); + + let observer: PerformanceObserver | null = null; + if (hasElementTiming) { + observer = new PerformanceObserver((list) => { + for (const entry of list.getEntries() as PerformanceElementTiming[]) { + elementEntries.push(entry); + const resolver = elementResolvers.get(entry.identifier); + if (resolver) { + elementResolvers.delete(entry.identifier); + resolver(); + } + } + }); + observer.observe({ type: 'element', buffered: false }); + } + + const waitForElementTiming = (identifier: string, timeout?: number): Promise => { + if (!hasElementTiming) { + console.warn( + `waitForElementTiming("${identifier}"): Element Timing API is not supported. ` + + 'Paint metrics will not be collected.', + ); + return Promise.resolve(); + } + if (elementEntries.some((entry) => entry.identifier === identifier)) { + return Promise.resolve(); + } + const { promise, resolve, reject } = Promise.withResolvers(); + const timeoutMs = timeout ?? 5000; + const timer = + timeoutMs > 0 && timeoutMs < Infinity + ? setTimeout(() => { + elementResolvers.delete(identifier); + reject( + new Error( + `waitForElementTiming("${identifier}"): timed out after ${timeoutMs}ms. ` + + 'Ensure the element has an `elementtiming` attribute and is visible in the viewport.', + ), + ); + }, timeoutMs) + : undefined; + elementResolvers.set(identifier, () => { + if (timer) { + clearTimeout(timer); + } + resolve(); + }); + return promise; + }; + + return { + elementEntries, + waitForElementTiming, + disconnect: () => observer?.disconnect(), + }; +} + +interface CreateCaseRuntimeOptions { + /** + * Produces the case to mount. Measurement passes a renderFn already wrapped in `` + * to collect render events; profiling passes the case bare so the React Profiler overhead stays + * out of the hand-captured DevTools trace. + */ + renderFn: () => React.ReactElement; + interaction?: BenchmarkInteraction; + /** + * Context handed to the interaction callback. The driver assembles it: measurement supplies the + * real React recording controls, profiling supplies no-ops since the user drives DevTools by hand. + */ + context: InteractionContext; + onUncaughtError?: (error: unknown) => void; +} + +export function createCaseRuntime({ + renderFn, + interaction, + context, + onUncaughtError, +}: CreateCaseRuntimeOptions): BenchmarkCaseRuntime { + let root: ReactDOMClient.Root | null = null; + let container: HTMLElement | null = null; + + return { + mount() { + if (root) { + return; + } + container = document.createElement('div'); + document.body.appendChild(container); + const newRoot = ReactDOMClient.createRoot(container, { onUncaughtError }); + root = newRoot; + ReactDOM.flushSync(() => { + newRoot.render( + + {renderFn()} + {/* Harness paint sentinel: emits the `default` Element Timing entry that + `waitForElementTiming('default')` waits on, in both measurement and profile mode. + Rendered as a sibling (outside renderFn / its BenchProfiler) so it isn't counted in + the measured render duration. */} + + , + ); + }); + }, + interact: interaction + ? async () => { + await interaction(context); + } + : undefined, + unmount() { + if (!root) { + return; + } + root.unmount(); + container?.remove(); + root = null; + container = null; + }, + isMounted: () => root !== null, + }; +} + +export interface PaintTiming { + /** The `elementtiming` identifier; `undefined` for the harness's `default` sentinel. */ + id: string | undefined; + /** `performance.now()` when the iteration started, before mounting. */ + start: number; + /** The element's `paintTime`. */ + end: number; +} + +export interface MeasuredIteration { + /** Renders captured while React recording was active. */ + renders: RenderEvent[]; + /** Element paints that happened while React recording was active. */ + paints: PaintTiming[]; + /** The error a render threw, if one did. The iteration stops at mount when it does. */ + renderError: unknown; + /** Whether a recording window was active yet captured no renders. */ + hadEmptyActiveWindow: boolean; +} + +/** + * Mounts, interacts with and unmounts a case once. Records nothing itself: renders and paints are + * returned, and the driver decides whether they count (they don't during warmup) and where they go. + */ +export async function measureIteration( + renderFn: () => React.ReactElement, + interaction: BenchmarkInteraction | undefined, + options: CaseOptions | undefined, + input: BenchmarkInput, +): Promise { + // Per-iteration switch for the harness's React render/paint recording. Starts paused when + // `reactRecordingPaused` is set; the interaction callback drives it from there. + const recording = createReactRecordingControls(!(options?.reactRecordingPaused ?? false)); + + // Drain event loop from previous unmount, then double GC for thorough cleanup + await settle(); + forceGC(); + + const captures: RenderEvent[] = []; + const timing = createElementTimingWaiter(); + let renderError: unknown = null; + + const runtime = createCaseRuntime({ + // Wrap the case in BenchProfiler so its renders are captured; the runtime mounts whatever + // renderFn returns (profiling passes the case bare). + renderFn: () => ( + + {renderFn()} + + ), + interaction, + context: { + waitForElementTiming: timing.waitForElementTiming, + pauseReactRecording: recording.pauseReactRecording, + resumeReactRecording: recording.resumeReactRecording, + input, + }, + onUncaughtError: (error) => { + renderError = error; + }, + }); + + const iterationStart = performance.now(); + + runtime.mount(); + + if (renderError) { + timing.disconnect(); + runtime.unmount(); + return { renders: captures, paints: [], renderError, hadEmptyActiveWindow: false }; + } + + await runtime.interact?.(); + + // Wait for the bench sentinel paint entry (relies on the driver's timeout) + await timing.waitForElementTiming('default', 0); + + // Close the final window so an active window that measured no renders is flagged. + recording.finalizeWindow(); + + timing.disconnect(); + + runtime.unmount(); + + const paints: PaintTiming[] = []; + for (const entry of timing.elementEntries) { + // Skip paints that happened while recording was paused. Attribute by the paint's + // `paintTime`, not by when the observer callback fired (which can lag the paint). + if (!recording.activeAt(entry.paintTime)) { + continue; + } + // The default sentinel is the base series; named markers become sub-series. + const id = entry.identifier === 'default' ? undefined : entry.identifier; + paints.push({ id, start: iterationStart, end: entry.paintTime }); + } + + if (options?.afterEach) { + await options.afterEach(); + } + + return { + renders: captures, + paints, + renderError, + hadEmptyActiveWindow: recording.hadEmptyActiveWindow, + }; +} diff --git a/packages/benchmark/src/cli/tachoRun.ts b/packages/benchmark/src/cli/tachoRun.ts index a6c6343e0d..40188a301b 100644 --- a/packages/benchmark/src/cli/tachoRun.ts +++ b/packages/benchmark/src/cli/tachoRun.ts @@ -48,6 +48,28 @@ const command: CommandModule<{}, Args> = { type: 'string', describe: 'Write the combined JSON report here. Default: .tachometer/results/report.json', }) + .option('engine', { + type: 'string', + choices: ['tachometer', 'interleaved'] as const, + default: 'tachometer' as const, + describe: + "What measures the pages: 'tachometer', or 'interleaved', which alternates variants round by round in Playwright, compares paired differences, and also runs *.bench.tsx files", + }) + .option('samples', { + type: 'number', + describe: + "Interleaved engine: measured rounds per case. Default: the case's sampleSize, else 30", + }) + .option('warmup', { + type: 'number', + describe: + 'Interleaved engine: discarded rounds before measuring. Default: 2 per page case, 5 per epoch for *.bench.tsx cases', + }) + .option('epoch-size', { + type: 'number', + describe: + 'Interleaved engine: measured rounds of a *.bench.tsx case before its pages are reopened in fresh processes. Default: 10', + }) .epilogue( 'Sampling (sampleSize, autoSampleConditions, timeout) is configured per case in its own tachometer.json — tachometer rejects those as CLI flags when a config file is used.', ); @@ -63,6 +85,10 @@ const command: CommandModule<{}, Args> = { install: argv.install, out: argv.out, upload: argv.upload, + engine: argv.engine, + samples: argv.samples, + warmup: argv.warmup, + epochSize: argv.epochSize, }); }, }; diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index 1c82c41d24..2a71d93d0b 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -1,28 +1,19 @@ -import * as React from 'react'; +import type * as React from 'react'; import { expect, it, TestRunner } from 'vitest'; import type { RunnerTestCase } from 'vitest'; -import * as ReactDOMClient from 'react-dom/client'; // aliased to react-dom/profiling by Vite -import * as ReactDOM from 'react-dom'; -import type { RenderEvent, IterationData, InteractionContext, BenchmarkCaseRuntime } from './types'; -import { ElementTiming } from './ElementTiming'; +import { cdp } from 'vitest/browser'; +import type { RenderEvent, IterationData } from './types'; import { ScalarMetric } from './ScalarMetric'; import { metricsGate } from './metricsGate'; -import { createReactRecordingControls } from './reactRecording'; -import type { ReactRecordingControls } from './reactRecording'; import { runProfileSession } from './profileSession'; +import { createInput } from './input'; +import { createCaseRuntime, createElementTimingWaiter, measureIteration } from './caseRuntime'; +import type { BenchmarkInteraction, CaseOptions } from './caseRuntime'; +// Installs the Vitest metric recorder. +import './Metric'; // Import for TaskMeta augmentation side effect import './taskMetaAugmentation'; -interface PerformanceElementTiming extends PerformanceEntry { - readonly entryType: 'element'; - // When the browser started painting the element (i.e. the render phase ended). Preferred over - // `renderTime`, which reports when the pixels reached the screen: that includes the wait for the - // next display refresh, adding variance and time unrelated to the CPU-bound render work these - // benchmarks optimize. - readonly paintTime: DOMHighResTimeStamp; - readonly identifier: string; -} - export type { RenderEvent, IterationData, InteractionContext } from './types'; export type { MetricKind, @@ -31,218 +22,25 @@ export type { MetricConfig, MetricDefinition, } from './types'; +export type { BenchmarkInput } from './input'; +export type { BenchmarkInteraction } from './caseRuntime'; export { ElementTiming } from './ElementTiming'; export { Metric, type MetricRecordOptions } from './Metric'; export { ScalarMetric }; export { DiscreteMetric } from './DiscreteMetric'; -function BenchProfiler({ - captures, - recording, - children, -}: { - captures: RenderEvent[]; - recording: ReactRecordingControls; - children: React.ReactNode; -}) { - const onRender = React.useCallback( - (id, phase, actualDuration, _baseDuration, startTime) => { - // Skip renders captured while React recording is paused (e.g. the mount when the benchmark - // starts paused, or a span the interaction explicitly excludes). - if (recording.active) { - captures.push({ id, phase, actualDuration, startTime }); - recording.markRendered(); - } - }, - [captures, recording], - ); - - return ( - - {children} - - ); -} - -// Double GC: the first pass collects garbage, the second catches weak refs -// and prevent leaking into the next iteration. -function forceGC() { - if (typeof window.gc === 'function') { - window.gc(); - window.gc(); - } -} - -declare global { - interface Window { - gc?: () => void; - } -} - -// Flush pending microtasks and React cleanup effects (e.g. from a previous unmount) -// so they don't interfere with the next iteration's timing. -function settle(): Promise { - return new Promise((resolve) => { - setTimeout(resolve, 0); - }); -} - -function supportsElementTiming(): boolean { - return PerformanceObserver.supportedEntryTypes.includes('element'); -} - -interface ElementTimingWaiter { - elementEntries: PerformanceElementTiming[]; - waitForElementTiming: (identifier: string, timeout?: number) => Promise; - disconnect: () => void; -} - -// Sets up a PerformanceObserver for the Element Timing API and exposes a promise-based -// `waitForElementTiming` helper. Used by the measurement loop (which also reads `elementEntries` -// to record paint metrics) and the interactive profiling session. -function createElementTimingWaiter(): ElementTimingWaiter { - const hasElementTiming = supportsElementTiming(); - const elementEntries: PerformanceElementTiming[] = []; - const elementResolvers = new Map void>(); - - let observer: PerformanceObserver | null = null; - if (hasElementTiming) { - observer = new PerformanceObserver((list) => { - for (const entry of list.getEntries() as PerformanceElementTiming[]) { - elementEntries.push(entry); - const resolver = elementResolvers.get(entry.identifier); - if (resolver) { - elementResolvers.delete(entry.identifier); - resolver(); - } - } - }); - observer.observe({ type: 'element', buffered: false }); - } - - const waitForElementTiming = (identifier: string, timeout?: number): Promise => { - if (!hasElementTiming) { - console.warn( - `waitForElementTiming("${identifier}"): Element Timing API is not supported. ` + - 'Paint metrics will not be collected.', - ); - return Promise.resolve(); - } - if (elementEntries.some((entry) => entry.identifier === identifier)) { - return Promise.resolve(); - } - const { promise, resolve, reject } = Promise.withResolvers(); - const timeoutMs = timeout ?? 5000; - const timer = - timeoutMs > 0 && timeoutMs < Infinity - ? setTimeout(() => { - elementResolvers.delete(identifier); - reject( - new Error( - `waitForElementTiming("${identifier}"): timed out after ${timeoutMs}ms. ` + - 'Ensure the element has an `elementtiming` attribute and is visible in the viewport.', - ), - ); - }, timeoutMs) - : undefined; - elementResolvers.set(identifier, () => { - if (timer) { - clearTimeout(timer); - } - resolve(); - }); - return promise; - }; - - return { - elementEntries, - waitForElementTiming, - disconnect: () => observer?.disconnect(), - }; -} - // When true, `benchmark()` opens an interactive profiling session in a headed // browser instead of running the automated measurement loop. Enabled by // `createBenchmarkVitestConfig({ profile: true })` or `BENCHMARK_PROFILE=true`, // both of which replace this expression at build time via Vite `define`. const PROFILE_MODE = process.env.BENCHMARK_PROFILE === 'true'; -interface CreateCaseRuntimeOptions { - /** - * Produces the case to mount. Measurement passes a renderFn already wrapped in `` - * to collect render events; profiling passes the case bare so the React Profiler overhead stays - * out of the hand-captured DevTools trace. - */ - renderFn: () => React.ReactElement; - interaction?: (ctx: InteractionContext) => Promise | void; - /** - * Context handed to the interaction callback. The driver assembles it: measurement supplies the - * real React recording controls, profiling supplies no-ops since the user drives DevTools by hand. - */ - context: InteractionContext; - onUncaughtError?: (error: unknown) => void; -} - -function createCaseRuntime({ - renderFn, - interaction, - context, - onUncaughtError, -}: CreateCaseRuntimeOptions): BenchmarkCaseRuntime { - let root: ReactDOMClient.Root | null = null; - let container: HTMLElement | null = null; - - return { - mount() { - if (root) { - return; - } - container = document.createElement('div'); - document.body.appendChild(container); - const newRoot = ReactDOMClient.createRoot(container, { onUncaughtError }); - root = newRoot; - ReactDOM.flushSync(() => { - newRoot.render( - - {renderFn()} - {/* Harness paint sentinel: emits the `default` Element Timing entry that - `waitForElementTiming('default')` waits on, in both measurement and profile mode. - Rendered as a sibling (outside renderFn / its BenchProfiler) so it isn't counted in - the measured render duration. */} - - , - ); - }); - }, - interact: interaction - ? async () => { - await interaction(context); - } - : undefined, - unmount() { - if (!root) { - return; - } - root.unmount(); - container?.remove(); - root = null; - container = null; - }, - isMounted: () => root !== null, - }; -} +type CdpMethod = Parameters['send']>[0]; -export type BenchmarkInteraction = (ctx: InteractionContext) => Promise | void; - -interface CaseOptions { - afterEach?: () => Promise | void; - /** - * Start each iteration with React render/paint recording paused. The interaction callback then - * calls `resumeReactRecording()` at the point it cares about — useful to exclude the mount and - * measure only the renders/paint of a later interaction. Defaults to `false` (mount recorded). - */ - reactRecordingPaused?: boolean; -} +// Gestures go through the CDP session Vitest's Playwright provider holds for the test page. The +// session's `send` only accepts protocol method names it knows; `input.send` takes any string and +// leaves unknown methods for Chrome to reject. +const input = createInput((method, params) => cdp().send(method as CdpMethod, params)); export interface BenchmarkOptions extends CaseOptions { runs?: number; @@ -276,82 +74,23 @@ const paint = new ScalarMetric({ format: { style: 'unit', unit: 'millisecond', maximumFractionDigits: 2 }, }); -async function measureIteration( +async function measureAndRecord( renderFn: () => React.ReactElement, interaction: BenchmarkInteraction | undefined, options: RunCaseOptions | undefined, ): Promise { - // Per-iteration switch for the harness's React render/paint recording. Starts paused when - // `reactRecordingPaused` is set; the interaction callback drives it from there. - const recording = createReactRecordingControls(!(options?.reactRecordingPaused ?? false)); - - // Drain event loop from previous unmount, then double GC for thorough cleanup - await settle(); - forceGC(); - - const captures: RenderEvent[] = []; - const timing = createElementTimingWaiter(); - let renderError: unknown = null; - - const runtime = createCaseRuntime({ - // Wrap the case in BenchProfiler so its renders are captured; the runtime mounts whatever - // renderFn returns (profiling passes the case bare). - renderFn: () => ( - - {renderFn()} - - ), + const { renders, paints, renderError, hadEmptyActiveWindow } = await measureIteration( + renderFn, interaction, - context: { - waitForElementTiming: timing.waitForElementTiming, - pauseReactRecording: recording.pauseReactRecording, - resumeReactRecording: recording.resumeReactRecording, - }, - onUncaughtError: (error) => { - renderError = error; - }, - }); - - const iterationStart = performance.now(); - - runtime.mount(); - - if (renderError) { - timing.disconnect(); - runtime.unmount(); - return { renders: captures, renderError, hadEmptyActiveWindow: false }; - } - - await runtime.interact?.(); - - // Wait for the bench sentinel paint entry (relies on test timeout) - await timing.waitForElementTiming('default', 0); - - // Close the final window so an active window that measured no renders is flagged. - recording.finalizeWindow(); - - timing.disconnect(); - - runtime.unmount(); - + options, + input, + ); if (!options?.warmup) { - for (const entry of timing.elementEntries) { - // Skip paints that happened while recording was paused. Attribute by the paint's - // `paintTime`, not by when the observer callback fired (which can lag the paint). - if (!recording.activeAt(entry.paintTime)) { - continue; - } - // The default sentinel is the base series; named markers become sub-series. - const id = entry.identifier === 'default' ? undefined : entry.identifier; - paint.record(entry.paintTime - iterationStart, id !== undefined ? { id } : undefined); + for (const { id, start, end } of paints) { + paint.record(end - start, id !== undefined ? { id } : undefined); } } - - if (options?.afterEach) { - await options.afterEach(); - } - - return { renders: captures, renderError, hadEmptyActiveWindow: recording.hadEmptyActiveWindow }; + return { renders, renderError, hadEmptyActiveWindow }; } /** @@ -378,11 +117,11 @@ export async function runCase( // and is re-enabled afterwards so metrics the driver records between iterations are kept. const test = TestRunner.getCurrentTest(); if (!test) { - return measureIteration(renderFn, interaction, options); + return measureAndRecord(renderFn, interaction, options); } metricsGate.setRecordingEnabled(test, !options?.warmup); try { - return await measureIteration(renderFn, interaction, options); + return await measureAndRecord(renderFn, interaction, options); } finally { metricsGate.setRecordingEnabled(test, true); } @@ -413,6 +152,7 @@ export function benchmark( waitForElementTiming: timing.waitForElementTiming, pauseReactRecording: () => {}, resumeReactRecording: () => {}, + input, }, }); await runProfileSession(name, runtime); diff --git a/packages/benchmark/src/input.ts b/packages/benchmark/src/input.ts new file mode 100644 index 0000000000..9c3f202d44 --- /dev/null +++ b/packages/benchmark/src/input.ts @@ -0,0 +1,71 @@ +/** Sends a raw Chrome DevTools Protocol command to the page under test. */ +export type CdpSend = (method: string, params?: Record) => Promise; + +export interface ScrollGestureOptions { + /** Viewport x coordinate (CSS pixels) where the gesture starts. */ + x: number; + /** Viewport y coordinate (CSS pixels) where the gesture starts. */ + y: number; + /** Horizontal distance to scroll, positive to scroll right. */ + deltaX?: number; + /** Vertical distance to scroll, positive to scroll down (like a wheel's `deltaY`). */ + deltaY?: number; + /** Gesture speed in pixels per second. Defaults to the browser's 800. */ + speed?: number; +} + +export interface PinchGestureOptions { + x: number; + y: number; + /** Relative scale at the end of the pinch: `> 1` zooms in, `< 1` zooms out. */ + scaleFactor: number; + /** Relative pointer speed in pixels per second. Defaults to the browser's 800. */ + relativeSpeed?: number; +} + +export interface TapGestureOptions { + x: number; + y: number; +} + +/** + * Trusted input for interactions. Each gesture is one command: the browser generates the whole + * event sequence itself at frame cadence, so the round trip to the driver is paid once per gesture, + * not once per event, and the page sees the event timing a real mouse or trackpad would produce. + */ +export interface BenchmarkInput { + scroll: (options: ScrollGestureOptions) => Promise; + pinch: (options: PinchGestureOptions) => Promise; + tap: (options: TapGestureOptions) => Promise; + /** Escape hatch for any other CDP command. */ + send: CdpSend; +} + +export function createInput(send: CdpSend): BenchmarkInput { + return { + scroll: async ({ x, y, deltaX = 0, deltaY = 0, speed }) => { + // CDP distances point the other way: positive `yDistance` scrolls up. + await send('Input.synthesizeScrollGesture', { + x, + y, + xDistance: -deltaX, + yDistance: -deltaY, + speed, + gestureSourceType: 'mouse', + }); + }, + pinch: async ({ x, y, scaleFactor, relativeSpeed }) => { + await send('Input.synthesizePinchGesture', { + x, + y, + scaleFactor, + relativeSpeed, + gestureSourceType: 'mouse', + }); + }, + tap: async ({ x, y }) => { + await send('Input.synthesizeTapGesture', { x, y, gestureSourceType: 'mouse' }); + }, + send, + }; +} diff --git a/packages/benchmark/src/launchArgs.ts b/packages/benchmark/src/launchArgs.ts new file mode 100644 index 0000000000..a129b683a8 --- /dev/null +++ b/packages/benchmark/src/launchArgs.ts @@ -0,0 +1,13 @@ +// Chromium/V8 launch args shared by every runner, kept intentionally minimal. `--expose-gc` is +// required: the harness forces GC between iterations for clean, comparable timings. The +// backgrounding flags stop Chrome from throttling the (headless or occluded) benchmark tab, which +// would otherwise add large variance. Heavier "determinism" flags (`--no-opt`, `--predictable`, +// `--hash-seed`/`--random-seed`, `--disable-gpu`, `--enable-benchmarking`) were measured to slow +// renders ~40% and distort paint timing without reducing variance, so they are omitted — add them +// per project if a specific workload needs them. +export const BENCHMARK_LAUNCH_ARGS = [ + '--js-flags=--expose-gc', + '--disable-background-timer-throttling', + '--disable-backgrounding-occluded-windows', + '--disable-renderer-backgrounding', +]; diff --git a/packages/benchmark/src/metricCore.ts b/packages/benchmark/src/metricCore.ts new file mode 100644 index 0000000000..76d08bcd2d --- /dev/null +++ b/packages/benchmark/src/metricCore.ts @@ -0,0 +1,58 @@ +import type { MetricConfig, MetricKind } from './types'; + +export interface MetricRecordOptions { + /** Sub-series label. Recorded under `${name}#${id}` in the report; omit for the base series. */ + id?: string; +} + +/** + * Where recorded values go. The runtime a benchmark runs under installs one: the Vitest harness + * accumulates them per running test, the A/B page emits them as `performance.measure` entries. + */ +export type MetricRecorder = ( + metric: Metric, + value: number, + options: MetricRecordOptions | undefined, +) => void; + +let recorder: MetricRecorder = (metric) => { + throw new Error( + `${metric.constructor.name}.record() was called outside a benchmark runtime. Import metrics from ` + + '`@mui/internal-benchmark` inside a benchmark file.', + ); +}; + +export function setMetricRecorder(next: MetricRecorder): void { + recorder = next; +} + +/** + * Base class for custom benchmark metrics. Use `ScalarMetric` or `DiscreteMetric`. + * + * A single instance can be declared at module scope and reused across tests and loop iterations, + * inside or outside React; the active runtime decides which test or sample a value belongs to. + */ +export abstract class Metric { + abstract readonly kind: MetricKind; + + readonly name: string; + + readonly config: MetricConfig; + + constructor(config: MetricConfig | string) { + this.config = typeof config === 'string' ? { name: config } : config; + this.name = this.config.name; + if (this.name.includes('#')) { + throw new Error( + `Metric name "${this.name}" must not contain "#" — it is reserved as the sub-series separator.`, + ); + } + } + + /** + * Records a single measured value. Pass `options.id` to split into a labeled sub-series. + */ + record(value: number, options?: MetricRecordOptions): void { + recorder(this, value, options); + } +} diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx new file mode 100644 index 0000000000..e664c95b04 --- /dev/null +++ b/packages/benchmark/src/page/page.tsx @@ -0,0 +1,178 @@ +import type * as React from 'react'; +import { createInput } from '../input'; +import { measureIteration } from '../caseRuntime'; +import type { BenchmarkInteraction, CaseOptions } from '../caseRuntime'; +import { setMetricRecorder } from '../metricCore'; + +// What `@mui/internal-benchmark` resolves to when benchmark files are built into an A/B page +// instead of run by Vitest. Same authoring API, different driver: `benchmark()` registers a case, +// and the runner calls `window.benchmarkPage.sample()` to run one iteration of it at a time. Every result +// leaves the page as a `performance.measure` entry, the same contract plain tachometer pages use. + +export type { RenderEvent, IterationData, InteractionContext } from '../types'; +export type { + MetricKind, + MetricDirection, + MetricAlarm, + MetricConfig, + MetricDefinition, +} from '../types'; +export type { BenchmarkInput } from '../input'; +export type { BenchmarkInteraction } from '../caseRuntime'; +export { ElementTiming } from '../ElementTiming'; +export { Metric, type MetricRecordOptions } from '../metricCore'; +export { ScalarMetric } from '../ScalarMetric'; +export { DiscreteMetric } from '../DiscreteMetric'; + +/** `benchmark()`'s options. The iteration counts are ignored here: the runner decides them. */ +export interface BenchmarkOptions extends CaseOptions { + runs?: number; + warmupRuns?: number; +} + +interface PageCase { + renderFn: () => React.ReactElement; + interaction: BenchmarkInteraction | undefined; + options: CaseOptions | undefined; +} + +const cases = new Map(); + +export function benchmark( + name: string, + renderFn: () => React.ReactElement, + interactionOrOptions?: BenchmarkInteraction | BenchmarkOptions, + maybeOptions?: BenchmarkOptions, +): void { + if (cases.has(name)) { + throw new Error(`Two benchmarks share the name "${name}". Benchmark names must be unique.`); + } + const interaction = typeof interactionOrOptions === 'function' ? interactionOrOptions : undefined; + const options = typeof interactionOrOptions === 'object' ? interactionOrOptions : maybeOptions; + cases.set(name, { + renderFn, + interaction, + options: options && { + afterEach: options.afterEach, + reactRecordingPaused: options.reactRecordingPaused, + }, + }); +} + +interface RecordedValue { + name: string; + value: number; +} + +// Values custom metrics record during a measured sample. `null` outside one (module scope, warmup), +// where recorded values are dropped. +let sampleValues: RecordedValue[] | null = null; + +setMetricRecorder((metric, value, options) => { + sampleValues?.push({ + name: options?.id === undefined ? metric.name : `${metric.name}#${options.id}`, + value, + }); +}); + +export interface BenchPage { + caseNames: () => string[]; + /** + * Runs one iteration of a case. A measured sample leaves its results as `performance.measure` + * entries for the runner to collect; a warmup sample leaves none. + */ + sample: (name: string, options: { warmup: boolean }) => Promise; +} + +declare global { + interface Window { + /** Set by `markPageReady()` once every benchmark file has registered its cases. */ + benchmarkPage?: BenchPage; + /** Exposed by the runner: forwards a CDP command to this page's session. */ + benchmarkCdp?: (method: string, params?: Record) => Promise; + } +} + +const input = createInput((method, params) => { + if (!window.benchmarkCdp) { + throw new Error('No CDP bridge: the runner must expose `benchmarkCdp` on this page.'); + } + return window.benchmarkCdp(method, params); +}); + +function emitSample( + result: Awaited>, + values: RecordedValue[], +): void { + // Render time is reported as totals rather than per render: an interaction's render count follows + // whatever the browser coalesced, so per-render entries would not line up across iterations. The + // per-phase split only adds information when there is more than one phase. + const byPhase = new Map(); + for (const render of result.renders) { + byPhase.set(render.phase, (byPhase.get(render.phase) ?? 0) + render.actualDuration); + } + if (result.renders.length > 0) { + const start = result.renders[0].startTime; + const total = [...byPhase.values()].reduce((sum, duration) => sum + duration, 0); + performance.measure('render', { start, duration: total, detail: { value: total } }); + if (byPhase.size > 1) { + for (const [phase, duration] of byPhase) { + performance.measure(`render:${phase}`, { start, duration, detail: { value: duration } }); + } + } + } + for (const { id, start, end } of result.paints) { + performance.measure(id === undefined ? 'bench:paint' : `bench:paint#${id}`, { start, end }); + } + // Custom metrics aren't necessarily durations, so the value travels in `detail`. + const now = performance.now(); + for (const { name, value } of values) { + performance.measure(name, { start: now, duration: 0, detail: { value } }); + } +} + +async function sample(name: string, { warmup }: { warmup: boolean }): Promise { + const benchCase = cases.get(name); + if (!benchCase) { + throw new Error(`No benchmark named "${name}". Known: ${[...cases.keys()].join(', ')}`); + } + performance.clearMarks(); + performance.clearMeasures(); + + const values: RecordedValue[] = []; + sampleValues = warmup ? null : values; + let result: Awaited>; + try { + result = await measureIteration( + benchCase.renderFn, + benchCase.interaction, + benchCase.options, + input, + ); + } finally { + sampleValues = null; + } + + if (result.renderError) { + throw result.renderError; + } + if (result.hadEmptyActiveWindow) { + throw new Error( + 'React recording was active but captured no renders. If you only measure imperative DOM ' + + 'updates or custom metrics, keep recording paused (reactRecordingPaused) instead of resuming.', + ); + } + if (!warmup) { + emitSample(result, values); + } +} + +/** Called by the generated page entry after it has imported every benchmark file. */ +export function markPageReady(): void { + if (typeof window.gc !== 'function') { + console.warn( + 'window.gc is not available. Run with --js-flags=--expose-gc for consistent GC between iterations.', + ); + } + window.benchmarkPage = { caseNames: () => [...cases.keys()], sample }; +} diff --git a/packages/benchmark/src/page/vitestStub.ts b/packages/benchmark/src/page/vitestStub.ts new file mode 100644 index 0000000000..1fda7c04a8 --- /dev/null +++ b/packages/benchmark/src/page/vitestStub.ts @@ -0,0 +1,26 @@ +// What `vitest` resolves to in an A/B page. Benchmark files sometimes mix `benchmark()` cases with +// plain Vitest tests; outside Vitest those tests are skipped, so the file still loads and its +// `benchmark()` cases still register. + +function skipped(api: string) { + return (name?: unknown) => { + console.warn(`${api}(${JSON.stringify(name)}) is skipped in A/B pages; only benchmark() runs.`); + }; +} + +export const it = skipped('it'); +export const test = skipped('test'); +export const bench = skipped('bench'); +export const beforeAll = skipped('beforeAll'); +export const afterAll = skipped('afterAll'); +export const beforeEach = skipped('beforeEach'); +export const afterEach = skipped('afterEach'); +export const onTestFinished = skipped('onTestFinished'); + +export function describe(_name: string, factory?: () => void): void { + factory?.(); +} + +export function expect(): never { + throw new Error('expect() is not available in A/B pages.'); +} diff --git a/packages/benchmark/src/tachometer/benchFiles.test.ts b/packages/benchmark/src/tachometer/benchFiles.test.ts new file mode 100644 index 0000000000..b2880abcad --- /dev/null +++ b/packages/benchmark/src/tachometer/benchFiles.test.ts @@ -0,0 +1,68 @@ +import * as path from 'node:path'; +import { mkdir, readFile, readdir, writeFile } from 'node:fs/promises'; +import { describe, expect, it } from 'vitest'; +import { makeTempDir } from '../utils/testUtils'; +import { discoverBenchFiles, writeBenchPages } from './benchFiles'; + +/** A throwaway harness holding the given files under `src/`. */ +async function makeHarness(files: string[]): Promise { + const harnessDir = await makeTempDir(); + await Promise.all( + files.map(async (file) => { + const absolute = path.join(harnessDir, 'src', file); + await mkdir(path.dirname(absolute), { recursive: true }); + await writeFile(absolute, ''); + }), + ); + return harnessDir; +} + +describe('discoverBenchFiles', () => { + it('finds benchmark files at any depth, each with its own page', async () => { + const harnessDir = await makeHarness([ + 'button/button.bench.tsx', + 'grid/rows/rows.bench.tsx', + 'button/index.html', + ]); + expect(await discoverBenchFiles({ harnessDir })).toEqual([ + { file: 'button/button.bench.tsx', page: '__bench__/button-button.html' }, + { file: 'grid/rows/rows.bench.tsx', page: '__bench__/grid-rows-rows.html' }, + ]); + }); + + it('filters by path, case-insensitively', async () => { + const harnessDir = await makeHarness(['button/button.bench.tsx', 'grid/grid.bench.tsx']); + const files = await discoverBenchFiles({ harnessDir, filters: ['GRID'] }); + expect(files.map((benchFile) => benchFile.file)).toEqual(['grid/grid.bench.tsx']); + }); +}); + +describe('writeBenchPages', () => { + it('writes a page that loads the benchmark file before reporting ready', async () => { + const harnessDir = await makeHarness(['button/button.bench.tsx']); + await writeBenchPages(harnessDir, await discoverBenchFiles({ harnessDir })); + + const pagesDir = path.join(harnessDir, 'src', '__bench__'); + expect((await readdir(pagesDir)).sort()).toEqual([ + '.gitignore', + 'button-button.entry.ts', + 'button-button.html', + ]); + expect(await readFile(path.join(pagesDir, 'button-button.entry.ts'), 'utf8')) + .toMatchInlineSnapshot(` + "import '../button/button.bench'; + import { markPageReady } from '@mui/internal-benchmark/page'; + + markPageReady(); + " + `); + }); + + it('drops pages of benchmark files that no longer exist', async () => { + const harnessDir = await makeHarness(['button/button.bench.tsx']); + await writeBenchPages(harnessDir, await discoverBenchFiles({ harnessDir })); + await writeBenchPages(harnessDir, []); + + await expect(readdir(path.join(harnessDir, 'src', '__bench__'))).rejects.toThrow(); + }); +}); diff --git a/packages/benchmark/src/tachometer/benchFiles.ts b/packages/benchmark/src/tachometer/benchFiles.ts new file mode 100644 index 0000000000..a6edd04aa4 --- /dev/null +++ b/packages/benchmark/src/tachometer/benchFiles.ts @@ -0,0 +1,96 @@ +import * as path from 'node:path'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import { globby } from 'globby'; +import { matchesFilters } from './discoverCases'; + +/** + * `*.bench.tsx` files: `benchmark()` cases, the same files Vitest runs, measured by the interleaved + * engine instead. Each file gets a generated page that loads it with `@mui/internal-benchmark` + * swapped for the page runtime, so the file itself needs no page, config or entry of its own. + */ + +/** Where the generated pages live, under `src/`. Ignored from the inside, like `.tachometer/`. */ +export const BENCH_PAGES_DIR = '__bench__'; + +const BENCH_FILE_PATTERN = '**/*.bench.tsx'; + +export interface BenchFile { + /** The benchmark file, posix, relative to `src`. */ + file: string; + /** The generated page that runs it, posix, relative to `src`. */ + page: string; +} + +function benchFileOf(file: string): BenchFile { + const slug = file.replace(/\.bench\.tsx$/, '').replace(/[^a-zA-Z0-9._-]+/g, '-'); + return { file, page: `${BENCH_PAGES_DIR}/${slug}.html` }; +} + +/** The `*.bench.tsx` files under `src/`, optionally filtered by path, sorted. */ +export async function discoverBenchFiles(options: { + harnessDir: string; + filters?: string[]; +}): Promise { + const { harnessDir, filters = [] } = options; + const files = await globby(BENCH_FILE_PATTERN, { + cwd: path.join(harnessDir, 'src'), + ignore: [`${BENCH_PAGES_DIR}/**`], + }); + return files + .filter((file) => matchesFilters(file, filters)) + .sort() + .map(benchFileOf); +} + +function pageHtml(benchFile: BenchFile, entry: string): string { + return ` + + + + ${benchFile.file} + + + + + +`; +} + +// The benchmark file first: its `benchmark()` calls register while it evaluates, and the page only +// reports itself ready once every case is known. +function pageEntry(benchFile: BenchFile): string { + // Extensionless, the way the harness's own imports are written; vite resolves the `.tsx`. + const relative = path.posix + .relative(path.posix.dirname(benchFile.page), benchFile.file) + .replace(/\.tsx$/, ''); + const specifier = relative.startsWith('../') ? relative : `./${relative}`; + return `import '${specifier}'; +import { markPageReady } from '@mui/internal-benchmark/page'; + +markPageReady(); +`; +} + +/** + * Writes a page per benchmark file into `src/__bench__/`, replacing whatever was there, so pages + * of deleted files do not linger as build inputs. + */ +export async function writeBenchPages(harnessDir: string, benchFiles: BenchFile[]): Promise { + const pagesDir = path.join(harnessDir, 'src', BENCH_PAGES_DIR); + await rm(pagesDir, { recursive: true, force: true }); + if (benchFiles.length === 0) { + return; + } + await mkdir(pagesDir, { recursive: true }); + await Promise.all([ + writeFile(path.join(pagesDir, '.gitignore'), '*\n'), + ...benchFiles.flatMap((benchFile) => { + const html = path.join(harnessDir, 'src', benchFile.page); + const entry = `${path.basename(benchFile.page, '.html')}.entry.ts`; + return [ + writeFile(html, pageHtml(benchFile, entry)), + writeFile(path.join(path.dirname(html), entry), pageEntry(benchFile)), + ]; + }), + ]); +} diff --git a/packages/benchmark/src/tachometer/discoverCases.ts b/packages/benchmark/src/tachometer/discoverCases.ts index dbf1b1ce47..9dd6e15b8b 100644 --- a/packages/benchmark/src/tachometer/discoverCases.ts +++ b/packages/benchmark/src/tachometer/discoverCases.ts @@ -60,6 +60,11 @@ export interface DiscoverCasesOptions { * case-insensitively. */ filters?: string[]; + /** + * Return an empty list instead of throwing when no case is selected, for callers that also run + * other kinds of cases and decide for themselves whether finding nothing is an error. + */ + allowEmpty?: boolean; } /** The file that marks a directory under `src/` as a benchmark case. */ @@ -150,6 +155,14 @@ async function parseLeafUrl( return { page: relative.split(path.sep).join('/'), suffix }; } +/** Whether a location under `src` is selected by the run's filters: all of them when there are none. */ +export function matchesFilters(location: string, filters: string[]): boolean { + return ( + filters.length === 0 || + filters.some((filter) => location.toLowerCase().includes(filter.toLowerCase())) + ); +} + /** * Every directory under `srcDir` that holds a `tachometer.json`, as a posix path relative to it. * @@ -179,19 +192,18 @@ async function findCaseLocations(srcDir: string): Promise { * which is the regression case. */ export async function discoverCases(options: DiscoverCasesOptions): Promise { - const { harnessDir, filters = [] } = options; + const { harnessDir, filters = [], allowEmpty = false } = options; const srcDir = path.join(harnessDir, 'src'); const located = await findCaseLocations(srcDir); // Matched case-insensitively against the location, the way vitest matches its file filters: a // folder name selects everything under it without the filter having to know what a folder means. - const selected = located.filter( - (location) => - filters.length === 0 || - filters.some((filter) => location.toLowerCase().includes(filter.toLowerCase())), - ); + const selected = located.filter((location) => matchesFilters(location, filters)); if (selected.length === 0) { + if (allowEmpty) { + return []; + } throw new Error( filters.length > 0 ? `No benchmark case under ${srcDir} matches ${filters.map((filter) => `"${filter}"`).join(', ')}.` diff --git a/packages/benchmark/src/tachometer/pairedStats.test.ts b/packages/benchmark/src/tachometer/pairedStats.test.ts new file mode 100644 index 0000000000..4f175f8d25 --- /dev/null +++ b/packages/benchmark/src/tachometer/pairedStats.test.ts @@ -0,0 +1,135 @@ +import { describe, expect, it } from 'vitest'; +import { + meanInterval, + shuffledIndices, + stableMeasurements, + tCritical95, + toTachometerJson, +} from './pairedStats'; +import type { Round } from './pairedStats'; +import { summarizeCase } from './summarizeCase'; +import type { BenchmarkCase } from './discoverCases'; + +/** Rounds where `b` is always `a + shift`, both riding the same per-round drift. */ +function driftingRounds(shift: number): Round[] { + const drift = [0, 40, -25, 60, -10, 35, -45, 20, 5, -30]; + return drift.map((offset) => [{ mount: 100 + offset }, { mount: 100 + offset + shift }]); +} + +function baselineCase(measurements: string[]): BenchmarkCase { + return { + name: 'Button', + configPath: '/harness/src/button/button.bench.tsx', + config: {}, + comparison: 'baseline', + leaves: [], + variants: ['Button [current]', 'Button [baseline]'], + measurements, + }; +} + +describe('tCritical95', () => { + it('reads small samples from the table', () => { + expect(tCritical95(1)).toBe(12.706); + expect(tCritical95(29)).toBe(2.045); + }); + + it('approaches the normal quantile for large samples', () => { + expect(tCritical95(60)).toBeCloseTo(2.0, 1); + expect(tCritical95(10_000)).toBeCloseTo(1.96, 2); + }); +}); + +describe('meanInterval', () => { + it('centers on the mean', () => { + const interval = meanInterval([1, 2, 3, 4, 5]); + expect((interval.low + interval.high) / 2).toBeCloseTo(3); + expect(interval.low).toBeLessThan(3); + }); + + it('is a point for a single value', () => { + expect(meanInterval([7])).toEqual({ low: 7, high: 7 }); + }); +}); + +describe('toTachometerJson', () => { + it('resolves a small shift that drift hides from the per-variant means', () => { + const json = toTachometerJson( + ['Button [current]', 'Button [baseline]'], + ['mount'], + driftingRounds(2), + ); + const [current, baseline] = json.benchmarks; + + // Each variant's own interval is wide enough to overlap the other's… + expect(current.mean.high).toBeGreaterThan(baseline.mean.low); + // …but the paired difference is exactly the shift, every round, relative to the baseline's + // mean of 107. + expect(current.differences[1]).toEqual({ + absolute: { low: -2, high: -2 }, + percentChange: { low: expect.closeTo(-1.87, 2), high: expect.closeTo(-1.87, 2) }, + }); + }); + + it('names entries so summarizeCase pairs them back by variant and measurement', () => { + const rounds: Round[] = [ + [ + { mount: 10, paint: 20 }, + { mount: 12, paint: 20 }, + ], + [ + { mount: 11, paint: 21 }, + { mount: 13, paint: 20 }, + ], + [ + { mount: 10, paint: 19 }, + { mount: 12, paint: 21 }, + ], + ]; + const summary = summarizeCase( + baselineCase(['mount', 'paint']), + toTachometerJson(['Button [current]', 'Button [baseline]'], ['mount', 'paint'], rounds), + ); + + expect(summary.measurements.map((measurement) => measurement.name)).toEqual(['mount', 'paint']); + const [mount, paint] = summary.measurements; + expect(mount.comparisons[0]).toMatchObject({ variant: 'Button [baseline]', verdict: 'faster' }); + expect(paint.comparisons[0]).toMatchObject({ variant: 'Button [baseline]', verdict: 'unsure' }); + }); + + it('never compares across measurements', () => { + const json = toTachometerJson( + ['a', 'b'], + ['mount', 'paint'], + [ + [ + { mount: 1, paint: 2 }, + { mount: 1, paint: 2 }, + ], + ], + ); + // Flat order is measurement-major: a/mount, b/mount, a/paint, b/paint. + expect(json.benchmarks[0].differences[2]).toBe(null); + expect(json.benchmarks[0].differences[3]).toBe(null); + expect(json.benchmarks[0].differences[1]).not.toBe(null); + }); +}); + +describe('stableMeasurements', () => { + it('keeps only measurements every variant reported every round', () => { + const rounds: Round[] = [ + [ + { render: 1, 'render:update': 1 }, + { render: 1, 'render:update': 1 }, + ], + [{ render: 1 }, { render: 1, 'render:update': 1 }], + ]; + expect(stableMeasurements(rounds)).toEqual(['render']); + }); +}); + +describe('shuffledIndices', () => { + it('returns a permutation', () => { + expect(shuffledIndices(5).sort()).toEqual([0, 1, 2, 3, 4]); + }); +}); diff --git a/packages/benchmark/src/tachometer/pairedStats.ts b/packages/benchmark/src/tachometer/pairedStats.ts new file mode 100644 index 0000000000..a850dd85ab --- /dev/null +++ b/packages/benchmark/src/tachometer/pairedStats.ts @@ -0,0 +1,142 @@ +import type { + ConfidenceInterval, + TachometerBenchmark, + TachometerDifference, + TachometerJson, +} from './summarizeCase'; + +/** + * Statistics for interleaved runs, in the shape tachometer reports so the rest of the pipeline + * (summaries, the report, the upload) reads them unchanged. + * + * The difference is in how differences are computed. Every variant is sampled once per round, in + * a shuffled order, so each round is a matched set: whatever the machine was doing during that + * round affected every variant in it. Intervals on a difference are therefore taken over the + * per-round differences rather than over two independent sample sets, which cancels that shared + * drift instead of counting it as noise. + */ + +/** One round: a variant's measured values by measurement name, indexed like the variants. */ +export type Round = Array | undefined>; + +// Two-sided 95% critical values of Student's t for 1–30 degrees of freedom. +const T_95 = [ + 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, + 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, + 2.045, 2.042, +]; + +/** The two-sided 95% critical value of Student's t, approximated past the table. */ +export function tCritical95(degreesOfFreedom: number): number { + if (degreesOfFreedom <= T_95.length) { + return T_95[Math.max(degreesOfFreedom, 1) - 1]; + } + // Cornish–Fisher expansion around the normal quantile; well within 0.1% past 30 degrees. + const z = 1.959964; + return z + (z ** 3 + z) / (4 * degreesOfFreedom); +} + +function meanOf(values: number[]): number { + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + +/** The 95% confidence interval of the mean. A single value is its own degenerate interval. */ +export function meanInterval(values: number[]): ConfidenceInterval { + const mean = meanOf(values); + if (values.length < 2) { + return { low: mean, high: mean }; + } + const variance = + values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (values.length - 1); + const halfWidth = tCritical95(values.length - 1) * Math.sqrt(variance / values.length); + return { low: mean - halfWidth, high: mean + halfWidth }; +} + +/** `a` against `b`, over the rounds where both were measured. */ +function pairedDifference( + rounds: Round[], + a: number, + b: number, + measurement: string, +): TachometerDifference | null { + const differences: number[] = []; + const baseline: number[] = []; + for (const round of rounds) { + const valueA = round[a]?.[measurement]; + const valueB = round[b]?.[measurement]; + if (valueA !== undefined && valueB !== undefined) { + differences.push(valueA - valueB); + baseline.push(valueB); + } + } + if (differences.length === 0) { + return null; + } + const absolute = meanInterval(differences); + const reference = meanOf(baseline); + return { + absolute, + percentChange: { + low: (absolute.low / reference) * 100, + high: (absolute.high / reference) * 100, + }, + }; +} + +/** + * Tachometer-shaped results for interleaved rounds: one entry per measurement × variant, named + * ` []`, with a difference matrix over the flat list (`null` across + * measurements, which are never compared with each other). + */ +export function toTachometerJson( + variants: string[], + measurements: string[], + rounds: Round[], +): TachometerJson { + const flat = measurements.flatMap((measurement) => + variants.map((variant, index) => ({ variant, index, measurement })), + ); + const benchmarks = flat.map(({ variant, index, measurement }): TachometerBenchmark => { + const samples = rounds.flatMap((round) => { + const value = round[index]?.[measurement]; + return value === undefined ? [] : [value]; + }); + return { + name: `${variant} [${measurement}]`, + measurement: { name: measurement }, + mean: meanInterval(samples), + samples, + differences: flat.map((other) => + other.measurement === measurement && other.index !== index + ? pairedDifference(rounds, index, other.index, measurement) + : null, + ), + }; + }); + return { benchmarks }; +} + +/** Measurement names every sample of every variant reported, in first-seen order. */ +export function stableMeasurements(rounds: Round[]): string[] { + const seen = new Set(); + for (const round of rounds) { + for (const values of round) { + for (const name of Object.keys(values ?? {})) { + seen.add(name); + } + } + } + return [...seen].filter((name) => + rounds.every((round) => round.every((values) => values?.[name] !== undefined)), + ); +} + +/** A random permutation of `0..length-1`, so no variant always runs first in its round. */ +export function shuffledIndices(length: number): number[] { + const indices = Array.from({ length }, (_, index) => index); + for (let index = indices.length - 1; index > 0; index -= 1) { + const swap = Math.floor(Math.random() * (index + 1)); + [indices[index], indices[swap]] = [indices[swap], indices[index]]; + } + return indices; +} diff --git a/packages/benchmark/src/tachometer/plugin.ts b/packages/benchmark/src/tachometer/plugin.ts index 0d2a7b72e6..3d97765133 100644 --- a/packages/benchmark/src/tachometer/plugin.ts +++ b/packages/benchmark/src/tachometer/plugin.ts @@ -1,7 +1,9 @@ import * as path from 'node:path'; +import { createRequire } from 'node:module'; import { readFile, realpath } from 'node:fs/promises'; import type { Plugin } from 'vite'; import { discoverCases, pagesOf } from './discoverCases'; +import { discoverBenchFiles, writeBenchPages } from './benchFiles'; import type { BenchmarkCase } from './discoverCases'; import { buildsDirOf, prepareOutputDir, OUTPUT_DIR } from './outputDir'; @@ -57,6 +59,20 @@ async function assertWorkspaceDepsBuilt( ); } +/** + * Where `@mui/internal-benchmark` really lives as seen from the harness, so imports made by the + * page runtime can be told apart from the harness's own. Undefined when the harness doesn't depend + * on it, in which case there is no page runtime to tell apart. + */ +async function resolveBenchmarkPackageDir(harnessDir: string): Promise { + try { + const require = createRequire(path.join(harnessDir, 'package.json')); + return await realpath(path.dirname(require.resolve('@mui/internal-benchmark/package.json'))); + } catch { + return undefined; + } +} + /** A case name and a query both come from a config file, so neither is trusted as markup. */ function escapeHtml(text: string): string { return text @@ -126,9 +142,34 @@ export function tachometer(options: TachometerPluginOptions = {}): Plugin { const harnessDir = options.harnessDir ?? process.cwd(); const srcDir = path.join(harnessDir, 'src'); let buildCases: BenchmarkCase[] = []; + let benchmarkPackageDir: string | undefined; return { name: 'mui-benchmark:tachometer', + // Ahead of vite's own resolver, so the redirects below win over the packages' `exports`. + enforce: 'pre', + + // Benchmark files are written against the Vitest harness. In a page they run under the page + // runtime instead: the same authoring API, driven one iteration at a time by the runner. + async resolveId(source, importer) { + if (source === '@mui/internal-benchmark') { + return this.resolve('@mui/internal-benchmark/page', importer, { skipSelf: true }); + } + if (source === 'vitest') { + return this.resolve('@mui/internal-benchmark/page/vitest', importer, { skipSelf: true }); + } + // React only reports render durations from its profiling build. Scoped to the page runtime's + // own import, so a plain tachometer page keeps measuring the production build it ships with. + if ( + source === 'react-dom/client' && + importer && + benchmarkPackageDir && + importer.startsWith(benchmarkPackageDir) + ) { + return this.resolve('react-dom/profiling', importer, { skipSelf: true }); + } + return null; + }, async config(userConfig, env) { const pkg = JSON.parse(await readFile(path.join(harnessDir, 'package.json'), 'utf8')); @@ -149,8 +190,15 @@ export function tachometer(options: TachometerPluginOptions = {}): Plugin { // `vite build --outDir`, which is exactly how `tacho run` directs each ref's build. const outDir = userConfig.build?.outDir ?? buildsDirOf(harnessDir, 'manual'); + // Written for every command, so the dev server serves a benchmark file's page as well. + const benchFiles = await discoverBenchFiles({ harnessDir }); + await writeBenchPages(harnessDir, benchFiles); + benchmarkPackageDir = await resolveBenchmarkPackageDir(harnessDir); + // One React for the harness, its benchmark files and the page runtime they run under. + const resolve = { dedupe: ['react', 'react-dom'] }; + if (env.command !== 'build') { - return { root: srcDir, appType: 'mpa', build: { outDir } }; + return { root: srcDir, appType: 'mpa', resolve, build: { outDir } }; } // Mark the output directory ignored from the inside before the build writes into it, so a @@ -162,19 +210,23 @@ export function tachometer(options: TachometerPluginOptions = {}): Plugin { await prepareOutputDir(harnessDir); } - const cases = await discoverCases({ harnessDir }); + // A harness may hold only benchmark files, so no `tachometer.json` is fine as long as there is + // something to build. + const cases = await discoverCases({ harnessDir, allowEmpty: benchFiles.length > 0 }); buildCases = cases; // Key the input map by the page path without its extension. Vite emits each page at its path // relative to `root` regardless, so `/index.html` lands at `//index.html`; // a distinct key per page is what lets one case folder hold several pages — a cross-library // comparison with `mosaic.html` and `tanstack.html` side by side — without them colliding. + const pages = [...pagesOf(cases), ...benchFiles.map((benchFile) => benchFile.page)]; const input = Object.fromEntries( - pagesOf(cases).map((page) => [page.replace(/\.html$/, ''), path.join(srcDir, page)]), + pages.map((page) => [page.replace(/\.html$/, ''), path.join(srcDir, page)]), ); return { root: srcDir, appType: 'mpa', + resolve, // Relative asset urls: tachometer serves the build directory and the pages live under // `//`, so absolute "/assets/…" paths would 404. base: './', diff --git a/packages/benchmark/src/tachometer/runInterleaved.ts b/packages/benchmark/src/tachometer/runInterleaved.ts new file mode 100644 index 0000000000..cb122b8185 --- /dev/null +++ b/packages/benchmark/src/tachometer/runInterleaved.ts @@ -0,0 +1,363 @@ +/* eslint-disable no-console */ + +import * as path from 'node:path'; +import chalk from 'chalk'; +import type { Browser, BrowserContext, CDPSession, Page } from '@playwright/test'; +import { BENCHMARK_LAUNCH_ARGS } from '../launchArgs'; +import { measurementNameOf } from './discoverCases'; +import type { BenchmarkCase, Leaf } from './discoverCases'; +import type { BenchFile } from './benchFiles'; +import type { ResolvedRef } from './refs'; +import { serveDirectory } from './serveDirectory'; +import { shuffledIndices, stableMeasurements, toTachometerJson } from './pairedStats'; +import type { Round } from './pairedStats'; +import type { TachometerJson } from './summarizeCase'; +// Types for `window.benchmarkPage`, which the evaluated functions below call into. +import type {} from '../page/page'; + +/** + * The interleaved engine: runs every case in Playwright instead of tachometer, sampling its variants + * round by round in a shuffled order and comparing them on paired differences. + * + * Two kinds of case run here: + * + * - A `tachometer.json` case keeps tachometer's page contract. Every sample is a fresh load of the + * page, measured by the `performance.measure` entry (or first contentful paint) it names. + * - A `*.bench.tsx` file runs its `benchmark()` cases in a page that stays open per build: every + * sample is one iteration, so module-scope data and the JIT stay warm, and interactions get + * trusted input through a CDP session the page is bridged to. + * + * Every variant gets a browser context of its own, so no two share a renderer process or a cache. + */ + +export interface InterleavedResult { + entry: BenchmarkCase; + json?: TachometerJson; + error?: string; +} + +export interface RunInterleavedOptions { + harnessDir: string; + /** Where each ref's pages were built, one directory per ref id. */ + buildsDir: string; + cases: BenchmarkCase[]; + benchFiles: BenchFile[]; + buildFor: (leaf: Leaf) => ResolvedRef; + /** The builds a benchmark file compares: the working tree first, then the baseline. */ + benchRefs: ResolvedRef[]; + browserBinary: string; + asRoot: boolean; + /** Measured rounds per case. Defaults to a `tachometer.json`'s `sampleSize`, else 30. */ + samples?: number; + /** + * Discarded rounds before measuring: once per case for page loads (default 2), once per epoch + * for `*.bench.tsx` iterations (default 5). + */ + warmup?: number; + /** `*.bench.tsx` only: measured rounds per set of pages before they are reopened. Default 10. */ + epochSize?: number; +} + +const DEFAULT_VIEWPORT = { width: 1920, height: 1080 }; +const DEFAULT_SAMPLES = 30; +const SAMPLE_TIMEOUT_MS = 120_000; + +type CdpMethod = Parameters[0]; + +function errorMessage(error: unknown): string { + return error instanceof Error ? (error.stack ?? error.message) : String(error); +} + +/** The Performance entry a tachometer measurement reads, under the name it reports it as. */ +interface EntrySpec { + name: string; + entryName: string; +} + +function entrySpecsOf(node: any): EntrySpec[] { + return [node.measurement ?? 'callback'].flat().map((measurement: any): EntrySpec => { + const name = measurementNameOf(measurement, node.measurementExpression); + if (measurement === 'fcp') { + return { name, entryName: 'first-contentful-paint' }; + } + if (typeof measurement === 'object' && measurement.mode === 'performance') { + return { name, entryName: measurement.entryName }; + } + throw new Error( + `The interleaved engine reads Performance entries only; "${name}" in "${node.name}" is a ` + + `${typeof measurement === 'string' ? measurement : measurement.mode} measurement.`, + ); + }); +} + +async function openContext(browser: Browser, viewport = DEFAULT_VIEWPORT) { + const context = await browser.newContext({ viewport }); + const page = await context.newPage(); + page.setDefaultTimeout(SAMPLE_TIMEOUT_MS); + page.on('pageerror', (error) => console.error(chalk.red(` page error: ${error.message}`))); + return { context, page }; +} + +async function runPageCase( + browser: Browser, + origin: string, + entry: BenchmarkCase, + options: RunInterleavedOptions, +): Promise { + const nodes: any[] = entry.config.benchmarks; + const targets = entry.leaves.map((leaf, index) => ({ + url: `${origin}/${options.buildFor(leaf).id}/${leaf.page}${leaf.suffix}`, + specs: entrySpecsOf(nodes[index]), + })); + // A renderer process keeps whatever speed it started with — on a machine with performance and + // efficiency cores, which kind the scheduler put it on — so two identical pages in two contexts + // can differ by several percent for as long as those processes live. Every sample is a fresh + // load anyway, so no variant is tied to a context: each round deals the variants out over the + // pool anew, and a slow process slows every variant equally often. + const pool = await Promise.all( + targets.map(() => openContext(browser, nodes[0]?.browser?.windowSize)), + ); + + try { + const sample = async (target: (typeof targets)[number], page: Page) => { + await page.goto(target.url); + const entryNames = target.specs.map((spec) => spec.entryName); + await page.waitForFunction( + (names) => names.every((name) => performance.getEntriesByName(name).length > 0), + entryNames, + ); + return page.evaluate( + (specs) => + Object.fromEntries( + specs.map(({ name, entryName }) => { + const [performanceEntry] = performance.getEntriesByName(entryName); + // A paint entry has no duration; tachometer reports its start time instead. + const value = + performanceEntry.entryType === 'paint' + ? performanceEntry.startTime + : performanceEntry.duration; + return [name, value]; + }), + ), + target.specs, + ); + }; + + const warmup = options.warmup ?? 2; + const samples = options.samples ?? entry.config.sampleSize ?? DEFAULT_SAMPLES; + const rounds: Round[] = []; + for (let roundIndex = 0; roundIndex < warmup + samples; roundIndex += 1) { + const round: Round = []; + const dealt = shuffledIndices(pool.length); + for (const index of shuffledIndices(targets.length)) { + // eslint-disable-next-line no-await-in-loop + round[index] = await sample(targets[index], pool[dealt[index]].page); + } + if (roundIndex >= warmup) { + rounds.push(round); + } + } + return toTachometerJson(entry.variants, entry.measurements, rounds); + } finally { + await Promise.all(pool.map((opened) => opened.context.close())); + } +} + +interface BenchTarget { + ref: ResolvedRef; + context: BrowserContext; + page: Page; + caseNames: string[]; +} + +async function openBenchPage( + browser: Browser, + origin: string, + benchFile: BenchFile, + ref: ResolvedRef, +): Promise { + const { context, page } = await openContext(browser); + const session = await context.newCDPSession(page); + // Interactions ask for trusted input through this bridge. `send` only accepts the protocol + // methods Playwright knows by name; the page may ask for any, and Chrome rejects unknown ones. + await page.exposeBinding( + 'benchmarkCdp', + (_source, method: string, params?: Record) => + session.send(method as CdpMethod, params), + ); + await page.goto(`${origin}/${ref.id}/${benchFile.page}`); + await page.waitForFunction(() => window.benchmarkPage !== undefined); + const visibility = await page.evaluate(() => document.visibilityState); + if (visibility !== 'visible') { + console.warn( + chalk.yellow(` ${benchFile.file} [${ref.id}] is ${visibility}; rAF-based waits may stall.`), + ); + } + const caseNames = await page.evaluate(() => window.benchmarkPage!.caseNames()); + return { ref, context, page, caseNames }; +} + +/** Runs one iteration; a measured one comes back as its `performance.measure` values by name. */ +async function sampleBenchCase( + target: BenchTarget, + name: string, + warmup: boolean, +): Promise | undefined> { + await target.page.evaluate((args) => window.benchmarkPage!.sample(args.name, args), { + name, + warmup, + }); + if (warmup) { + return undefined; + } + return target.page.evaluate(() => { + const values: Record = {}; + for (const measure of performance.getEntriesByType('measure') as PerformanceMeasure[]) { + const detailValue: unknown = measure.detail?.value; + const value = typeof detailValue === 'number' ? detailValue : measure.duration; + // A metric recorded more than once in an iteration counts its total for that iteration. + values[measure.name] = (values[measure.name] ?? 0) + value; + } + return values; + }); +} + +/** Opens a page per build, in a random order so no build always gets the first process. */ +async function openBenchPages( + browser: Browser, + origin: string, + benchFile: BenchFile, + refs: ResolvedRef[], +): Promise { + const targets: BenchTarget[] = []; + for (const index of shuffledIndices(refs.length)) { + // eslint-disable-next-line no-await-in-loop + targets[index] = await openBenchPage(browser, origin, benchFile, refs[index]); + } + return targets; +} + +async function closeBenchPages(targets: BenchTarget[]): Promise { + await Promise.all(targets.map((target) => target.context.close())); +} + +/** + * Measures one case in epochs. A page has to stay open for iterations to be warm, which ties each + * build to one renderer process for as long as it does — and a process keeps whatever speed it + * started with (a performance or an efficiency core, typically), enough to show up as a difference + * between identical builds. So every epoch opens fresh pages, warms them up, and measures a slice + * of the rounds: the process each build lands on is drawn again per epoch instead of once per case. + */ +async function measureBenchCase( + browser: Browser, + origin: string, + benchFile: BenchFile, + name: string, + options: RunInterleavedOptions, +): Promise { + const warmup = options.warmup ?? 5; + const samples = options.samples ?? DEFAULT_SAMPLES; + const epochSize = options.epochSize ?? 10; + const rounds: Round[] = []; + while (rounds.length < samples) { + // eslint-disable-next-line no-await-in-loop + const targets = await openBenchPages(browser, origin, benchFile, options.benchRefs); + try { + const measured = Math.min(epochSize, samples - rounds.length); + for (let roundIndex = 0; roundIndex < warmup + measured; roundIndex += 1) { + const round: Round = []; + for (const index of shuffledIndices(targets.length)) { + // eslint-disable-next-line no-await-in-loop + round[index] = await sampleBenchCase(targets[index], name, roundIndex < warmup); + } + if (roundIndex >= warmup) { + rounds.push(round); + } + } + } finally { + // eslint-disable-next-line no-await-in-loop + await closeBenchPages(targets); + } + } + return rounds; +} + +async function runBenchFile( + browser: Browser, + origin: string, + benchFile: BenchFile, + options: RunInterleavedOptions, +): Promise { + // Opened once just to learn which cases each build defines. + const listed = await openBenchPages(browser, origin, benchFile, options.benchRefs); + await closeBenchPages(listed); + const [current, ...others] = listed; + + const results: InterleavedResult[] = []; + for (const name of current.caseNames) { + const entry: BenchmarkCase = { + name, + configPath: path.join(options.harnessDir, 'src', benchFile.file), + config: {}, + comparison: 'baseline', + leaves: [], + variants: listed.map((target) => + target.ref.kind === 'worktree' ? `${name} [current]` : `${name} [baseline]`, + ), + measurements: [], + }; + const missing = others.find((target) => !target.caseNames.includes(name)); + if (missing) { + // A case added by the change under test has nothing to be compared against yet. + results.push({ entry, error: `"${name}" does not exist in ${missing.ref.id}.` }); + continue; + } + + console.log(chalk.cyan(`\nRunning "${name}" (${benchFile.file})…`)); + try { + // eslint-disable-next-line no-await-in-loop + const rounds = await measureBenchCase(browser, origin, benchFile, name, options); + // A measurement some iteration did not produce (an interaction whose render count varies, + // say) cannot be paired round by round, so only those every iteration reported are kept. + entry.measurements = stableMeasurements(rounds); + results.push({ entry, json: toTachometerJson(entry.variants, entry.measurements, rounds) }); + } catch (error) { + console.error(chalk.red(` ${errorMessage(error)}`)); + results.push({ entry, error: errorMessage(error) }); + } + } + return results; +} + +export async function runInterleaved(options: RunInterleavedOptions): Promise { + const { chromium } = await import('@playwright/test'); + const server = await serveDirectory(options.buildsDir); + const browser = await chromium.launch({ + executablePath: options.browserBinary, + headless: true, + args: [...BENCHMARK_LAUNCH_ARGS, ...(options.asRoot ? ['--no-sandbox'] : [])], + }); + + try { + const results: InterleavedResult[] = []; + // Sequential on purpose: concurrent cases would contend for the same machine. + for (const entry of options.cases) { + console.log(chalk.cyan(`\nRunning "${entry.name}"…`)); + try { + // eslint-disable-next-line no-await-in-loop + results.push({ entry, json: await runPageCase(browser, server.origin, entry, options) }); + } catch (error) { + console.error(chalk.red(` ${errorMessage(error)}`)); + results.push({ entry, error: errorMessage(error) }); + } + } + for (const benchFile of options.benchFiles) { + // eslint-disable-next-line no-await-in-loop + results.push(...(await runBenchFile(browser, server.origin, benchFile, options))); + } + return results; + } finally { + await browser.close(); + await server.close(); + } +} diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts index 45ffd3ccc2..ad01290828 100644 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -11,6 +11,8 @@ import type { PackedPackage } from '../utils/packWorkspace'; import { resolveBaselineRef, WORKTREE_REF } from './refs'; import type { ResolvedRef } from './refs'; import { discoverCases, pagesOf } from './discoverCases'; +import { discoverBenchFiles } from './benchFiles'; +import { runInterleaved } from './runInterleaved'; import type { BenchmarkCase, Leaf } from './discoverCases'; import { isSummarized, refLabel } from './format'; import { buildRefPages, restoreWorkspace } from './buildPages'; @@ -42,11 +44,11 @@ function fileSlugOf(name: string): string { * rather than being a case every later step has to allow for. */ async function resolveBuilds( - cases: BenchmarkCase[], + needsBaseline: boolean, baseline: string | undefined, repoRoot: string, ): Promise<{ refs: ResolvedRef[]; buildFor: (leaf: Leaf) => ResolvedRef }> { - if (!cases.some((entry) => entry.comparison === 'baseline')) { + if (!needsBaseline) { return { refs: [WORKTREE_REF], buildFor: () => WORKTREE_REF }; } const baselineRef = await resolveBaselineRef(baseline, repoRoot); @@ -82,8 +84,22 @@ export interface RunTachometerOptions { upload?: boolean; /** How a ref's pages find the library under test. Defaults to `in-place`. */ resolveMode?: ResolveMode; + /** + * What measures the pages. `tachometer` hands each case to tachometer. `interleaved` runs them + * in Playwright, alternating variants round by round and comparing paired differences — and is + * the engine that runs `*.bench.tsx` files. Defaults to `tachometer`. + */ + engine?: Engine; + /** Interleaved engine only: measured rounds per case, overriding `sampleSize`. */ + samples?: number; + /** Interleaved engine only: discarded rounds before measuring. */ + warmup?: number; + /** Interleaved engine only: measured rounds of a `*.bench.tsx` case per set of pages. */ + epochSize?: number; } +export type Engine = 'tachometer' | 'interleaved'; + /** * Benchmarks the harness's pages across one or more builds of the workspace with tachometer, and * writes a combined JSON report. @@ -103,6 +119,10 @@ export async function runTachometer(options: RunTachometerOptions): Promise 0) { + console.warn( + chalk.yellow( + `Skipping ${benchFiles.length} *.bench.tsx file(s): only the interleaved engine runs them (--engine interleaved).`, + ), + ); + benchFiles = []; + } + if (cases.length === 0 && benchFiles.length === 0) { + // Nothing selected: let discovery say what it looked for. + await discoverCases({ harnessDir, filters }); + } + // A benchmark file always compares the working tree against the baseline. + const { refs, buildFor } = await resolveBuilds( + benchFiles.length > 0 || cases.some((entry) => entry.comparison === 'baseline'), + baseline, + repoRoot, + ); // The commit follows the name, so a revision like `HEAD~1` still says which commit it landed on. const describeRef = (ref: ResolvedRef) => @@ -136,7 +174,10 @@ export async function runTachometer(options: RunTachometerOptions): Promise benchFile.file).join(', ')}`)); console.log(chalk.cyan(`Pages: ${pagesOf(cases).join(', ')}`)); console.log(chalk.cyan(`Refs: ${refs.map(describeRef).join(', ')}`)); console.log( @@ -195,11 +238,31 @@ export async function runTachometer(options: RunTachometerOptions): Promise = []; + if (engine === 'interleaved') { + const worktreeRef = refs.find((ref) => ref.kind === 'worktree') ?? WORKTREE_REF; + results.push( + ...(await runInterleaved({ + harnessDir, + buildsDir, + cases, + benchFiles, + buildFor, + // The working tree first: the first variant is the reference a case is judged against. + benchRefs: [worktreeRef, ...refs.filter((ref) => ref !== worktreeRef)], + browserBinary, + asRoot, + samples, + warmup, + epochSize, + })), + ); + } + // Rewrite each leaf url to its ref's built page, then run each case's own config. Tachometer // resolves relative urls against the config file's directory and these configs are written to a // temp dir, so the rewritten urls are absolute. - const results: Array<{ entry: BenchmarkCase; json: any }> = []; - for (const entry of cases) { + for (const entry of engine === 'tachometer' ? cases : []) { for (const leaf of entry.leaves) { leaf.node.url = `${path.join(buildsDir, buildFor(leaf).id, leaf.page)}${leaf.suffix}`; } @@ -253,7 +316,10 @@ export async function runTachometer(options: RunTachometerOptions): Promise { + cases: results.map(({ entry, json, error: runError }): CaseResult => { + if (runError !== undefined) { + return { name: entry.name, comparison: entry.comparison, error: runError }; + } try { return summarizeCase(entry, json); } catch (error) { @@ -262,7 +328,9 @@ export async function runTachometer(options: RunTachometerOptions): Promise [entry.name, json])), + raw: Object.fromEntries( + results.flatMap(({ entry, json }) => (json === undefined ? [] : [[entry.name, json]])), + ), }; await mkdir(path.dirname(outPath), { recursive: true }); await writeFile(outPath, `${JSON.stringify(report, null, 2)}\n`); diff --git a/packages/benchmark/src/tachometer/serveDirectory.ts b/packages/benchmark/src/tachometer/serveDirectory.ts new file mode 100644 index 0000000000..5112f55196 --- /dev/null +++ b/packages/benchmark/src/tachometer/serveDirectory.ts @@ -0,0 +1,64 @@ +import * as http from 'node:http'; +import * as path from 'node:path'; +import { createReadStream } from 'node:fs'; +import { stat } from 'node:fs/promises'; +import type { AddressInfo } from 'node:net'; + +const CONTENT_TYPES: Record = { + '.html': 'text/html; charset=utf-8', + '.js': 'text/javascript; charset=utf-8', + '.mjs': 'text/javascript; charset=utf-8', + '.css': 'text/css; charset=utf-8', + '.json': 'application/json; charset=utf-8', + '.map': 'application/json; charset=utf-8', + '.svg': 'image/svg+xml', + '.png': 'image/png', + '.jpg': 'image/jpeg', + '.woff2': 'font/woff2', + '.wasm': 'application/wasm', +}; + +export interface StaticServer { + /** `http://127.0.0.1:`, with no trailing slash. */ + origin: string; + close: () => Promise; +} + +/** + * Serves a directory of built pages on a free local port. Nothing is cached by the server; the + * browser's own cache is per context, so each side of a comparison warms its own. + */ +export async function serveDirectory(root: string): Promise { + const server = http.createServer(async (request, response) => { + const pathname = decodeURIComponent(new URL(request.url ?? '/', 'http://localhost').pathname); + const file = path.join(root, pathname.endsWith('/') ? `${pathname}index.html` : pathname); + if (path.relative(root, file).startsWith('..')) { + response.writeHead(403).end(); + return; + } + try { + if (!(await stat(file)).isFile()) { + throw new Error('Not a file'); + } + } catch { + response.writeHead(404).end(); + return; + } + response.writeHead(200, { + 'Content-Type': CONTENT_TYPES[path.extname(file)] ?? 'application/octet-stream', + }); + createReadStream(file).pipe(response); + }); + + await new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }); + const { port } = server.address() as AddressInfo; + return { + origin: `http://127.0.0.1:${port}`, + close: () => + new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }), + }; +} diff --git a/packages/benchmark/src/types.ts b/packages/benchmark/src/types.ts index b4a19a2ad7..7aba51eee7 100644 --- a/packages/benchmark/src/types.ts +++ b/packages/benchmark/src/types.ts @@ -1,3 +1,5 @@ +import type { BenchmarkInput } from './input'; + export interface RenderEvent { id: string; /** @@ -35,6 +37,11 @@ export interface InteractionContext { * already active. */ resumeReactRecording: () => void; + /** + * Trusted input gestures (scroll, pinch, tap) dispatched by the browser itself, for interactions + * that should behave like a real mouse or trackpad rather than synthetic DOM events. + */ + input: BenchmarkInput; } // One benchmark case's lifecycle. The measurement loop creates one per iteration and drives it diff --git a/packages/benchmark/src/vitest.ts b/packages/benchmark/src/vitest.ts index 0eacd8caa5..4d470d995b 100644 --- a/packages/benchmark/src/vitest.ts +++ b/packages/benchmark/src/vitest.ts @@ -1,6 +1,7 @@ import react from '@vitejs/plugin-react'; import { playwright } from '@vitest/browser-playwright'; import type { ViteUserConfig } from 'vitest/config'; +import { BENCHMARK_LAUNCH_ARGS } from './launchArgs'; export interface CreateBenchmarkVitestConfigOptions { /** @@ -55,20 +56,8 @@ function resolveViewport(option?: { return undefined; } -// Chromium/V8 launch args shared by measurement and profiling, kept intentionally minimal. -// `--expose-gc` is required: the harness forces GC between iterations for clean, comparable -// timings. The backgrounding flags stop Chrome from throttling the (headless or occluded) -// benchmark tab, which would otherwise add large variance. Heavier "determinism" flags -// (`--no-opt`, `--predictable`, `--hash-seed`/`--random-seed`, `--disable-gpu`, -// `--enable-benchmarking`) were measured to slow renders ~40% and distort paint timing without -// reducing variance, so they are omitted — add them per project via `launchArgs` if a specific -// workload needs them. -const LAUNCH_ARGS = [ - '--js-flags=--expose-gc', - '--disable-background-timer-throttling', - '--disable-backgrounding-occluded-windows', - '--disable-renderer-backgrounding', -]; +// Shared by measurement and profiling; add project-specific flags via `launchArgs`. +const LAUNCH_ARGS = BENCHMARK_LAUNCH_ARGS; export function createBenchmarkVitestConfig( options?: CreateBenchmarkVitestConfigOptions, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3eb3108d15..ad88ae0c46 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1247,6 +1247,9 @@ importers: react: specifier: 19.2.8 version: 19.2.8 + react-dom: + specifier: 19.2.8 + version: 19.2.8(react@19.2.8) devDependencies: '@mui/internal-benchmark': specifier: workspace:* @@ -1257,6 +1260,9 @@ importers: '@types/react': specifier: 19.2.18 version: 19.2.18 + '@types/react-dom': + specifier: 19.2.7 + version: 19.2.7(@types/react@19.2.18) tachometer: specifier: ^0.7.2 version: 0.7.2(supports-color@10.2.2) diff --git a/test/tachometer/package.json b/test/tachometer/package.json index cca8712442..57afb031b5 100644 --- a/test/tachometer/package.json +++ b/test/tachometer/package.json @@ -10,12 +10,14 @@ }, "dependencies": { "@mui/internal-test-utils": "workspace:*", - "react": "19.2.8" + "react": "19.2.8", + "react-dom": "19.2.8" }, "devDependencies": { "@mui/internal-benchmark": "workspace:*", "@playwright/test": "1.62.1", "@types/react": "19.2.18", + "@types/react-dom": "19.2.7", "tachometer": "^0.7.2", "vite": "^8.2.2" } diff --git a/test/tachometer/src/list/list.bench.tsx b/test/tachometer/src/list/list.bench.tsx new file mode 100644 index 0000000000..66846ee109 --- /dev/null +++ b/test/tachometer/src/list/list.bench.tsx @@ -0,0 +1,59 @@ +import * as React from 'react'; +import { benchmark, ScalarMetric } from '@mui/internal-benchmark'; +import { reactMajor } from '@mui/internal-test-utils/env'; + +/** + * `benchmark()` cases, the same API Vitest runs. Under `tacho run --engine interleaved` each case + * runs one iteration at a time in a page that stays open per build, with the builds alternating + * round by round. + * + * Like the `workload` page, it imports a workspace package so both builds resolve it from their own + * packed tarball. + */ + +function List({ count }: { count: number }) { + return ( +
      + {Array.from({ length: count }, (_, index) => ( +
    • Item {index}
      • + ))} +
    + ); +} + +benchmark('list mount', () => ); + +const scrollDuration = new ScalarMetric({ + name: 'scroll_gesture', + format: { style: 'unit', unit: 'millisecond', maximumFractionDigits: 2 }, +}); + +function ScrollingList() { + const [scrollTop, setScrollTop] = React.useState(0); + return ( +
    setScrollTop(event.currentTarget.scrollTop)} + > +

    + Scrolled {Math.round(scrollTop)}px +

    + +
    + ); +} + +// A trusted wheel-style scroll: the browser generates the whole gesture at frame cadence, so the +// list sees the event timing a real mouse wheel would produce. +benchmark( + 'list scroll', + () => , + async ({ input, resumeReactRecording }) => { + resumeReactRecording(); + scrollDuration.time(); + await input.scroll({ x: 200, y: 150, deltaY: 1200, speed: 2400 }); + scrollDuration.timeEnd(); + }, + { reactRecordingPaused: true }, +); diff --git a/test/tachometer/tsconfig.json b/test/tachometer/tsconfig.json index cc3ed604af..2061e93f0e 100644 --- a/test/tachometer/tsconfig.json +++ b/test/tachometer/tsconfig.json @@ -4,6 +4,7 @@ "module": "ESNext", "moduleResolution": "bundler", "lib": ["ES2022", "DOM", "DOM.Iterable"], + "jsx": "react-jsx", "types": ["vite/client"], "strict": true, "noEmit": true, From 50d8359e18b9fb229d12186d1587fd51f3476423 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:20:58 +0200 Subject: [PATCH 072/173] [benchmark] Add compare() for comparing implementations compare(name, { ours: () => import('./x.ours'), other: () => import('./x.other') }) groups variant modules of plain benchmark() calls. Cases are paired across variants by name; the first variant is the reference. The interleaved engine opens a page per variant (?compare=&variant=) that loads only that variant's module, so no variant's code, styles or module state is present while another is measured, and reports each paired case as a variants comparison on the working tree. Under Vitest, compare() is an async describe that loads each variant, naming its tests `mount [other]`. The demo interactions now wait for the mount's paint before scrolling: a gesture sent before the scroller is painted may not reach it. --- packages/benchmark/README.md | 40 +++- packages/benchmark/src/index.tsx | 33 ++- packages/benchmark/src/page/page.tsx | 84 +++++++- .../src/tachometer/benchFiles.test.ts | 2 +- .../benchmark/src/tachometer/benchFiles.ts | 2 +- .../src/tachometer/runInterleaved.ts | 197 +++++++++++++----- test/tachometer/src/list/list.bench.tsx | 4 +- test/tachometer/src/lists/lists.bench.tsx | 14 ++ test/tachometer/src/lists/lists.table.tsx | 48 +++++ test/tachometer/src/lists/lists.ul.tsx | 45 ++++ 10 files changed, 401 insertions(+), 68 deletions(-) create mode 100644 test/tachometer/src/lists/lists.bench.tsx create mode 100644 test/tachometer/src/lists/lists.table.tsx create mode 100644 test/tachometer/src/lists/lists.ul.tsx diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 90cf295a48..5f1397d6ff 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -85,7 +85,44 @@ benchmark( ``` A gesture's event count follows what the browser coalesced, so its render count can vary from one -iteration to the next. +iteration to the next. Wait for the mount to be painted — `await waitForElementTiming('default')` — +before starting a gesture on something the case just rendered; until then, the browser may not route +the gesture to it. + +### Comparing implementations + +`compare()` measures implementations against each other — one library against another — instead of +one build against another. Each variant is a module of ordinary `benchmark()` calls, loaded lazily; +cases are paired across variants by name, and the first variant is the reference: + +```tsx +// scatter.bench.tsx +import { compare } from '@mui/internal-benchmark'; + +compare('scatter', { + ours: () => import('./scatter.ours'), + recharts: () => import('./scatter.recharts'), +}); +``` + +```tsx +// scatter.recharts.tsx +import { benchmark } from '@mui/internal-benchmark'; + +benchmark('mount', () => ); +benchmark( + 'zoom', + () => , + async ({ input }) => { + await input.pinch({ x: 400, y: 300, scaleFactor: 2 }); + }, +); +``` + +Under the interleaved engine every variant gets a page of its own that loads only its own module, so +no variant's code, styles or module state is present while another is measured; all variants come +from the working tree. Under Vitest, which has no notion of variants, each variant's cases simply run +as tests of their own, named `mount [recharts]`. ### Scoping which renders are measured @@ -444,6 +481,7 @@ is drawn once per epoch rather than once per case. ## API - `benchmark` — define a benchmark test case +- `compare` — compare implementations, each a lazily loaded module of `benchmark()` cases - `runCase` — run a single iteration of a case, for custom drivers - `ElementTiming` — invisible marker component for paint timing (renders a `` tracked by the Element Timing API) - `ScalarMetric` — record a continuous custom measurement (with a `console.time`-style timing helper) diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index 2a71d93d0b..5187b32848 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -1,5 +1,5 @@ import type * as React from 'react'; -import { expect, it, TestRunner } from 'vitest'; +import { describe, expect, it, TestRunner } from 'vitest'; import type { RunnerTestCase } from 'vitest'; import { cdp } from 'vitest/browser'; import type { RenderEvent, IterationData } from './types'; @@ -127,14 +127,43 @@ export async function runCase( } } +/** Loads a variant's module, whose `benchmark()` calls define that variant's cases. */ +export type VariantLoader = () => Promise; + +// The `compare()` variant whose module is loading, so its `benchmark()` calls can say whose they are. +let loadingVariant: string | undefined; + +/** + * Compares implementations against each other — one library against another, say. Each variant is + * a module of ordinary `benchmark()` calls; cases are paired across variants by name, and the first + * variant is the reference. + * + * Vitest has no notion of variants, so here every variant's cases simply run as tests of their own, + * suffixed with the variant: `mount [recharts]`. The interleaved engine is what compares them. + */ +export function compare(name: string, variants: Record): void { + describe(name, async () => { + for (const [key, load] of Object.entries(variants)) { + loadingVariant = key; + try { + // eslint-disable-next-line no-await-in-loop + await load(); + } finally { + loadingVariant = undefined; + } + } + }); +} + export function benchmark( - name: string, + caseName: string, renderFn: () => React.ReactElement, interactionOrOptions?: BenchmarkInteraction | BenchmarkOptions, maybeOptions?: BenchmarkOptions, ) { const interaction = typeof interactionOrOptions === 'function' ? interactionOrOptions : undefined; const options = typeof interactionOrOptions === 'object' ? interactionOrOptions : maybeOptions; + const name = loadingVariant === undefined ? caseName : `${caseName} [${loadingVariant}]`; // In profile mode, skip the automated measurement loop entirely: build a bare case runtime (no // BenchProfiler wrapper, no-op recording since the user drives DevTools by hand) and hand it to diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index e664c95b04..c5210e8667 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -36,7 +36,13 @@ interface PageCase { options: CaseOptions | undefined; } -const cases = new Map(); +/** The file's own `benchmark()` cases. */ +const fileCases = new Map(); +// Where `benchmark()` registers: the file's own cases, except while a `compare()` variant module is +// being loaded, whose cases belong to that variant. +let registering = fileCases; +// The cases this page runs: the file's own, or the variant the url selects. +let pageCases = fileCases; export function benchmark( name: string, @@ -44,12 +50,12 @@ export function benchmark( interactionOrOptions?: BenchmarkInteraction | BenchmarkOptions, maybeOptions?: BenchmarkOptions, ): void { - if (cases.has(name)) { + if (registering.has(name)) { throw new Error(`Two benchmarks share the name "${name}". Benchmark names must be unique.`); } const interaction = typeof interactionOrOptions === 'function' ? interactionOrOptions : undefined; const options = typeof interactionOrOptions === 'object' ? interactionOrOptions : maybeOptions; - cases.set(name, { + registering.set(name, { renderFn, interaction, options: options && { @@ -59,6 +65,49 @@ export function benchmark( }); } +/** Loads a variant's module, whose `benchmark()` calls define that variant's cases. */ +export type VariantLoader = () => Promise; + +const comparisons = new Map>(); + +/** + * Compares implementations against each other — one library against another, say. Each variant is + * a module of ordinary `benchmark()` calls; cases are paired across variants by name, and the first + * variant is the reference. A variant's page loads only its own module, so no variant's code, + * styles or module state is present while another is measured. + */ +export function compare(name: string, variants: Record): void { + if (comparisons.has(name)) { + throw new Error(`Two comparisons share the name "${name}". Comparison names must be unique.`); + } + if (Object.keys(variants).length < 2) { + throw new Error(`Comparison "${name}" needs at least two variants.`); + } + comparisons.set(name, variants); +} + +/** Loads the variant `?compare=&variant=` selects, if any, and runs its cases instead. */ +async function selectVariant(): Promise { + const params = new URLSearchParams(window.location.search); + const comparison = params.get('compare'); + const variant = params.get('variant'); + if (comparison === null || variant === null) { + return; + } + const load = comparisons.get(comparison)?.[variant]; + if (!load) { + throw new Error(`No variant "${variant}" in comparison "${comparison}".`); + } + const variantCases = new Map(); + registering = variantCases; + try { + await load(); + } finally { + registering = fileCases; + } + pageCases = variantCases; +} + interface RecordedValue { name: string; value: number; @@ -77,6 +126,8 @@ setMetricRecorder((metric, value, options) => { export interface BenchPage { caseNames: () => string[]; + /** The file's `compare()` calls, with their variant keys in order. */ + comparisons: () => Array<{ name: string; variants: string[] }>; /** * Runs one iteration of a case. A measured sample leaves its results as `performance.measure` * entries for the runner to collect; a warmup sample leaves none. @@ -88,6 +139,8 @@ declare global { interface Window { /** Set by `markPageReady()` once every benchmark file has registered its cases. */ benchmarkPage?: BenchPage; + /** Set instead of `benchmarkPage` when the page could not get ready. */ + benchmarkPageError?: string; /** Exposed by the runner: forwards a CDP command to this page's session. */ benchmarkCdp?: (method: string, params?: Record) => Promise; } @@ -132,9 +185,9 @@ function emitSample( } async function sample(name: string, { warmup }: { warmup: boolean }): Promise { - const benchCase = cases.get(name); + const benchCase = pageCases.get(name); if (!benchCase) { - throw new Error(`No benchmark named "${name}". Known: ${[...cases.keys()].join(', ')}`); + throw new Error(`No benchmark named "${name}". Known: ${[...pageCases.keys()].join(', ')}`); } performance.clearMarks(); performance.clearMeasures(); @@ -167,12 +220,27 @@ async function sample(name: string, { warmup }: { warmup: boolean }): Promise { if (typeof window.gc !== 'function') { console.warn( 'window.gc is not available. Run with --js-flags=--expose-gc for consistent GC between iterations.', ); } - window.benchmarkPage = { caseNames: () => [...cases.keys()], sample }; + try { + await selectVariant(); + } catch (error) { + window.benchmarkPageError = + error instanceof Error ? (error.stack ?? error.message) : String(error); + return; + } + window.benchmarkPage = { + caseNames: () => [...pageCases.keys()], + comparisons: () => + [...comparisons].map(([name, variants]) => ({ name, variants: Object.keys(variants) })), + sample, + }; } diff --git a/packages/benchmark/src/tachometer/benchFiles.test.ts b/packages/benchmark/src/tachometer/benchFiles.test.ts index b2880abcad..c896cebf94 100644 --- a/packages/benchmark/src/tachometer/benchFiles.test.ts +++ b/packages/benchmark/src/tachometer/benchFiles.test.ts @@ -53,7 +53,7 @@ describe('writeBenchPages', () => { "import '../button/button.bench'; import { markPageReady } from '@mui/internal-benchmark/page'; - markPageReady(); + void markPageReady(); " `); }); diff --git a/packages/benchmark/src/tachometer/benchFiles.ts b/packages/benchmark/src/tachometer/benchFiles.ts index a6edd04aa4..4127d20b22 100644 --- a/packages/benchmark/src/tachometer/benchFiles.ts +++ b/packages/benchmark/src/tachometer/benchFiles.ts @@ -67,7 +67,7 @@ function pageEntry(benchFile: BenchFile): string { return `import '${specifier}'; import { markPageReady } from '@mui/internal-benchmark/page'; -markPageReady(); +void markPageReady(); `; } diff --git a/packages/benchmark/src/tachometer/runInterleaved.ts b/packages/benchmark/src/tachometer/runInterleaved.ts index cb122b8185..94d9e1fdfb 100644 --- a/packages/benchmark/src/tachometer/runInterleaved.ts +++ b/packages/benchmark/src/tachometer/runInterleaved.ts @@ -5,7 +5,7 @@ import chalk from 'chalk'; import type { Browser, BrowserContext, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS } from '../launchArgs'; import { measurementNameOf } from './discoverCases'; -import type { BenchmarkCase, Leaf } from './discoverCases'; +import type { BenchmarkCase, CaseComparison, Leaf } from './discoverCases'; import type { BenchFile } from './benchFiles'; import type { ResolvedRef } from './refs'; import { serveDirectory } from './serveDirectory'; @@ -163,19 +163,25 @@ async function runPageCase( } } -interface BenchTarget { - ref: ResolvedRef; +/** + * A page a benchmark file's cases are measured in: one build of the file, or one variant of a + * `compare()` in it. + */ +interface PageSlot { + /** How the slot is named in the report: `current`, `baseline`, or the variant's key. */ + variant: string; + url: string; +} + +interface OpenedPage { + slot: PageSlot; context: BrowserContext; page: Page; caseNames: string[]; + comparisons: Array<{ name: string; variants: string[] }>; } -async function openBenchPage( - browser: Browser, - origin: string, - benchFile: BenchFile, - ref: ResolvedRef, -): Promise { +async function openBenchPage(browser: Browser, slot: PageSlot): Promise { const { context, page } = await openContext(browser); const session = await context.newCDPSession(page); // Interactions ask for trusted input through this bridge. `send` only accepts the protocol @@ -185,21 +191,29 @@ async function openBenchPage( (_source, method: string, params?: Record) => session.send(method as CdpMethod, params), ); - await page.goto(`${origin}/${ref.id}/${benchFile.page}`); - await page.waitForFunction(() => window.benchmarkPage !== undefined); + await page.goto(slot.url); + await page.waitForFunction( + () => window.benchmarkPage !== undefined || window.benchmarkPageError !== undefined, + ); + const pageError = await page.evaluate(() => window.benchmarkPageError); + if (pageError !== undefined) { + await context.close(); + throw new Error(`${slot.url} could not get ready: ${pageError}`); + } const visibility = await page.evaluate(() => document.visibilityState); if (visibility !== 'visible') { - console.warn( - chalk.yellow(` ${benchFile.file} [${ref.id}] is ${visibility}; rAF-based waits may stall.`), - ); + console.warn(chalk.yellow(` ${slot.url} is ${visibility}; rAF-based waits may stall.`)); } - const caseNames = await page.evaluate(() => window.benchmarkPage!.caseNames()); - return { ref, context, page, caseNames }; + const { caseNames, comparisons } = await page.evaluate(() => ({ + caseNames: window.benchmarkPage!.caseNames(), + comparisons: window.benchmarkPage!.comparisons(), + })); + return { slot, context, page, caseNames, comparisons }; } /** Runs one iteration; a measured one comes back as its `performance.measure` values by name. */ async function sampleBenchCase( - target: BenchTarget, + target: OpenedPage, name: string, warmup: boolean, ): Promise | undefined> { @@ -222,36 +236,35 @@ async function sampleBenchCase( }); } -/** Opens a page per build, in a random order so no build always gets the first process. */ -async function openBenchPages( - browser: Browser, - origin: string, - benchFile: BenchFile, - refs: ResolvedRef[], -): Promise { - const targets: BenchTarget[] = []; - for (const index of shuffledIndices(refs.length)) { - // eslint-disable-next-line no-await-in-loop - targets[index] = await openBenchPage(browser, origin, benchFile, refs[index]); +/** Opens a page per slot, in a random order so no slot always gets the first process. */ +async function openBenchPages(browser: Browser, slots: PageSlot[]): Promise { + const opened: OpenedPage[] = []; + try { + for (const index of shuffledIndices(slots.length)) { + // eslint-disable-next-line no-await-in-loop + opened[index] = await openBenchPage(browser, slots[index]); + } + } catch (error) { + await closeBenchPages(opened.filter(Boolean)); + throw error; } - return targets; + return opened; } -async function closeBenchPages(targets: BenchTarget[]): Promise { +async function closeBenchPages(targets: OpenedPage[]): Promise { await Promise.all(targets.map((target) => target.context.close())); } /** * Measures one case in epochs. A page has to stay open for iterations to be warm, which ties each - * build to one renderer process for as long as it does — and a process keeps whatever speed it + * slot to one renderer process for as long as it does — and a process keeps whatever speed it * started with (a performance or an efficiency core, typically), enough to show up as a difference * between identical builds. So every epoch opens fresh pages, warms them up, and measures a slice - * of the rounds: the process each build lands on is drawn again per epoch instead of once per case. + * of the rounds: the process each slot lands on is drawn again per epoch instead of once per case. */ async function measureBenchCase( browser: Browser, - origin: string, - benchFile: BenchFile, + slots: PageSlot[], name: string, options: RunInterleavedOptions, ): Promise { @@ -261,7 +274,7 @@ async function measureBenchCase( const rounds: Round[] = []; while (rounds.length < samples) { // eslint-disable-next-line no-await-in-loop - const targets = await openBenchPages(browser, origin, benchFile, options.benchRefs); + const targets = await openBenchPages(browser, slots); try { const measured = Math.min(epochSize, samples - rounds.length); for (let roundIndex = 0; roundIndex < warmup + measured; roundIndex += 1) { @@ -282,41 +295,41 @@ async function measureBenchCase( return rounds; } -async function runBenchFile( +/** + * Runs every case the first slot defines across all slots, paired by name. The first slot is the + * reference; a case another slot lacks is reported as an error rather than compared with nothing. + */ +async function runSlotCases( browser: Browser, - origin: string, - benchFile: BenchFile, + slots: PageSlot[], + listed: OpenedPage[], + describe: { comparison: CaseComparison; nameOf: (caseName: string) => string; file: string }, options: RunInterleavedOptions, ): Promise { - // Opened once just to learn which cases each build defines. - const listed = await openBenchPages(browser, origin, benchFile, options.benchRefs); - await closeBenchPages(listed); - const [current, ...others] = listed; - + const [reference, ...others] = listed; const results: InterleavedResult[] = []; - for (const name of current.caseNames) { + for (const caseName of reference.caseNames) { + const name = describe.nameOf(caseName); const entry: BenchmarkCase = { name, - configPath: path.join(options.harnessDir, 'src', benchFile.file), + configPath: path.join(options.harnessDir, 'src', describe.file), config: {}, - comparison: 'baseline', + comparison: describe.comparison, leaves: [], - variants: listed.map((target) => - target.ref.kind === 'worktree' ? `${name} [current]` : `${name} [baseline]`, - ), + variants: slots.map((slot) => `${name} [${slot.variant}]`), measurements: [], }; - const missing = others.find((target) => !target.caseNames.includes(name)); + const missing = others.find((target) => !target.caseNames.includes(caseName)); if (missing) { - // A case added by the change under test has nothing to be compared against yet. - results.push({ entry, error: `"${name}" does not exist in ${missing.ref.id}.` }); + // A case added by the change under test, or one library lacks, has nothing to compare with. + results.push({ entry, error: `"${caseName}" does not exist in [${missing.slot.variant}].` }); continue; } - console.log(chalk.cyan(`\nRunning "${name}" (${benchFile.file})…`)); + console.log(chalk.cyan(`\nRunning "${name}" (${describe.file})…`)); try { // eslint-disable-next-line no-await-in-loop - const rounds = await measureBenchCase(browser, origin, benchFile, name, options); + const rounds = await measureBenchCase(browser, slots, caseName, options); // A measurement some iteration did not produce (an interaction whose render count varies, // say) cannot be paired round by round, so only those every iteration reported are kept. entry.measurements = stableMeasurements(rounds); @@ -329,6 +342,82 @@ async function runBenchFile( return results; } +function benchPageUrl(origin: string, ref: ResolvedRef, benchFile: BenchFile): URL { + return new URL(`${ref.id}/${benchFile.page}`, `${origin}/`); +} + +/** + * Runs a benchmark file: its own `benchmark()` cases across the builds, and each `compare()` in it + * across its variants, every variant built from the working tree. + */ +async function runBenchFile( + browser: Browser, + origin: string, + benchFile: BenchFile, + options: RunInterleavedOptions, +): Promise { + const buildSlots = options.benchRefs.map((ref): PageSlot => ({ + variant: ref.kind === 'worktree' ? 'current' : 'baseline', + url: benchPageUrl(origin, ref, benchFile).href, + })); + // Opened once just to learn which cases and comparisons each build defines. + const listed = await openBenchPages(browser, buildSlots); + await closeBenchPages(listed); + + const results = await runSlotCases( + browser, + buildSlots, + listed, + { comparison: 'baseline', nameOf: (caseName) => caseName, file: benchFile.file }, + options, + ); + + const [worktreeRef] = options.benchRefs; + for (const { name: comparison, variants } of listed[0].comparisons) { + const variantSlots = variants.map((variant): PageSlot => { + const url = benchPageUrl(origin, worktreeRef, benchFile); + url.searchParams.set('compare', comparison); + url.searchParams.set('variant', variant); + return { variant, url: url.href }; + }); + try { + // eslint-disable-next-line no-await-in-loop + const variantPages = await openBenchPages(browser, variantSlots); + // eslint-disable-next-line no-await-in-loop + await closeBenchPages(variantPages); + results.push( + // eslint-disable-next-line no-await-in-loop + ...(await runSlotCases( + browser, + variantSlots, + variantPages, + { + comparison: 'variants', + nameOf: (caseName) => `${comparison} / ${caseName}`, + file: benchFile.file, + }, + options, + )), + ); + } catch (error) { + console.error(chalk.red(` ${errorMessage(error)}`)); + results.push({ + entry: { + name: comparison, + configPath: path.join(options.harnessDir, 'src', benchFile.file), + config: {}, + comparison: 'variants', + leaves: [], + variants: [], + measurements: [], + }, + error: errorMessage(error), + }); + } + } + return results; +} + export async function runInterleaved(options: RunInterleavedOptions): Promise { const { chromium } = await import('@playwright/test'); const server = await serveDirectory(options.buildsDir); diff --git a/test/tachometer/src/list/list.bench.tsx b/test/tachometer/src/list/list.bench.tsx index 66846ee109..f85d5c9739 100644 --- a/test/tachometer/src/list/list.bench.tsx +++ b/test/tachometer/src/list/list.bench.tsx @@ -49,7 +49,9 @@ function ScrollingList() { benchmark( 'list scroll', () => , - async ({ input, resumeReactRecording }) => { + async ({ input, resumeReactRecording, waitForElementTiming }) => { + // The scroller has to be painted before a gesture can hit it. + await waitForElementTiming('default'); resumeReactRecording(); scrollDuration.time(); await input.scroll({ x: 200, y: 150, deltaY: 1200, speed: 2400 }); diff --git a/test/tachometer/src/lists/lists.bench.tsx b/test/tachometer/src/lists/lists.bench.tsx new file mode 100644 index 0000000000..7a66d17e64 --- /dev/null +++ b/test/tachometer/src/lists/lists.bench.tsx @@ -0,0 +1,14 @@ +import { compare } from '@mui/internal-benchmark'; + +/** + * Two implementations of the same list, compared with each other rather than across builds — the + * shape a comparison between libraries takes. Each variant is a module of plain `benchmark()` cases; + * cases with the same name are paired, and the first variant is the reference. + * + * Every variant gets a page of its own that loads only its own module, so neither implementation's + * code or styles are present while the other is measured. + */ +compare('lists', { + ul: () => import('./lists.ul'), + table: () => import('./lists.table'), +}); diff --git a/test/tachometer/src/lists/lists.table.tsx b/test/tachometer/src/lists/lists.table.tsx new file mode 100644 index 0000000000..024fae26c0 --- /dev/null +++ b/test/tachometer/src/lists/lists.table.tsx @@ -0,0 +1,48 @@ +import * as React from 'react'; +import { benchmark } from '@mui/internal-benchmark'; + +const ROWS = 1000; + +function List() { + return ( + + + {Array.from({ length: ROWS }, (_, index) => ( + + + + + ))} + +
    Row {index}{index % 7}
    + ); +} + +function ScrollingList() { + const [scrollTop, setScrollTop] = React.useState(0); + return ( +
    setScrollTop(event.currentTarget.scrollTop)} + > +

    + Scrolled {Math.round(scrollTop)}px +

    + +
    + ); +} + +benchmark('mount', () => ); + +benchmark( + 'scroll', + () => , + async ({ input, resumeReactRecording, waitForElementTiming }) => { + // The scroller has to be painted before a gesture can hit it. + await waitForElementTiming('default'); + resumeReactRecording(); + await input.scroll({ x: 200, y: 150, deltaY: 1200, speed: 2400 }); + }, + { reactRecordingPaused: true }, +); diff --git a/test/tachometer/src/lists/lists.ul.tsx b/test/tachometer/src/lists/lists.ul.tsx new file mode 100644 index 0000000000..441d6fb5a7 --- /dev/null +++ b/test/tachometer/src/lists/lists.ul.tsx @@ -0,0 +1,45 @@ +import * as React from 'react'; +import { benchmark } from '@mui/internal-benchmark'; + +const ROWS = 1000; + +function List() { + return ( +
      + {Array.from({ length: ROWS }, (_, index) => ( +
    • + Row {index} {index % 7} +
      • + ))} +
    + ); +} + +function ScrollingList() { + const [scrollTop, setScrollTop] = React.useState(0); + return ( +
    setScrollTop(event.currentTarget.scrollTop)} + > +

    + Scrolled {Math.round(scrollTop)}px +

    + +
    + ); +} + +benchmark('mount', () => ); + +benchmark( + 'scroll', + () => , + async ({ input, resumeReactRecording, waitForElementTiming }) => { + // The scroller has to be painted before a gesture can hit it. + await waitForElementTiming('default'); + resumeReactRecording(); + await input.scroll({ x: 200, y: 150, deltaY: 1200, speed: 2400 }); + }, + { reactRecordingPaused: true }, +); From ae0ac34d9c84dc06b8ff14eaa492691df22a32bc Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Fri, 25 Sep 2026 17:09:38 +0200 Subject: [PATCH 073/173] [benchmark] Remove epochs from the interleaved engine Epochs reopened a bench case's pages every few rounds to redraw which renderer process each variant ran in, on the premise that a process keeps a stable speed offset for its lifetime. A/A probes refuted that: with two contexts each tied to one variant for a whole run, the paired false positive rate was the nominal 5% (1/20, 0/20, 1/20 across CPU-only, rAF-inclusive and the built workload page). A bench case now opens one page per variant once, discards `--warmup` rounds (default 10), then measures. `--epoch-size` is gone. --- packages/benchmark/README.md | 10 ++-- packages/benchmark/src/cli/tachoRun.ts | 8 +--- .../src/tachometer/runInterleaved.ts | 46 ++++++++----------- .../benchmark/src/tachometer/runTachometer.ts | 4 -- 4 files changed, 22 insertions(+), 46 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 5f1397d6ff..c6ecc43704 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -470,13 +470,9 @@ It runs two kinds of case side by side: benchmark tacho run --engine interleaved --baseline "$(code-infra baseline)" --samples 30 ``` -Every variant runs in a browser context of its own, but a context's renderer process keeps whatever -speed it started with — on a machine with performance and efficiency cores, which kind it landed on -— and two identical builds can differ by several percent for as long as their processes live. Page -loads are therefore dealt out over the contexts anew every round. A `*.bench.tsx` page has to stay -open to stay warm, so its rounds run in epochs instead (`--epoch-size`, 10 rounds by default): each -epoch reopens the pages in fresh processes and warms them up again, so the process a build lands on -is drawn once per epoch rather than once per case. +Every variant runs in a browser context of its own. A `*.bench.tsx` case opens one page per variant +and keeps it open for the whole case: `--warmup` rounds (10 by default) are discarded once, then the +measured rounds follow. ## API diff --git a/packages/benchmark/src/cli/tachoRun.ts b/packages/benchmark/src/cli/tachoRun.ts index 40188a301b..f54e2b98bb 100644 --- a/packages/benchmark/src/cli/tachoRun.ts +++ b/packages/benchmark/src/cli/tachoRun.ts @@ -63,12 +63,7 @@ const command: CommandModule<{}, Args> = { .option('warmup', { type: 'number', describe: - 'Interleaved engine: discarded rounds before measuring. Default: 2 per page case, 5 per epoch for *.bench.tsx cases', - }) - .option('epoch-size', { - type: 'number', - describe: - 'Interleaved engine: measured rounds of a *.bench.tsx case before its pages are reopened in fresh processes. Default: 10', + 'Interleaved engine: discarded rounds before measuring. Default: 2 for page cases, 10 for *.bench.tsx cases', }) .epilogue( 'Sampling (sampleSize, autoSampleConditions, timeout) is configured per case in its own tachometer.json — tachometer rejects those as CLI flags when a config file is used.', @@ -88,7 +83,6 @@ const command: CommandModule<{}, Args> = { engine: argv.engine, samples: argv.samples, warmup: argv.warmup, - epochSize: argv.epochSize, }); }, }; diff --git a/packages/benchmark/src/tachometer/runInterleaved.ts b/packages/benchmark/src/tachometer/runInterleaved.ts index 94d9e1fdfb..707186cade 100644 --- a/packages/benchmark/src/tachometer/runInterleaved.ts +++ b/packages/benchmark/src/tachometer/runInterleaved.ts @@ -50,12 +50,10 @@ export interface RunInterleavedOptions { /** Measured rounds per case. Defaults to a `tachometer.json`'s `sampleSize`, else 30. */ samples?: number; /** - * Discarded rounds before measuring: once per case for page loads (default 2), once per epoch - * for `*.bench.tsx` iterations (default 5). + * Discarded rounds before measuring, once per case: default 2 for page loads, 10 for + * `*.bench.tsx` iterations. */ warmup?: number; - /** `*.bench.tsx` only: measured rounds per set of pages before they are reopened. Default 10. */ - epochSize?: number; } const DEFAULT_VIEWPORT = { width: 1920, height: 1080 }; @@ -256,11 +254,9 @@ async function closeBenchPages(targets: OpenedPage[]): Promise { } /** - * Measures one case in epochs. A page has to stay open for iterations to be warm, which ties each - * slot to one renderer process for as long as it does — and a process keeps whatever speed it - * started with (a performance or an efficiency core, typically), enough to show up as a difference - * between identical builds. So every epoch opens fresh pages, warms them up, and measures a slice - * of the rounds: the process each slot lands on is drawn again per epoch instead of once per case. + * Measures one case: a page per slot stays open for the whole case, so iterations stay warm and + * module-scope data is built once. Warmup and measured rounds alike run every slot once, in a + * shuffled order. */ async function measureBenchCase( browser: Browser, @@ -268,29 +264,23 @@ async function measureBenchCase( name: string, options: RunInterleavedOptions, ): Promise { - const warmup = options.warmup ?? 5; + const warmup = options.warmup ?? 10; const samples = options.samples ?? DEFAULT_SAMPLES; - const epochSize = options.epochSize ?? 10; const rounds: Round[] = []; - while (rounds.length < samples) { - // eslint-disable-next-line no-await-in-loop - const targets = await openBenchPages(browser, slots); - try { - const measured = Math.min(epochSize, samples - rounds.length); - for (let roundIndex = 0; roundIndex < warmup + measured; roundIndex += 1) { - const round: Round = []; - for (const index of shuffledIndices(targets.length)) { - // eslint-disable-next-line no-await-in-loop - round[index] = await sampleBenchCase(targets[index], name, roundIndex < warmup); - } - if (roundIndex >= warmup) { - rounds.push(round); - } + const targets = await openBenchPages(browser, slots); + try { + for (let roundIndex = 0; roundIndex < warmup + samples; roundIndex += 1) { + const round: Round = []; + for (const index of shuffledIndices(targets.length)) { + // eslint-disable-next-line no-await-in-loop + round[index] = await sampleBenchCase(targets[index], name, roundIndex < warmup); + } + if (roundIndex >= warmup) { + rounds.push(round); } - } finally { - // eslint-disable-next-line no-await-in-loop - await closeBenchPages(targets); } + } finally { + await closeBenchPages(targets); } return rounds; } diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts index ad01290828..a0ed2f0911 100644 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -94,8 +94,6 @@ export interface RunTachometerOptions { samples?: number; /** Interleaved engine only: discarded rounds before measuring. */ warmup?: number; - /** Interleaved engine only: measured rounds of a `*.bench.tsx` case per set of pages. */ - epochSize?: number; } export type Engine = 'tachometer' | 'interleaved'; @@ -122,7 +120,6 @@ export async function runTachometer(options: RunTachometerOptions): Promise Date: Fri, 25 Sep 2026 20:08:55 +0200 Subject: [PATCH 074/173] [benchmark] Give each page-case variant its own context again Page loads were dealt over the contexts anew every round to spread a per-process speed offset across variants. A/A probes showed no such offset, so each variant keeps one context, loading its page fresh for every sample. --- .../src/tachometer/runInterleaved.ts | 34 +++++++++---------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/packages/benchmark/src/tachometer/runInterleaved.ts b/packages/benchmark/src/tachometer/runInterleaved.ts index 707186cade..e6cd5f5663 100644 --- a/packages/benchmark/src/tachometer/runInterleaved.ts +++ b/packages/benchmark/src/tachometer/runInterleaved.ts @@ -103,23 +103,22 @@ async function runPageCase( options: RunInterleavedOptions, ): Promise { const nodes: any[] = entry.config.benchmarks; - const targets = entry.leaves.map((leaf, index) => ({ - url: `${origin}/${options.buildFor(leaf).id}/${leaf.page}${leaf.suffix}`, - specs: entrySpecsOf(nodes[index]), - })); - // A renderer process keeps whatever speed it started with — on a machine with performance and - // efficiency cores, which kind the scheduler put it on — so two identical pages in two contexts - // can differ by several percent for as long as those processes live. Every sample is a fresh - // load anyway, so no variant is tied to a context: each round deals the variants out over the - // pool anew, and a slow process slows every variant equally often. - const pool = await Promise.all( - targets.map(() => openContext(browser, nodes[0]?.browser?.windowSize)), + const targets = await Promise.all( + entry.leaves.map(async (leaf, index) => { + const node = nodes[index]; + const opened = await openContext(browser, node.browser?.windowSize); + return { + ...opened, + url: `${origin}/${options.buildFor(leaf).id}/${leaf.page}${leaf.suffix}`, + specs: entrySpecsOf(node), + }; + }), ); try { - const sample = async (target: (typeof targets)[number], page: Page) => { - await page.goto(target.url); - const entryNames = target.specs.map((spec) => spec.entryName); + const sample = async ({ page, url, specs }: (typeof targets)[number]) => { + await page.goto(url); + const entryNames = specs.map((spec) => spec.entryName); await page.waitForFunction( (names) => names.every((name) => performance.getEntriesByName(name).length > 0), entryNames, @@ -137,7 +136,7 @@ async function runPageCase( return [name, value]; }), ), - target.specs, + specs, ); }; @@ -146,10 +145,9 @@ async function runPageCase( const rounds: Round[] = []; for (let roundIndex = 0; roundIndex < warmup + samples; roundIndex += 1) { const round: Round = []; - const dealt = shuffledIndices(pool.length); for (const index of shuffledIndices(targets.length)) { // eslint-disable-next-line no-await-in-loop - round[index] = await sample(targets[index], pool[dealt[index]].page); + round[index] = await sample(targets[index]); } if (roundIndex >= warmup) { rounds.push(round); @@ -157,7 +155,7 @@ async function runPageCase( } return toTachometerJson(entry.variants, entry.measurements, rounds); } finally { - await Promise.all(pool.map((opened) => opened.context.close())); + await Promise.all(targets.map((target) => target.context.close())); } } From e9fe4a0918115fe1b3b59cd448c5bd97d6edfb92 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 11:48:06 +0200 Subject: [PATCH 075/173] [benchmark] Run only *.bench.tsx files in the interleaved engine tachometer.json cases stay with the tachometer engine; the interleaved engine now only runs benchmark files, so each engine discovers its own kind of case and the page-load sampling path, its warmup default and the second discovery walk go away. Cleanups from review: - summarizeCase takes the four fields it reads, so interleaved results no longer invent tachometer config; CaseRun is a results-or-error union shared by both engines - benchmark()/runCase() argument parsing, BenchmarkOptions, VariantLoader and the gc warning live once in caseRuntime.tsx - runCase() has one path with or without a running test - pages open in parallel, readiness is read in one evaluate, and the static server and browser start together - pairedStats uses the package's calculateMean --- packages/benchmark/README.md | 35 +- packages/benchmark/src/caseRuntime.tsx | 27 ++ packages/benchmark/src/cli/tachoRun.ts | 8 +- packages/benchmark/src/index.tsx | 84 ++--- packages/benchmark/src/page/page.tsx | 45 +-- .../benchmark/src/tachometer/pairedStats.ts | 9 +- .../src/tachometer/runInterleaved.ts | 318 ++++++------------ .../benchmark/src/tachometer/runTachometer.ts | 77 ++--- .../benchmark/src/tachometer/summarizeCase.ts | 13 +- packages/benchmark/src/vitest.ts | 7 +- 10 files changed, 251 insertions(+), 372 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index c6ecc43704..673dd0ebab 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -448,31 +448,26 @@ resolution there, leaving the repository untouched, at the cost of a second inst ### Interleaved engine -`--engine interleaved` measures the same builds without tachometer: it drives Chromium through -Playwright, samples every variant once per round in a shuffled order, and judges a difference on the -per-round differences rather than on two independent sets of samples. Whatever the machine was doing -during a round — thermal throttling, a background process — then affects both sides of it and -cancels out, instead of counting as noise or, worse, as a difference. The report, the upload and the -PR comment are the same as tachometer's. - -It runs two kinds of case side by side: - -- **`tachometer.json` cases**, unchanged. Every sample is a fresh page load, measured by the - `performance.measure` entry (or `fcp`) the case names. Other measurement modes are tachometer-only. -- **`*.bench.tsx` files** under `src/`, written with `benchmark()` exactly as for Vitest. The plugin - generates a page per file under `src/__bench__/`, where `@mui/internal-benchmark` resolves to a page - runtime instead of Vitest. The page stays open, so every sample is one warm iteration and - module-scope data is built once; renders, `bench:paint` and custom metrics are reported as - measurements. The iteration counts in `benchmark()`'s options are ignored — `--samples` and - `--warmup` decide. +`--engine interleaved` measures the same builds without tachometer, and runs a different kind of +case: the `*.bench.tsx` files under `src/`, written with `benchmark()` exactly as for Vitest. +`tachometer.json` cases stay with the tachometer engine. + +The plugin generates a page per benchmark file under `src/__bench__/`, where +`@mui/internal-benchmark` resolves to a page runtime instead of Vitest. The runner drives Chromium +through Playwright and gives every variant a page of its own, in a browser context of its own, that +stays open for the whole case: every sample is one warm iteration, and module-scope data is built +once. Variants are sampled once per round in a shuffled order, and a difference is judged on the +per-round differences rather than on two independent sets of samples, so whatever the machine was +doing during a round — thermal throttling, a background process — affects both sides of it and +cancels out. Renders, `bench:paint` and custom metrics are reported as measurements; the report, the +upload and the PR comment are the same as tachometer's. ```bash benchmark tacho run --engine interleaved --baseline "$(code-infra baseline)" --samples 30 ``` -Every variant runs in a browser context of its own. A `*.bench.tsx` case opens one page per variant -and keeps it open for the whole case: `--warmup` rounds (10 by default) are discarded once, then the -measured rounds follow. +The iteration counts in `benchmark()`'s options are ignored: `--warmup` rounds (10 by default) are +discarded once per case, then `--samples` rounds (30 by default) are measured. ## API diff --git a/packages/benchmark/src/caseRuntime.tsx b/packages/benchmark/src/caseRuntime.tsx index cc472d6c43..d7a25dd0f5 100644 --- a/packages/benchmark/src/caseRuntime.tsx +++ b/packages/benchmark/src/caseRuntime.tsx @@ -32,6 +32,33 @@ export interface CaseOptions { reactRecordingPaused?: boolean; } +/** `benchmark()`'s options. The page runtime ignores the iteration counts: the runner decides them. */ +export interface BenchmarkOptions extends CaseOptions { + runs?: number; + warmupRuns?: number; +} + +/** Loads a `compare()` variant's module, whose `benchmark()` calls define that variant's cases. */ +export type VariantLoader = () => Promise; + +/** Splits `benchmark()`-style arguments, where the interaction may be left out. */ +export function splitCaseArgs( + interactionOrOptions: BenchmarkInteraction | Options | undefined, + maybeOptions: Options | undefined, +): { interaction: BenchmarkInteraction | undefined; options: Options | undefined } { + return typeof interactionOrOptions === 'function' + ? { interaction: interactionOrOptions, options: maybeOptions } + : { interaction: undefined, options: interactionOrOptions ?? maybeOptions }; +} + +export function warnIfNoGc(): void { + if (typeof window.gc !== 'function') { + console.warn( + 'window.gc is not available. Run with --js-flags=--expose-gc for consistent GC between iterations.', + ); + } +} + function BenchProfiler({ captures, recording, diff --git a/packages/benchmark/src/cli/tachoRun.ts b/packages/benchmark/src/cli/tachoRun.ts index f54e2b98bb..22d9a9bfe7 100644 --- a/packages/benchmark/src/cli/tachoRun.ts +++ b/packages/benchmark/src/cli/tachoRun.ts @@ -53,17 +53,15 @@ const command: CommandModule<{}, Args> = { choices: ['tachometer', 'interleaved'] as const, default: 'tachometer' as const, describe: - "What measures the pages: 'tachometer', or 'interleaved', which alternates variants round by round in Playwright, compares paired differences, and also runs *.bench.tsx files", + "What measures the harness: 'tachometer' runs its tachometer.json cases; 'interleaved' runs its *.bench.tsx files in Playwright, alternating variants round by round and comparing paired differences", }) .option('samples', { type: 'number', - describe: - "Interleaved engine: measured rounds per case. Default: the case's sampleSize, else 30", + describe: 'Interleaved engine: measured rounds per case. Default: 30', }) .option('warmup', { type: 'number', - describe: - 'Interleaved engine: discarded rounds before measuring. Default: 2 for page cases, 10 for *.bench.tsx cases', + describe: 'Interleaved engine: discarded rounds before measuring. Default: 10', }) .epilogue( 'Sampling (sampleSize, autoSampleConditions, timeout) is configured per case in its own tachometer.json — tachometer rejects those as CLI flags when a config file is used.', diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index 5187b32848..a59151ac81 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -7,8 +7,19 @@ import { ScalarMetric } from './ScalarMetric'; import { metricsGate } from './metricsGate'; import { runProfileSession } from './profileSession'; import { createInput } from './input'; -import { createCaseRuntime, createElementTimingWaiter, measureIteration } from './caseRuntime'; -import type { BenchmarkInteraction, CaseOptions } from './caseRuntime'; +import { + createCaseRuntime, + createElementTimingWaiter, + measureIteration, + splitCaseArgs, + warnIfNoGc, +} from './caseRuntime'; +import type { + BenchmarkInteraction, + BenchmarkOptions, + CaseOptions, + VariantLoader, +} from './caseRuntime'; // Installs the Vitest metric recorder. import './Metric'; // Import for TaskMeta augmentation side effect @@ -23,7 +34,7 @@ export type { MetricDefinition, } from './types'; export type { BenchmarkInput } from './input'; -export type { BenchmarkInteraction } from './caseRuntime'; +export type { BenchmarkInteraction, BenchmarkOptions, VariantLoader } from './caseRuntime'; export { ElementTiming } from './ElementTiming'; export { Metric, type MetricRecordOptions } from './Metric'; export { ScalarMetric }; @@ -42,11 +53,6 @@ type CdpMethod = Parameters['send']>[0]; // leaves unknown methods for Chrome to reject. const input = createInput((method, params) => cdp().send(method as CdpMethod, params)); -export interface BenchmarkOptions extends CaseOptions { - runs?: number; - warmupRuns?: number; -} - export interface RunCaseOptions extends CaseOptions { /** * Run the iteration without recording anything: no `bench:paint`, and custom metrics recorded @@ -74,25 +80,6 @@ const paint = new ScalarMetric({ format: { style: 'unit', unit: 'millisecond', maximumFractionDigits: 2 }, }); -async function measureAndRecord( - renderFn: () => React.ReactElement, - interaction: BenchmarkInteraction | undefined, - options: RunCaseOptions | undefined, -): Promise { - const { renders, paints, renderError, hadEmptyActiveWindow } = await measureIteration( - renderFn, - interaction, - options, - input, - ); - if (!options?.warmup) { - for (const { id, start, end } of paints) { - paint.record(end - start, id !== undefined ? { id } : undefined); - } - } - return { renders, renderError, hadEmptyActiveWindow }; -} - /** * Mounts, interacts with and unmounts a case exactly once, recording its renders and paint. * @@ -109,27 +96,35 @@ export async function runCase( interactionOrOptions?: BenchmarkInteraction | RunCaseOptions, maybeOptions?: RunCaseOptions, ): Promise { - const interaction = typeof interactionOrOptions === 'function' ? interactionOrOptions : undefined; - const options = typeof interactionOrOptions === 'object' ? interactionOrOptions : maybeOptions; + const { interaction, options } = splitCaseArgs(interactionOrOptions, maybeOptions); // Custom metrics recorded inside the case honor warmup exclusion through the gate, the same way // renders and `bench:paint` are excluded during warmup. The gate is keyed on the running test, // and is re-enabled afterwards so metrics the driver records between iterations are kept. const test = TestRunner.getCurrentTest(); - if (!test) { - return measureAndRecord(renderFn, interaction, options); + if (test) { + metricsGate.setRecordingEnabled(test, !options?.warmup); } - metricsGate.setRecordingEnabled(test, !options?.warmup); try { - return await measureAndRecord(renderFn, interaction, options); + const { renders, paints, renderError, hadEmptyActiveWindow } = await measureIteration( + renderFn, + interaction, + options, + input, + ); + if (!options?.warmup) { + for (const { id, start, end } of paints) { + paint.record(end - start, id !== undefined ? { id } : undefined); + } + } + return { renders, renderError, hadEmptyActiveWindow }; } finally { - metricsGate.setRecordingEnabled(test, true); + if (test) { + metricsGate.setRecordingEnabled(test, true); + } } } -/** Loads a variant's module, whose `benchmark()` calls define that variant's cases. */ -export type VariantLoader = () => Promise; - // The `compare()` variant whose module is loading, so its `benchmark()` calls can say whose they are. let loadingVariant: string | undefined; @@ -161,8 +156,7 @@ export function benchmark( interactionOrOptions?: BenchmarkInteraction | BenchmarkOptions, maybeOptions?: BenchmarkOptions, ) { - const interaction = typeof interactionOrOptions === 'function' ? interactionOrOptions : undefined; - const options = typeof interactionOrOptions === 'object' ? interactionOrOptions : maybeOptions; + const { interaction, options } = splitCaseArgs(interactionOrOptions, maybeOptions); const name = loadingVariant === undefined ? caseName : `${caseName} [${loadingVariant}]`; // In profile mode, skip the automated measurement loop entirely: build a bare case runtime (no @@ -194,11 +188,7 @@ export function benchmark( const runs = options?.runs ?? 20; const warmupRuns = options?.warmupRuns ?? 10; - if (typeof window.gc !== 'function') { - console.warn( - 'window.gc is not available. Run with --js-flags=--expose-gc for consistent GC between iterations.', - ); - } + warnIfNoGc(); const iterations: IterationData[] = []; let renderError: unknown = null; @@ -208,11 +198,7 @@ export function benchmark( for (let i = 0; i < warmupRuns + runs; i += 1) { const warmup = i < warmupRuns; // eslint-disable-next-line no-await-in-loop - const result = await runCase(renderFn, interaction, { - afterEach: options?.afterEach, - reactRecordingPaused: options?.reactRecordingPaused, - warmup, - }); + const result = await runCase(renderFn, interaction, { ...options, warmup }); if (result.renderError) { renderError = result.renderError; break; diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index c5210e8667..717197da5a 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -1,13 +1,18 @@ import type * as React from 'react'; import { createInput } from '../input'; -import { measureIteration } from '../caseRuntime'; -import type { BenchmarkInteraction, CaseOptions } from '../caseRuntime'; +import { measureIteration, splitCaseArgs, warnIfNoGc } from '../caseRuntime'; +import type { + BenchmarkInteraction, + BenchmarkOptions, + CaseOptions, + VariantLoader, +} from '../caseRuntime'; import { setMetricRecorder } from '../metricCore'; // What `@mui/internal-benchmark` resolves to when benchmark files are built into an A/B page // instead of run by Vitest. Same authoring API, different driver: `benchmark()` registers a case, -// and the runner calls `window.benchmarkPage.sample()` to run one iteration of it at a time. Every result -// leaves the page as a `performance.measure` entry, the same contract plain tachometer pages use. +// and the runner calls `window.benchmarkPage.sample()` to run one iteration of it at a time. Every +// result leaves the page as a `performance.measure` entry, so it also shows up in a DevTools trace. export type { RenderEvent, IterationData, InteractionContext } from '../types'; export type { @@ -18,18 +23,12 @@ export type { MetricDefinition, } from '../types'; export type { BenchmarkInput } from '../input'; -export type { BenchmarkInteraction } from '../caseRuntime'; +export type { BenchmarkInteraction, BenchmarkOptions, VariantLoader } from '../caseRuntime'; export { ElementTiming } from '../ElementTiming'; export { Metric, type MetricRecordOptions } from '../metricCore'; export { ScalarMetric } from '../ScalarMetric'; export { DiscreteMetric } from '../DiscreteMetric'; -/** `benchmark()`'s options. The iteration counts are ignored here: the runner decides them. */ -export interface BenchmarkOptions extends CaseOptions { - runs?: number; - warmupRuns?: number; -} - interface PageCase { renderFn: () => React.ReactElement; interaction: BenchmarkInteraction | undefined; @@ -53,21 +52,10 @@ export function benchmark( if (registering.has(name)) { throw new Error(`Two benchmarks share the name "${name}". Benchmark names must be unique.`); } - const interaction = typeof interactionOrOptions === 'function' ? interactionOrOptions : undefined; - const options = typeof interactionOrOptions === 'object' ? interactionOrOptions : maybeOptions; - registering.set(name, { - renderFn, - interaction, - options: options && { - afterEach: options.afterEach, - reactRecordingPaused: options.reactRecordingPaused, - }, - }); + // The iteration counts in `options` are ignored here: the runner decides them. + registering.set(name, { renderFn, ...splitCaseArgs(interactionOrOptions, maybeOptions) }); } -/** Loads a variant's module, whose `benchmark()` calls define that variant's cases. */ -export type VariantLoader = () => Promise; - const comparisons = new Map>(); /** @@ -161,12 +149,13 @@ function emitSample( // whatever the browser coalesced, so per-render entries would not line up across iterations. The // per-phase split only adds information when there is more than one phase. const byPhase = new Map(); + let total = 0; for (const render of result.renders) { byPhase.set(render.phase, (byPhase.get(render.phase) ?? 0) + render.actualDuration); + total += render.actualDuration; } if (result.renders.length > 0) { const start = result.renders[0].startTime; - const total = [...byPhase.values()].reduce((sum, duration) => sum + duration, 0); performance.measure('render', { start, duration: total, detail: { value: total } }); if (byPhase.size > 1) { for (const [phase, duration] of byPhase) { @@ -225,11 +214,7 @@ async function sample(name: string, { warmup }: { warmup: boolean }): Promise { - if (typeof window.gc !== 'function') { - console.warn( - 'window.gc is not available. Run with --js-flags=--expose-gc for consistent GC between iterations.', - ); - } + warnIfNoGc(); try { await selectVariant(); } catch (error) { diff --git a/packages/benchmark/src/tachometer/pairedStats.ts b/packages/benchmark/src/tachometer/pairedStats.ts index a850dd85ab..90ea9bd0cf 100644 --- a/packages/benchmark/src/tachometer/pairedStats.ts +++ b/packages/benchmark/src/tachometer/pairedStats.ts @@ -1,3 +1,4 @@ +import { calculateMean } from '../stats'; import type { ConfidenceInterval, TachometerBenchmark, @@ -36,13 +37,9 @@ export function tCritical95(degreesOfFreedom: number): number { return z + (z ** 3 + z) / (4 * degreesOfFreedom); } -function meanOf(values: number[]): number { - return values.reduce((sum, value) => sum + value, 0) / values.length; -} - /** The 95% confidence interval of the mean. A single value is its own degenerate interval. */ export function meanInterval(values: number[]): ConfidenceInterval { - const mean = meanOf(values); + const mean = calculateMean(values); if (values.length < 2) { return { low: mean, high: mean }; } @@ -73,7 +70,7 @@ function pairedDifference( return null; } const absolute = meanInterval(differences); - const reference = meanOf(baseline); + const reference = calculateMean(baseline); return { absolute, percentChange: { diff --git a/packages/benchmark/src/tachometer/runInterleaved.ts b/packages/benchmark/src/tachometer/runInterleaved.ts index e6cd5f5663..9ea91791be 100644 --- a/packages/benchmark/src/tachometer/runInterleaved.ts +++ b/packages/benchmark/src/tachometer/runInterleaved.ts @@ -1,63 +1,43 @@ /* eslint-disable no-console */ -import * as path from 'node:path'; import chalk from 'chalk'; import type { Browser, BrowserContext, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS } from '../launchArgs'; -import { measurementNameOf } from './discoverCases'; -import type { BenchmarkCase, CaseComparison, Leaf } from './discoverCases'; +import type { CaseComparison } from './discoverCases'; import type { BenchFile } from './benchFiles'; import type { ResolvedRef } from './refs'; import { serveDirectory } from './serveDirectory'; import { shuffledIndices, stableMeasurements, toTachometerJson } from './pairedStats'; import type { Round } from './pairedStats'; -import type { TachometerJson } from './summarizeCase'; +import type { CaseRun, SummarizedCaseShape } from './summarizeCase'; // Types for `window.benchmarkPage`, which the evaluated functions below call into. import type {} from '../page/page'; /** - * The interleaved engine: runs every case in Playwright instead of tachometer, sampling its variants - * round by round in a shuffled order and comparing them on paired differences. - * - * Two kinds of case run here: - * - * - A `tachometer.json` case keeps tachometer's page contract. Every sample is a fresh load of the - * page, measured by the `performance.measure` entry (or first contentful paint) it names. - * - A `*.bench.tsx` file runs its `benchmark()` cases in a page that stays open per build: every - * sample is one iteration, so module-scope data and the JIT stay warm, and interactions get - * trusted input through a CDP session the page is bridged to. - * - * Every variant gets a browser context of its own, so no two share a renderer process or a cache. + * The interleaved engine: runs the `benchmark()` cases of `*.bench.tsx` files in Playwright instead + * of tachometer. Every variant gets a page of its own, in a browser context of its own, that stays + * open for the whole case: every sample is one iteration, so module-scope data and the JIT stay + * warm, and interactions get trusted input through a CDP session the page is bridged to. Variants + * are sampled once per round in a shuffled order and compared on paired differences. */ -export interface InterleavedResult { - entry: BenchmarkCase; - json?: TachometerJson; - error?: string; -} - export interface RunInterleavedOptions { - harnessDir: string; /** Where each ref's pages were built, one directory per ref id. */ buildsDir: string; - cases: BenchmarkCase[]; benchFiles: BenchFile[]; - buildFor: (leaf: Leaf) => ResolvedRef; /** The builds a benchmark file compares: the working tree first, then the baseline. */ benchRefs: ResolvedRef[]; browserBinary: string; asRoot: boolean; - /** Measured rounds per case. Defaults to a `tachometer.json`'s `sampleSize`, else 30. */ + /** Measured rounds per case. Defaults to 30. */ samples?: number; - /** - * Discarded rounds before measuring, once per case: default 2 for page loads, 10 for - * `*.bench.tsx` iterations. - */ + /** Discarded rounds before measuring, once per case. Defaults to 10. */ warmup?: number; } -const DEFAULT_VIEWPORT = { width: 1920, height: 1080 }; +const VIEWPORT = { width: 1920, height: 1080 }; const DEFAULT_SAMPLES = 30; +const DEFAULT_WARMUP = 10; const SAMPLE_TIMEOUT_MS = 120_000; type CdpMethod = Parameters[0]; @@ -66,99 +46,6 @@ function errorMessage(error: unknown): string { return error instanceof Error ? (error.stack ?? error.message) : String(error); } -/** The Performance entry a tachometer measurement reads, under the name it reports it as. */ -interface EntrySpec { - name: string; - entryName: string; -} - -function entrySpecsOf(node: any): EntrySpec[] { - return [node.measurement ?? 'callback'].flat().map((measurement: any): EntrySpec => { - const name = measurementNameOf(measurement, node.measurementExpression); - if (measurement === 'fcp') { - return { name, entryName: 'first-contentful-paint' }; - } - if (typeof measurement === 'object' && measurement.mode === 'performance') { - return { name, entryName: measurement.entryName }; - } - throw new Error( - `The interleaved engine reads Performance entries only; "${name}" in "${node.name}" is a ` + - `${typeof measurement === 'string' ? measurement : measurement.mode} measurement.`, - ); - }); -} - -async function openContext(browser: Browser, viewport = DEFAULT_VIEWPORT) { - const context = await browser.newContext({ viewport }); - const page = await context.newPage(); - page.setDefaultTimeout(SAMPLE_TIMEOUT_MS); - page.on('pageerror', (error) => console.error(chalk.red(` page error: ${error.message}`))); - return { context, page }; -} - -async function runPageCase( - browser: Browser, - origin: string, - entry: BenchmarkCase, - options: RunInterleavedOptions, -): Promise { - const nodes: any[] = entry.config.benchmarks; - const targets = await Promise.all( - entry.leaves.map(async (leaf, index) => { - const node = nodes[index]; - const opened = await openContext(browser, node.browser?.windowSize); - return { - ...opened, - url: `${origin}/${options.buildFor(leaf).id}/${leaf.page}${leaf.suffix}`, - specs: entrySpecsOf(node), - }; - }), - ); - - try { - const sample = async ({ page, url, specs }: (typeof targets)[number]) => { - await page.goto(url); - const entryNames = specs.map((spec) => spec.entryName); - await page.waitForFunction( - (names) => names.every((name) => performance.getEntriesByName(name).length > 0), - entryNames, - ); - return page.evaluate( - (specs) => - Object.fromEntries( - specs.map(({ name, entryName }) => { - const [performanceEntry] = performance.getEntriesByName(entryName); - // A paint entry has no duration; tachometer reports its start time instead. - const value = - performanceEntry.entryType === 'paint' - ? performanceEntry.startTime - : performanceEntry.duration; - return [name, value]; - }), - ), - specs, - ); - }; - - const warmup = options.warmup ?? 2; - const samples = options.samples ?? entry.config.sampleSize ?? DEFAULT_SAMPLES; - const rounds: Round[] = []; - for (let roundIndex = 0; roundIndex < warmup + samples; roundIndex += 1) { - const round: Round = []; - for (const index of shuffledIndices(targets.length)) { - // eslint-disable-next-line no-await-in-loop - round[index] = await sample(targets[index]); - } - if (roundIndex >= warmup) { - rounds.push(round); - } - } - return toTachometerJson(entry.variants, entry.measurements, rounds); - } finally { - await Promise.all(targets.map((target) => target.context.close())); - } -} - /** * A page a benchmark file's cases are measured in: one build of the file, or one variant of a * `compare()` in it. @@ -178,7 +65,10 @@ interface OpenedPage { } async function openBenchPage(browser: Browser, slot: PageSlot): Promise { - const { context, page } = await openContext(browser); + const context = await browser.newContext({ viewport: VIEWPORT }); + const page = await context.newPage(); + page.setDefaultTimeout(SAMPLE_TIMEOUT_MS); + page.on('pageerror', (error) => console.error(chalk.red(` page error: ${error.message}`))); const session = await context.newCDPSession(page); // Interactions ask for trusted input through this bridge. `send` only accepts the protocol // methods Playwright knows by name; the page may ask for any, and Chrome rejects unknown ones. @@ -188,23 +78,46 @@ async function openBenchPage(browser: Browser, slot: PageSlot): Promise window.benchmarkPage !== undefined || window.benchmarkPageError !== undefined, - ); - const pageError = await page.evaluate(() => window.benchmarkPageError); - if (pageError !== undefined) { + const ready = await page.waitForFunction(() => { + if (window.benchmarkPageError !== undefined) { + return { error: window.benchmarkPageError }; + } + if (window.benchmarkPage === undefined) { + return false; + } + return { + visibility: document.visibilityState, + caseNames: window.benchmarkPage.caseNames(), + comparisons: window.benchmarkPage.comparisons(), + }; + }); + const state = await ready.jsonValue(); + if (!state) { + throw new Error(`${slot.url} never got ready.`); + } + if ('error' in state) { await context.close(); - throw new Error(`${slot.url} could not get ready: ${pageError}`); + throw new Error(`${slot.url} could not get ready: ${state.error}`); } - const visibility = await page.evaluate(() => document.visibilityState); - if (visibility !== 'visible') { - console.warn(chalk.yellow(` ${slot.url} is ${visibility}; rAF-based waits may stall.`)); + if (state.visibility !== 'visible') { + console.warn(chalk.yellow(` ${slot.url} is ${state.visibility}; rAF-based waits may stall.`)); } - const { caseNames, comparisons } = await page.evaluate(() => ({ - caseNames: window.benchmarkPage!.caseNames(), - comparisons: window.benchmarkPage!.comparisons(), - })); - return { slot, context, page, caseNames, comparisons }; + return { slot, context, page, caseNames: state.caseNames, comparisons: state.comparisons }; +} + +async function closeBenchPages(targets: OpenedPage[]): Promise { + await Promise.all(targets.map((target) => target.context.close())); +} + +async function openBenchPages(browser: Browser, slots: PageSlot[]): Promise { + const settled = await Promise.allSettled(slots.map((slot) => openBenchPage(browser, slot))); + const opened = settled.flatMap((result) => (result.status === 'fulfilled' ? [result.value] : [])); + const failed = settled.find((result) => result.status === 'rejected'); + if (failed) { + await closeBenchPages(opened); + throw failed.reason; + } + return opened; } /** Runs one iteration; a measured one comes back as its `performance.measure` values by name. */ @@ -213,9 +126,9 @@ async function sampleBenchCase( name: string, warmup: boolean, ): Promise | undefined> { - await target.page.evaluate((args) => window.benchmarkPage!.sample(args.name, args), { + await target.page.evaluate((args) => window.benchmarkPage!.sample(args.name, args.options), { name, - warmup, + options: { warmup }, }); if (warmup) { return undefined; @@ -232,27 +145,8 @@ async function sampleBenchCase( }); } -/** Opens a page per slot, in a random order so no slot always gets the first process. */ -async function openBenchPages(browser: Browser, slots: PageSlot[]): Promise { - const opened: OpenedPage[] = []; - try { - for (const index of shuffledIndices(slots.length)) { - // eslint-disable-next-line no-await-in-loop - opened[index] = await openBenchPage(browser, slots[index]); - } - } catch (error) { - await closeBenchPages(opened.filter(Boolean)); - throw error; - } - return opened; -} - -async function closeBenchPages(targets: OpenedPage[]): Promise { - await Promise.all(targets.map((target) => target.context.close())); -} - /** - * Measures one case: a page per slot stays open for the whole case, so iterations stay warm and + * Measures one case in fresh pages that stay open for the whole case, so iterations stay warm and * module-scope data is built once. Warmup and measured rounds alike run every slot once, in a * shuffled order. */ @@ -262,7 +156,7 @@ async function measureBenchCase( name: string, options: RunInterleavedOptions, ): Promise { - const warmup = options.warmup ?? 10; + const warmup = options.warmup ?? DEFAULT_WARMUP; const samples = options.samples ?? DEFAULT_SAMPLES; const rounds: Round[] = []; const targets = await openBenchPages(browser, slots); @@ -283,34 +177,43 @@ async function measureBenchCase( return rounds; } +function entryOf( + name: string, + comparison: CaseComparison, + slots: PageSlot[], + measurements: string[] = [], +): SummarizedCaseShape { + return { + name, + comparison, + variants: slots.map((slot) => `${name} [${slot.variant}]`), + measurements, + }; +} + /** - * Runs every case the first slot defines across all slots, paired by name. The first slot is the - * reference; a case another slot lacks is reported as an error rather than compared with nothing. + * Runs every case the first listed page defines across all of them, paired by name. The first page + * is the reference; a case another page lacks is reported as an error rather than compared with + * nothing. */ -async function runSlotCases( +async function runListedCases( browser: Browser, - slots: PageSlot[], listed: OpenedPage[], describe: { comparison: CaseComparison; nameOf: (caseName: string) => string; file: string }, options: RunInterleavedOptions, -): Promise { +): Promise { + const slots = listed.map((target) => target.slot); const [reference, ...others] = listed; - const results: InterleavedResult[] = []; + const results: CaseRun[] = []; for (const caseName of reference.caseNames) { const name = describe.nameOf(caseName); - const entry: BenchmarkCase = { - name, - configPath: path.join(options.harnessDir, 'src', describe.file), - config: {}, - comparison: describe.comparison, - leaves: [], - variants: slots.map((slot) => `${name} [${slot.variant}]`), - measurements: [], - }; const missing = others.find((target) => !target.caseNames.includes(caseName)); if (missing) { // A case added by the change under test, or one library lacks, has nothing to compare with. - results.push({ entry, error: `"${caseName}" does not exist in [${missing.slot.variant}].` }); + results.push({ + entry: entryOf(name, describe.comparison, slots), + error: `"${caseName}" does not exist in [${missing.slot.variant}].`, + }); continue; } @@ -320,11 +223,14 @@ async function runSlotCases( const rounds = await measureBenchCase(browser, slots, caseName, options); // A measurement some iteration did not produce (an interaction whose render count varies, // say) cannot be paired round by round, so only those every iteration reported are kept. - entry.measurements = stableMeasurements(rounds); + const entry = entryOf(name, describe.comparison, slots, stableMeasurements(rounds)); results.push({ entry, json: toTachometerJson(entry.variants, entry.measurements, rounds) }); } catch (error) { console.error(chalk.red(` ${errorMessage(error)}`)); - results.push({ entry, error: errorMessage(error) }); + results.push({ + entry: entryOf(name, describe.comparison, slots), + error: errorMessage(error), + }); } } return results; @@ -343,18 +249,18 @@ async function runBenchFile( origin: string, benchFile: BenchFile, options: RunInterleavedOptions, -): Promise { +): Promise { const buildSlots = options.benchRefs.map((ref): PageSlot => ({ variant: ref.kind === 'worktree' ? 'current' : 'baseline', url: benchPageUrl(origin, ref, benchFile).href, })); - // Opened once just to learn which cases and comparisons each build defines. + // Opened once to learn which cases and comparisons each build defines; every case then measures + // in fresh pages of its own. const listed = await openBenchPages(browser, buildSlots); await closeBenchPages(listed); - const results = await runSlotCases( + const results = await runListedCases( browser, - buildSlots, listed, { comparison: 'baseline', nameOf: (caseName) => caseName, file: benchFile.file }, options, @@ -375,9 +281,8 @@ async function runBenchFile( await closeBenchPages(variantPages); results.push( // eslint-disable-next-line no-await-in-loop - ...(await runSlotCases( + ...(await runListedCases( browser, - variantSlots, variantPages, { comparison: 'variants', @@ -389,45 +294,26 @@ async function runBenchFile( ); } catch (error) { console.error(chalk.red(` ${errorMessage(error)}`)); - results.push({ - entry: { - name: comparison, - configPath: path.join(options.harnessDir, 'src', benchFile.file), - config: {}, - comparison: 'variants', - leaves: [], - variants: [], - measurements: [], - }, - error: errorMessage(error), - }); + results.push({ entry: entryOf(comparison, 'variants', []), error: errorMessage(error) }); } } return results; } -export async function runInterleaved(options: RunInterleavedOptions): Promise { +export async function runInterleaved(options: RunInterleavedOptions): Promise { const { chromium } = await import('@playwright/test'); - const server = await serveDirectory(options.buildsDir); - const browser = await chromium.launch({ - executablePath: options.browserBinary, - headless: true, - args: [...BENCHMARK_LAUNCH_ARGS, ...(options.asRoot ? ['--no-sandbox'] : [])], - }); + const [server, browser] = await Promise.all([ + serveDirectory(options.buildsDir), + chromium.launch({ + executablePath: options.browserBinary, + headless: true, + args: [...BENCHMARK_LAUNCH_ARGS, ...(options.asRoot ? ['--no-sandbox'] : [])], + }), + ]); try { - const results: InterleavedResult[] = []; + const results: CaseRun[] = []; // Sequential on purpose: concurrent cases would contend for the same machine. - for (const entry of options.cases) { - console.log(chalk.cyan(`\nRunning "${entry.name}"…`)); - try { - // eslint-disable-next-line no-await-in-loop - results.push({ entry, json: await runPageCase(browser, server.origin, entry, options) }); - } catch (error) { - console.error(chalk.red(` ${errorMessage(error)}`)); - results.push({ entry, error: errorMessage(error) }); - } - } for (const benchFile of options.benchFiles) { // eslint-disable-next-line no-await-in-loop results.push(...(await runBenchFile(browser, server.origin, benchFile, options))); diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts index a0ed2f0911..2cd07261f3 100644 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -19,6 +19,7 @@ import { buildRefPages, restoreWorkspace } from './buildPages'; import type { ResolveMode } from './buildPages'; import { assertDriverMatchesBrowser, resolveBrowserBinary, withBrowserDefaults } from './browser'; import { summarizeCase } from './summarizeCase'; +import type { CaseRun } from './summarizeCase'; import { renderTachometerReport } from './renderReport'; import { getCiMetadata } from '../ciReport'; import { syncPrComment } from '../syncPrComment'; @@ -143,19 +144,17 @@ export async function runTachometer(options: RunTachometerOptions): Promise 0) { - console.warn( - chalk.yellow( - `Skipping ${benchFiles.length} *.bench.tsx file(s): only the interleaved engine runs them (--engine interleaved).`, - ), + // Each engine runs its own kind of case: tachometer the `tachometer.json` cases, the interleaved + // engine the `*.bench.tsx` files. + const cases = engine === 'tachometer' ? await discoverCases({ harnessDir, filters }) : []; + const benchFiles = + engine === 'interleaved' ? await discoverBenchFiles({ harnessDir, filters }) : []; + if (engine === 'interleaved' && benchFiles.length === 0) { + throw new Error( + filters.length > 0 + ? `No *.bench.tsx file under ${path.join(harnessDir, 'src')} matches ${filters.map((filter) => `"${filter}"`).join(', ')}.` + : `No *.bench.tsx file found under ${path.join(harnessDir, 'src')}.`, ); - benchFiles = []; - } - if (cases.length === 0 && benchFiles.length === 0) { - // Nothing selected: let discovery say what it looked for. - await discoverCases({ harnessDir, filters }); } // A benchmark file always compares the working tree against the baseline. const { refs, buildFor } = await resolveBuilds( @@ -186,9 +185,12 @@ export async function runTachometer(options: RunTachometerOptions): Promise benchFile.file).join(', ')}`)); - console.log(chalk.cyan(`Pages: ${pagesOf(cases).join(', ')}`)); + if (engine === 'tachometer') { + console.log(chalk.cyan(`Cases: ${cases.map(describeCase).join(', ')}`)); + console.log(chalk.cyan(`Pages: ${pagesOf(cases).join(', ')}`)); + } else { + console.log(chalk.cyan(`Files: ${benchFiles.map((benchFile) => benchFile.file).join(', ')}`)); + } console.log(chalk.cyan(`Refs: ${refs.map(describeRef).join(', ')}`)); console.log( chalk.cyan(`Browser: ${browserBinary}${asRoot ? ' (as root, so without its sandbox)' : ''}`), @@ -235,30 +237,24 @@ export async function runTachometer(options: RunTachometerOptions): Promise = []; - if (engine === 'interleaved') { - const worktreeRef = refs.find((ref) => ref.kind === 'worktree') ?? WORKTREE_REF; - results.push( - ...(await runInterleaved({ - harnessDir, - buildsDir, - cases, - benchFiles, - buildFor, - // The working tree first: the first variant is the reference a case is judged against. - benchRefs: [worktreeRef, ...refs.filter((ref) => ref !== worktreeRef)], - browserBinary, - asRoot, - samples, - warmup, - })), - ); - } + const results: CaseRun[] = + engine === 'interleaved' + ? await runInterleaved({ + buildsDir, + benchFiles, + // `resolveBuilds` lists the working tree first, which makes it the reference. + benchRefs: refs, + browserBinary, + asRoot, + samples, + warmup, + }) + : []; // Rewrite each leaf url to its ref's built page, then run each case's own config. Tachometer // resolves relative urls against the config file's directory and these configs are written to a // temp dir, so the rewritten urls are absolute. - for (const entry of engine === 'tachometer' ? cases : []) { + for (const entry of cases) { for (const leaf of entry.leaves) { leaf.node.url = `${path.join(buildsDir, buildFor(leaf).id, leaf.page)}${leaf.suffix}`; } @@ -312,12 +308,13 @@ export async function runTachometer(options: RunTachometerOptions): Promise { - if (runError !== undefined) { - return { name: entry.name, comparison: entry.comparison, error: runError }; + cases: results.map((result): CaseResult => { + const { entry } = result; + if ('error' in result) { + return { name: entry.name, comparison: entry.comparison, error: result.error }; } try { - return summarizeCase(entry, json); + return summarizeCase(entry, result.json); } catch (error) { const message = error instanceof Error ? error.message : String(error); console.error(chalk.yellow(`Could not summarize "${entry.name}": ${message}`)); @@ -325,7 +322,7 @@ export async function runTachometer(options: RunTachometerOptions): Promise (json === undefined ? [] : [[entry.name, json]])), + results.flatMap((result) => ('json' in result ? [[result.entry.name, result.json]] : [])), ), }; await mkdir(path.dirname(outPath), { recursive: true }); diff --git a/packages/benchmark/src/tachometer/summarizeCase.ts b/packages/benchmark/src/tachometer/summarizeCase.ts index 19ab0c9da6..e01bca7b2f 100644 --- a/packages/benchmark/src/tachometer/summarizeCase.ts +++ b/packages/benchmark/src/tachometer/summarizeCase.ts @@ -119,7 +119,18 @@ function indexByMeasurement( * The first declared variant is the reference, because that is the one under test: for the * auto-expanded regression case it is `[current]`. */ -export function summarizeCase(entry: BenchmarkCase, json: TachometerJson): CaseSummary { +/** What a summary needs to know about a case, whichever engine measured it. */ +export type SummarizedCaseShape = Pick< + BenchmarkCase, + 'name' | 'comparison' | 'variants' | 'measurements' +>; + +/** One case as an engine ran it: its raw results, or the error that stopped it. */ +export type CaseRun = + | { entry: SummarizedCaseShape; json: TachometerJson } + | { entry: SummarizedCaseShape; error: string }; + +export function summarizeCase(entry: SummarizedCaseShape, json: TachometerJson): CaseSummary { const [reference] = entry.variants; if (!reference) { throw new Error(`Case "${entry.name}" declares no variants.`); diff --git a/packages/benchmark/src/vitest.ts b/packages/benchmark/src/vitest.ts index 4d470d995b..aed1189ff4 100644 --- a/packages/benchmark/src/vitest.ts +++ b/packages/benchmark/src/vitest.ts @@ -56,9 +56,6 @@ function resolveViewport(option?: { return undefined; } -// Shared by measurement and profiling; add project-specific flags via `launchArgs`. -const LAUNCH_ARGS = BENCHMARK_LAUNCH_ARGS; - export function createBenchmarkVitestConfig( options?: CreateBenchmarkVitestConfigOptions, ): ViteUserConfig { @@ -70,7 +67,7 @@ export function createBenchmarkVitestConfig( // Vitest's `viewport` only sizes the iframe, so otherwise it's cropped/scrolled in the headed // window instead of filling it. (Measurement is headless, so it has no window to size.) const profileArgs = [ - ...LAUNCH_ARGS, + ...BENCHMARK_LAUNCH_ARGS, '--auto-open-devtools-for-tabs', `--window-size=${viewport.width},${viewport.height}`, ]; @@ -105,7 +102,7 @@ export function createBenchmarkVitestConfig( ], provider: playwright({ launchOptions: { - args: [...(profile ? profileArgs : LAUNCH_ARGS), ...launchArgs], + args: [...(profile ? profileArgs : BENCHMARK_LAUNCH_ARGS), ...launchArgs], }, }), }, From 3ccbcce85ddb47709b9c5ff92cab93b850083b0d Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 12:19:10 +0200 Subject: [PATCH 076/173] [benchmark] Replace tacho run with benchmark run and a raw-sample report `benchmark run` runs a harness's *.bench.tsx files, the working tree against a baseline (default HEAD~1, so on master every commit is compared with its parent), with the interleaved Playwright runner. The tachometer engine, tachometer.json cases, chromedriver checks and the tachometer report are gone. The new report (`benchmark-run`, schema in runReport/) holds raw samples, round-aligned across variants, plus metric definitions, builds and the environment. analyzeRun computes paired confidence intervals, direction aware changes and alarm severities from it; the runner's table uses it and the dashboard will too. Also: - the plugin builds only generated bench-file pages and makes a `vitest` import a build error; the page runtime's Vitest stub is gone - the page runtime reports metric definitions and a render count - caseRuntime imports react-dom/profiling directly - --launch-arg passes Chromium flags (e.g. a hardware GPU backend) - the harness moves to test/benchmark, with the workload and sort demos ported to benchmark files (sort as a compare()) --- .circleci/config.yml | 22 +- .gitignore | 10 +- package.json | 3 +- packages/benchmark/package.json | 6 +- packages/benchmark/src/caseRuntime.tsx | 9 +- packages/benchmark/src/cli/index.ts | 14 +- packages/benchmark/src/cli/report.ts | 47 + .../benchmark/src/cli/{tachoRun.ts => run.ts} | 64 +- packages/benchmark/src/cli/tachoReport.ts | 51 - packages/benchmark/src/index.tsx | 3 +- packages/benchmark/src/metricCore.ts | 5 + packages/benchmark/src/page/page.tsx | 69 +- packages/benchmark/src/page/vitestStub.ts | 26 - packages/benchmark/src/reactDomProfiling.d.ts | 4 + .../src/runReport/analyzeRun.test.ts | 258 +++ .../benchmark/src/runReport/analyzeRun.ts | 234 +++ packages/benchmark/src/runReport/format.ts | 29 + packages/benchmark/src/runReport/index.ts | 3 + packages/benchmark/src/runReport/schema.ts | 79 + .../{tachometer => runner}/benchFiles.test.ts | 0 .../src/{tachometer => runner}/benchFiles.ts | 18 +- packages/benchmark/src/runner/browser.ts | 26 + .../src/{tachometer => runner}/buildPages.ts | 19 +- .../src/{tachometer => runner}/outputDir.ts | 6 +- .../src/{tachometer => runner}/plugin.test.ts | 111 +- packages/benchmark/src/runner/plugin.ts | 217 +++ packages/benchmark/src/runner/printReport.ts | 74 + .../src/{tachometer => runner}/refs.test.ts | 0 .../src/{tachometer => runner}/refs.ts | 5 + .../benchmark/src/runner/runBenchmarks.ts | 221 +++ .../{tachometer => runner}/runInterleaved.ts | 278 +-- .../{tachometer => runner}/serveDirectory.ts | 0 packages/benchmark/src/runner/upload.ts | 60 + .../benchmark/src/tachometer/browser.test.ts | 172 -- packages/benchmark/src/tachometer/browser.ts | 160 -- .../benchmark/src/tachometer/ciReport.test.ts | 141 -- packages/benchmark/src/tachometer/ciReport.ts | 124 -- .../src/tachometer/discoverCases.test.ts | 577 ------ .../benchmark/src/tachometer/discoverCases.ts | 333 ---- .../benchmark/src/tachometer/format.test.ts | 100 -- packages/benchmark/src/tachometer/format.ts | 93 - .../src/tachometer/pairedStats.test.ts | 135 -- .../benchmark/src/tachometer/pairedStats.ts | 139 -- packages/benchmark/src/tachometer/plugin.ts | 294 ---- .../benchmark/src/tachometer/renderReport.ts | 241 --- .../benchmark/src/tachometer/runTachometer.ts | 383 ---- .../src/tachometer/summarizeCase.test.ts | 203 --- .../benchmark/src/tachometer/summarizeCase.ts | 195 --- packages/benchmark/src/utils/html.ts | 8 + pnpm-lock.yaml | 1551 +---------------- pnpm-workspace.yaml | 1 - test/{tachometer => benchmark}/README.md | 0 test/{tachometer => benchmark}/package.json | 5 +- .../scripts/runBenchmark.mjs | 0 test/benchmark/src/_shared/ScrollingList.tsx | 21 + .../src/libs/sort/shared.ts | 0 test/benchmark/src/libs/sort/sort.alpha.tsx | 12 + test/benchmark/src/libs/sort/sort.bench.tsx | 12 + .../src/libs/sort/sort.beta.tsx} | 11 +- test/benchmark/src/libs/sort/sort.ours.tsx | 12 + .../src/list/list.bench.tsx | 33 +- .../src/lists/lists.bench.tsx | 0 .../src/lists/lists.table.tsx | 22 +- .../src/lists/lists.ul.tsx | 22 +- .../benchmark/src/workload/workload.bench.tsx | 47 + test/{tachometer => benchmark}/tsconfig.json | 0 test/benchmark/vite.config.mjs | 6 + test/tachometer/src/_shared/measure.ts | 51 - test/tachometer/src/libs/sort/alpha.html | 11 - test/tachometer/src/libs/sort/alpha.ts | 10 - test/tachometer/src/libs/sort/beta.html | 11 - test/tachometer/src/libs/sort/ours.html | 11 - test/tachometer/src/libs/sort/ours.ts | 13 - test/tachometer/src/libs/sort/tachometer.json | 22 - .../src/workload-large/tachometer.json | 18 - test/tachometer/src/workload/index.html | 11 - test/tachometer/src/workload/index.ts | 43 - test/tachometer/src/workload/tachometer.json | 18 - test/tachometer/vite.config.mjs | 6 - tsconfig.json | 2 +- 80 files changed, 1788 insertions(+), 5463 deletions(-) create mode 100644 packages/benchmark/src/cli/report.ts rename packages/benchmark/src/cli/{tachoRun.ts => run.ts} (54%) delete mode 100644 packages/benchmark/src/cli/tachoReport.ts delete mode 100644 packages/benchmark/src/page/vitestStub.ts create mode 100644 packages/benchmark/src/reactDomProfiling.d.ts create mode 100644 packages/benchmark/src/runReport/analyzeRun.test.ts create mode 100644 packages/benchmark/src/runReport/analyzeRun.ts create mode 100644 packages/benchmark/src/runReport/format.ts create mode 100644 packages/benchmark/src/runReport/index.ts create mode 100644 packages/benchmark/src/runReport/schema.ts rename packages/benchmark/src/{tachometer => runner}/benchFiles.test.ts (100%) rename packages/benchmark/src/{tachometer => runner}/benchFiles.ts (85%) create mode 100644 packages/benchmark/src/runner/browser.ts rename packages/benchmark/src/{tachometer => runner}/buildPages.ts (94%) rename packages/benchmark/src/{tachometer => runner}/outputDir.ts (86%) rename packages/benchmark/src/{tachometer => runner}/plugin.test.ts (54%) create mode 100644 packages/benchmark/src/runner/plugin.ts create mode 100644 packages/benchmark/src/runner/printReport.ts rename packages/benchmark/src/{tachometer => runner}/refs.test.ts (100%) rename packages/benchmark/src/{tachometer => runner}/refs.ts (93%) create mode 100644 packages/benchmark/src/runner/runBenchmarks.ts rename packages/benchmark/src/{tachometer => runner}/runInterleaved.ts (52%) rename packages/benchmark/src/{tachometer => runner}/serveDirectory.ts (100%) create mode 100644 packages/benchmark/src/runner/upload.ts delete mode 100644 packages/benchmark/src/tachometer/browser.test.ts delete mode 100644 packages/benchmark/src/tachometer/browser.ts delete mode 100644 packages/benchmark/src/tachometer/ciReport.test.ts delete mode 100644 packages/benchmark/src/tachometer/ciReport.ts delete mode 100644 packages/benchmark/src/tachometer/discoverCases.test.ts delete mode 100644 packages/benchmark/src/tachometer/discoverCases.ts delete mode 100644 packages/benchmark/src/tachometer/format.test.ts delete mode 100644 packages/benchmark/src/tachometer/format.ts delete mode 100644 packages/benchmark/src/tachometer/pairedStats.test.ts delete mode 100644 packages/benchmark/src/tachometer/pairedStats.ts delete mode 100644 packages/benchmark/src/tachometer/plugin.ts delete mode 100644 packages/benchmark/src/tachometer/renderReport.ts delete mode 100644 packages/benchmark/src/tachometer/runTachometer.ts delete mode 100644 packages/benchmark/src/tachometer/summarizeCase.test.ts delete mode 100644 packages/benchmark/src/tachometer/summarizeCase.ts create mode 100644 packages/benchmark/src/utils/html.ts rename test/{tachometer => benchmark}/README.md (100%) rename test/{tachometer => benchmark}/package.json (71%) rename test/{tachometer => benchmark}/scripts/runBenchmark.mjs (100%) create mode 100644 test/benchmark/src/_shared/ScrollingList.tsx rename test/{tachometer => benchmark}/src/libs/sort/shared.ts (100%) create mode 100644 test/benchmark/src/libs/sort/sort.alpha.tsx create mode 100644 test/benchmark/src/libs/sort/sort.bench.tsx rename test/{tachometer/src/libs/sort/beta.ts => benchmark/src/libs/sort/sort.beta.tsx} (80%) create mode 100644 test/benchmark/src/libs/sort/sort.ours.tsx rename test/{tachometer => benchmark}/src/list/list.bench.tsx (59%) rename test/{tachometer => benchmark}/src/lists/lists.bench.tsx (100%) rename test/{tachometer => benchmark}/src/lists/lists.table.tsx (62%) rename test/{tachometer => benchmark}/src/lists/lists.ul.tsx (60%) create mode 100644 test/benchmark/src/workload/workload.bench.tsx rename test/{tachometer => benchmark}/tsconfig.json (100%) create mode 100644 test/benchmark/vite.config.mjs delete mode 100644 test/tachometer/src/_shared/measure.ts delete mode 100644 test/tachometer/src/libs/sort/alpha.html delete mode 100644 test/tachometer/src/libs/sort/alpha.ts delete mode 100644 test/tachometer/src/libs/sort/beta.html delete mode 100644 test/tachometer/src/libs/sort/ours.html delete mode 100644 test/tachometer/src/libs/sort/ours.ts delete mode 100644 test/tachometer/src/libs/sort/tachometer.json delete mode 100644 test/tachometer/src/workload-large/tachometer.json delete mode 100644 test/tachometer/src/workload/index.html delete mode 100644 test/tachometer/src/workload/index.ts delete mode 100644 test/tachometer/src/workload/tachometer.json delete mode 100644 test/tachometer/vite.config.mjs diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ae37c8d95..e2a7a21d1c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -114,7 +114,7 @@ jobs: BENCHMARK_UPLOAD: 'true' command: pnpm -F ./test/performance benchmark - test_tachometer: + test_benchmark_run: <<: *default-job executor: name: code-infra/mui-node-browser @@ -133,26 +133,26 @@ jobs: # is the command's own job, so there is nothing to keep in step with it here. BASE_SHA=$(pnpm code-infra baseline --base-branch master) echo "Baseline commit: $BASE_SHA" - echo "$BASE_SHA" > .tachometer-baseline-sha + echo "$BASE_SHA" > .benchmark-baseline-sha - restore_cache: name: Restore the packed-baseline cache keys: - - tachometer-cache-v1-{{ checksum ".tachometer-baseline-sha" }} + - benchmark-run-cache-v1-{{ checksum ".benchmark-baseline-sha" }} - run: - name: Run the tachometer smoke test + name: Run the benchmark smoke test no_output_timeout: 30m environment: - TACHO_UPLOAD: 'true' - # TEMPORARY: production does not know the tachometer report type until this pull + BENCHMARK_UPLOAD: 'true' + # TEMPORARY: production does not know the benchmark-run report type until this pull # request lands, so it would reject the upload. Points at this pull request's own # dashboard preview until then — remove the line before merging. - CI_REPORT_API_URL: https://code-infra-dashboard-pr-1829.onrender.com - command: pnpm test:tacho --baseline "$(cat .tachometer-baseline-sha)" + CI_REPORT_API_URL: https://code-infra-dashboard-pr-1898.onrender.com + command: pnpm test:benchmark-run --baseline "$(cat .benchmark-baseline-sha)" - save_cache: name: Save the packed-baseline cache - key: tachometer-cache-v1-{{ checksum ".tachometer-baseline-sha" }} + key: benchmark-run-cache-v1-{{ checksum ".benchmark-baseline-sha" }} paths: - - test/tachometer/.tachometer/packed + - test/benchmark/.benchmark/packed test_browser: <<: *default-job @@ -195,7 +195,7 @@ workflows: <<: *default-context - test_benchmark: <<: *default-context - - test_tachometer: + - test_benchmark_run: <<: *default-context - test_browser: <<: *default-context diff --git a/.gitignore b/.gitignore index bae08dce3b..a5d602a451 100644 --- a/.gitignore +++ b/.gitignore @@ -61,7 +61,9 @@ next-env.d.ts .claude/worktrees/ .claude/cache/ -# Everything the tachometer harness writes: built pages per ref, packed workspace tarballs, the -# isolated installs and the run reports. The directory ignores itself from the inside too, but the -# entry is listed here because ESLint is fed from the root .gitignore and never reads nested ones. -/test/tachometer/.tachometer/ +# Everything the benchmark harness writes: built pages per ref, packed workspace tarballs, the +# isolated installs, the run reports, and the pages generated per benchmark file. Both directories +# ignore themselves from the inside too, but are listed here because ESLint is fed from the root +# .gitignore and never reads nested ones. +/test/benchmark/.benchmark/ +/test/benchmark/src/__bench__/ diff --git a/package.json b/package.json index 3411b8df27..30d813cd59 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "docs:lint": "pnpm -F \"./docs\" run lint", "docs:lib": "pnpm -F \"./packages/docs-infra\" run build", "test:benchmark": "pnpm -F ./test/performance benchmark", - "test:tacho": "pnpm -F ./test/tachometer test:tacho", + "test:benchmark-run": "pnpm -F ./test/benchmark benchmark:run", "org-versions": "tsx scripts/orgVersions.mts" }, "dependencies": { @@ -63,7 +63,6 @@ "@vitest/browser-playwright": "4.1.11", "@vitest/coverage-v8": "4.1.11", "baseline-browser-mapping": "2.11.21", - "chromedriver": "^151.0.5", "eslint-plugin-n": "18.3.0", "pkg-pr-new": "0.0.88", "playwright": "1.63.0", diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index c8317d56b7..bbe06f6b50 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -15,10 +15,8 @@ "./reporter": "./src/reporter.ts", "./vitest": "./src/vitest.ts", "./page": "./src/page/page.tsx", - "./page/vitest": "./src/page/vitestStub.ts", - "./tachometerPlugin": "./src/tachometer/plugin.ts", - "./tachometerReport": "./src/tachometer/renderReport.ts", - "./tachometerFormat": "./src/tachometer/format.ts", + "./vitePlugin": "./src/runner/plugin.ts", + "./runReport": "./src/runReport/index.ts", "./cli": "./src/cli/index.ts" }, "scripts": { diff --git a/packages/benchmark/src/caseRuntime.tsx b/packages/benchmark/src/caseRuntime.tsx index d7a25dd0f5..acc59b8d4e 100644 --- a/packages/benchmark/src/caseRuntime.tsx +++ b/packages/benchmark/src/caseRuntime.tsx @@ -1,5 +1,6 @@ import * as React from 'react'; -import * as ReactDOMClient from 'react-dom/client'; // aliased to react-dom/profiling by Vite +// React only reports render durations from its profiling build. +import * as ReactDOMClient from 'react-dom/profiling'; import * as ReactDOM from 'react-dom'; import type { RenderEvent, InteractionContext, BenchmarkCaseRuntime } from './types'; import type { BenchmarkInput } from './input'; @@ -22,6 +23,12 @@ interface PerformanceElementTiming extends PerformanceEntry { export type BenchmarkInteraction = (ctx: InteractionContext) => Promise | void; +/** + * The harness-owned paint metric: the default sentinel is the base series (`bench:paint`) and named + * `elementtiming` markers are sub-series (`bench:paint#grid-header`, …). + */ +export const PAINT_METRIC_NAME = 'bench:paint'; + export interface CaseOptions { afterEach?: () => Promise | void; /** diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index a67b7bf25d..05284741b3 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -8,8 +8,8 @@ import { hideBin } from 'yargs/helpers'; // repository than in the published package, which is built from `build/`, so a relative path // cannot reach the manifest from both — and the repository's workspace symlink hides the break. import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; -import tachoReport from './tachoReport'; -import tachoRun from './tachoRun'; +import report from './report'; +import run from './run'; let globalArgv: { verbose?: boolean } = {}; @@ -26,14 +26,8 @@ await yargs(hideBin(process.argv)) .middleware((argv) => { globalArgv = argv; }, true) - .command({ - command: 'tacho ', - describe: 'Benchmark a tachometer harness, and read the reports it writes.', - builder: (group) => - group.command(tachoRun).command(tachoReport).demandCommand(1, 'Specify a tacho subcommand.'), - // The group itself does nothing; `demandCommand` above means a subcommand always runs. - handler: () => {}, - }) + .command(run) + .command(report) .fail((msg, err, yargsInstance) => { if (msg) { yargsInstance.showHelp(); diff --git a/packages/benchmark/src/cli/report.ts b/packages/benchmark/src/cli/report.ts new file mode 100644 index 0000000000..c7ac7f7327 --- /dev/null +++ b/packages/benchmark/src/cli/report.ts @@ -0,0 +1,47 @@ +import * as path from 'node:path'; +import { readFile } from 'node:fs/promises'; +import { z } from 'zod/v4'; +import type { CommandModule } from 'yargs'; +import { OUTPUT_DIR } from '../runner/outputDir'; + +interface Args { + /** Path to a JSON report written by `benchmark run`. */ + file: string; +} + +const DEFAULT_REPORT = path.join(OUTPUT_DIR, 'results', 'report.json'); + +const command: CommandModule<{}, Args> = { + command: 'report [file]', + describe: + 'Print the tables for a JSON report written by `benchmark run`. Reads the last run of the harness in the current directory when no path is given.', + builder: (yargs) => { + return yargs.positional('file', { + type: 'string', + default: DEFAULT_REPORT, + describe: 'Path to the JSON report', + }); + }, + handler: async (argv) => { + const reportPath = path.resolve(argv.file); + const raw = await readFile(reportPath, 'utf8').catch(() => { + throw new Error( + `No report at ${reportPath}. Pass a path, or run \`benchmark run\` from a harness first.`, + ); + }); + + const { benchmarkRunReportSchema } = await import('../runReport'); + const parsed = benchmarkRunReportSchema.safeParse(JSON.parse(raw)); + if (!parsed.success) { + throw new Error( + `${reportPath} is not a benchmark report this version can read.\n` + + `${z.prettifyError(parsed.error)}`, + ); + } + + const { printRunReport } = await import('../runner/printReport'); + printRunReport(parsed.data); + }, +}; + +export default command; diff --git a/packages/benchmark/src/cli/tachoRun.ts b/packages/benchmark/src/cli/run.ts similarity index 54% rename from packages/benchmark/src/cli/tachoRun.ts rename to packages/benchmark/src/cli/run.ts index 22d9a9bfe7..e439ceec00 100644 --- a/packages/benchmark/src/cli/tachoRun.ts +++ b/packages/benchmark/src/cli/run.ts @@ -1,24 +1,40 @@ import type { CommandModule } from 'yargs'; -import type { RunTachometerOptions } from '../tachometer/runTachometer'; +import type { RunBenchmarksOptions } from '../runner/runBenchmarks'; -/** Everything `runTachometer` takes except the directory, which is where the command was run. */ -type Args = Omit; +/** Everything `runBenchmarks` takes except the directory, which is where the command was run. */ +type Args = Omit & { launchArg?: string[] }; const command: CommandModule<{}, Args> = { command: 'run [filters...]', describe: - 'Benchmark a tachometer harness across builds of this workspace and write a JSON report. Run from the harness package.', + "Benchmark a harness's *.bench.tsx files, the working tree against a baseline build, and write a JSON report. Run from the harness package.", builder: (yargs) => { return yargs .positional('filters', { type: 'string', array: true, - describe: 'Only run cases whose path under src contains this substring', + describe: 'Only run benchmark files whose path under src contains this substring', }) .option('baseline', { type: 'string', describe: - 'The build to compare against: a revision, on its own or as git:. Defaults to HEAD~1; `code-infra baseline` resolves a fork point to pass here', + 'The build to compare against: a revision, on its own or as git:. Defaults to HEAD~1, the parent commit; `code-infra baseline` resolves a fork point to pass here', + }) + .option('samples', { + type: 'number', + default: 30, + describe: 'Measured rounds per benchmark; each round samples every variant once', + }) + .option('warmup', { + type: 'number', + default: 10, + describe: 'Discarded rounds before measuring, once per benchmark', + }) + .option('launch-arg', { + type: 'string', + array: true, + describe: + 'Extra Chromium launch argument, repeatable — e.g. --use-angle=metal for hardware-rendered paint timings', }) .option('resolve-mode', { type: 'string', @@ -40,47 +56,29 @@ const command: CommandModule<{}, Args> = { }) .option('upload', { type: 'boolean', - default: process.env.TACHO_UPLOAD === 'true', + default: process.env.BENCHMARK_UPLOAD === 'true', describe: - 'Upload the report and refresh the PR comment. Defaults to TACHO_UPLOAD=true, so CI can switch it on by environment. Requires CIRCLE_OIDC_TOKEN_V2', + 'Upload the report and refresh the PR comment. Defaults to BENCHMARK_UPLOAD=true, so CI can switch it on by environment. Requires CIRCLE_OIDC_TOKEN_V2', }) .option('out', { type: 'string', - describe: 'Write the combined JSON report here. Default: .tachometer/results/report.json', - }) - .option('engine', { - type: 'string', - choices: ['tachometer', 'interleaved'] as const, - default: 'tachometer' as const, - describe: - "What measures the harness: 'tachometer' runs its tachometer.json cases; 'interleaved' runs its *.bench.tsx files in Playwright, alternating variants round by round and comparing paired differences", - }) - .option('samples', { - type: 'number', - describe: 'Interleaved engine: measured rounds per case. Default: 30', - }) - .option('warmup', { - type: 'number', - describe: 'Interleaved engine: discarded rounds before measuring. Default: 10', - }) - .epilogue( - 'Sampling (sampleSize, autoSampleConditions, timeout) is configured per case in its own tachometer.json — tachometer rejects those as CLI flags when a config file is used.', - ); + describe: 'Write the JSON report here. Default: .benchmark/results/report.json', + }); }, handler: async (argv) => { - const { runTachometer } = await import('../tachometer/runTachometer'); - await runTachometer({ + const { runBenchmarks } = await import('../runner/runBenchmarks'); + await runBenchmarks({ harnessDir: process.cwd(), filters: argv.filters ?? [], baseline: argv.baseline, + samples: argv.samples, + warmup: argv.warmup, + launchArgs: argv.launchArg ?? [], resolveMode: argv.resolveMode, buildCmd: argv.buildCmd, install: argv.install, out: argv.out, upload: argv.upload, - engine: argv.engine, - samples: argv.samples, - warmup: argv.warmup, }); }, }; diff --git a/packages/benchmark/src/cli/tachoReport.ts b/packages/benchmark/src/cli/tachoReport.ts deleted file mode 100644 index 49b59569ec..0000000000 --- a/packages/benchmark/src/cli/tachoReport.ts +++ /dev/null @@ -1,51 +0,0 @@ -import * as path from 'node:path'; -import { readFile } from 'node:fs/promises'; -import { z } from 'zod/v4'; -import type { CommandModule } from 'yargs'; -import { OUTPUT_DIR } from '../tachometer/outputDir'; - -interface Args { - /** Path to a JSON report written by `tacho run`. */ - file: string; -} - -const DEFAULT_REPORT = path.join(OUTPUT_DIR, 'results', 'report.json'); - -const command: CommandModule<{}, Args> = { - command: 'report [file]', - describe: - 'Print the table for a JSON report written by `tacho run`. Reads the last run of the harness in the current directory when no path is given.', - builder: (yargs) => { - return yargs - .positional('file', { - type: 'string', - default: DEFAULT_REPORT, - describe: 'Path to the JSON report', - }) - .epilogue( - 'The same table `tacho run` prints when it finishes, so a report kept from an earlier run — or downloaded from CI — can be read without sampling again.', - ); - }, - handler: async (argv) => { - const reportPath = path.resolve(argv.file); - const raw = await readFile(reportPath, 'utf8').catch(() => { - throw new Error( - `No report at ${reportPath}. Pass a path, or run \`tacho run\` from a harness first.`, - ); - }); - - const { tachometerReportSchema } = await import('../tachometer/ciReport'); - const parsed = tachometerReportSchema.safeParse(JSON.parse(raw)); - if (!parsed.success) { - throw new Error( - `${reportPath} is not a tachometer report this version can read.\n` + - `${z.prettifyError(parsed.error)}`, - ); - } - - const { renderTachometerReport } = await import('../tachometer/renderReport'); - renderTachometerReport(parsed.data); - }, -}; - -export default command; diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index a59151ac81..823c31ba01 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -11,6 +11,7 @@ import { createCaseRuntime, createElementTimingWaiter, measureIteration, + PAINT_METRIC_NAME, splitCaseArgs, warnIfNoGc, } from './caseRuntime'; @@ -76,7 +77,7 @@ export interface RunCaseResult { // alarm): it dominates each test's total duration, so a per-test paint alarm just duplicates the // Duration regression signal and floods the report on any broadly-regressed run. const paint = new ScalarMetric({ - name: 'bench:paint', + name: PAINT_METRIC_NAME, format: { style: 'unit', unit: 'millisecond', maximumFractionDigits: 2 }, }); diff --git a/packages/benchmark/src/metricCore.ts b/packages/benchmark/src/metricCore.ts index 76d08bcd2d..2549844964 100644 --- a/packages/benchmark/src/metricCore.ts +++ b/packages/benchmark/src/metricCore.ts @@ -9,6 +9,11 @@ export interface MetricRecordOptions { * Where recorded values go. The runtime a benchmark runs under installs one: the Vitest harness * accumulates them per running test, the A/B page emits them as `performance.measure` entries. */ +/** The name a sub-series is reported under: `name#id`, or `name` for the base series. */ +export function seriesName(name: string, id: string | undefined): string { + return id === undefined ? name : `${name}#${id}`; +} + export type MetricRecorder = ( metric: Metric, value: number, diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 717197da5a..bbf79769f8 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -1,18 +1,20 @@ import type * as React from 'react'; import { createInput } from '../input'; -import { measureIteration, splitCaseArgs, warnIfNoGc } from '../caseRuntime'; +import { measureIteration, PAINT_METRIC_NAME, splitCaseArgs, warnIfNoGc } from '../caseRuntime'; +import type { MeasuredIteration } from '../caseRuntime'; import type { BenchmarkInteraction, BenchmarkOptions, CaseOptions, VariantLoader, } from '../caseRuntime'; -import { setMetricRecorder } from '../metricCore'; +import { seriesName, setMetricRecorder } from '../metricCore'; +import type { MetricConfig, MetricKind } from '../types'; -// What `@mui/internal-benchmark` resolves to when benchmark files are built into an A/B page -// instead of run by Vitest. Same authoring API, different driver: `benchmark()` registers a case, -// and the runner calls `window.benchmarkPage.sample()` to run one iteration of it at a time. Every -// result leaves the page as a `performance.measure` entry, so it also shows up in a DevTools trace. +// What `@mui/internal-benchmark` resolves to when benchmark files are built into a page for +// `benchmark run`. Same authoring API, different driver: `benchmark()` registers a case, and the +// runner calls `window.benchmarkPage.sample()` to run one iteration of it at a time. Every result +// leaves the page as a `performance.measure` entry, so it also shows up in a DevTools trace. export type { RenderEvent, IterationData, InteractionContext } from '../types'; export type { @@ -105,17 +107,47 @@ interface RecordedValue { // where recorded values are dropped. let sampleValues: RecordedValue[] | null = null; +/** How the runner is told to read a metric: what its report carries per metric name. */ +export interface PageMetricDefinition { + kind: MetricKind; + format?: Intl.NumberFormatOptions; + alarm?: MetricConfig['alarm']; +} + +const MILLISECONDS: Intl.NumberFormatOptions = { + style: 'unit', + unit: 'millisecond', + maximumFractionDigits: 2, +}; + +// The harness's own metrics. Render time and render count alarm on any resolved change for the +// worse; paint dominates each case's duration and duplicates that signal, so it is informational. +const metricDefinitions = new Map([ + ['render', { kind: 'scalar', format: MILLISECONDS, alarm: {} }], + ['render:count', { kind: 'discrete', alarm: {} }], + ['render:mount', { kind: 'scalar', format: MILLISECONDS }], + ['render:update', { kind: 'scalar', format: MILLISECONDS }], + ['render:nested-update', { kind: 'scalar', format: MILLISECONDS }], + [PAINT_METRIC_NAME, { kind: 'scalar', format: MILLISECONDS }], +]); + setMetricRecorder((metric, value, options) => { - sampleValues?.push({ - name: options?.id === undefined ? metric.name : `${metric.name}#${options.id}`, - value, - }); + if (!metricDefinitions.has(metric.name)) { + metricDefinitions.set(metric.name, { + kind: metric.kind, + format: metric.config.format, + alarm: metric.config.alarm, + }); + } + sampleValues?.push({ name: seriesName(metric.name, options?.id), value }); }); export interface BenchPage { caseNames: () => string[]; /** The file's `compare()` calls, with their variant keys in order. */ comparisons: () => Array<{ name: string; variants: string[] }>; + /** Every metric reported so far, by name: the harness's own and those the cases recorded. */ + metricDefinitions: () => Record; /** * Runs one iteration of a case. A measured sample leaves its results as `performance.measure` * entries for the runner to collect; a warmup sample leaves none. @@ -125,7 +157,7 @@ export interface BenchPage { declare global { interface Window { - /** Set by `markPageReady()` once every benchmark file has registered its cases. */ + /** Set by `markPageReady()` once the benchmark file has registered its cases. */ benchmarkPage?: BenchPage; /** Set instead of `benchmarkPage` when the page could not get ready. */ benchmarkPageError?: string; @@ -141,10 +173,7 @@ const input = createInput((method, params) => { return window.benchmarkCdp(method, params); }); -function emitSample( - result: Awaited>, - values: RecordedValue[], -): void { +function emitSample(result: MeasuredIteration, values: RecordedValue[]): void { // Render time is reported as totals rather than per render: an interaction's render count follows // whatever the browser coalesced, so per-render entries would not line up across iterations. The // per-phase split only adds information when there is more than one phase. @@ -157,6 +186,11 @@ function emitSample( if (result.renders.length > 0) { const start = result.renders[0].startTime; performance.measure('render', { start, duration: total, detail: { value: total } }); + performance.measure('render:count', { + start, + duration: 0, + detail: { value: result.renders.length }, + }); if (byPhase.size > 1) { for (const [phase, duration] of byPhase) { performance.measure(`render:${phase}`, { start, duration, detail: { value: duration } }); @@ -164,7 +198,7 @@ function emitSample( } } for (const { id, start, end } of result.paints) { - performance.measure(id === undefined ? 'bench:paint' : `bench:paint#${id}`, { start, end }); + performance.measure(seriesName(PAINT_METRIC_NAME, id), { start, end }); } // Custom metrics aren't necessarily durations, so the value travels in `detail`. const now = performance.now(); @@ -183,7 +217,7 @@ async function sample(name: string, { warmup }: { warmup: boolean }): Promise>; + let result: MeasuredIteration; try { result = await measureIteration( benchCase.renderFn, @@ -226,6 +260,7 @@ export async function markPageReady(): Promise { caseNames: () => [...pageCases.keys()], comparisons: () => [...comparisons].map(([name, variants]) => ({ name, variants: Object.keys(variants) })), + metricDefinitions: () => Object.fromEntries(metricDefinitions), sample, }; } diff --git a/packages/benchmark/src/page/vitestStub.ts b/packages/benchmark/src/page/vitestStub.ts deleted file mode 100644 index 1fda7c04a8..0000000000 --- a/packages/benchmark/src/page/vitestStub.ts +++ /dev/null @@ -1,26 +0,0 @@ -// What `vitest` resolves to in an A/B page. Benchmark files sometimes mix `benchmark()` cases with -// plain Vitest tests; outside Vitest those tests are skipped, so the file still loads and its -// `benchmark()` cases still register. - -function skipped(api: string) { - return (name?: unknown) => { - console.warn(`${api}(${JSON.stringify(name)}) is skipped in A/B pages; only benchmark() runs.`); - }; -} - -export const it = skipped('it'); -export const test = skipped('test'); -export const bench = skipped('bench'); -export const beforeAll = skipped('beforeAll'); -export const afterAll = skipped('afterAll'); -export const beforeEach = skipped('beforeEach'); -export const afterEach = skipped('afterEach'); -export const onTestFinished = skipped('onTestFinished'); - -export function describe(_name: string, factory?: () => void): void { - factory?.(); -} - -export function expect(): never { - throw new Error('expect() is not available in A/B pages.'); -} diff --git a/packages/benchmark/src/reactDomProfiling.d.ts b/packages/benchmark/src/reactDomProfiling.d.ts new file mode 100644 index 0000000000..6ea3abcec3 --- /dev/null +++ b/packages/benchmark/src/reactDomProfiling.d.ts @@ -0,0 +1,4 @@ +// `@types/react-dom` types `react-dom/client` but not the profiling build, which has the same API. +declare module 'react-dom/profiling' { + export * from 'react-dom/client'; +} diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts new file mode 100644 index 0000000000..061162fae5 --- /dev/null +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -0,0 +1,258 @@ +import { describe, expect, it } from 'vitest'; +import { + analyzeRun, + compareSamples, + findRegressions, + meanInterval, + median, + tCritical95, +} from './analyzeRun'; +import type { BenchmarkRunReport, RunBenchmark, RunMetricDefinition } from './schema'; + +const SCALAR: RunMetricDefinition = { kind: 'scalar', alarm: {} }; + +/** Rounds where `shifted` is always `base + shift`, both riding the same per-round drift. */ +function drifting(shift: number) { + const drift = [0, 40, -25, 60, -10, 35, -45, 20, 5, -30]; + return { + base: drift.map((offset) => 100 + offset), + shifted: drift.map((offset) => 100 + offset + shift), + }; +} + +function reportOf(benchmarks: RunBenchmark[], metrics: BenchmarkRunReport['metrics'] = {}) { + const report: BenchmarkRunReport = { + version: 1, + generatedAt: '2026-01-01T00:00:00.000Z', + head: { sha: 'abc' }, + environment: { browser: 'Chromium', platform: 'linux', arch: 'x64', launchArgs: [] }, + sampling: { samples: 10, warmup: 2 }, + builds: { current: { label: 'working tree' }, baseline: { sha: 'def', label: 'HEAD~1' } }, + metrics, + benchmarks, + }; + return report; +} + +describe('tCritical95', () => { + it('reads small samples from the table', () => { + expect(tCritical95(1)).toBe(12.706); + expect(tCritical95(29)).toBe(2.045); + }); + + it('approaches the normal quantile for large samples', () => { + expect(tCritical95(10_000)).toBeCloseTo(1.96, 2); + }); +}); + +describe('meanInterval', () => { + it('centers on the mean', () => { + const interval = meanInterval([1, 2, 3, 4, 5]); + expect((interval.low + interval.high) / 2).toBeCloseTo(3); + expect(interval.low).toBeLessThan(3); + }); + + it('is a point for a single value', () => { + expect(meanInterval([7])).toEqual({ low: 7, high: 7 }); + }); +}); + +describe('median', () => { + it('takes the middle value, or the mean of the middle two', () => { + expect(median([3, 1, 2])).toBe(2); + expect(median([4, 1, 3, 2])).toBe(2.5); + }); +}); + +describe('compareSamples', () => { + it('resolves a small shift that per-round drift hides from the variant means', () => { + const { base, shifted } = drifting(2); + const comparison = compareSamples( + { name: 'current', values: shifted }, + { name: 'baseline', values: base }, + SCALAR, + ); + + expect(comparison.absolute).toEqual({ low: 2, high: 2 }); + expect(comparison.change).toBe('worse'); + expect(comparison.rounds).toBe(10); + }); + + it('reads the direction from the alarm', () => { + const { base, shifted } = drifting(2); + const comparison = compareSamples( + { name: 'current', values: shifted }, + { name: 'baseline', values: base }, + { kind: 'scalar', alarm: { direction: 'higherIsBetter' } }, + ); + + expect(comparison.change).toBe('better'); + expect(comparison.severity).toBe('none'); + }); + + it('is unsure when the interval straddles zero', () => { + const comparison = compareSamples( + { name: 'current', values: [10, 12, 9, 11] }, + { name: 'baseline', values: [11, 10, 10, 12] }, + SCALAR, + ); + + expect(comparison.change).toBe('unsure'); + }); + + describe('severity', () => { + const { base, shifted } = drifting(10); + + it('is none for a metric without an alarm, however large the change', () => { + const comparison = compareSamples( + { name: 'current', values: shifted }, + { name: 'baseline', values: base }, + { kind: 'scalar' }, + ); + expect(comparison.severity).toBe('none'); + }); + + it('grades a change against the warn and error thresholds', () => { + const compareWith = (alarm: RunMetricDefinition['alarm']) => + compareSamples( + { name: 'current', values: shifted }, + { name: 'baseline', values: base }, + { kind: 'scalar', alarm }, + ).severity; + + // The change is +10 on a mean of 105, about +9.5%. + expect(compareWith({ warn: 0.05, error: 0.2 })).toBe('warning'); + expect(compareWith({ warn: 0.05, error: 0.09 })).toBe('error'); + expect(compareWith({ warn: 0.2 })).toBe('none'); + }); + + it('compares a discrete metric in absolute counts', () => { + const comparison = compareSamples( + { name: 'current', values: [5, 5, 5] }, + { name: 'baseline', values: [3, 3, 3] }, + { kind: 'discrete', alarm: { error: 2 } }, + ); + expect(comparison.severity).toBe('error'); + }); + }); +}); + +describe('analyzeRun', () => { + it('compares the current build against the baseline', () => { + const { base, shifted } = drifting(2); + const [analysis] = analyzeRun( + reportOf( + [ + { + name: 'Button mount', + file: 'button.bench.tsx', + kind: 'baseline', + variants: ['current', 'baseline'], + samples: { current: { render: shifted }, baseline: { render: base } }, + }, + ], + { render: SCALAR }, + ), + ); + + expect(analysis.metrics.map((metric) => metric.metric)).toEqual(['render']); + expect(analysis.metrics[0].comparisons).toMatchObject([ + { subject: 'current', against: 'baseline', change: 'worse', severity: 'error' }, + ]); + }); + + it('compares every other variant of a comparison against the first', () => { + const [analysis] = analyzeRun( + reportOf([ + { + name: 'lists / mount', + file: 'lists.bench.tsx', + kind: 'compare', + variants: ['ul', 'table', 'grid'], + samples: { + ul: { render: [1, 1] }, + table: { render: [2, 2] }, + grid: { render: [3, 3] }, + }, + }, + ]), + ); + + expect( + analysis.metrics[0].comparisons.map(({ subject, against }) => `${subject} vs ${against}`), + ).toEqual(['table vs ul', 'grid vs ul']); + }); + + it('defines a sub-series by its base metric', () => { + const [analysis] = analyzeRun( + reportOf( + [ + { + name: 'Grid mount', + file: 'grid.bench.tsx', + kind: 'baseline', + variants: ['current', 'baseline'], + samples: { + current: { 'bench:paint#header': [5, 5] }, + baseline: { 'bench:paint#header': [5, 5] }, + }, + }, + ], + { 'bench:paint': { kind: 'scalar', format: { style: 'unit', unit: 'millisecond' } } }, + ), + ); + + expect(analysis.metrics[0].definition.format).toEqual({ + style: 'unit', + unit: 'millisecond', + }); + }); + + it('keeps a failed benchmark, with no metrics', () => { + const [analysis] = analyzeRun( + reportOf([ + { + name: 'Broken', + file: 'broken.bench.tsx', + kind: 'baseline', + variants: ['current', 'baseline'], + error: 'Render failed', + }, + ]), + ); + + expect(analysis.metrics).toEqual([]); + }); +}); + +describe('findRegressions', () => { + it('reports alarmed changes for the worse, on baseline benchmarks only', () => { + const { base, shifted } = drifting(2); + const samples = { render: shifted }; + const analyses = analyzeRun( + reportOf( + [ + { + name: 'Button mount', + file: 'button.bench.tsx', + kind: 'baseline', + variants: ['current', 'baseline'], + samples: { current: samples, baseline: { render: base } }, + }, + { + name: 'libs / mount', + file: 'libs.bench.tsx', + kind: 'compare', + variants: ['ours', 'theirs'], + samples: { ours: { render: base }, theirs: samples }, + }, + ], + { render: SCALAR }, + ), + ); + + expect(findRegressions(analyses)).toMatchObject([ + { benchmark: 'Button mount', metric: 'render' }, + ]); + }); +}); diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts new file mode 100644 index 0000000000..1e9a777137 --- /dev/null +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -0,0 +1,234 @@ +import { calculateMean } from '../stats'; +import type { BenchmarkRunReport, RunBenchmark, RunMetricDefinition } from './schema'; + +/** + * The one analysis of a run report, shared by the runner's table, the pull request comment and the + * dashboard, so they cannot disagree about what a run found. + * + * Variants are compared on paired differences: each round measured every variant once, so the + * per-round difference cancels whatever the machine was doing during that round, and its + * confidence interval is far tighter than one computed from two independent sets of samples. + */ + +export interface Interval { + low: number; + high: number; +} + +export interface SampleSummary { + count: number; + median: number; + /** 95% confidence interval of the mean. */ + mean: Interval; +} + +/** Which way a resolved change went, given the metric's direction; `unsure` if it did not resolve. */ +export type Change = 'better' | 'worse' | 'unsure'; + +/** How much a change for the worse matters, per the metric's alarm. */ +export type Severity = 'error' | 'warning' | 'none'; + +export interface MetricComparison { + /** The variant the difference is about. */ + subject: string; + /** The variant it is measured against. */ + against: string; + /** `subject − against`, in the metric's unit: 95% confidence interval of the mean difference. */ + absolute: Interval; + /** The same, as a percentage of `against`'s mean. */ + relative: Interval; + change: Change; + severity: Severity; + /** Rounds both variants were measured in. */ + rounds: number; +} + +export interface MetricAnalysis { + metric: string; + definition: RunMetricDefinition; + variants: Record; + comparisons: MetricComparison[]; +} + +export interface BenchmarkAnalysis { + benchmark: RunBenchmark; + /** Empty for a benchmark that failed. */ + metrics: MetricAnalysis[]; +} + +// Two-sided 95% critical values of Student's t for 1–30 degrees of freedom. +const T_95 = [ + 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, + 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, + 2.045, 2.042, +]; + +/** The two-sided 95% critical value of Student's t, approximated past the table. */ +export function tCritical95(degreesOfFreedom: number): number { + if (degreesOfFreedom <= T_95.length) { + return T_95[Math.max(degreesOfFreedom, 1) - 1]; + } + // Cornish–Fisher expansion around the normal quantile; well within 0.1% past 30 degrees. + const z = 1.959964; + return z + (z ** 3 + z) / (4 * degreesOfFreedom); +} + +/** The 95% confidence interval of the mean. A single value is its own degenerate interval. */ +export function meanInterval(values: number[]): Interval { + const mean = calculateMean(values); + if (values.length < 2) { + return { low: mean, high: mean }; + } + const variance = + values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (values.length - 1); + const halfWidth = tCritical95(values.length - 1) * Math.sqrt(variance / values.length); + return { low: mean - halfWidth, high: mean + halfWidth }; +} + +export function median(values: number[]): number { + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]; +} + +export function summarizeSamples(values: number[]): SampleSummary { + return { count: values.length, median: median(values), mean: meanInterval(values) }; +} + +const DEFAULT_DEFINITION: RunMetricDefinition = { kind: 'scalar' }; + +/** A metric's definition; a `name#id` sub-series is defined by its base metric. */ +export function definitionOf(report: BenchmarkRunReport, metric: string): RunMetricDefinition { + const [base] = metric.split('#'); + return report.metrics[metric] ?? report.metrics[base] ?? DEFAULT_DEFINITION; +} + +/** + * How far a change went in the bad direction, at the least: the interval bound nearest zero, in + * the unit the alarm thresholds use — a fraction for a scalar metric, a count for a discrete one. + */ +function worseningAtLeast( + comparison: Pick, + definition: RunMetricDefinition, +): number { + const lowerIsBetter = definition.alarm?.direction !== 'higherIsBetter'; + if (definition.kind === 'discrete') { + return lowerIsBetter ? comparison.absolute.low : -comparison.absolute.high; + } + return (lowerIsBetter ? comparison.relative.low : -comparison.relative.high) / 100; +} + +export function severityOf( + comparison: Pick, + definition: RunMetricDefinition, +): Severity { + const { alarm } = definition; + if (comparison.change !== 'worse' || !alarm) { + return 'none'; + } + if (alarm.error === undefined && alarm.warn === undefined) { + return 'error'; + } + const worsening = worseningAtLeast(comparison, definition); + if (alarm.error !== undefined && worsening >= alarm.error) { + return 'error'; + } + if (alarm.warn !== undefined && worsening >= alarm.warn) { + return 'warning'; + } + return 'none'; +} + +export function compareSamples( + subject: { name: string; values: number[] }, + against: { name: string; values: number[] }, + definition: RunMetricDefinition, +): MetricComparison { + const rounds = Math.min(subject.values.length, against.values.length); + const differences = Array.from( + { length: rounds }, + (_, round) => subject.values[round] - against.values[round], + ); + const absolute = meanInterval(differences); + const reference = calculateMean(against.values.slice(0, rounds)); + const relative = { + low: (absolute.low / reference) * 100, + high: (absolute.high / reference) * 100, + }; + + let change: Change = 'unsure'; + if (absolute.low > 0 || absolute.high < 0) { + const increased = absolute.low > 0; + const lowerIsBetter = definition.alarm?.direction !== 'higherIsBetter'; + change = increased === lowerIsBetter ? 'worse' : 'better'; + } + + const comparison = { subject: subject.name, against: against.name, absolute, relative, change }; + return { ...comparison, severity: severityOf(comparison, definition), rounds }; +} + +/** The pairs a benchmark's kind compares: the current build against the baseline, or every variant against the reference. */ +function pairsOf(benchmark: RunBenchmark): Array<{ subject: string; against: string }> { + const [reference, ...others] = benchmark.variants; + return benchmark.kind === 'baseline' + ? others.map((other) => ({ subject: reference, against: other })) + : others.map((other) => ({ subject: other, against: reference })); +} + +export function analyzeBenchmark( + report: BenchmarkRunReport, + benchmark: RunBenchmark, +): BenchmarkAnalysis { + const { samples } = benchmark; + if (!samples) { + return { benchmark, metrics: [] }; + } + const [reference] = benchmark.variants; + const metricNames = Object.keys(samples[reference] ?? {}); + + const metrics = metricNames.map((metric): MetricAnalysis => { + const definition = definitionOf(report, metric); + const valuesOf = (variant: string) => samples[variant]?.[metric] ?? []; + return { + metric, + definition, + variants: Object.fromEntries( + benchmark.variants.map((variant) => [variant, summarizeSamples(valuesOf(variant))]), + ), + comparisons: pairsOf(benchmark).map(({ subject, against }) => + compareSamples( + { name: subject, values: valuesOf(subject) }, + { name: against, values: valuesOf(against) }, + definition, + ), + ), + }; + }); + return { benchmark, metrics }; +} + +export function analyzeRun(report: BenchmarkRunReport): BenchmarkAnalysis[] { + return report.benchmarks.map((benchmark) => analyzeBenchmark(report, benchmark)); +} + +export interface Regression { + benchmark: string; + metric: string; + comparison: MetricComparison; +} + +/** + * The changes that say something is wrong with the code under test: only a `baseline` benchmark + * measures that, and only an alarmed metric's change for the worse counts. + */ +export function findRegressions(analyses: BenchmarkAnalysis[]): Regression[] { + return analyses.flatMap(({ benchmark, metrics }) => + benchmark.kind !== 'baseline' + ? [] + : metrics.flatMap(({ metric, comparisons }) => + comparisons + .filter((comparison) => comparison.severity !== 'none') + .map((comparison) => ({ benchmark: benchmark.name, metric, comparison })), + ), + ); +} diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts new file mode 100644 index 0000000000..d469a168f7 --- /dev/null +++ b/packages/benchmark/src/runReport/format.ts @@ -0,0 +1,29 @@ +import type { Interval, MetricComparison } from './analyzeRun'; +import type { RunMetricDefinition } from './schema'; + +/** + * How a run report's numbers read, for every renderer of one: the terminal table, the pull request + * comment and the dashboard show the same run, so they format it the same way. + */ + +const DEFAULT_SCALAR_FORMAT: Intl.NumberFormatOptions = { maximumFractionDigits: 2 }; +const DEFAULT_DISCRETE_FORMAT: Intl.NumberFormatOptions = { maximumFractionDigits: 1 }; + +/** A value in its metric's own format — `12.3 ms`, `4`. */ +export function formatValue(value: number, definition: RunMetricDefinition): string { + const options = + definition.format ?? + (definition.kind === 'discrete' ? DEFAULT_DISCRETE_FORMAT : DEFAULT_SCALAR_FORMAT); + return new Intl.NumberFormat('en-US', options).format(value); +} + +/** Both bounds signed, so the direction of a difference reads without the verdict beside it. */ +export function formatPercent(interval: Interval): string { + const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(1)}%`; + return `${signed(interval.low)} – ${signed(interval.high)}`; +} + +/** A comparison as one phrase: `worse +2.7% – +6.8%`, `unsure -1.2% – +0.9%`. */ +export function formatComparison(comparison: MetricComparison): string { + return `${comparison.change} ${formatPercent(comparison.relative)}`; +} diff --git a/packages/benchmark/src/runReport/index.ts b/packages/benchmark/src/runReport/index.ts new file mode 100644 index 0000000000..d1ca83e745 --- /dev/null +++ b/packages/benchmark/src/runReport/index.ts @@ -0,0 +1,3 @@ +export * from './schema'; +export * from './analyzeRun'; +export * from './format'; diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts new file mode 100644 index 0000000000..d5074e5fb5 --- /dev/null +++ b/packages/benchmark/src/runReport/schema.ts @@ -0,0 +1,79 @@ +import { z } from 'zod/v4'; + +/** + * The report `benchmark run` produces, uploads and the dashboard reads. + * + * It holds raw samples rather than conclusions: every variant of a benchmark is sampled once per + * round, and index `i` of every sample array is round `i`, so any statistic — paired differences + * included — can be computed from the report later, by whichever reader needs it, without running + * the benchmarks again. `analyzeRun` is the one analysis the runner and the dashboard share. + * + * Pure and node-free, so a browser can import it. + */ + +/** + * The same shape a metric's `alarm` config has. `warn` and `error` are the smallest changes that + * count at each level — a relative fraction (`0.1` = 10%) for a scalar metric, an absolute count + * for a discrete one. With both omitted, any resolved change for the worse is an error. + */ +const alarmSchema = z.object({ + direction: z.enum(['lowerIsBetter', 'higherIsBetter']).optional(), + warn: z.number().min(0).optional(), + error: z.number().min(0).optional(), +}); + +const metricDefinitionSchema = z.object({ + kind: z.enum(['scalar', 'discrete']), + format: z.custom().optional(), + /** Present when a change in this metric is a regression; without it the metric is informational. */ + alarm: alarmSchema.optional(), +}); + +/** A build the benchmark pages were loaded from. */ +const buildSchema = z.object({ + /** The commit it was built from; absent for the working tree. */ + sha: z.string().optional(), + /** How the run was given it: `working tree`, `HEAD~1`, a tag, a SHA. */ + label: z.string(), +}); + +const benchmarkSchema = z.object({ + name: z.string(), + /** The `*.bench.tsx` file it is defined in, relative to the harness's `src/`. */ + file: z.string(), + /** + * `baseline` measures one benchmark across the run's builds — `current` against `baseline` — so a + * difference is a change made by the code under test. `compare` measures the variants of a + * `compare()` against each other on the current build, where a difference is the point. + */ + kind: z.enum(['baseline', 'compare']), + /** Variant names, the reference first: `current` for `baseline`, the first variant for `compare`. */ + variants: z.array(z.string()), + /** Per variant, per metric: one value per round, round-aligned across variants and metrics. */ + samples: z.record(z.string(), z.record(z.string(), z.array(z.number()))).optional(), + /** Why the benchmark produced no samples. */ + error: z.string().optional(), +}); + +export const benchmarkRunReportSchema = z.object({ + version: z.literal(1), + generatedAt: z.string(), + head: z.object({ sha: z.string(), branch: z.string().optional() }), + environment: z.object({ + browser: z.string(), + platform: z.string(), + arch: z.string(), + launchArgs: z.array(z.string()), + }), + sampling: z.object({ samples: z.number(), warmup: z.number() }), + /** What the `baseline`-kind variant names refer to. */ + builds: z.record(z.string(), buildSchema), + /** Every metric the benchmarks report, keyed by name; `name#id` sub-series share `name`'s entry. */ + metrics: z.record(z.string(), metricDefinitionSchema), + benchmarks: z.array(benchmarkSchema), +}); + +export type BenchmarkRunReport = z.infer; +export type RunMetricDefinition = z.infer; +export type RunBenchmark = z.infer; +export type RunBuild = z.infer; diff --git a/packages/benchmark/src/tachometer/benchFiles.test.ts b/packages/benchmark/src/runner/benchFiles.test.ts similarity index 100% rename from packages/benchmark/src/tachometer/benchFiles.test.ts rename to packages/benchmark/src/runner/benchFiles.test.ts diff --git a/packages/benchmark/src/tachometer/benchFiles.ts b/packages/benchmark/src/runner/benchFiles.ts similarity index 85% rename from packages/benchmark/src/tachometer/benchFiles.ts rename to packages/benchmark/src/runner/benchFiles.ts index 4127d20b22..0a67d0d3cb 100644 --- a/packages/benchmark/src/tachometer/benchFiles.ts +++ b/packages/benchmark/src/runner/benchFiles.ts @@ -1,7 +1,7 @@ import * as path from 'node:path'; import { mkdir, rm, writeFile } from 'node:fs/promises'; import { globby } from 'globby'; -import { matchesFilters } from './discoverCases'; +import { escapeHtml } from '../utils/html'; /** * `*.bench.tsx` files: `benchmark()` cases, the same files Vitest runs, measured by the interleaved @@ -9,7 +9,7 @@ import { matchesFilters } from './discoverCases'; * swapped for the page runtime, so the file itself needs no page, config or entry of its own. */ -/** Where the generated pages live, under `src/`. Ignored from the inside, like `.tachometer/`. */ +/** Where the generated pages live, under `src/`. Ignored from the inside, like `.benchmark/`. */ export const BENCH_PAGES_DIR = '__bench__'; const BENCH_FILE_PATTERN = '**/*.bench.tsx'; @@ -26,6 +26,18 @@ function benchFileOf(file: string): BenchFile { return { file, page: `${BENCH_PAGES_DIR}/${slug}.html` }; } +/** + * Whether a file is selected by the run's filters: all of them when there are none. Matched + * case-insensitively, the way vitest matches its file filters, so a folder name selects everything + * under it. + */ +function matchesFilters(file: string, filters: string[]): boolean { + return ( + filters.length === 0 || + filters.some((filter) => file.toLowerCase().includes(filter.toLowerCase())) + ); +} + /** The `*.bench.tsx` files under `src/`, optionally filtered by path, sorted. */ export async function discoverBenchFiles(options: { harnessDir: string; @@ -47,7 +59,7 @@ function pageHtml(benchFile: BenchFile, entry: string): string { - ${benchFile.file} + ${escapeHtml(benchFile.file)} diff --git a/packages/benchmark/src/runner/browser.ts b/packages/benchmark/src/runner/browser.ts new file mode 100644 index 0000000000..d770685040 --- /dev/null +++ b/packages/benchmark/src/runner/browser.ts @@ -0,0 +1,26 @@ +import type * as PlaywrightTest from '@playwright/test'; +import { pathExists } from '../utils/path'; + +/** + * Resolves the browser every benchmark runs on: Playwright's pinned Chrome for Testing, rather than + * whatever Chrome the machine has auto-updated to. + */ +export async function resolveBrowserBinary(harnessDir: string): Promise { + let playwright: typeof PlaywrightTest; + try { + playwright = await import('@playwright/test'); + } catch { + // An optional peer, so it installs cleanly when absent and only fails here. + throw new Error( + `"@playwright/test" is not installed. Add it as a devDependency of ${harnessDir}.`, + ); + } + + const binary = playwright.chromium.executablePath(); + if (!binary || !(await pathExists(binary))) { + throw new Error( + `Playwright's Chromium is not installed at "${binary}". Run \`pnpm exec playwright install chromium\`.`, + ); + } + return binary; +} diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/runner/buildPages.ts similarity index 94% rename from packages/benchmark/src/tachometer/buildPages.ts rename to packages/benchmark/src/runner/buildPages.ts index 4b910cae47..64890e8782 100644 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ b/packages/benchmark/src/runner/buildPages.ts @@ -14,7 +14,7 @@ import { run } from '../utils/exec'; import { tarballFor } from '../utils/packWorkspace'; import type { PackedPackage } from '../utils/packWorkspace'; import { installedVersions, readPackageJson, writePackageJson } from '../utils/pnpm'; -import { refLabel } from './format'; +import { refLabel } from './refs'; import type { ResolvedRef } from './refs'; import { pathExists } from '../utils/path'; @@ -32,13 +32,12 @@ export type ResolveMode = 'isolated' | 'in-place'; * Dependencies the run needs but a page never imports, so a ref's tree does without them — * whichever section of the harness's manifest they sit in, since neither reason below turns on that. * - * Every one is resolved from the harness rather than from here: the browser and its driver by the - * runner, tachometer by the command that samples, and this package by the harness's own vite config - * — which is loaded from the harness, since the build happens in place. Every one is also expensive - * to install: tachometer pulls a chromedriver, `@playwright/test` a browser, and this package a - * dependency tree many times the size of a harness. + * Both are resolved from the harness rather than from here: the browser by the runner, and this + * package by the harness's own vite config — which is loaded from the harness, since the build + * happens in place. Both are also expensive to install: `@playwright/test` pulls a browser, and this + * package a dependency tree many times the size of a harness. */ -const RUNNER_ONLY_DEPS = ['tachometer', 'chromedriver', '@playwright/test', pkgJson.name]; +const RUNNER_ONLY_DEPS = ['@playwright/test', pkgJson.name]; /** * A dependency map for a ref's tree: every `workspace:` entry points at that ref's packed tarball, @@ -109,7 +108,7 @@ function resolveFromTree(treeDir: string): Vite.Plugin { const importer = path.join(treeDir, 'package.json'); return { - name: 'mui-benchmark:tachometer-resolve', + name: 'mui-benchmark:resolve-from-tree', enforce: 'pre', async resolveId(id, _importer, options) { if (id.startsWith('.') || path.isAbsolute(id) || id.startsWith('\0') || id.includes(':')) { @@ -210,7 +209,7 @@ async function installRefTree({ ]); await writePackageJson(treeDir, { - name: 'tacho-resolve-tree', + name: 'mui-benchmark:resolve-tree', private: true, version: '0.0.0', dependencies: rewriteWorkspaceDeps( @@ -256,7 +255,7 @@ async function installRefTree({ * Builds the benchmark pages against one ref's packed build. * * The pages are built **where they live**, from the harness's own directory and its own vite config, - * so a `tacho run` build sees exactly what a plain `vite build` sees — the same tsconfig, the same + * so a `benchmark run` build sees exactly what a plain `vite build` sees — the same tsconfig, the same * postcss config, the same everything. Only *resolution* differs: a plugin redirects every bare * import into a small tree installed for this ref, where the library under test is that ref's packed * build. diff --git a/packages/benchmark/src/tachometer/outputDir.ts b/packages/benchmark/src/runner/outputDir.ts similarity index 86% rename from packages/benchmark/src/tachometer/outputDir.ts rename to packages/benchmark/src/runner/outputDir.ts index fee71d0e3e..b6c0b9c203 100644 --- a/packages/benchmark/src/tachometer/outputDir.ts +++ b/packages/benchmark/src/runner/outputDir.ts @@ -6,7 +6,7 @@ import { mkdir, writeFile } from 'node:fs/promises'; * under this one directory inside the harness, so a repository has a single thing to ignore and * deleting it is the whole reset story. */ -export const OUTPUT_DIR = '.tachometer'; +export const OUTPUT_DIR = '.benchmark'; /** * Creates the output directory, marking it ignored from the inside. @@ -27,8 +27,8 @@ export async function prepareOutputDir(harnessDir: string): Promise { * Where the pages built for `refId` go, or the root of every ref's pages when it is omitted. * * Named here rather than composed at each call site because the runner and the vite plugin have to - * agree on it: the runner points tachometer's static server at the output directory and rewrites - * each url under this path, and a plain `vite build` has to land somewhere that does not collide. + * agree on it: the runner serves every ref's pages from under this path, and a plain `vite build` + * has to land somewhere that does not collide. */ export function buildsDirOf(harnessDir: string, refId?: string): string { const builds = path.join(harnessDir, OUTPUT_DIR, 'builds'); diff --git a/packages/benchmark/src/tachometer/plugin.test.ts b/packages/benchmark/src/runner/plugin.test.ts similarity index 54% rename from packages/benchmark/src/tachometer/plugin.test.ts rename to packages/benchmark/src/runner/plugin.test.ts index 49175b5582..a1a4403244 100644 --- a/packages/benchmark/src/tachometer/plugin.test.ts +++ b/packages/benchmark/src/runner/plugin.test.ts @@ -2,35 +2,26 @@ import * as path from 'node:path'; import { mkdir, symlink, writeFile } from 'node:fs/promises'; import { describe, expect, it } from 'vitest'; import { makeTempDir } from '../utils/testUtils'; -import { tachometer } from './plugin'; - -/** A case folder to create in a throwaway harness. */ -interface CaseFixture { - config: any; - pages?: string[]; -} +import { benchmarkPlugin } from './plugin'; /** Builds a throwaway harness with a package.json, so the plugin can read its dependencies. */ async function makeHarness({ - cases, + benchFiles = ['button/button.bench.tsx'], dependencies = {}, }: { - cases: Record; + benchFiles?: string[]; dependencies?: Record; -}): Promise { +} = {}): Promise { const harnessDir = await makeTempDir(); await writeFile( path.join(harnessDir, 'package.json'), JSON.stringify({ name: 'harness', private: true, dependencies }, null, 2), ); await Promise.all( - Object.entries(cases).map(async ([name, { config, pages = [] }]) => { - const caseDir = path.join(harnessDir, 'src', name); - await mkdir(caseDir, { recursive: true }); - await writeFile(path.join(caseDir, 'tachometer.json'), JSON.stringify(config, null, 2)); - await Promise.all( - pages.map((page) => writeFile(path.join(caseDir, page), '\n')), - ); + benchFiles.map(async (file) => { + const absolute = path.join(harnessDir, 'src', file); + await mkdir(path.dirname(absolute), { recursive: true }); + await writeFile(absolute, ''); }), ); return harnessDir; @@ -42,22 +33,15 @@ async function callConfig( command: 'build' | 'serve', userConfig: any = {}, ): Promise { - const plugin = tachometer({ harnessDir }); + const plugin = benchmarkPlugin({ harnessDir }); const hook = plugin.config as any; return hook(userConfig, { command, mode: command === 'build' ? 'production' : 'development' }); } -const oneCase = { - alpha: { - config: { benchmarks: [{ name: 'alpha', url: './index.html' }] }, - pages: ['index.html'], - }, -}; - -describe('tachometer plugin', () => { +describe('benchmarkPlugin', () => { describe('config hook', () => { it('roots the app at src and declares it multi-page', async () => { - const harnessDir = await makeHarness({ cases: oneCase }); + const harnessDir = await makeHarness(); const config = await callConfig(harnessDir, 'serve'); @@ -68,7 +52,7 @@ describe('tachometer plugin', () => { }); it('derives no build input while serving', async () => { - const harnessDir = await makeHarness({ cases: oneCase }); + const harnessDir = await makeHarness(); const config = await callConfig(harnessDir, 'serve'); @@ -78,37 +62,28 @@ describe('tachometer plugin', () => { it('points serving at the output directory too, so preview finds the built pages', async () => { // `vite preview` runs as `serve` but serves `build.outDir`. Leaving it unset there sends // preview to `/dist` — inside `src/` — and it exits instead of serving the build. - const harnessDir = await makeHarness({ cases: oneCase }); + const harnessDir = await makeHarness(); const config = await callConfig(harnessDir, 'serve'); - expect(config.build?.outDir).toBe(path.join(harnessDir, '.tachometer', 'builds', 'manual')); + expect(config.build?.outDir).toBe(path.join(harnessDir, '.benchmark', 'builds', 'manual')); }); it('uses relative asset urls for a build', async () => { - // Tachometer serves the build directory and pages live under //, so absolute - // "/assets/…" paths would 404. - const harnessDir = await makeHarness({ cases: oneCase }); + // The runner serves each ref's build under its own path, so absolute "/assets/…" paths + // would 404. + const harnessDir = await makeHarness(); const config = await callConfig(harnessDir, 'build'); expect(config.base).toBe('./'); }); - it('defaults the output directory to the run output folder, outside the source root', async () => { - // Vite would otherwise emit into /dist, i.e. inside src/ next to the case sources. - const harnessDir = await makeHarness({ cases: oneCase }); - - const config = await callConfig(harnessDir, 'build'); - - expect(config.build.outDir).toBe(path.join(harnessDir, '.tachometer', 'builds', 'manual')); - }); - it('leaves an output directory the caller chose alone', async () => { // A plugin's returned config is merged *over* the inline config, so defaulting this - // unconditionally would silently override `vite build --outDir` — which is how `tacho run` - // directs each ref's build into its own directory. - const harnessDir = await makeHarness({ cases: oneCase }); + // unconditionally would silently override `vite build --outDir` — which is how + // `benchmark run` directs each ref's build into its own directory. + const harnessDir = await makeHarness(); const config = await callConfig(harnessDir, 'build', { build: { outDir: '/somewhere/builds/current' }, @@ -117,50 +92,29 @@ describe('tachometer plugin', () => { expect(config.build.outDir).toBe('/somewhere/builds/current'); }); - it('derives the build input from the discovered cases', async () => { - const harnessDir = await makeHarness({ cases: oneCase }); + it('builds a generated page per benchmark file', async () => { + const harnessDir = await makeHarness({ + benchFiles: ['button/button.bench.tsx', 'grid/grid.bench.tsx'], + }); const config = await callConfig(harnessDir, 'build'); expect(config.build.rollupOptions.input).toEqual({ - 'alpha/index': path.join(harnessDir, 'src', 'alpha', 'index.html'), + '__bench__/button-button': path.join(harnessDir, 'src', '__bench__', 'button-button.html'), + '__bench__/grid-grid': path.join(harnessDir, 'src', '__bench__', 'grid-grid.html'), }); }); - it('keys several pages in one case folder separately', async () => { - // A distinct key per page is what stops a cross-library case's two pages colliding into one - // entry. - const harnessDir = await makeHarness({ - cases: { - libs: { - config: { - benchmarks: [ - { - name: 'libs', - expand: [ - { name: 'libs [ours]', url: './ours.html' }, - { name: 'libs [theirs]', url: './theirs.html' }, - ], - }, - ], - }, - pages: ['ours.html', 'theirs.html'], - }, - }, - }); - - const config = await callConfig(harnessDir, 'build'); + it('refuses a harness without benchmark files', async () => { + const harnessDir = await makeHarness({ benchFiles: [] }); - expect(Object.keys(config.build.rollupOptions.input).sort()).toEqual([ - 'libs/ours', - 'libs/theirs', - ]); + await expect(callConfig(harnessDir, 'build')).rejects.toThrow(/No \*\.bench\.tsx file/); }); }); describe('workspace link check', () => { it('passes when the harness links to nothing', async () => { - const harnessDir = await makeHarness({ cases: oneCase }); + const harnessDir = await makeHarness(); await expect(callConfig(harnessDir, 'serve')).resolves.toBeTruthy(); }); @@ -168,10 +122,7 @@ describe('tachometer plugin', () => { it('explains that the library has to be built when its link dangles', async () => { // A harness resolves the library through its build output, so before the first build the // link points at a directory that does not exist yet. - const harnessDir = await makeHarness({ - cases: oneCase, - dependencies: { '@scope/library': 'workspace:*' }, - }); + const harnessDir = await makeHarness({ dependencies: { '@scope/library': 'workspace:*' } }); await mkdir(path.join(harnessDir, 'node_modules', '@scope'), { recursive: true }); await symlink( path.join(harnessDir, 'packages', 'library', 'build'), diff --git a/packages/benchmark/src/runner/plugin.ts b/packages/benchmark/src/runner/plugin.ts new file mode 100644 index 0000000000..14b71e3a26 --- /dev/null +++ b/packages/benchmark/src/runner/plugin.ts @@ -0,0 +1,217 @@ +import * as path from 'node:path'; +import { readFile, realpath } from 'node:fs/promises'; +import type { Plugin } from 'vite'; +import { escapeHtml } from '../utils/html'; +import { discoverBenchFiles, writeBenchPages } from './benchFiles'; +import type { BenchFile } from './benchFiles'; +import { buildsDirOf, prepareOutputDir, OUTPUT_DIR } from './outputDir'; + +/** + * Vite plugin for a benchmark harness. + * + * The harness keeps its own vite config and runs plain `vite` / `vite build` / `vite preview`; this + * plugin only contributes what is derived from its `*.bench.tsx` files: + * + * - A generated page per benchmark file, under `src/__bench__/`, and those pages as the build's + * entry points. + * - `@mui/internal-benchmark` resolved to the page runtime, which runs the file's `benchmark()` + * cases one iteration at a time for `benchmark run`. + * - `root` at the harness's `src/`, as an MPA (vite defaults to `spa`, whose fallback would rewrite + * unmatched urls toward a root `index.html` that does not exist here), and an index page listing + * the benchmark files. + * + * It deliberately never resolves refs, so a plain `vite build` needs no git history. + */ + +export interface BenchmarkPluginOptions { + /** + * The harness package directory. Defaults to the current working directory, which is where vite + * is run from. + */ + harnessDir?: string; +} + +/** + * Checks that the workspace packages the harness links to have actually been built. + * + * A harness depends on the library under test via `workspace:*`, and `publishConfig.directory` + * makes that link point at the package's build output — which does not exist until the library is + * built. The symlink is then dangling, and vite's failure ("failed to resolve import") does not + * say why. + */ +async function assertWorkspaceDepsBuilt( + harnessDir: string, + deps: Record, +): Promise { + const linked = Object.entries(deps).filter(([, version]) => version.startsWith('workspace:')); + await Promise.all( + linked.map(async ([name]) => { + try { + await realpath(path.join(harnessDir, 'node_modules', ...name.split('/'))); + } catch { + throw new Error( + `"${name}" is linked into ${harnessDir} but its target does not exist. ` + + `Build the workspace packages first (e.g. \`pnpm release:build\`) — the harness ` + + `resolves the library through its build output, not its source.`, + ); + } + }), + ); +} + +/** The index page: one link per benchmark file's page. */ +function renderIndex(benchFiles: BenchFile[], note: string): string { + const items = benchFiles + .map( + (benchFile) => + `
  • ${escapeHtml(benchFile.file)}
    • `, + ) + .join('\n'); + + return ` + + + + Benchmarks + + + +

    Benchmarks

    +

    ${note}

    +
      +${items} +
    + + +`; +} + +/** Creates the benchmark harness plugin. */ +export function benchmarkPlugin(options: BenchmarkPluginOptions = {}): Plugin { + const harnessDir = options.harnessDir ?? process.cwd(); + const srcDir = path.join(harnessDir, 'src'); + let benchFiles: BenchFile[] = []; + + return { + name: 'mui-benchmark:pages', + // Ahead of vite's own resolver, so the redirect below wins over the package's `exports`. + enforce: 'pre', + + // Benchmark files are written against `@mui/internal-benchmark`; in a page that is the page + // runtime, which the runner drives one iteration at a time. Tests are not benchmarks, so a page + // has no Vitest to import. + async resolveId(source, importer) { + if (source === '@mui/internal-benchmark') { + return this.resolve('@mui/internal-benchmark/page', importer, { skipSelf: true }); + } + if (source === 'vitest' || source.startsWith('vitest/')) { + this.error( + `${importer ?? 'A benchmark page'} imports "${source}". Benchmark files run outside ` + + 'Vitest: define cases with benchmark() or compare(), and use the interaction context ' + + '(input, waitForElementTiming, …) instead of Vitest APIs.', + ); + } + return null; + }, + + async config(userConfig, env) { + const pkg = JSON.parse(await readFile(path.join(harnessDir, 'package.json'), 'utf8')); + await assertWorkspaceDepsBuilt(harnessDir, { + ...pkg.dependencies, + ...pkg.devDependencies, + }); + + // The run output directory, next to the per-ref directories the runner writes — vite would + // otherwise default to `/dist`, i.e. inside `src/` next to the benchmark sources. + // + // Resolved for every command, not just `build`: `vite preview` runs as `serve` but serves + // `build.outDir`, so leaving it unset there sends preview looking for `/dist` and it + // exits rather than serving the pages that were just built. + // + // Only fill it in when the caller said nothing: a plugin's returned config is merged *over* + // the inline config, so unconditionally setting it here would silently override + // `vite build --outDir`, which is exactly how `benchmark run` directs each ref's build. + const outDir = userConfig.build?.outDir ?? buildsDirOf(harnessDir, 'manual'); + + // Written for every command, so the dev server serves a benchmark file's page as well. + benchFiles = await discoverBenchFiles({ harnessDir }); + if (benchFiles.length === 0) { + throw new Error(`No *.bench.tsx file found under ${srcDir}.`); + } + await writeBenchPages(harnessDir, benchFiles); + // One React for the harness, its benchmark files and the page runtime they run under. + const resolve = { dedupe: ['react', 'react-dom'] }; + + if (env.command !== 'build') { + return { root: srcDir, appType: 'mpa', resolve, build: { outDir } }; + } + + // Mark the output directory ignored from the inside before the build writes into it, so a + // harness that does not list `.benchmark` is not offered its built pages for commit. Keyed on + // where the output actually lands rather than on who chose it: `benchmark run` passes its own + // `outDir` under the same directory and prepares it itself, and any other caller that writes + // there deserves the marker too. Vite resolves a relative `outDir` against `root`. + if (path.resolve(srcDir, outDir).startsWith(path.join(harnessDir, OUTPUT_DIR))) { + await prepareOutputDir(harnessDir); + } + + const input = Object.fromEntries( + benchFiles.map((benchFile) => [ + benchFile.page.replace(/\.html$/, ''), + path.join(srcDir, benchFile.page), + ]), + ); + + return { + root: srcDir, + appType: 'mpa', + resolve, + // Relative asset urls: the runner serves each ref's build under its own path, so absolute + // "/assets/…" paths would 404. + base: './', + build: { + outDir, + // The output directory sits outside `root`, which is `src/`; allow vite to clean it anyway. + emptyOutDir: true, + chunkSizeWarningLimit: 9999, + rollupOptions: { input }, + }, + }; + }, + + // The same index the dev server renders, emitted as a real file so the built directory is + // navigable too — under `vite preview`, under any static server, or opened from disk. + generateBundle() { + this.emitFile({ + type: 'asset', + fileName: 'index.html', + source: renderIndex( + benchFiles, + 'These are production bundles — the same build `benchmark run` measures.', + ), + }); + }, + + configureServer(server) { + server.middlewares.use((req, res, next) => { + const url = (req.url ?? '/').split('?')[0]; + if (url !== '/' && url !== '/index.html') { + next(); + return; + } + res.setHeader('Content-Type', 'text/html; charset=utf-8'); + res.end( + renderIndex( + benchFiles, + "Pages are served from the working tree's build of the library. Rebuild it and reload to pick up a source change.", + ), + ); + }); + }, + }; +} diff --git a/packages/benchmark/src/runner/printReport.ts b/packages/benchmark/src/runner/printReport.ts new file mode 100644 index 0000000000..31ab8770f5 --- /dev/null +++ b/packages/benchmark/src/runner/printReport.ts @@ -0,0 +1,74 @@ +/* eslint-disable no-console */ + +import { dim, green, printTable, red, yellow } from '../format'; +import { analyzeRun, findRegressions, formatComparison, formatValue } from '../runReport'; +import type { BenchmarkAnalysis, BenchmarkRunReport } from '../runReport'; + +function printBenchmark({ benchmark, metrics }: BenchmarkAnalysis): void { + const [reference] = benchmark.variants; + const comparedTo = (subject: string, against: string) => + benchmark.kind === 'baseline' ? `Δ vs ${against}` : `${subject} vs ${against}`; + + const headers = [ + 'Metric', + ...benchmark.variants.map((variant) => `${variant} (median)`), + ...(metrics[0]?.comparisons ?? []).map(({ subject, against }) => comparedTo(subject, against)), + 'Rounds', + ]; + const rows = metrics.map(({ metric, definition, variants, comparisons }) => [ + metric, + ...benchmark.variants.map((variant) => formatValue(variants[variant].median, definition)), + ...comparisons.map(formatComparison), + String(variants[reference].count), + ]); + const widths = headers.map((header, column) => + Math.max(header.length, ...rows.map((row) => row[column].length)), + ); + + printTable( + headers.map((header, column) => ({ header, width: widths[column] })), + rows, + undefined, + `${benchmark.name} (${benchmark.file})`, + ); +} + +/** + * Prints a run report as the terminal tables, and says up front what regressed — the same analysis + * the pull request comment and the dashboard show. + */ +export function printRunReport(report: BenchmarkRunReport): void { + const analyses = analyzeRun(report); + + for (const analysis of analyses) { + if (analysis.benchmark.error) { + console.log(red(`✖ ${analysis.benchmark.name}: ${analysis.benchmark.error}`)); + continue; + } + printBenchmark(analysis); + } + + const regressions = findRegressions(analyses); + console.log(''); + for (const { benchmark, metric, comparison } of regressions) { + const color = comparison.severity === 'error' ? red : yellow; + console.log(color(`⚠ ${benchmark} · ${metric} · ${formatComparison(comparison)}`)); + } + const measured = analyses.filter((analysis) => !analysis.benchmark.error).length; + const failed = analyses.length - measured; + console.log( + [ + `${measured} benchmark${measured === 1 ? '' : 's'} measured`, + regressions.length > 0 ? red(`${regressions.length} regression(s)`) : green('no regressions'), + failed > 0 ? red(`${failed} failed`) : null, + ] + .filter(Boolean) + .join(' · '), + ); + console.log( + dim( + 'Each Δ is a 95% confidence interval on the paired per-round difference, relative to the ' + + 'variant it is measured against; "unsure" means it straddles zero.', + ), + ); +} diff --git a/packages/benchmark/src/tachometer/refs.test.ts b/packages/benchmark/src/runner/refs.test.ts similarity index 100% rename from packages/benchmark/src/tachometer/refs.test.ts rename to packages/benchmark/src/runner/refs.test.ts diff --git a/packages/benchmark/src/tachometer/refs.ts b/packages/benchmark/src/runner/refs.ts similarity index 93% rename from packages/benchmark/src/tachometer/refs.ts rename to packages/benchmark/src/runner/refs.ts index ac94c1c6c8..b563008718 100644 --- a/packages/benchmark/src/tachometer/refs.ts +++ b/packages/benchmark/src/runner/refs.ts @@ -20,6 +20,11 @@ export type ResolvedRef = | { kind: 'worktree'; id: string; sha?: undefined; requested?: undefined } | { kind: 'git'; id: string; sha: string; requested: string }; +/** What a ref is called in a report: the working tree, or the revision the run was given. */ +export function refLabel(ref: ResolvedRef): string { + return ref.kind === 'worktree' ? 'working tree' : ref.requested; +} + /** The working tree, which every run builds. Never cached, having no immutable identity. */ export const WORKTREE_REF: ResolvedRef = { kind: 'worktree', id: 'current' }; diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts new file mode 100644 index 0000000000..4d3282c6c9 --- /dev/null +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -0,0 +1,221 @@ +/* eslint-disable no-console */ + +import * as path from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; +import chalk from 'chalk'; +import { execaSync } from 'execa'; +import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; +import { packRef, packWorkingTree } from '../utils/packWorkspace'; +import type { PackedPackage } from '../utils/packWorkspace'; +import type { BenchmarkRunReport } from '../runReport'; +import { refLabel, resolveBaselineRef, WORKTREE_REF } from './refs'; +import type { ResolvedRef } from './refs'; +import { discoverBenchFiles } from './benchFiles'; +import { runInterleaved } from './runInterleaved'; +import { buildRefPages, restoreWorkspace } from './buildPages'; +import type { ResolveMode } from './buildPages'; +import { resolveBrowserBinary } from './browser'; +import { printRunReport } from './printReport'; +import { publishRunReport } from './upload'; +import { buildsDirOf, prepareOutputDir } from './outputDir'; + +export interface RunBenchmarksOptions { + /** The harness package directory (where the command was run). */ + harnessDir: string; + /** Only run benchmark files whose path under `src` contains one of these, case-insensitively. */ + filters?: string[]; + /** + * The build the working tree is compared against: a revision, on its own or as `git:`. + * Defaults to `HEAD~1`, the parent commit. + */ + baseline?: string; + /** + * Command that builds the publishable workspace packages. Run for every ref — in each ref's own + * checkout, and in the working tree — so both sides of a comparison are built the same way. + * Defaults to `pnpm release:build`. + */ + buildCmd?: string; + /** Whether to install inside a ref's checkout. Defaults to true. */ + install?: boolean; + /** Where to write the JSON report. Defaults to `.benchmark/results/report.json`. */ + out?: string; + /** Upload the report and refresh the pull request comment. */ + upload?: boolean; + /** How a ref's pages find the library under test. Defaults to `in-place`. */ + resolveMode?: ResolveMode; + /** Measured rounds per case. Defaults to 30. */ + samples?: number; + /** Discarded rounds before measuring, once per case. Defaults to 10. */ + warmup?: number; + /** Extra Chromium launch arguments, e.g. to select a hardware GPU backend for paint timings. */ + launchArgs?: string[]; +} + +/** + * Benchmarks a harness's `*.bench.tsx` files across two builds of the workspace — the working tree + * and a baseline — and writes, prints and optionally uploads the run report. + * + * Both builds are packed from their own commit and installed into the harness the same way, so + * neither resolves the library through a workspace link. + */ +export async function runBenchmarks(options: RunBenchmarksOptions): Promise { + const { + harnessDir, + filters = [], + baseline, + buildCmd = 'pnpm release:build', + install = true, + out, + upload = false, + resolveMode = 'in-place', + samples = 30, + warmup = 10, + launchArgs = [], + } = options; + + const repoRoot = await findWorkspaceDir(harnessDir); + if (!repoRoot) { + throw new Error(`Could not find a pnpm workspace root above ${harnessDir}.`); + } + + // Everything a run writes goes under one directory, so a harness has one thing to ignore and + // deleting it is the whole reset story. `packed` holds tarballs — a ref's keyed by commit SHA, + // the working tree's by content hash — and is the one worth caching in CI; `trees` holds the + // install each ref's pages resolve through, cheap to recreate and not portable between + // containers, since its links point into a pnpm store; `builds` holds the pages built per ref; + // `results` the report. + const outputDir = await prepareOutputDir(harnessDir); + const buildsDir = buildsDirOf(harnessDir); + const packedDir = path.join(outputDir, 'packed'); + const treesDir = path.join(outputDir, 'trees'); + + // A run killed outright — Ctrl-C, not a thrown error — never reaches the restore below, leaving + // the repository pinned to tarballs. The copy it left behind is how that is noticed, so put it + // back before doing anything else, whatever mode this run is in. + await restoreWorkspace(repoRoot, outputDir); + + const benchFiles = await discoverBenchFiles({ harnessDir, filters }); + if (benchFiles.length === 0) { + const srcDir = path.join(harnessDir, 'src'); + throw new Error( + filters.length > 0 + ? `No *.bench.tsx file under ${srcDir} matches ${filters.map((filter) => `"${filter}"`).join(', ')}.` + : `No *.bench.tsx file found under ${srcDir}.`, + ); + } + // The working tree first: it is the reference every comparison is judged from. + const refs: ResolvedRef[] = [WORKTREE_REF, await resolveBaselineRef(baseline, repoRoot)]; + + // Before any build: a browser that is not installed fails the run either way, and finding out now + // costs seconds instead of minutes of packing and installing. + const browserBinary = await resolveBrowserBinary(harnessDir); + // `getuid` is POSIX-only; on Windows nobody is root. Chrome cannot enter its sandbox as root. + const allLaunchArgs = process.getuid?.() === 0 ? [...launchArgs, '--no-sandbox'] : launchArgs; + + // The commit follows the name, so a revision like `HEAD~1` still says which commit it landed on. + const describeRef = (ref: ResolvedRef) => + ref.sha ? `${refLabel(ref)} (${ref.sha.slice(0, 9)})` : refLabel(ref); + console.log(chalk.cyan(`Files: ${benchFiles.map((benchFile) => benchFile.file).join(', ')}`)); + console.log(chalk.cyan(`Refs: ${refs.map(describeRef).join(', ')}`)); + console.log(chalk.cyan(`Browser: ${browserBinary} ${allLaunchArgs.join(' ')}`)); + + try { + // Build one variant per ref, each resolving through its own install so both sides of a + // comparison resolve the library identically. + for (const ref of refs) { + let packages: PackedPackage[]; + if (ref.kind === 'worktree') { + // eslint-disable-next-line no-await-in-loop + packages = await packWorkingTree({ + repoRoot, + outRoot: path.join(packedDir, 'current'), + buildCmd, + }); + } else { + // packRef caches a ref's tarballs by SHA; a hit skips the checkout, install, and build. + // eslint-disable-next-line no-await-in-loop + const packed = await packRef({ + repoRoot, + ref: ref.sha, + outRoot: packedDir, + // An empty install command means "skip"; otherwise packRef's default install runs. + installCmd: install ? undefined : '', + buildCmd, + }); + packages = packed.packages; + } + + // eslint-disable-next-line no-await-in-loop + await buildRefPages({ + harnessDir, + repoRoot, + ref, + packages, + treeDir: path.join(treesDir, ref.id), + outputDir, + outDir: path.join(buildsDir, ref.id), + resolveMode, + }); + } + + const results = await runInterleaved({ + buildsDir, + benchFiles, + benchRefs: refs, + browserBinary, + launchArgs: allLaunchArgs, + samples, + warmup, + }); + + const report: BenchmarkRunReport = { + version: 1, + generatedAt: new Date().toISOString(), + head: { + sha: execaSync('git', ['rev-parse', 'HEAD'], { cwd: repoRoot }).stdout.trim(), + branch: execaSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { + cwd: repoRoot, + }).stdout.trim(), + }, + environment: { + browser: `Chromium ${results.browserVersion}`, + platform: process.platform, + arch: process.arch, + launchArgs: allLaunchArgs, + }, + sampling: { samples, warmup }, + builds: Object.fromEntries( + refs.map((ref) => [ + ref.kind === 'worktree' ? 'current' : 'baseline', + { sha: ref.sha, label: refLabel(ref) }, + ]), + ), + metrics: results.metrics, + benchmarks: results.benchmarks, + }; + + const outPath = out ? path.resolve(out) : path.join(outputDir, 'results', 'report.json'); + await mkdir(path.dirname(outPath), { recursive: true }); + await writeFile(outPath, `${JSON.stringify(report, null, 2)}\n`); + console.log(''); + printRunReport(report); + console.log(chalk.green(`\nWrote JSON report to ${outPath}`)); + + if (upload) { + await publishRunReport(report); + } + + // Written and uploaded first, so the errors are readable and the comment says what happened — + // then fail, because a harness that measured nothing at all otherwise reports a passing job. + if (report.benchmarks.every((benchmark) => benchmark.error !== undefined)) { + throw new Error('No benchmark produced any measurement.'); + } + return report; + } finally { + // Always, including after a failure: an in-place run leaves the repository resolving the library + // to a tarball until this puts it back. + if (resolveMode === 'in-place') { + await restoreWorkspace(repoRoot, outputDir); + } + } +} diff --git a/packages/benchmark/src/tachometer/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts similarity index 52% rename from packages/benchmark/src/tachometer/runInterleaved.ts rename to packages/benchmark/src/runner/runInterleaved.ts index 9ea91791be..a495502f2a 100644 --- a/packages/benchmark/src/tachometer/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -3,22 +3,19 @@ import chalk from 'chalk'; import type { Browser, BrowserContext, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS } from '../launchArgs'; -import type { CaseComparison } from './discoverCases'; +import type { RunBenchmark, RunMetricDefinition } from '../runReport'; import type { BenchFile } from './benchFiles'; import type { ResolvedRef } from './refs'; import { serveDirectory } from './serveDirectory'; -import { shuffledIndices, stableMeasurements, toTachometerJson } from './pairedStats'; -import type { Round } from './pairedStats'; -import type { CaseRun, SummarizedCaseShape } from './summarizeCase'; // Types for `window.benchmarkPage`, which the evaluated functions below call into. import type {} from '../page/page'; /** - * The interleaved engine: runs the `benchmark()` cases of `*.bench.tsx` files in Playwright instead - * of tachometer. Every variant gets a page of its own, in a browser context of its own, that stays - * open for the whole case: every sample is one iteration, so module-scope data and the JIT stay - * warm, and interactions get trusted input through a CDP session the page is bridged to. Variants - * are sampled once per round in a shuffled order and compared on paired differences. + * Runs the `benchmark()` cases of `*.bench.tsx` files in Playwright. Every variant gets a page of + * its own, in a browser context of its own, that stays open for the whole case: every sample is one + * iteration, so module-scope data and the JIT stay warm, and interactions get trusted input through + * a CDP session the page is bridged to. Variants are sampled once per round in a shuffled order, so + * the samples are round-aligned and can be compared on paired differences. */ export interface RunInterleavedOptions { @@ -28,24 +25,53 @@ export interface RunInterleavedOptions { /** The builds a benchmark file compares: the working tree first, then the baseline. */ benchRefs: ResolvedRef[]; browserBinary: string; - asRoot: boolean; - /** Measured rounds per case. Defaults to 30. */ - samples?: number; - /** Discarded rounds before measuring, once per case. Defaults to 10. */ - warmup?: number; + launchArgs: string[]; + samples: number; + /** Discarded rounds before measuring, once per case. */ + warmup: number; +} + +export interface InterleavedResults { + benchmarks: RunBenchmark[]; + metrics: Record; + /** The browser as it reports itself, e.g. `151.0.7922.34`. */ + browserVersion: string; } const VIEWPORT = { width: 1920, height: 1080 }; -const DEFAULT_SAMPLES = 30; -const DEFAULT_WARMUP = 10; const SAMPLE_TIMEOUT_MS = 120_000; type CdpMethod = Parameters[0]; +/** One measured round: every variant's values by metric name, in slot order. */ +type Round = Array>; + function errorMessage(error: unknown): string { return error instanceof Error ? (error.stack ?? error.message) : String(error); } +/** A random permutation of `0..length-1`, so no variant always runs first in its round. */ +function shuffledIndices(length: number): number[] { + const indices = Array.from({ length }, (_, index) => index); + for (let index = indices.length - 1; index > 0; index -= 1) { + const swap = Math.floor(Math.random() * (index + 1)); + [indices[index], indices[swap]] = [indices[swap], indices[index]]; + } + return indices; +} + +/** + * The metrics every variant reported in every round. One some iteration did not produce — a + * per-phase render split that only sometimes has a second phase, say — cannot be paired round by + * round, so it is left out. + */ +function stableMetrics(rounds: Round[]): string[] { + const [first] = rounds; + return Object.keys(first?.[0] ?? {}).filter((metric) => + rounds.every((round) => round.every((values) => values[metric] !== undefined)), + ); +} + /** * A page a benchmark file's cases are measured in: one build of the file, or one variant of a * `compare()` in it. @@ -120,29 +146,37 @@ async function openBenchPages(browser: Browser, slots: PageSlot[]): Promise | undefined> { - await target.page.evaluate((args) => window.benchmarkPage!.sample(args.name, args.options), { - name, - options: { warmup }, - }); - if (warmup) { - return undefined; - } - return target.page.evaluate(() => { - const values: Record = {}; - for (const measure of performance.getEntriesByType('measure') as PerformanceMeasure[]) { - const detailValue: unknown = measure.detail?.value; - const value = typeof detailValue === 'number' ? detailValue : measure.duration; - // A metric recorded more than once in an iteration counts its total for that iteration. - values[measure.name] = (values[measure.name] ?? 0) + value; - } - return values; - }); +): Promise | null> { + return target.page.evaluate( + async (args) => { + await window.benchmarkPage!.sample(args.name, { warmup: args.warmup }); + if (args.warmup) { + return null; + } + const values: Record = {}; + for (const measure of performance.getEntriesByType('measure') as PerformanceMeasure[]) { + const detailValue: unknown = measure.detail?.value; + const value = typeof detailValue === 'number' ? detailValue : measure.duration; + // A metric recorded more than once in an iteration counts its total for that iteration. + values[measure.name] = (values[measure.name] ?? 0) + value; + } + return values; + }, + { name, warmup }, + ); +} + +interface CaseMeasurement { + rounds: Round[]; + metrics: Record; } /** @@ -155,40 +189,56 @@ async function measureBenchCase( slots: PageSlot[], name: string, options: RunInterleavedOptions, -): Promise { - const warmup = options.warmup ?? DEFAULT_WARMUP; - const samples = options.samples ?? DEFAULT_SAMPLES; - const rounds: Round[] = []; +): Promise { const targets = await openBenchPages(browser, slots); try { - for (let roundIndex = 0; roundIndex < warmup + samples; roundIndex += 1) { - const round: Round = []; + for (let roundIndex = 0; roundIndex < options.warmup; roundIndex += 1) { for (const index of shuffledIndices(targets.length)) { // eslint-disable-next-line no-await-in-loop - round[index] = await sampleBenchCase(targets[index], name, roundIndex < warmup); + await sampleBenchCase(targets[index], name, true); } - if (roundIndex >= warmup) { - rounds.push(round); + } + const rounds: Round[] = []; + for (let roundIndex = 0; roundIndex < options.samples; roundIndex += 1) { + const round: Round = []; + for (const index of shuffledIndices(targets.length)) { + // eslint-disable-next-line no-await-in-loop + round[index] = (await sampleBenchCase(targets[index], name, false)) ?? {}; } + rounds.push(round); } + // Every page defines the same harness metrics, and a custom metric in whichever case records it. + const metrics = Object.assign( + {}, + ...(await Promise.all( + targets.map((target) => + target.page.evaluate(() => window.benchmarkPage!.metricDefinitions()), + ), + )), + ); + return { rounds, metrics }; } finally { await closeBenchPages(targets); } - return rounds; } -function entryOf( - name: string, - comparison: CaseComparison, - slots: PageSlot[], - measurements: string[] = [], -): SummarizedCaseShape { - return { - name, - comparison, - variants: slots.map((slot) => `${name} [${slot.variant}]`), - measurements, - }; +/** The round-aligned samples of a case, per variant and metric. */ +function samplesOf(slots: PageSlot[], rounds: Round[]): RunBenchmark['samples'] { + const metrics = stableMetrics(rounds); + return Object.fromEntries( + slots.map((slot, index) => [ + slot.variant, + Object.fromEntries( + metrics.map((metric) => [metric, rounds.map((round) => round[index][metric])]), + ), + ]), + ); +} + +interface CaseGroup { + kind: RunBenchmark['kind']; + file: string; + nameOf: (caseName: string) => string; } /** @@ -199,41 +249,55 @@ function entryOf( async function runListedCases( browser: Browser, listed: OpenedPage[], - describe: { comparison: CaseComparison; nameOf: (caseName: string) => string; file: string }, + group: CaseGroup, options: RunInterleavedOptions, -): Promise { + metrics: Record, +): Promise { const slots = listed.map((target) => target.slot); + const variants = slots.map((slot) => slot.variant); const [reference, ...others] = listed; - const results: CaseRun[] = []; + const benchmarks: RunBenchmark[] = []; for (const caseName of reference.caseNames) { - const name = describe.nameOf(caseName); + const entry = { name: group.nameOf(caseName), file: group.file, kind: group.kind, variants }; const missing = others.find((target) => !target.caseNames.includes(caseName)); if (missing) { // A case added by the change under test, or one library lacks, has nothing to compare with. - results.push({ - entry: entryOf(name, describe.comparison, slots), + benchmarks.push({ + ...entry, error: `"${caseName}" does not exist in [${missing.slot.variant}].`, }); continue; } - console.log(chalk.cyan(`\nRunning "${name}" (${describe.file})…`)); + console.log(chalk.cyan(`\nRunning "${entry.name}" (${group.file})…`)); try { // eslint-disable-next-line no-await-in-loop - const rounds = await measureBenchCase(browser, slots, caseName, options); - // A measurement some iteration did not produce (an interaction whose render count varies, - // say) cannot be paired round by round, so only those every iteration reported are kept. - const entry = entryOf(name, describe.comparison, slots, stableMeasurements(rounds)); - results.push({ entry, json: toTachometerJson(entry.variants, entry.measurements, rounds) }); + const measured = await measureBenchCase(browser, slots, caseName, options); + Object.assign(metrics, measured.metrics); + benchmarks.push({ ...entry, samples: samplesOf(slots, measured.rounds) }); } catch (error) { console.error(chalk.red(` ${errorMessage(error)}`)); - results.push({ - entry: entryOf(name, describe.comparison, slots), - error: errorMessage(error), - }); + benchmarks.push({ ...entry, error: errorMessage(error) }); } } - return results; + return benchmarks; +} + +/** Opens a set of pages just to learn what they define, then runs their cases. */ +async function listAndRun( + browser: Browser, + slots: PageSlot[], + group: CaseGroup, + options: RunInterleavedOptions, + metrics: Record, +): Promise<{ benchmarks: RunBenchmark[]; comparisons: OpenedPage['comparisons'] }> { + // Every case then measures in fresh pages of its own. + const listed = await openBenchPages(browser, slots); + await closeBenchPages(listed); + return { + benchmarks: await runListedCases(browser, listed, group, options, metrics), + comparisons: listed[0].comparisons, + }; } function benchPageUrl(origin: string, ref: ResolvedRef, benchFile: BenchFile): URL { @@ -249,25 +313,22 @@ async function runBenchFile( origin: string, benchFile: BenchFile, options: RunInterleavedOptions, -): Promise { + metrics: Record, +): Promise { const buildSlots = options.benchRefs.map((ref): PageSlot => ({ variant: ref.kind === 'worktree' ? 'current' : 'baseline', url: benchPageUrl(origin, ref, benchFile).href, })); - // Opened once to learn which cases and comparisons each build defines; every case then measures - // in fresh pages of its own. - const listed = await openBenchPages(browser, buildSlots); - await closeBenchPages(listed); - - const results = await runListedCases( + const { benchmarks, comparisons } = await listAndRun( browser, - listed, - { comparison: 'baseline', nameOf: (caseName) => caseName, file: benchFile.file }, + buildSlots, + { kind: 'baseline', file: benchFile.file, nameOf: (caseName) => caseName }, options, + metrics, ); const [worktreeRef] = options.benchRefs; - for (const { name: comparison, variants } of listed[0].comparisons) { + for (const { name: comparison, variants } of comparisons) { const variantSlots = variants.map((variant): PageSlot => { const url = benchPageUrl(origin, worktreeRef, benchFile); url.searchParams.set('compare', comparison); @@ -276,49 +337,52 @@ async function runBenchFile( }); try { // eslint-disable-next-line no-await-in-loop - const variantPages = await openBenchPages(browser, variantSlots); - // eslint-disable-next-line no-await-in-loop - await closeBenchPages(variantPages); - results.push( - // eslint-disable-next-line no-await-in-loop - ...(await runListedCases( - browser, - variantPages, - { - comparison: 'variants', - nameOf: (caseName) => `${comparison} / ${caseName}`, - file: benchFile.file, - }, - options, - )), + const compared = await listAndRun( + browser, + variantSlots, + { + kind: 'compare', + file: benchFile.file, + nameOf: (caseName) => `${comparison} / ${caseName}`, + }, + options, + metrics, ); + benchmarks.push(...compared.benchmarks); } catch (error) { console.error(chalk.red(` ${errorMessage(error)}`)); - results.push({ entry: entryOf(comparison, 'variants', []), error: errorMessage(error) }); + benchmarks.push({ + name: comparison, + file: benchFile.file, + kind: 'compare', + variants, + error: errorMessage(error), + }); } } - return results; + return benchmarks; } -export async function runInterleaved(options: RunInterleavedOptions): Promise { +export async function runInterleaved(options: RunInterleavedOptions): Promise { const { chromium } = await import('@playwright/test'); const [server, browser] = await Promise.all([ serveDirectory(options.buildsDir), chromium.launch({ executablePath: options.browserBinary, headless: true, - args: [...BENCHMARK_LAUNCH_ARGS, ...(options.asRoot ? ['--no-sandbox'] : [])], + args: [...BENCHMARK_LAUNCH_ARGS, ...options.launchArgs], }), ]); try { - const results: CaseRun[] = []; + const benchmarks: RunBenchmark[] = []; + const metrics: Record = {}; // Sequential on purpose: concurrent cases would contend for the same machine. for (const benchFile of options.benchFiles) { // eslint-disable-next-line no-await-in-loop - results.push(...(await runBenchFile(browser, server.origin, benchFile, options))); + benchmarks.push(...(await runBenchFile(browser, server.origin, benchFile, options, metrics))); } - return results; + return { benchmarks, metrics, browserVersion: browser.version() }; } finally { await browser.close(); await server.close(); diff --git a/packages/benchmark/src/tachometer/serveDirectory.ts b/packages/benchmark/src/runner/serveDirectory.ts similarity index 100% rename from packages/benchmark/src/tachometer/serveDirectory.ts rename to packages/benchmark/src/runner/serveDirectory.ts diff --git a/packages/benchmark/src/runner/upload.ts b/packages/benchmark/src/runner/upload.ts new file mode 100644 index 0000000000..d2317c998c --- /dev/null +++ b/packages/benchmark/src/runner/upload.ts @@ -0,0 +1,60 @@ +/* eslint-disable no-console -- progress belongs in the CI log. */ + +import chalk from 'chalk'; +import { postToDashboard } from '../ciApi'; +import { ciReportUploadSchema, getCiMetadata } from '../ciReport'; +import { syncPrComment } from '../syncPrComment'; +import { benchmarkRunReportSchema } from '../runReport'; +import type { BenchmarkRunReport } from '../runReport'; + +export const BENCHMARK_RUN_REPORT_TYPE = 'benchmark-run'; + +const benchmarkRunUploadSchema = ciReportUploadSchema( + BENCHMARK_RUN_REPORT_TYPE, + 1, + benchmarkRunReportSchema, +); + +/** + * Uploads a report and refreshes the pull request comment. + * + * Validates before sending because the server stores `report` without inspecting it. A failure to + * refresh the comment does not fail the run: the numbers are already measured, written and + * uploaded, and losing them to a comment API hiccup would waste the whole run. + */ +export async function publishRunReport(report: BenchmarkRunReport): Promise { + const metadata = await getCiMetadata(); + if (!metadata.repo) { + console.warn( + chalk.yellow('Skipping upload: no repository detected, which usually means this is not CI.'), + ); + return; + } + + const upload = benchmarkRunUploadSchema.parse({ + version: 1, + reportType: BENCHMARK_RUN_REPORT_TYPE, + ...metadata, + report, + }); + const responseText = await postToDashboard( + '/api/ci-reports/upload', + upload, + 'The benchmark upload', + ); + console.log(`Benchmark report uploaded. S3 key: ${JSON.parse(responseText).key}`); + + try { + console.log('Syncing PR comment via the dashboard API…'); + const result = await syncPrComment(metadata.repo); + console.log( + result.skipped ? 'No open PR found for this branch, skipping.' : 'PR comment synced.', + ); + } catch (error) { + console.error( + chalk.yellow( + `Failed to sync the PR comment: ${error instanceof Error ? error.message : error}`, + ), + ); + } +} diff --git a/packages/benchmark/src/tachometer/browser.test.ts b/packages/benchmark/src/tachometer/browser.test.ts deleted file mode 100644 index 29f970ea1d..0000000000 --- a/packages/benchmark/src/tachometer/browser.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -import * as path from 'node:path'; -import { chmod, mkdir, writeFile } from 'node:fs/promises'; -import { describe, expect, it } from 'vitest'; -import { assertDriverMatchesBrowser, majorVersionOf, withBrowserDefaults } from './browser'; -import { makeTempDir } from '../utils/testUtils'; - -describe('majorVersionOf', () => { - it('reads the major from Chrome for Testing output', () => { - expect(majorVersionOf('Google Chrome for Testing 151.0.7922.34')).toBe(151); - }); - - it('reads the major from plain Chrome output', () => { - expect(majorVersionOf('Google Chrome 152.0.6367.60')).toBe(152); - }); - - it('reads the major from a bare package version', () => { - // chromedriver's package.json version, which is what the driver side is read from. - expect(majorVersionOf('151.0.5')).toBe(151); - }); - - it('returns undefined when there is no version to read', () => { - expect(majorVersionOf('')).toBeUndefined(); - expect(majorVersionOf('no version here')).toBeUndefined(); - }); - - it('ignores a leading number that is not part of a version', () => { - // Requires three dot-separated components, so a stray count cannot be mistaken for a version. - expect(majorVersionOf('2 browsers found: Chrome 151.0.7922.34')).toBe(151); - }); -}); - -describe('withBrowserDefaults', () => { - const BINARY = '/path/to/Chrome for Testing'; - - it('fills in the binary for a case that does not pin one', () => { - expect(withBrowserDefaults({ name: 'chrome', headless: true }, BINARY, false)).toEqual({ - name: 'chrome', - headless: true, - binary: BINARY, - }); - }); - - it('leaves a case that pins its own binary alone', () => { - const pinned = withBrowserDefaults({ binary: '/opt/chrome' }, BINARY, false); - - expect(pinned.binary).toBe('/opt/chrome'); - }); - - it('keeps the sandbox when the run is not root', () => { - // The sandbox is a real protection for the pages a benchmark loads; only root cannot have it. - expect(withBrowserDefaults({ name: 'chrome' }, BINARY, false)).not.toHaveProperty( - 'addArguments', - ); - }); - - it('disables the sandbox when the run is root', () => { - expect(withBrowserDefaults({ name: 'chrome' }, BINARY, true).addArguments).toEqual([ - '--no-sandbox', - ]); - }); - - it("keeps a case's own arguments when it disables the sandbox", () => { - // Cases pass flags that hold their measurements still; dropping them would move the numbers. - const merged = withBrowserDefaults( - { addArguments: ['--disable-renderer-backgrounding'] }, - BINARY, - true, - ); - - expect(merged.addArguments).toEqual(['--disable-renderer-backgrounding', '--no-sandbox']); - }); - - it('does not repeat an argument a case already passes', () => { - const merged = withBrowserDefaults({ addArguments: ['--no-sandbox'] }, BINARY, true); - - expect(merged.addArguments).toEqual(['--no-sandbox']); - }); - - it('names the browser for a case that declares none', () => { - // Tachometer defaults an absent `browser` to chrome, but validates the config against a schema - // where every browser variant requires `name` — so injecting the binary alone is rejected - // before a single sample is taken. - expect(withBrowserDefaults(undefined, BINARY, true)).toEqual({ - name: 'chrome', - binary: BINARY, - addArguments: ['--no-sandbox'], - }); - }); - - it('expands the string shorthand instead of spreading it into characters', () => { - expect(withBrowserDefaults('chrome-headless', BINARY, false)).toEqual({ - name: 'chrome', - headless: true, - binary: BINARY, - }); - }); - - it('keeps the remote url out of the browser name', () => { - expect(withBrowserDefaults('firefox@http://localhost:4444', BINARY, false)).toEqual({ - name: 'firefox', - headless: false, - remoteUrl: 'http://localhost:4444', - binary: BINARY, - }); - }); - - it('names the browser for a case whose object leaves it out', () => { - expect(withBrowserDefaults({ headless: true }, BINARY, false).name).toBe('chrome'); - }); -}); - -describe('assertDriverMatchesBrowser', () => { - /** - * A harness with tachometer installed, and optionally a chromedriver where tachometer can reach - * it — one directory up from its own, the way a workspace root is reachable in a real install. - * - * Real directories and real module resolution rather than a stand-in: what is being checked is - * which chromedriver node finds from tachometer, and faking that would assume the answer. - */ - async function makeHarness(driverVersion?: string): Promise { - const harnessDir = await makeTempDir(); - const modules = path.join(harnessDir, 'node_modules'); - - await writeFile(path.join(harnessDir, 'package.json'), JSON.stringify({ name: 'harness' })); - await mkdir(path.join(modules, 'tachometer'), { recursive: true }); - await writeFile( - path.join(modules, 'tachometer', 'package.json'), - JSON.stringify({ name: 'tachometer', version: '0.7.2' }), - ); - - if (driverVersion) { - await mkdir(path.join(modules, 'chromedriver'), { recursive: true }); - await writeFile( - path.join(modules, 'chromedriver', 'package.json'), - JSON.stringify({ name: 'chromedriver', version: driverVersion }), - ); - } - - return harnessDir; - } - - /** An executable standing in for the browser, reporting `version` to `--version`. */ - async function makeBrowser(version: string): Promise { - const dir = await makeTempDir(); - const binary = path.join(dir, 'chrome'); - await writeFile(binary, `#!/bin/sh\necho "Google Chrome for Testing ${version}"\n`); - await chmod(binary, 0o755); - return binary; - } - - it('passes when the driver and the browser share a major', async () => { - const harnessDir = await makeHarness('151.0.5'); - const binary = await makeBrowser('151.0.7922.34'); - - expect(() => assertDriverMatchesBrowser(harnessDir, binary)).not.toThrow(); - }); - - it('fails when the driver is on a different major', async () => { - const harnessDir = await makeHarness('150.0.3'); - const binary = await makeBrowser('151.0.7922.34'); - - expect(() => assertDriverMatchesBrowser(harnessDir, binary)).toThrow(/cannot drive Chrome/); - }); - - it('fails when no driver is reachable, naming the major to pin', async () => { - // Tachometer would install the newest chromedriver here, which does not drive a pinned Chrome. - const harnessDir = await makeHarness(); - const binary = await makeBrowser('151.0.7922.34'); - - expect(() => assertDriverMatchesBrowser(harnessDir, binary)).toThrow(/"chromedriver": "\^151"/); - }); -}); diff --git a/packages/benchmark/src/tachometer/browser.ts b/packages/benchmark/src/tachometer/browser.ts deleted file mode 100644 index 95e539eebc..0000000000 --- a/packages/benchmark/src/tachometer/browser.ts +++ /dev/null @@ -1,160 +0,0 @@ -import * as path from 'node:path'; -import { createRequire } from 'node:module'; -import { execFileSync } from 'node:child_process'; -import chalk from 'chalk'; -import type * as PlaywrightTest from '@playwright/test'; -import { pathExists } from '../utils/path'; - -/** - * Picking the browser to benchmark in, and checking it can actually be driven. - * - * Two separate programs are involved. The browser itself comes from Playwright, which downloads a - * fixed Chrome build. Tachometer needs `chromedriver` to drive that browser, and supplies it itself: - * it uses whichever one it can find next to itself, and downloads the newest if it finds none. - * - * A chromedriver only drives the Chrome version it was built for. Since Playwright's Chrome is - * deliberately a little behind the latest, letting tachometer fetch the newest driver usually gets a - * pair that cannot work together. - * - * So the consumer supplies it instead: a `chromedriver` devDependency at the workspace root, which - * is the only place tachometer resolves one from. Declared beside `@playwright/test`, so one place - * governs both versions and Renovate moves them together. All this module does is compare the - * version tachometer would resolve against the browser's own — which version that is stays the - * consumer's to choose. - */ - -/** - * Resolves the browser every case runs on: Playwright's pinned Chrome for Testing, rather than - * whatever Chrome the machine has auto-updated to. - * - * The path is machine-specific, so callers inject it into each config instead of committing it. - */ -export async function resolveBrowserBinary(harnessDir: string): Promise { - let playwright: typeof PlaywrightTest; - try { - playwright = await import('@playwright/test'); - } catch { - // An optional peer, so it installs cleanly when absent and only fails here. - throw new Error( - `"@playwright/test" is not installed. Add it as a devDependency of ${harnessDir}.`, - ); - } - - const binary = playwright.chromium.executablePath(); - if (!binary || !(await pathExists(binary))) { - throw new Error( - `Playwright's Chromium is not installed at "${binary}". Run \`pnpm exec playwright install chromium\`.`, - ); - } - return binary; -} - -/** - * The major version in output like `Google Chrome for Testing 151.0.7922.34`, or undefined when - * there is no version in it. - * - * At least three components are required, so a stray count in the same line is not read as one. - */ -export function majorVersionOf(versionOutput: string): number | undefined { - const match = /(\d+)\.\d+\.\d+/.exec(versionOutput); - return match ? Number(match[1]) : undefined; -} - -/** - * Throws when the chromedriver tachometer will use cannot drive `binary`. - */ -export function assertDriverMatchesBrowser(harnessDir: string, binary: string): void { - const browserMajor = majorVersionOf( - execFileSync(binary, ['--version'], { encoding: 'utf8' }).trim(), - ); - - // The same lookup tachometer does before launching: it uses whichever chromedriver resolves from - // its own package root, and installs the newest one when none does. - let driverVersion: string | undefined; - try { - const fromHarness = createRequire(path.join(harnessDir, 'package.json')); - const fromTachometer = createRequire(fromHarness.resolve('tachometer/package.json')); - driverVersion = fromTachometer('chromedriver/package.json').version; - } catch { - throw new Error( - `No chromedriver is reachable from tachometer, so it will install the newest one — which ` + - `will not drive Chrome for Testing ${browserMajor}, since Playwright pins a Chrome behind ` + - `the current release. Add "chromedriver": "^${browserMajor}" to the workspace root, where ` + - `tachometer can resolve it.`, - ); - } - - const driverMajor = majorVersionOf(driverVersion ?? ''); - if (browserMajor === undefined || driverMajor === undefined) { - console.warn( - chalk.yellow('Could not read the browser or chromedriver version; skipping the check.'), - ); - return; - } - if (browserMajor !== driverMajor) { - throw new Error( - `chromedriver ${driverVersion} cannot drive Chrome for Testing ${browserMajor}. ` + - `Update the "chromedriver" devDependency to ${browserMajor}.x, or align ` + - `"@playwright/test" with it — bump the two together.`, - ); - } -} - -/** - * A benchmark's `browser` config. Only the two fields a run touches are named; everything else - * tachometer accepts there (`name`, `headless`, `windowSize`, …) passes through untouched. - */ -export type BrowserConfig = { - binary?: string; - addArguments?: string[]; -} & Record; - -/** Chrome cannot enter its sandbox as root, and refusing to start is all it does about it. */ -const NO_SANDBOX = '--no-sandbox'; - -/** Tachometer's default when a benchmark names no browser. */ -const DEFAULT_BROWSER = 'chrome'; - -/** The suffix tachometer's browser shorthand uses to ask for a headless run. */ -const HEADLESS_SUFFIX = '-headless'; - -/** - * Whatever a case put in `browser` as the object form tachometer's schema requires: `name` is - * mandatory there and unknown keys are rejected, so the string shorthand - * `"[-headless][@]"` and an absent `browser` are both expanded here. - */ -function asBrowserObject(browser: string | BrowserConfig | undefined): BrowserConfig { - if (browser === undefined) { - return { name: DEFAULT_BROWSER }; - } - if (typeof browser !== 'string') { - // An object of its own may still leave out `name`, which the schema requires. - return { name: DEFAULT_BROWSER, ...browser }; - } - const at = browser.indexOf('@'); - const named = at === -1 ? browser : browser.slice(0, at); - const headless = named.endsWith(HEADLESS_SUFFIX); - const base: BrowserConfig = { - name: headless ? named.slice(0, -HEADLESS_SUFFIX.length) : named, - headless, - }; - return at === -1 ? base : { ...base, remoteUrl: browser.slice(at + 1) }; -} - -/** - * Merges what the run decides into a case's own `browser` config: the binary, which is - * machine-specific, and the sandbox flag, which depends on the user the run happens under. - */ -export function withBrowserDefaults( - browser: string | BrowserConfig | undefined, - binary: string, - asRoot: boolean, -): BrowserConfig { - const base = asBrowserObject(browser); - const merged: BrowserConfig = { ...base, binary: base.binary ?? binary }; - const addArguments = merged.addArguments ?? []; - if (asRoot && !addArguments.includes(NO_SANDBOX)) { - merged.addArguments = [...addArguments, NO_SANDBOX]; - } - return merged; -} diff --git a/packages/benchmark/src/tachometer/ciReport.test.ts b/packages/benchmark/src/tachometer/ciReport.test.ts deleted file mode 100644 index 0e10d03b22..0000000000 --- a/packages/benchmark/src/tachometer/ciReport.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { tachometerUploadSchema } from './ciReport'; - -/** - * A report of the shape `runTachometer` writes, trimmed to one case. - * - * @param {Partial} [overrides] - Fields to replace - * @returns {any} - */ -function report(overrides = {}) { - return { - version: 1, - reportType: 'tachometer', - generatedAt: '2026-09-02T00:00:00.000Z', - head: { ref: 'HEAD', sha: 'a'.repeat(40), branch: 'feature' }, - browser: '/path/to/Chrome for Testing', - refs: [ - { id: 'current', kind: 'worktree' }, - { id: 'git-abc123456', kind: 'git', requested: 'HEAD~1', sha: 'b'.repeat(40) }, - ], - cases: [ - { - name: 'workload', - comparison: 'baseline', - reference: 'workload [current]', - measurements: [ - { - name: 'mount', - variants: [ - { - variant: 'workload [current]', - meanMs: { low: 20.5, high: 22.7 }, - samples: 120, - }, - ], - comparisons: [ - { - variant: 'workload [baseline]', - verdict: 'unsure', - absoluteMs: { low: -1.2, high: 2.1 }, - percentChange: { low: -5.2, high: 9.3 }, - versusReference: { - verdict: 'unsure', - absoluteMs: { low: -2.1, high: 1.2 }, - percentChange: { low: -9.3, high: 5.2 }, - }, - }, - ], - }, - ], - }, - ], - raw: { workload: { benchmarks: [] } }, - ...overrides, - }; -} - -/** - * @param {Partial} [overrides] - Envelope fields to replace - * @returns {any} - */ -function envelope(overrides = {}) { - return { - version: 1, - timestamp: 1_770_000_000_000, - commitSha: 'a'.repeat(40), - repo: 'mui/mui-public', - reportType: 'tachometer', - prNumber: 1829, - branch: 'feature', - report: report(), - ...overrides, - }; -} - -describe('tachometerUploadSchema', () => { - it('accepts a report of the shape the runner writes', () => { - expect(tachometerUploadSchema.safeParse(envelope()).success).toBe(true); - }); - - it('carries the envelope metadata through', () => { - const parsed = tachometerUploadSchema.parse(envelope()); - - expect(parsed).toMatchObject({ - repo: 'mui/mui-public', - prNumber: 1829, - branch: 'feature', - reportType: 'tachometer', - }); - }); - - it('accepts a run with no pull request', () => { - // A push to the base branch uploads too; it just has no PR to comment on. - const { prNumber, ...withoutPr } = envelope(); - - expect(tachometerUploadSchema.safeParse(withoutPr).success).toBe(true); - }); - - it('accepts a case that failed to summarize', () => { - // Kept in the report rather than dropped, so the comment can say the case produced nothing. - const withFailure = envelope({ - report: report({ - cases: [{ name: 'broken', comparison: 'baseline', error: 'produced no benchmarks' }], - }), - }); - - expect(tachometerUploadSchema.safeParse(withFailure).success).toBe(true); - }); - - describe('rejects', () => { - it('a report from another benchmark axis', () => { - // The dashboard routes on reportType, so the wrong one would render as the wrong thing. - const wrongType = envelope({ reportType: 'benchmark' }); - - expect(tachometerUploadSchema.safeParse(wrongType).success).toBe(false); - }); - - it('a commit that is not a full sha', () => { - // The artifact key is built from it, and a short sha would not match what the dashboard - // looks up for the head commit. - expect(tachometerUploadSchema.safeParse(envelope({ commitSha: 'abc1234' })).success).toBe( - false, - ); - }); - - it('a repo that is not owner/name', () => { - expect(tachometerUploadSchema.safeParse(envelope({ repo: 'mui-public' })).success).toBe( - false, - ); - }); - - it('a verdict outside the known set', () => { - const badVerdict = report(); - badVerdict.cases[0].measurements[0].comparisons[0].verdict = 'regressed'; - - expect(tachometerUploadSchema.safeParse(envelope({ report: badVerdict })).success).toBe( - false, - ); - }); - }); -}); diff --git a/packages/benchmark/src/tachometer/ciReport.ts b/packages/benchmark/src/tachometer/ciReport.ts deleted file mode 100644 index 8fe6cb09d4..0000000000 --- a/packages/benchmark/src/tachometer/ciReport.ts +++ /dev/null @@ -1,124 +0,0 @@ -/* eslint-disable no-console -- progress belongs in the CI log. */ - -import { z } from 'zod/v4'; -import { postToDashboard } from '../ciApi'; -import { ciReportUploadSchema } from '../ciReport'; - -/** - * The tachometer axis's upload envelope. It differs from the Vitest one only in the shape of - * `report`; the transport, the CI metadata and the pull request sync are shared. - */ - -const confidenceIntervalSchema = z.object({ - low: z.number(), - high: z.number(), -}); - -const differenceSchema = z.object({ - verdict: z.enum(['faster', 'slower', 'unsure']), - absoluteMs: confidenceIntervalSchema, - percentChange: confidenceIntervalSchema, -}); - -const variantResultSchema = z.object({ - variant: z.string(), - meanMs: confidenceIntervalSchema, - samples: z.number(), -}); - -const comparisonSchema = differenceSchema.extend({ - variant: z.string(), - versusReference: differenceSchema.optional(), -}); - -const measurementResultSchema = z.object({ - name: z.string(), - variants: z.array(variantResultSchema), - comparisons: z.array(comparisonSchema), -}); - -// A case that failed to summarize carries an `error` and no measurements. It is kept in the report -// rather than dropped, so the comment can say a case produced nothing instead of quietly omitting it. -const caseResultSchema = z.object({ - name: z.string(), - // `baseline` compares the working tree against the baseline build, so a difference there is a - // regression; `variants` compares pages built from the same tree, where a difference is the - // point. Carried per case because that is where the two are told apart. - comparison: z.enum(['baseline', 'variants']), - reference: z.string().optional(), - measurements: z.array(measurementResultSchema).optional(), - error: z.string().optional(), -}); - -/** - * A build the pages were loaded from. The working tree has no commit behind it and the baseline - * always does, so the two carry different fields. - * - * `requested` is the revision as the run was given it — `HEAD~1`, a tag, a SHA — kept instead of a - * ready-made display string so a renderer decides how it reads. - */ -const refSchema = z.discriminatedUnion('kind', [ - z.object({ kind: z.literal('worktree'), id: z.string() }), - z.object({ - kind: z.literal('git'), - id: z.string(), - sha: z.string(), - requested: z.string(), - }), -]); - -const tachometerReportV1Schema = z.object({ - version: z.literal(1), - reportType: z.literal('tachometer'), - generatedAt: z.string(), - head: z.object({ ref: z.string(), sha: z.string(), branch: z.string().optional() }), - browser: z.string().optional(), - refs: z.array(refSchema), - cases: z.array(caseResultSchema), - // Every sample tachometer produced. Not read by the comment, but kept so a richer view can be - // built later without changing what CI uploads. - raw: z.record(z.string(), z.unknown()), -}); - -/** - * Every report shape the store may hold, keyed on `version`. - * - * Reports written months ago by an older version of this tooling are still in S3 and still get - * rendered. A new shape joins this union with its own literal version rather than replacing the old - * one, so a reader narrows on `version` and the compiler makes it handle each. - */ -export const tachometerReportSchema = z.discriminatedUnion('version', [tachometerReportV1Schema]); - -/** - * The upload envelope. Built from the same factory the Vitest axis uses, so both report types reach - * the dashboard's upload route in the shape it expects. - */ -export const tachometerUploadSchema = ciReportUploadSchema('tachometer', 1, tachometerReportSchema); - -export type TachometerUpload = z.infer; -export type TachometerReport = z.infer; -export type ReportRef = z.infer; -export type CaseResult = z.infer; -export type ConfidenceInterval = z.infer; -export type Verdict = z.infer['verdict']; -export type MeasurementResult = z.infer; -export type VariantResult = z.infer; -export type Comparison = z.infer; - -/** - * Uploads a report to the CI report store. - * - * Validates before sending because the server stores `report` without inspecting it — a malformed - * report would be accepted here and only fail much later, when the dashboard tried to render it. - */ -export async function uploadCiReport(upload: TachometerUpload): Promise { - tachometerUploadSchema.parse(upload); - - const responseText = await postToDashboard( - '/api/ci-reports/upload', - upload, - 'The tachometer upload', - ); - const result = JSON.parse(responseText); - console.log(`Tachometer report uploaded. S3 key: ${result.key}`); -} diff --git a/packages/benchmark/src/tachometer/discoverCases.test.ts b/packages/benchmark/src/tachometer/discoverCases.test.ts deleted file mode 100644 index 08d3285563..0000000000 --- a/packages/benchmark/src/tachometer/discoverCases.test.ts +++ /dev/null @@ -1,577 +0,0 @@ -import * as path from 'node:path'; -import { mkdir, writeFile } from 'node:fs/promises'; -import { describe, expect, it } from 'vitest'; -import { makeTempDir } from '../utils/testUtils'; -import { discoverCases, measurementNameOf, pagesOf } from './discoverCases'; - -/** A case folder to create in a throwaway harness. */ -interface CaseFixture { - config: any; - pages?: string[]; -} - -/** Builds a throwaway harness: `src//tachometer.json` plus whatever pages each case owns. */ -async function makeHarness(cases: Record): Promise { - const harnessDir = await makeTempDir(); - await Promise.all( - Object.entries(cases).map(async ([name, { config, pages = [] }]) => { - const caseDir = path.join(harnessDir, 'src', name); - await mkdir(caseDir, { recursive: true }); - await writeFile(path.join(caseDir, 'tachometer.json'), JSON.stringify(config, null, 2)); - await Promise.all( - pages.map((page) => writeFile(path.join(caseDir, page), '\n')), - ); - }), - ); - return harnessDir; -} - -/** A minimal single-benchmark config. */ -function config(name: string, url: string): any { - return { - $schema: '../../node_modules/tachometer/config.schema.json', - benchmarks: [{ name, url }], - }; -} - -describe('discoverCases', () => { - it('finds every case folder holding a tachometer.json', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - beta: { config: config('beta', './index.html'), pages: ['index.html'] }, - }); - - const cases = await discoverCases({ harnessDir }); - - expect(cases.map((entry) => entry.name)).toEqual(['alpha', 'beta']); - }); - - it('narrows to cases matching a filter', async () => { - const harnessDir = await makeHarness({ - 'grid-init': { config: config('grid-init', './index.html'), pages: ['index.html'] }, - 'grid-scroll': { config: config('grid-scroll', './index.html'), pages: ['index.html'] }, - }); - - const cases = await discoverCases({ harnessDir, filters: ['scroll'] }); - - expect(cases.map((entry) => entry.name)).toEqual(['grid-scroll']); - }); - - it('finds a case nested in a folder', async () => { - const harnessDir = await makeHarness({ - 'libs/mount': { config: config('libs-mount', './index.html'), pages: ['index.html'] }, - }); - - const cases = await discoverCases({ harnessDir }); - - expect(cases.map((entry) => entry.name)).toEqual(['libs-mount']); - }); - - it('takes a case name from its benchmark rather than its folder', async () => { - // The folder is where a case lives; the config is what it calls itself. Keeping them separate - // is what lets a case move between folders without being renamed in every report. - const harnessDir = await makeHarness({ - 'libs/mount': { config: config('libs-mount', './index.html'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.name).toBe('libs-mount'); - }); - - it('selects a whole folder by filtering on the path', async () => { - const harnessDir = await makeHarness({ - 'libs/mount': { config: config('libs-mount', './index.html'), pages: ['index.html'] }, - 'libs/scroll': { config: config('libs-scroll', './index.html'), pages: ['index.html'] }, - regression: { config: config('regression', './index.html'), pages: ['index.html'] }, - }); - - const cases = await discoverCases({ harnessDir, filters: ['libs'] }); - - expect(cases.map((entry) => entry.name)).toEqual(['libs-mount', 'libs-scroll']); - }); - - it('matches a filter case-insensitively', async () => { - const harnessDir = await makeHarness({ - GridScroll: { config: config('grid-scroll', './index.html'), pages: ['index.html'] }, - }); - - const cases = await discoverCases({ harnessDir, filters: ['gridscroll'] }); - - expect(cases.map((entry) => entry.name)).toEqual(['grid-scroll']); - }); - - it('refuses two cases that call themselves the same thing', async () => { - // The name identifies a case in the report and in the file its results are written to, so a - // duplicate would silently overwrite rather than fail. - const harnessDir = await makeHarness({ - 'libs/scroll': { config: config('scroll', './index.html'), pages: ['index.html'] }, - scroll: { config: config('scroll', './index.html'), pages: ['index.html'] }, - }); - - await expect(discoverCases({ harnessDir })).rejects.toThrow( - /Two benchmark cases are both named "scroll"/, - ); - }); - - it('strips $schema, which tachometer rejects in a config it is handed', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.config.$schema).toBeUndefined(); - }); - - it('throws when no case matches', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - await expect(discoverCases({ harnessDir, filters: ['nope'] })).rejects.toThrow( - /No benchmark case .* matches "nope"/, - ); - }); - - describe('auto-expansion', () => { - it('expands a benchmark with no variants into current versus baseline', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.config.benchmarks).toEqual([ - { name: 'alpha [current]', url: './index.html' }, - { name: 'alpha [baseline]', url: './index.html' }, - ]); - }); - - it('points the two variants at the two builds', async () => { - // The temp harness is no repository at all, so anything reaching for git would fail here. - // Which build a variant loads follows from the case's shape, which is what lets a plain - // `vite build` run in a checkout with no history. - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.comparison).toBe('baseline'); - expect(entry.leaves.map((leaf) => leaf.ref)).toEqual(['current', 'baseline']); - }); - - it('hands tachometer benchmarks it cannot expand again', async () => { - // The config is parsed afresh, so a benchmark that still carried `expand` would be expanded a - // second time — and an un-rewritten parent url would point at a page that was never built. - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - for (const benchmark of entry.config.benchmarks) { - expect(benchmark).not.toHaveProperty('expand'); - } - }); - - it('gives both variants the same page, query and fragment', async () => { - // Everything after the page path belongs to the page. A parameterised case that lost it on one - // side would compare the parameters against the defaults and call the difference a regression. - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html?rows=100#state'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - const target = 'alpha/index.html?rows=100#state'; - - expect(entry.leaves.map((leaf) => `${leaf.page}${leaf.suffix}`)).toEqual([target, target]); - }); - - it('merges an expansion over its parent, nearest value winning', async () => { - // Tachometer merges an expansion over its parent before it reads anything, so a field set - // deeper wins and one set only on the parent is inherited. Flattening here has to match, or - // the config handed over would describe different benchmarks than the ones discovered. - const harnessDir = await makeHarness({ - libs: { - config: { - benchmarks: [ - { - name: 'libs', - measurement: { mode: 'performance', entryName: 'mount' }, - expand: [{ name: 'libs [ours]', url: './ours.html' }, { url: './theirs.html' }], - }, - ], - }, - pages: ['ours.html', 'theirs.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.config.benchmarks).toEqual([ - { - name: 'libs [ours]', - url: './ours.html', - measurement: { mode: 'performance', entryName: 'mount' }, - }, - { - name: 'libs', - url: './theirs.html', - measurement: { mode: 'performance', entryName: 'mount' }, - }, - ]); - }); - - it('leaves a benchmark that declares its own variants alone', async () => { - const harnessDir = await makeHarness({ - libs: { - config: { - benchmarks: [ - { - name: 'libs', - expand: [ - { name: 'libs [ours]', url: './ours.html' }, - { name: 'libs [theirs]', url: './theirs.html' }, - ], - }, - ], - }, - pages: ['ours.html', 'theirs.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.leaves.map((leaf) => leaf.page)).toEqual(['libs/ours.html', 'libs/theirs.html']); - // Its own variants, so they are compared with each other and all load the working tree. - expect(entry.comparison).toBe('variants'); - expect(entry.leaves.map((leaf) => leaf.ref)).toEqual(['current', 'current']); - }); - }); - - describe('leaf urls', () => { - it('inherits a parent url through a nested expand', async () => { - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [ - { - name: 'alpha', - url: './index.html', - expand: [ - { name: 'alpha [one]', expand: [{ name: 'alpha [one/a]' }] }, - { name: 'alpha [two]', url: './other.html' }, - ], - }, - ], - }, - pages: ['index.html', 'other.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.leaves.map((leaf) => leaf.page)).toEqual([ - 'alpha/index.html', - 'alpha/other.html', - ]); - }); - - it('carries the query and the fragment through as written', async () => { - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [ - { name: 'alpha', expand: [{ name: 'a', url: './index.html?rows=100&cols=8#x' }] }, - ], - }, - pages: ['index.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.leaves[0].suffix).toBe('?rows=100&cols=8#x'); - }); - - it('resolves a case that owns no page but parameterises a sibling', async () => { - const harnessDir = await makeHarness({ - updates: { config: config('updates', './index.html'), pages: ['index.html'] }, - 'updates-throttled': { - config: { - benchmarks: [ - { - name: 'updates-throttled', - expand: [{ name: 'throttled', url: '../updates/index.html?throttle=16' }], - }, - ], - }, - }, - }); - - const cases = await discoverCases({ harnessDir }); - const throttled = cases.find((entry) => entry.name === 'updates-throttled'); - - expect(throttled?.leaves[0]).toMatchObject({ - page: 'updates/index.html', - suffix: '?throttle=16', - }); - }); - - it('rejects a url resolving outside src', async () => { - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [{ name: 'alpha', expand: [{ name: 'a', url: '../../outside.html' }] }], - }, - }, - }); - - await expect(discoverCases({ harnessDir })).rejects.toThrow(/resolves outside/); - }); - - it('rejects a benchmark that names no url', async () => { - // Named for the benchmark the author wrote rather than for a variant they did not, which is - // what the auto-expanded pair would otherwise be reported as. - const harnessDir = await makeHarness({ - alpha: { config: { benchmarks: [{ name: 'alpha' }] }, pages: ['index.html'] }, - }); - - await expect(discoverCases({ harnessDir })).rejects.toThrow( - /Benchmark "alpha" in .* has no "url"/, - ); - }); - - it('rejects a url pointing at a missing page', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './missing.html') }, - }); - - await expect(discoverCases({ harnessDir })).rejects.toThrow(/missing page/); - }); - }); -}); - -describe('pagesOf', () => { - it('deduplicates the pages a case references from several variants', async () => { - // The auto-expanded [current] and [baseline] variants point at the same page. - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - expect(pagesOf(await discoverCases({ harnessDir }))).toEqual(['alpha/index.html']); - }); - - it('unions the pages of every case, sorted', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - libs: { - config: { - benchmarks: [ - { - name: 'libs', - expand: [ - { name: 'libs [ours]', url: './ours.html' }, - { name: 'libs [theirs]', url: './theirs.html' }, - ], - }, - ], - }, - pages: ['ours.html', 'theirs.html'], - }, - }); - - expect(pagesOf(await discoverCases({ harnessDir }))).toEqual([ - 'alpha/index.html', - 'libs/ours.html', - 'libs/theirs.html', - ]); - }); -}); - -describe('measurementNameOf', () => { - it('uses an explicit name', () => { - expect(measurementNameOf({ mode: 'performance', entryName: 'mount', name: 'paint' })).toBe( - 'paint', - ); - }); - - it('uses the expression for an expression measurement', () => { - expect(measurementNameOf({ mode: 'expression', expression: 'window.total' })).toBe( - 'window.total', - ); - }); - - it('names a callback measurement', () => { - expect(measurementNameOf({ mode: 'callback' })).toBe('callback'); - expect(measurementNameOf('callback')).toBe('callback'); - }); - - it('names the "global" shorthand by the expression tachometer expands it to', () => { - // `global` becomes an expression measurement, and a result is named by the expression — not by - // the shorthand — so taking the shorthand at face value would never match its own result. - expect(measurementNameOf('global')).toBe('window.tachometerResult'); - expect(measurementNameOf('global', 'window.myResult')).toBe('window.myResult'); - }); - - it('falls back to the entry name', () => { - expect(measurementNameOf({ mode: 'performance', entryName: 'mount' })).toBe('mount'); - }); - - it('abbreviates first-contentful-paint the way tachometer does', () => { - expect(measurementNameOf({ mode: 'performance', entryName: 'first-contentful-paint' })).toBe( - 'fcp', - ); - }); -}); - -describe('case variants and measurements', () => { - it('lists the auto-expanded variants in order, the reference first', async () => { - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [ - { - name: 'alpha', - url: './index.html', - measurement: { mode: 'performance', entryName: 'mount' }, - }, - ], - }, - pages: ['index.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.variants).toEqual(['alpha [current]', 'alpha [baseline]']); - expect(entry.measurements).toEqual(['mount']); - }); - - it('falls back to the case name when a benchmark names neither itself nor its variants', async () => { - // The case name is already the fallback for a nameless benchmark, so the variants read - // "alpha [current]" rather than "undefined [current]". - const harnessDir = await makeHarness({ - alpha: { - config: { benchmarks: [{ url: './index.html' }] }, - pages: ['index.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.variants).toEqual(['alpha [current]', 'alpha [baseline]']); - }); - - it('takes a variant name from the nearest node that sets one', async () => { - // Tachometer merges an expansion over its parent, so a name set partway down the tree is what - // it reports — reading only the leaf and the benchmark would look up a name it never used. - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [ - { - name: 'alpha', - expand: [ - { name: 'alpha [ours]', expand: [{ url: './index.html' }] }, - { name: 'alpha [theirs]', expand: [{ url: './other.html' }] }, - ], - }, - ], - }, - pages: ['index.html', 'other.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.variants).toEqual(['alpha [ours]', 'alpha [theirs]']); - }); - - it('collects every measurement a benchmark declares', async () => { - // A page with several measurements is exactly the case whose results must be paired by name. - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [ - { - name: 'alpha', - url: './index.html', - measurement: [ - { mode: 'performance', entryName: 'mount' }, - { mode: 'performance', entryName: 'scroll' }, - ], - }, - ], - }, - pages: ['index.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.measurements).toEqual(['mount', 'scroll']); - }); - - it('collects the measurements an expansion declares for itself', async () => { - // Read from the parent, a case whose children each measure something different reports the - // parent's default while tachometer reports theirs, and pairing the two then finds nothing. - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [ - { - name: 'alpha', - expand: [ - { - name: 'alpha [one]', - url: './index.html', - measurement: { mode: 'performance', entryName: 'mount' }, - }, - { - name: 'alpha [two]', - url: './other.html', - measurement: { mode: 'performance', entryName: 'paint' }, - }, - ], - }, - ], - }, - pages: ['index.html', 'other.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.measurements).toEqual(['mount', 'paint']); - }); - - it('refuses two variants that end up with the same name', async () => { - // An expansion that names itself nothing inherits its parent's name, so this is reachable from - // a config that looks fine. Results are paired back by name, so one would shadow the other. - const harnessDir = await makeHarness({ - libs: { - config: { - benchmarks: [ - { name: 'libs', expand: [{ url: './ours.html' }, { url: './theirs.html' }] }, - ], - }, - pages: ['ours.html', 'theirs.html'], - }, - }); - - await expect(discoverCases({ harnessDir })).rejects.toThrow(/has two variants named "libs"/); - }); - - it('defaults to the callback measurement when none is declared', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.measurements).toEqual(['callback']); - }); -}); diff --git a/packages/benchmark/src/tachometer/discoverCases.ts b/packages/benchmark/src/tachometer/discoverCases.ts deleted file mode 100644 index 9dd6e15b8b..0000000000 --- a/packages/benchmark/src/tachometer/discoverCases.ts +++ /dev/null @@ -1,333 +0,0 @@ -import * as path from 'node:path'; -import { readFile } from 'node:fs/promises'; -import { globby } from 'globby'; -import { pathExists } from '../utils/path'; - -/** - * Case discovery: the `tachometer.json` files under a harness's `src/` are the source of truth for - * which benchmark cases exist and which pages they reference. - * - * Both the runner and the vite plugin read them through here, so there is exactly one notion of - * "what pages exist". Discovery reads only the filesystem — which builds a case loads follows from - * its shape rather than from git, so a plain `vite build` needs no checkout with history. - */ - -/** Which of a run's two builds a variant loads. */ -export type BuildRef = 'current' | 'baseline'; - -/** - * What a case compares. - * - * `baseline` is the working tree against the baseline build: the regression case, and what a case - * gets when it declares no variants of its own. `variants` is the pages the case names compared - * against each other, every one of them built from the working tree. - */ -export type CaseComparison = 'baseline' | 'variants'; - -export interface Leaf { - /** The config node whose `url` this is; rewritten in place once builds exist. */ - node: { url?: string }; - /** Page path relative to `src`, posix, e.g. `data-grid-init/index.html`. */ - page: string; - /** The build this variant loads. */ - ref: BuildRef; - /** Everything after the page path: query and fragment, as written. */ - suffix: string; -} - -export interface BenchmarkCase { - /** The case name, as its benchmark declares it. */ - name: string; - /** Absolute path to its `tachometer.json`. */ - configPath: string; - /** The parsed config, mutated in place as urls are rewritten. */ - config: any; - /** What the case compares. */ - comparison: CaseComparison; - /** Every node that selects a page. */ - leaves: Leaf[]; - /** Variant names in order, as tachometer will report them; the first is the reference. */ - variants: string[]; - /** Measurement names, as tachometer will name them in its output. */ - measurements: string[]; -} - -export interface DiscoverCasesOptions { - /** The harness package directory; cases live under its `src/`. */ - harnessDir: string; - /** - * Only include cases whose path under `src` contains one of these substrings, - * case-insensitively. - */ - filters?: string[]; - /** - * Return an empty list instead of throwing when no case is selected, for callers that also run - * other kinds of cases and decide for themselves whether finding nothing is an error. - */ - allowEmpty?: boolean; -} - -/** The file that marks a directory under `src/` as a benchmark case. */ -const CONFIG_NAME = 'tachometer.json'; - -/** Tachometer's expression when a `global` measurement does not name one. */ -const DEFAULT_MEASUREMENT_EXPRESSION = 'window.tachometerResult'; - -/** - * The name tachometer gives a measurement: an explicit `name`, else the expression, else the entry - * name. - * - * With more than one measurement on a page, tachometer appends ` []` to each benchmark's - * name, and results have to be paired back by that name — pairing by position would compare - * unrelated measurements. - */ -export function measurementNameOf(measurement: any, measurementExpression?: string): string { - if (typeof measurement === 'string') { - // Tachometer's string shorthands. `callback` and `fcp` name their results the same way the - // shorthand reads, but `global` expands to an *expression* measurement that is named by the - // expression itself — so taking the shorthand at face value would look for a result that is - // never emitted under that name. - return measurement === 'global' - ? (measurementExpression ?? DEFAULT_MEASUREMENT_EXPRESSION) - : measurement; - } - if (measurement.name) { - return measurement.name; - } - if (measurement.mode === 'expression') { - return measurement.expression; - } - if (measurement.mode === 'callback') { - return 'callback'; - } - return measurement.entryName === 'first-contentful-paint' ? 'fcp' : measurement.entryName; -} - -/** - * Flattens a benchmark's `expand` tree into the list of benchmarks it stands for, the way tachometer - * does before it reads anything else. - * - * `expand` is recursive and an expansion is merged over its parent, so the nearest value of every - * field wins — `url`, `name`, `measurement`, `browser` alike. Doing that merge once, here, is what - * keeps the rest of discovery dealing in whole benchmarks: per-field inheritance is a rule that only - * has value while it agrees with tachometer's exactly, and re-deriving it once per field is how the - * two drift apart. - * - * `expand` itself is dropped. Tachometer leaves the parent's on the merged object, where it is inert - * because expansion has already happened — but the config written from this is parsed afresh, and a - * benchmark still carrying `expand` would be expanded a second time. - */ -function flattenExpansions(benchmark: any): any[] { - const { expand, ...rest } = benchmark; - if (!Array.isArray(expand) || expand.length === 0) { - return [rest]; - } - return expand.flatMap((child) => flattenExpansions(child).map((leaf) => ({ ...rest, ...leaf }))); -} - -/** - * Resolves one variant's url into the source page it references and whatever follows it. - * - * The path is carried through as written rather than through `URL`: these urls are relative to their - * own config's directory and traverse out of it (`../row-updates/index.html?throttle=16`), and `URL` - * would clamp that traversal against whatever base it was given. Everything after the path belongs - * to the page and survives the rewrite verbatim, or a parameterised benchmark would silently run its - * defaults. - */ -async function parseLeafUrl( - url: string, - configDir: string, - srcDir: string, -): Promise<{ page: string; suffix: string }> { - const [pathPart] = url.split(/[?#]/); - const suffix = url.slice(pathPart.length); - const absolute = path.resolve(configDir, decodeURIComponent(pathPart)); - - const relative = path.relative(srcDir, absolute); - if (relative.startsWith('..') || path.isAbsolute(relative)) { - throw new Error(`Benchmark url "${url}" resolves outside ${srcDir}.`); - } - if (!(await pathExists(absolute))) { - throw new Error(`Benchmark url "${url}" points at a missing page (${absolute}).`); - } - // Posix, because a page path is a url as often as it is a path — the case index links to it and - // rollup names a chunk after it — and node takes forward slashes on Windows for the rest. - return { page: relative.split(path.sep).join('/'), suffix }; -} - -/** Whether a location under `src` is selected by the run's filters: all of them when there are none. */ -export function matchesFilters(location: string, filters: string[]): boolean { - return ( - filters.length === 0 || - filters.some((filter) => location.toLowerCase().includes(filter.toLowerCase())) - ); -} - -/** - * Every directory under `srcDir` that holds a `tachometer.json`, as a posix path relative to it. - * - * Nested rather than flat, so a harness can group its cases in folders — which is the whole of the - * partitioning story: the filter matches these paths, and nothing else has to understand what a - * folder means. - */ -async function findCaseLocations(srcDir: string): Promise { - const configs = await globby(`**/${CONFIG_NAME}`, { cwd: srcDir }); - return ( - configs - // `dirname` gives `.` for a config sitting directly in `src`; that case's location is the root. - .map((config) => (path.posix.dirname(config) === '.' ? '' : path.posix.dirname(config))) - // Sorted so a run's case order — and therefore its report — does not depend on the order the - // filesystem happened to return. - .sort() - ); -} - -/** - * Reads every `tachometer.json` under `src/` (optionally filtered by location) and resolves the page - * each of its variants loads. - * - * `expand` decides what a case compares. A benchmark that declares one owns its variant axis, and - * every variant loads the working tree — a comparison between pages, such as one library against - * another. A benchmark without one is expanded into `[current]` and `[baseline]` over the same page, - * which is the regression case. - */ -export async function discoverCases(options: DiscoverCasesOptions): Promise { - const { harnessDir, filters = [], allowEmpty = false } = options; - const srcDir = path.join(harnessDir, 'src'); - - const located = await findCaseLocations(srcDir); - // Matched case-insensitively against the location, the way vitest matches its file filters: a - // folder name selects everything under it without the filter having to know what a folder means. - const selected = located.filter((location) => matchesFilters(location, filters)); - - if (selected.length === 0) { - if (allowEmpty) { - return []; - } - throw new Error( - filters.length > 0 - ? `No benchmark case under ${srcDir} matches ${filters.map((filter) => `"${filter}"`).join(', ')}.` - : `No benchmark case with a ${CONFIG_NAME} found under ${srcDir}.`, - ); - } - - const configs = await Promise.all( - selected.map(async (location) => { - const configPath = path.join(srcDir, location, CONFIG_NAME); - const config = JSON.parse(await readFile(configPath, 'utf8')); - // `$schema` is for editors; tachometer rejects it in a config it is handed. - delete config.$schema; - // The benchmark names itself, so a case keeps its identity wherever its folder is moved to, - // and the ` []` prefix that tachometer builds from that name is strippable for - // display by construction rather than by the folder happening to agree with it. - const name: string = config.benchmarks?.[0]?.name ?? path.basename(location); - return { name, configPath, config }; - }), - ); - - const byName = new Map(); - for (const entry of configs) { - const clash = byName.get(entry.name); - if (clash) { - throw new Error( - `Two benchmark cases are both named "${entry.name}": ${clash} and ${entry.configPath}. ` + - `Names identify a case in the report and in its results file, so they have to be unique.`, - ); - } - byName.set(entry.name, entry.configPath); - } - - const cases: BenchmarkCase[] = []; - for (const { name, configPath, config } of configs) { - const configDir = path.dirname(configPath); - - const measurements = new Set(); - const nodes: Array<{ node: any; ref: BuildRef }> = []; - for (const benchmark of config.benchmarks ?? []) { - // A benchmark need not name itself, and the case name is the same fallback its own name gets - // — without it the auto-expanded variants would read "undefined [current]". Set before - // flattening so every benchmark it expands to inherits it. - benchmark.name ??= name; - - const declaresVariants = Array.isArray(benchmark.expand) && benchmark.expand.length > 0; - const expanded = flattenExpansions(benchmark); - - // Read from the flattened nodes, not the parent: an expansion may set its own `measurement`, - // and a parent that names none would otherwise contribute `callback` while tachometer reports - // what the child asked for. `measurement` may also be a single entry or a list, and a page - // with several is exactly the case whose results are paired by name rather than by position. - for (const node of expanded) { - for (const measurement of [node.measurement ?? 'callback'].flat()) { - measurements.add(measurementNameOf(measurement, node.measurementExpression)); - } - } - if (declaresVariants) { - for (const node of expanded) { - nodes.push({ node, ref: 'current' }); - } - } else { - // Without `expand` exactly one benchmark comes back, and the same page is measured twice. - const [base] = expanded; - if (base.url === undefined) { - throw new Error(`Benchmark "${benchmark.name}" in ${configPath} has no "url".`); - } - nodes.push( - { node: { ...base, name: `${benchmark.name} [current]` }, ref: 'current' }, - { node: { ...base, name: `${benchmark.name} [baseline]` }, ref: 'baseline' }, - ); - } - } - - // A declared `expand` tree may bottom out in a variant that names no url and inherits none. - for (const { node } of nodes) { - if (node.url === undefined) { - throw new Error( - `Benchmark "${node.name}" in ${configPath} expands to a variant with no "url".`, - ); - } - } - - // Results are paired back to what was declared by variant name, and an expansion that sets no - // name of its own inherits its parent's — so two variants can end up sharing one, which would - // quietly report the last one's numbers for both. - const named = new Set(); - for (const { node } of nodes) { - if (named.has(node.name)) { - throw new Error( - `Benchmark case "${name}" has two variants named "${node.name}" (${configPath}). ` + - `Names identify a variant in the report and in its results, so they have to be unique.`, - ); - } - named.add(node.name); - } - - // The flattened list is what tachometer is handed: it expands to exactly this, and every later - // step — the url rewrite, the browser defaults — then addresses one whole benchmark at a time. - config.benchmarks = nodes.map((entry) => entry.node); - - // eslint-disable-next-line no-await-in-loop - const leaves: Leaf[] = await Promise.all( - nodes.map(async ({ node, ref }) => ({ - node, - ref, - ...(await parseLeafUrl(node.url, configDir, srcDir)), - })), - ); - - cases.push({ - name, - configPath, - config, - comparison: leaves.some((leaf) => leaf.ref === 'baseline') ? 'baseline' : 'variants', - leaves, - variants: nodes.map((entry) => entry.node.name as string), - measurements: [...measurements], - }); - } - return cases; -} - -/** The distinct pages referenced by a set of cases, as paths relative to `src`, sorted. */ -export function pagesOf(cases: BenchmarkCase[]): string[] { - return [...new Set(cases.flatMap((entry) => entry.leaves.map((leaf) => leaf.page)))].sort(); -} diff --git a/packages/benchmark/src/tachometer/format.test.ts b/packages/benchmark/src/tachometer/format.test.ts deleted file mode 100644 index 19cca7d4a0..0000000000 --- a/packages/benchmark/src/tachometer/format.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { groupCasesByVariantSet, shortNameOf, variantsOf } from './format'; -import type { SummarizedCase } from './format'; - -function variant(name: string) { - return { - variant: name, - meanMs: { low: 20, high: 22 }, - samples: 120, - }; -} - -function caseWith(name: string, variantNames: string[], measurements = ['mount']): SummarizedCase { - return { - name, - comparison: 'baseline', - reference: `${name} ${variantNames[0]}`, - measurements: measurements.map((measurement) => ({ - name: measurement, - variants: variantNames.map((short) => variant(`${name} ${short}`)), - comparisons: [], - })), - }; -} - -describe('shortNameOf', () => { - it('drops the case prefix tachometer puts on a variant', () => { - expect(shortNameOf('libs-mount', 'libs-mount [ag-grid]')).toBe('[ag-grid]'); - }); - - it('leaves a name that does not carry the prefix alone', () => { - expect(shortNameOf('scroll', 'row-updates [current]')).toBe('row-updates [current]'); - }); -}); - -describe('variantsOf', () => { - it('lists the variants in the order they first appear', () => { - expect(variantsOf(caseWith('libs-mount', ['[mosaic]', '[ag-grid]', '[mui-x]']))).toEqual([ - '[mosaic]', - '[ag-grid]', - '[mui-x]', - ]); - }); - - it('counts a variant once across measurements', () => { - const entry = caseWith('workload', ['[current]', '[baseline]'], ['mount', 'cold-start']); - - expect(variantsOf(entry)).toEqual(['[current]', '[baseline]']); - }); -}); - -describe('groupCasesByVariantSet', () => { - it('puts cases sharing a variant set in one group', () => { - // Auto-expanded cases all carry [current]/[baseline], so they belong in one table. - const groups = groupCasesByVariantSet([ - caseWith('workload', ['[current]', '[baseline]']), - caseWith('scroll', ['[current]', '[baseline]']), - ]); - - expect(groups).toHaveLength(1); - expect(groups[0].cases.map((entry) => entry.name)).toEqual(['workload', 'scroll']); - }); - - it('gives a case with a different variant set its own group', () => { - // A table over the union would leave a blank cell in every row, and could name only one - // reference for both. - const groups = groupCasesByVariantSet([ - caseWith('workload', ['[current]', '[baseline]']), - caseWith('libs-mount', ['[mosaic]', '[ag-grid]']), - ]); - - expect(groups.map((group) => group.variants)).toEqual([ - ['[current]', '[baseline]'], - ['[mosaic]', '[ag-grid]'], - ]); - }); - - it('separates variant sets that differ only in order', () => { - // The first variant is the reference, so the same names in another order is another table. - const groups = groupCasesByVariantSet([ - caseWith('a', ['[current]', '[baseline]']), - caseWith('b', ['[baseline]', '[current]']), - ]); - - expect(groups).toHaveLength(2); - }); - - it('keeps the groups in order of first appearance', () => { - const groups = groupCasesByVariantSet([ - caseWith('libs-mount', ['[mosaic]', '[ag-grid]']), - caseWith('workload', ['[current]', '[baseline]']), - caseWith('libs-scroll', ['[mosaic]', '[ag-grid]']), - ]); - - expect(groups.map((group) => group.cases.map((entry) => entry.name))).toEqual([ - ['libs-mount', 'libs-scroll'], - ['workload'], - ]); - }); -}); diff --git a/packages/benchmark/src/tachometer/format.ts b/packages/benchmark/src/tachometer/format.ts deleted file mode 100644 index 6bdd4f56b8..0000000000 --- a/packages/benchmark/src/tachometer/format.ts +++ /dev/null @@ -1,93 +0,0 @@ -import type { CaseResult, MeasurementResult, ReportRef, ConfidenceInterval } from './ciReport'; - -/** - * How a report's numbers and names are written, for every renderer of one. - * - * The terminal table and the pull request comment show the same run, so they format it here rather - * than each their own way. Pure, and free of anything node-only, so a browser can import it. - */ - -/** A case that produced results, as opposed to one carrying only the error that stopped it. */ -export type SummarizedCase = CaseResult & { measurements: MeasurementResult[] }; - -export interface VariantGroup { - /** The variant set the group's cases share, reference first. */ - variants: string[]; - cases: SummarizedCase[]; -} - -/** `20.50 – 22.67 ms`, with the unit written once. */ -export function formatMean(interval: ConfidenceInterval): string { - return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; -} - -/** Both bounds signed, so the direction of a difference reads without the verdict beside it. */ -function signedInterval(interval: ConfidenceInterval, unit: string, digits: number): string { - const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(digits)}${unit}`; - return `${signed(interval.low)} – ${signed(interval.high)}`; -} - -/** `+2.7% – +6.8%`. */ -export function formatPercent(interval: ConfidenceInterval): string { - return signedInterval(interval, '%', 1); -} - -/** `+6.10 ms – +15.20 ms`. */ -export function formatSignedMs(interval: ConfidenceInterval): string { - return signedInterval(interval, ' ms', 2); -} - -/** What a ref is called in a report: the working tree, or the revision the run was given. */ -export function refLabel(ref: ReportRef): string { - return ref.kind === 'worktree' ? 'working tree' : ref.requested; -} - -/** - * Tachometer names each variant ` []`, and the case is named by the table itself, so - * the prefix is dropped for display. Dropping it is also what lets cases share a table: every - * auto-expanded case then contributes the same `[current]`/`[baseline]` pair rather than a pair of - * its own. - */ -export function shortNameOf(caseName: string, variant: string): string { - return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; -} - -/** Whether a case produced results, as opposed to carrying only the error that stopped it. */ -export function isSummarized(entry: CaseResult): entry is SummarizedCase { - return entry.measurements !== undefined && entry.measurements.length > 0; -} - -/** The distinct variants a case reports, in order of first appearance. */ -export function variantsOf(entry: SummarizedCase): string[] { - const variants: string[] = []; - for (const measurement of entry.measurements) { - for (const variant of measurement.variants) { - const short = shortNameOf(entry.name, variant.variant); - if (!variants.includes(short)) { - variants.push(short); - } - } - } - return variants; -} - -/** - * Splits cases into the groups that each get their own table. - * - * A regression case carries `[current]`/`[baseline]`, a cross-library case `[ours]`/`[theirs]`, and - * each set has its own reference, the first variant. One table over the union of the columns would - * leave a blank cell in every row, and could name only one reference for all of them. - */ -export function groupCasesByVariantSet(cases: SummarizedCase[]): VariantGroup[] { - const groups = new Map(); - - for (const entry of cases) { - const variants = variantsOf(entry); - const key = JSON.stringify(variants); - const group = groups.get(key) ?? { variants, cases: [] }; - group.cases.push(entry); - groups.set(key, group); - } - - return [...groups.values()]; -} diff --git a/packages/benchmark/src/tachometer/pairedStats.test.ts b/packages/benchmark/src/tachometer/pairedStats.test.ts deleted file mode 100644 index 4f175f8d25..0000000000 --- a/packages/benchmark/src/tachometer/pairedStats.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - meanInterval, - shuffledIndices, - stableMeasurements, - tCritical95, - toTachometerJson, -} from './pairedStats'; -import type { Round } from './pairedStats'; -import { summarizeCase } from './summarizeCase'; -import type { BenchmarkCase } from './discoverCases'; - -/** Rounds where `b` is always `a + shift`, both riding the same per-round drift. */ -function driftingRounds(shift: number): Round[] { - const drift = [0, 40, -25, 60, -10, 35, -45, 20, 5, -30]; - return drift.map((offset) => [{ mount: 100 + offset }, { mount: 100 + offset + shift }]); -} - -function baselineCase(measurements: string[]): BenchmarkCase { - return { - name: 'Button', - configPath: '/harness/src/button/button.bench.tsx', - config: {}, - comparison: 'baseline', - leaves: [], - variants: ['Button [current]', 'Button [baseline]'], - measurements, - }; -} - -describe('tCritical95', () => { - it('reads small samples from the table', () => { - expect(tCritical95(1)).toBe(12.706); - expect(tCritical95(29)).toBe(2.045); - }); - - it('approaches the normal quantile for large samples', () => { - expect(tCritical95(60)).toBeCloseTo(2.0, 1); - expect(tCritical95(10_000)).toBeCloseTo(1.96, 2); - }); -}); - -describe('meanInterval', () => { - it('centers on the mean', () => { - const interval = meanInterval([1, 2, 3, 4, 5]); - expect((interval.low + interval.high) / 2).toBeCloseTo(3); - expect(interval.low).toBeLessThan(3); - }); - - it('is a point for a single value', () => { - expect(meanInterval([7])).toEqual({ low: 7, high: 7 }); - }); -}); - -describe('toTachometerJson', () => { - it('resolves a small shift that drift hides from the per-variant means', () => { - const json = toTachometerJson( - ['Button [current]', 'Button [baseline]'], - ['mount'], - driftingRounds(2), - ); - const [current, baseline] = json.benchmarks; - - // Each variant's own interval is wide enough to overlap the other's… - expect(current.mean.high).toBeGreaterThan(baseline.mean.low); - // …but the paired difference is exactly the shift, every round, relative to the baseline's - // mean of 107. - expect(current.differences[1]).toEqual({ - absolute: { low: -2, high: -2 }, - percentChange: { low: expect.closeTo(-1.87, 2), high: expect.closeTo(-1.87, 2) }, - }); - }); - - it('names entries so summarizeCase pairs them back by variant and measurement', () => { - const rounds: Round[] = [ - [ - { mount: 10, paint: 20 }, - { mount: 12, paint: 20 }, - ], - [ - { mount: 11, paint: 21 }, - { mount: 13, paint: 20 }, - ], - [ - { mount: 10, paint: 19 }, - { mount: 12, paint: 21 }, - ], - ]; - const summary = summarizeCase( - baselineCase(['mount', 'paint']), - toTachometerJson(['Button [current]', 'Button [baseline]'], ['mount', 'paint'], rounds), - ); - - expect(summary.measurements.map((measurement) => measurement.name)).toEqual(['mount', 'paint']); - const [mount, paint] = summary.measurements; - expect(mount.comparisons[0]).toMatchObject({ variant: 'Button [baseline]', verdict: 'faster' }); - expect(paint.comparisons[0]).toMatchObject({ variant: 'Button [baseline]', verdict: 'unsure' }); - }); - - it('never compares across measurements', () => { - const json = toTachometerJson( - ['a', 'b'], - ['mount', 'paint'], - [ - [ - { mount: 1, paint: 2 }, - { mount: 1, paint: 2 }, - ], - ], - ); - // Flat order is measurement-major: a/mount, b/mount, a/paint, b/paint. - expect(json.benchmarks[0].differences[2]).toBe(null); - expect(json.benchmarks[0].differences[3]).toBe(null); - expect(json.benchmarks[0].differences[1]).not.toBe(null); - }); -}); - -describe('stableMeasurements', () => { - it('keeps only measurements every variant reported every round', () => { - const rounds: Round[] = [ - [ - { render: 1, 'render:update': 1 }, - { render: 1, 'render:update': 1 }, - ], - [{ render: 1 }, { render: 1, 'render:update': 1 }], - ]; - expect(stableMeasurements(rounds)).toEqual(['render']); - }); -}); - -describe('shuffledIndices', () => { - it('returns a permutation', () => { - expect(shuffledIndices(5).sort()).toEqual([0, 1, 2, 3, 4]); - }); -}); diff --git a/packages/benchmark/src/tachometer/pairedStats.ts b/packages/benchmark/src/tachometer/pairedStats.ts deleted file mode 100644 index 90ea9bd0cf..0000000000 --- a/packages/benchmark/src/tachometer/pairedStats.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { calculateMean } from '../stats'; -import type { - ConfidenceInterval, - TachometerBenchmark, - TachometerDifference, - TachometerJson, -} from './summarizeCase'; - -/** - * Statistics for interleaved runs, in the shape tachometer reports so the rest of the pipeline - * (summaries, the report, the upload) reads them unchanged. - * - * The difference is in how differences are computed. Every variant is sampled once per round, in - * a shuffled order, so each round is a matched set: whatever the machine was doing during that - * round affected every variant in it. Intervals on a difference are therefore taken over the - * per-round differences rather than over two independent sample sets, which cancels that shared - * drift instead of counting it as noise. - */ - -/** One round: a variant's measured values by measurement name, indexed like the variants. */ -export type Round = Array | undefined>; - -// Two-sided 95% critical values of Student's t for 1–30 degrees of freedom. -const T_95 = [ - 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, - 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, - 2.045, 2.042, -]; - -/** The two-sided 95% critical value of Student's t, approximated past the table. */ -export function tCritical95(degreesOfFreedom: number): number { - if (degreesOfFreedom <= T_95.length) { - return T_95[Math.max(degreesOfFreedom, 1) - 1]; - } - // Cornish–Fisher expansion around the normal quantile; well within 0.1% past 30 degrees. - const z = 1.959964; - return z + (z ** 3 + z) / (4 * degreesOfFreedom); -} - -/** The 95% confidence interval of the mean. A single value is its own degenerate interval. */ -export function meanInterval(values: number[]): ConfidenceInterval { - const mean = calculateMean(values); - if (values.length < 2) { - return { low: mean, high: mean }; - } - const variance = - values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (values.length - 1); - const halfWidth = tCritical95(values.length - 1) * Math.sqrt(variance / values.length); - return { low: mean - halfWidth, high: mean + halfWidth }; -} - -/** `a` against `b`, over the rounds where both were measured. */ -function pairedDifference( - rounds: Round[], - a: number, - b: number, - measurement: string, -): TachometerDifference | null { - const differences: number[] = []; - const baseline: number[] = []; - for (const round of rounds) { - const valueA = round[a]?.[measurement]; - const valueB = round[b]?.[measurement]; - if (valueA !== undefined && valueB !== undefined) { - differences.push(valueA - valueB); - baseline.push(valueB); - } - } - if (differences.length === 0) { - return null; - } - const absolute = meanInterval(differences); - const reference = calculateMean(baseline); - return { - absolute, - percentChange: { - low: (absolute.low / reference) * 100, - high: (absolute.high / reference) * 100, - }, - }; -} - -/** - * Tachometer-shaped results for interleaved rounds: one entry per measurement × variant, named - * ` []`, with a difference matrix over the flat list (`null` across - * measurements, which are never compared with each other). - */ -export function toTachometerJson( - variants: string[], - measurements: string[], - rounds: Round[], -): TachometerJson { - const flat = measurements.flatMap((measurement) => - variants.map((variant, index) => ({ variant, index, measurement })), - ); - const benchmarks = flat.map(({ variant, index, measurement }): TachometerBenchmark => { - const samples = rounds.flatMap((round) => { - const value = round[index]?.[measurement]; - return value === undefined ? [] : [value]; - }); - return { - name: `${variant} [${measurement}]`, - measurement: { name: measurement }, - mean: meanInterval(samples), - samples, - differences: flat.map((other) => - other.measurement === measurement && other.index !== index - ? pairedDifference(rounds, index, other.index, measurement) - : null, - ), - }; - }); - return { benchmarks }; -} - -/** Measurement names every sample of every variant reported, in first-seen order. */ -export function stableMeasurements(rounds: Round[]): string[] { - const seen = new Set(); - for (const round of rounds) { - for (const values of round) { - for (const name of Object.keys(values ?? {})) { - seen.add(name); - } - } - } - return [...seen].filter((name) => - rounds.every((round) => round.every((values) => values?.[name] !== undefined)), - ); -} - -/** A random permutation of `0..length-1`, so no variant always runs first in its round. */ -export function shuffledIndices(length: number): number[] { - const indices = Array.from({ length }, (_, index) => index); - for (let index = indices.length - 1; index > 0; index -= 1) { - const swap = Math.floor(Math.random() * (index + 1)); - [indices[index], indices[swap]] = [indices[swap], indices[index]]; - } - return indices; -} diff --git a/packages/benchmark/src/tachometer/plugin.ts b/packages/benchmark/src/tachometer/plugin.ts deleted file mode 100644 index 3d97765133..0000000000 --- a/packages/benchmark/src/tachometer/plugin.ts +++ /dev/null @@ -1,294 +0,0 @@ -import * as path from 'node:path'; -import { createRequire } from 'node:module'; -import { readFile, realpath } from 'node:fs/promises'; -import type { Plugin } from 'vite'; -import { discoverCases, pagesOf } from './discoverCases'; -import { discoverBenchFiles, writeBenchPages } from './benchFiles'; -import type { BenchmarkCase } from './discoverCases'; -import { buildsDirOf, prepareOutputDir, OUTPUT_DIR } from './outputDir'; - -/** - * Vite plugin for a tachometer benchmark harness. - * - * The harness keeps its own vite config and runs plain `vite` / `vite build` / `vite preview`; this - * plugin only contributes what is derived from the benchmark cases: - * - * - `root` is the harness's `src/`, and the app is an MPA (vite defaults to `spa`, whose fallback - * would rewrite unmatched urls toward a root `index.html` that does not exist here). - * - The build's entry points come from the `tachometer.json` files, so the configs are the single - * source of truth for which pages exist. - * - The dev server serves a generated index listing every case, including cases that own no page of - * their own and only parameterise a sibling's. - * - * It deliberately never resolves refs, so a plain `vite build` needs no git history. - */ - -export interface TachometerPluginOptions { - /** - * The harness package directory. Defaults to the current working directory, which is where vite - * is run from. - */ - harnessDir?: string; -} - -/** - * Checks that the workspace packages the harness links to have actually been built. - * - * A harness depends on the library under test via `workspace:*`, and `publishConfig.directory` - * makes that link point at the package's build output — which does not exist until the library is - * built. The symlink is then dangling, and vite's failure ("failed to resolve import") does not - * say why. - */ -async function assertWorkspaceDepsBuilt( - harnessDir: string, - deps: Record, -): Promise { - const linked = Object.entries(deps).filter(([, version]) => version.startsWith('workspace:')); - await Promise.all( - linked.map(async ([name]) => { - try { - await realpath(path.join(harnessDir, 'node_modules', ...name.split('/'))); - } catch { - throw new Error( - `"${name}" is linked into ${harnessDir} but its target does not exist. ` + - `Build the workspace packages first (e.g. \`pnpm release:build\`) — the harness ` + - `resolves the library through its build output, not its source.`, - ); - } - }), - ); -} - -/** - * Where `@mui/internal-benchmark` really lives as seen from the harness, so imports made by the - * page runtime can be told apart from the harness's own. Undefined when the harness doesn't depend - * on it, in which case there is no page runtime to tell apart. - */ -async function resolveBenchmarkPackageDir(harnessDir: string): Promise { - try { - const require = createRequire(path.join(harnessDir, 'package.json')); - return await realpath(path.dirname(require.resolve('@mui/internal-benchmark/package.json'))); - } catch { - return undefined; - } -} - -/** A case name and a query both come from a config file, so neither is trusted as markup. */ -function escapeHtml(text: string): string { - return text - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"'); -} - -/** - * Renders the case index: one link per case, carrying whatever query the case parameterises its - * page with. - */ -function renderIndex(cases: BenchmarkCase[], note: string): string { - const items = cases - .map((entry) => { - // A case's variants can point at several pages (a cross-library comparison) or at one page - // with different queries (a parameterised case), so dedupe on the pair. - const targets = [...new Set(entry.leaves.map((leaf) => `${leaf.page}${leaf.suffix}`))].sort(); - const links = targets - .map( - (target) => - `
  • ${escapeHtml(target)}
    • `, - ) - .join('\n '); - return `
  • \n ${escapeHtml(entry.name)}\n
      \n ${links}\n
    \n
    • `; - }) - .join('\n'); - - return ` - - - - Tachometer benchmark cases - - - -

    Tachometer benchmark cases

    -

    ${note}

    -
      -${items} -
    - - -`; -} - -/** - * Whether a case's own page occupies the `index.html` the case list would be served at. - * - * A `tachometer.json` directly in `src/` is a supported layout, and such a case may own the root - * page. Asked in one place because the build and the dev server both have to give the same answer: - * otherwise `/` shows the case under one command and the list under the other. - */ -function aCaseOwnsIndex(cases: BenchmarkCase[]): boolean { - return pagesOf(cases).includes('index.html'); -} - -/** Creates the tachometer harness plugin. */ -export function tachometer(options: TachometerPluginOptions = {}): Plugin { - const harnessDir = options.harnessDir ?? process.cwd(); - const srcDir = path.join(harnessDir, 'src'); - let buildCases: BenchmarkCase[] = []; - let benchmarkPackageDir: string | undefined; - - return { - name: 'mui-benchmark:tachometer', - // Ahead of vite's own resolver, so the redirects below win over the packages' `exports`. - enforce: 'pre', - - // Benchmark files are written against the Vitest harness. In a page they run under the page - // runtime instead: the same authoring API, driven one iteration at a time by the runner. - async resolveId(source, importer) { - if (source === '@mui/internal-benchmark') { - return this.resolve('@mui/internal-benchmark/page', importer, { skipSelf: true }); - } - if (source === 'vitest') { - return this.resolve('@mui/internal-benchmark/page/vitest', importer, { skipSelf: true }); - } - // React only reports render durations from its profiling build. Scoped to the page runtime's - // own import, so a plain tachometer page keeps measuring the production build it ships with. - if ( - source === 'react-dom/client' && - importer && - benchmarkPackageDir && - importer.startsWith(benchmarkPackageDir) - ) { - return this.resolve('react-dom/profiling', importer, { skipSelf: true }); - } - return null; - }, - - async config(userConfig, env) { - const pkg = JSON.parse(await readFile(path.join(harnessDir, 'package.json'), 'utf8')); - await assertWorkspaceDepsBuilt(harnessDir, { - ...pkg.dependencies, - ...pkg.devDependencies, - }); - - // The run output directory, next to the per-ref directories the runner writes — vite would - // otherwise default to `/dist`, i.e. inside `src/` next to the case sources. - // - // Resolved for every command, not just `build`: `vite preview` runs as `serve` but serves - // `build.outDir`, so leaving it unset there sends preview looking for `/dist` and it - // exits rather than serving the pages that were just built. - // - // Only fill it in when the caller said nothing: a plugin's returned config is merged *over* - // the inline config, so unconditionally setting it here would silently override - // `vite build --outDir`, which is exactly how `tacho run` directs each ref's build. - const outDir = userConfig.build?.outDir ?? buildsDirOf(harnessDir, 'manual'); - - // Written for every command, so the dev server serves a benchmark file's page as well. - const benchFiles = await discoverBenchFiles({ harnessDir }); - await writeBenchPages(harnessDir, benchFiles); - benchmarkPackageDir = await resolveBenchmarkPackageDir(harnessDir); - // One React for the harness, its benchmark files and the page runtime they run under. - const resolve = { dedupe: ['react', 'react-dom'] }; - - if (env.command !== 'build') { - return { root: srcDir, appType: 'mpa', resolve, build: { outDir } }; - } - - // Mark the output directory ignored from the inside before the build writes into it, so a - // harness that does not list `.tachometer` is not offered its built pages for commit. Keyed on - // where the output actually lands rather than on who chose it: `tacho run` passes its own - // `outDir` under the same directory and prepares it itself, and any other caller that writes - // there deserves the marker too. Vite resolves a relative `outDir` against `root`. - if (path.resolve(srcDir, outDir).startsWith(path.join(harnessDir, OUTPUT_DIR))) { - await prepareOutputDir(harnessDir); - } - - // A harness may hold only benchmark files, so no `tachometer.json` is fine as long as there is - // something to build. - const cases = await discoverCases({ harnessDir, allowEmpty: benchFiles.length > 0 }); - buildCases = cases; - // Key the input map by the page path without its extension. Vite emits each page at its path - // relative to `root` regardless, so `/index.html` lands at `//index.html`; - // a distinct key per page is what lets one case folder hold several pages — a cross-library - // comparison with `mosaic.html` and `tanstack.html` side by side — without them colliding. - const pages = [...pagesOf(cases), ...benchFiles.map((benchFile) => benchFile.page)]; - const input = Object.fromEntries( - pages.map((page) => [page.replace(/\.html$/, ''), path.join(srcDir, page)]), - ); - - return { - root: srcDir, - appType: 'mpa', - resolve, - // Relative asset urls: tachometer serves the build directory and the pages live under - // `//`, so absolute "/assets/…" paths would 404. - base: './', - build: { - outDir, - // The output directory sits outside `root`, which is `src/`; allow vite to clean it anyway. - emptyOutDir: true, - chunkSizeWarningLimit: 9999, - rollupOptions: { input }, - }, - }; - }, - - // The same index the dev server renders, emitted as a real file so the built directory is - // navigable too — under `vite preview`, under any static server, or opened from disk. A page-less - // case is only reachable through this list, so a build without it can only be entered by typing - // urls from memory. - generateBundle() { - // Rollup drops an emit whose name a bundle output already claims, with no error and no - // warning, so the list would simply vanish. Say so instead. - if (aCaseOwnsIndex(buildCases)) { - this.warn( - 'A case owns index.html, so the case list was not emitted. Move the case into a subfolder of src/ to get it back.', - ); - return; - } - - this.emitFile({ - type: 'asset', - fileName: 'index.html', - source: renderIndex( - buildCases, - 'These are production bundles — the same build tachometer measures.', - ), - }); - }, - - configureServer(server) { - server.middlewares.use(async (req, res, next) => { - const url = (req.url ?? '/').split('?')[0]; - if (url !== '/' && url !== '/index.html') { - next(); - return; - } - try { - const cases = await discoverCases({ harnessDir }); - if (aCaseOwnsIndex(cases)) { - // Let vite serve the case's own page, the way the build does. - next(); - return; - } - res.setHeader('Content-Type', 'text/html; charset=utf-8'); - res.end( - renderIndex( - cases, - "Pages are served from the working tree's build of the library. Rebuild it and reload to pick up a source change.", - ), - ); - } catch (error) { - next(error); - } - }); - }, - }; -} diff --git a/packages/benchmark/src/tachometer/renderReport.ts b/packages/benchmark/src/tachometer/renderReport.ts deleted file mode 100644 index 1ad46cd643..0000000000 --- a/packages/benchmark/src/tachometer/renderReport.ts +++ /dev/null @@ -1,241 +0,0 @@ -/* eslint-disable no-console -- a reporter: printing the report is its output. */ - -// Column widths have to measure visible characters, so chalk's escapes come off first. -import { stripVTControlCharacters as stripAnsi } from 'node:util'; -import chalk from 'chalk'; -import { - formatMean, - formatPercent, - groupCasesByVariantSet, - isSummarized, - refLabel, - shortNameOf, -} from './format'; -import type { SummarizedCase } from './format'; -import type { TachometerReport, Verdict } from './ciReport'; - -/** The report's types, from the schema that defines it. */ -export type { - TachometerReport, - CaseResult, - MeasurementResult, - VariantResult, - Comparison, - ReportRef, - ConfidenceInterval, - Verdict, -} from './ciReport'; - -/** Prints a table, padding each column to its widest visible cell. */ -function printTable(headers: string[], rows: string[][], leftColumns: number): void { - const widths = headers.map((header, index) => - Math.max(stripAnsi(header).length, ...rows.map((row) => stripAnsi(row[index] ?? '').length)), - ); - const pad = (text: string, width: number, left: boolean) => { - const fill = ' '.repeat(Math.max(0, width - stripAnsi(text).length)); - return left ? text + fill : fill + text; - }; - const line = (cells: string[]) => - cells - .map((cell, index) => pad(cell ?? '', widths[index], index < leftColumns)) - .join(' ') - .trimEnd(); - - console.log(line(headers)); - console.log(widths.map((width) => '─'.repeat(width)).join(' ')); - for (const row of rows) { - console.log(line(row)); - } -} - -function colorVerdict(verdict: Verdict, text: string): string { - if (verdict === 'faster') { - return chalk.green(text); - } - if (verdict === 'slower') { - return chalk.red(text); - } - return chalk.dim(text); -} - -/** - * Renders one case as a table of variants, for a comparison with more than two of them. - * - * Rows are variants, not cases: with several libraries, a column per variant plus a Δ column per - * pair would run off the screen. Each measurement then shows the variant's interval next to its - * difference *relative to the reference* — the direction a row about that library reads in — taken - * from the variant's own row of tachometer's matrix. - */ -function printVariantTable(entry: SummarizedCase, variants: string[]): void { - const [reference] = variants; - const headers = [ - entry.name, - ...entry.measurements.flatMap((measurement) => [measurement.name, `vs ${reference}`]), - 'Samples', - ]; - - const rows = variants.map((variant) => { - // Looked up once per row rather than accumulated while the measurement cells render, so the - // last two columns do not depend on the cell loop having run. - const perMeasurement = entry.measurements.map((measurement) => ({ - found: measurement.variants.find( - (candidate) => shortNameOf(entry.name, candidate.variant) === variant, - ), - comparison: measurement.comparisons.find( - (candidate) => shortNameOf(entry.name, candidate.variant) === variant, - ), - })); - const sampleCounts = new Set( - perMeasurement.flatMap(({ found }) => (found ? [found.samples] : [])), - ); - - const cells = perMeasurement.flatMap(({ found, comparison }) => { - let delta = chalk.dim('—'); - if (variant === reference) { - delta = chalk.dim('reference'); - } else if (comparison?.versusReference) { - const { verdict, percentChange } = comparison.versusReference; - delta = colorVerdict(verdict, `${verdict} ${formatPercent(percentChange)}`); - } - return [found ? formatMean(found.meanMs) : chalk.dim('—'), delta]; - }); - - return [variant, ...cells, [...sampleCounts].join('/')]; - }); - - printTable(headers, rows, 1); - console.log( - chalk.dim( - `vs ${reference} is tachometer's confidence interval on the difference, that variant ` + - `relative to ${reference} — negative is faster than ${reference}.`, - ), - ); -} - -/** Renders a group of cases sharing a variant set, one row per case and measurement. */ -function printCaseTable(cases: SummarizedCase[], variants: string[]): void { - const [reference, ...others] = variants; - const headers = [ - 'Case', - 'Measurement', - ...variants, - ...others.map((variant) => `Δ vs ${variant}`), - 'Samples', - ]; - - const rows: string[][] = []; - for (const [caseIndex, entry] of cases.entries()) { - if (caseIndex > 0) { - rows.push(headers.map(() => '')); - } - let firstRow = true; - for (const measurement of entry.measurements) { - const byVariant = new Map( - measurement.variants.map((variant) => [shortNameOf(entry.name, variant.variant), variant]), - ); - const byComparison = new Map( - measurement.comparisons.map((comparison) => [ - shortNameOf(entry.name, comparison.variant), - comparison, - ]), - ); - // Auto-sampling stops per case, not per variant, so the counts are normally equal; when they - // are not, saying so beats printing one number that describes neither side. - const sampleCounts = [...new Set(measurement.variants.map((variant) => variant.samples))]; - - rows.push([ - firstRow ? entry.name : '', - measurement.name, - ...variants.map((variant) => { - const found = byVariant.get(variant); - return found ? formatMean(found.meanMs) : chalk.dim('—'); - }), - ...others.map((variant) => { - const comparison = byComparison.get(variant); - if (!comparison) { - return chalk.dim('—'); - } - return colorVerdict( - comparison.verdict, - `${comparison.verdict} ${formatPercent(comparison.percentChange)}`, - ); - }), - sampleCounts.join('/'), - ]); - firstRow = false; - } - } - - printTable(headers, rows, 2); - console.log( - chalk.dim( - `Δ is tachometer's confidence interval on the difference, ${reference ?? 'the reference'} ` + - `relative to the other variant${others.length > 1 ? 's' : ''} — negative is faster.`, - ), - ); -} - -/** - * Prints a tachometer report: one table per variant set, with a column per variant holding its 95% - * confidence interval and a Δ column per non-reference variant. - * - * The Δ column is **tachometer's own confidence interval on the difference**, not a delta of the - * two means. That is the whole reason this suite exists next to a profiling one: variants are - * sampled round-robin in a single session and auto-sampled until that interval clears the case's - * `autoSampleConditions`, so the interval — not a point estimate plus a post-hoc test — is the - * result. `unsure` means it never resolved within the sampling budget, which for two builds that - * really are equivalent is the expected outcome rather than a failure. - * - * Signs are the reference relative to the variant, so a faster reference reads negative; the - * verdict word carries the direction so the sign never has to be read twice. - */ -export function renderTachometerReport(report: TachometerReport): void { - // One predicate for both halves: partitioning on two conditions that have to agree left a case - // with an empty `measurements` array rendered nowhere and reported nowhere. - const usable = report.cases.filter(isSummarized); - const failed = report.cases.filter((entry) => !isSummarized(entry)); - - if (usable.length === 0) { - console.log(chalk.dim('No tachometer results to show.')); - for (const entry of failed) { - console.log(chalk.red(` ${entry.name}: ${entry.error ?? 'no result'}`)); - } - return; - } - - for (const [groupIndex, group] of groupCasesByVariantSet(usable).entries()) { - if (groupIndex > 0) { - console.log(''); - } - if (group.variants.length > 2) { - for (const [caseIndex, entry] of group.cases.entries()) { - if (caseIndex > 0) { - console.log(''); - } - printVariantTable(entry, group.variants); - } - continue; - } - printCaseTable(group.cases, group.variants); - } - - const notes = [ - 'Each cell is a 95% confidence interval for the mean, in milliseconds; one sample = one page load.', - '"unsure" means the interval still straddles zero: the difference did not resolve within the case\'s sampling budget.', - ]; - - // No short SHA appended: `id` already ends in one. - const refLabels = report.refs.map((ref) => `${ref.id} = ${refLabel(ref)}`); - if (refLabels.length > 0) { - notes.push(`Builds: ${refLabels.join(' · ')}`); - } - notes.push( - `head: ${report.head.sha.slice(0, 9)} (${report.head.branch || '?'}) · ` + - `measured on the production bundle, installed from a packed tarball.`, - ); - console.log(chalk.dim(`\n${notes.join('\n')}`)); - - for (const entry of failed) { - console.log(chalk.red(`\n${entry.name}: ${entry.error ?? 'no result'}`)); - } -} diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts deleted file mode 100644 index 2cd07261f3..0000000000 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ /dev/null @@ -1,383 +0,0 @@ -/* eslint-disable no-console */ - -import * as path from 'node:path'; -import * as os from 'node:os'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import chalk from 'chalk'; -import { execaSync } from 'execa'; -import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; -import { packRef, packWorkingTree } from '../utils/packWorkspace'; -import type { PackedPackage } from '../utils/packWorkspace'; -import { resolveBaselineRef, WORKTREE_REF } from './refs'; -import type { ResolvedRef } from './refs'; -import { discoverCases, pagesOf } from './discoverCases'; -import { discoverBenchFiles } from './benchFiles'; -import { runInterleaved } from './runInterleaved'; -import type { BenchmarkCase, Leaf } from './discoverCases'; -import { isSummarized, refLabel } from './format'; -import { buildRefPages, restoreWorkspace } from './buildPages'; -import type { ResolveMode } from './buildPages'; -import { assertDriverMatchesBrowser, resolveBrowserBinary, withBrowserDefaults } from './browser'; -import { summarizeCase } from './summarizeCase'; -import type { CaseRun } from './summarizeCase'; -import { renderTachometerReport } from './renderReport'; -import { getCiMetadata } from '../ciReport'; -import { syncPrComment } from '../syncPrComment'; -import { uploadCiReport } from './ciReport'; -import type { CaseResult, TachometerReport } from './ciReport'; -import { buildsDirOf, prepareOutputDir } from './outputDir'; -import { run } from '../utils/exec'; - -/** - * A case's name as a filename component. Names come from the benchmark's own `name`, so they are - * prose rather than identifiers and may hold anything — a slash included. - */ -function fileSlugOf(name: string): string { - return name.replace(/[^a-zA-Z0-9._-]+/g, '-'); -} - -/** - * The builds a run measures, and which of them a given variant loads. - * - * The baseline is resolved only when a case compares against it: a harness that only puts pages - * side by side needs no second commit, and asking for one would fail in a clone too shallow to hold - * it. The lookup comes back with the list so that "there is no baseline" is settled once, here, - * rather than being a case every later step has to allow for. - */ -async function resolveBuilds( - needsBaseline: boolean, - baseline: string | undefined, - repoRoot: string, -): Promise<{ refs: ResolvedRef[]; buildFor: (leaf: Leaf) => ResolvedRef }> { - if (!needsBaseline) { - return { refs: [WORKTREE_REF], buildFor: () => WORKTREE_REF }; - } - const baselineRef = await resolveBaselineRef(baseline, repoRoot); - return { - refs: [WORKTREE_REF, baselineRef], - buildFor: (leaf) => (leaf.ref === 'baseline' ? baselineRef : WORKTREE_REF), - }; -} - -export interface RunTachometerOptions { - /** The harness package directory (where the command was run). */ - harnessDir: string; - /** - * Only run cases whose path under `src` contains one of these substrings, case-insensitively. - */ - filters?: string[]; - /** - * The build every `[baseline]` variant loads: a revision, on its own or as `git:`. Defaults - * to `HEAD~1`. - */ - baseline?: string; - /** - * Command that builds the publishable workspace packages. Run for every ref — in each ref's own - * checkout, and in the working tree — so both sides of a comparison are built the same way. - * Defaults to `pnpm release:build`. - */ - buildCmd?: string; - /** Whether to install inside a ref's checkout. Defaults to true. */ - install?: boolean; - /** Where to write the combined JSON report. Defaults to `.tachometer/results/report.json`. */ - out?: string; - /** Upload the report and refresh the pull request comment. */ - upload?: boolean; - /** How a ref's pages find the library under test. Defaults to `in-place`. */ - resolveMode?: ResolveMode; - /** - * What measures the pages. `tachometer` hands each case to tachometer. `interleaved` runs them - * in Playwright, alternating variants round by round and comparing paired differences — and is - * the engine that runs `*.bench.tsx` files. Defaults to `tachometer`. - */ - engine?: Engine; - /** Interleaved engine only: measured rounds per case, overriding `sampleSize`. */ - samples?: number; - /** Interleaved engine only: discarded rounds before measuring. */ - warmup?: number; -} - -export type Engine = 'tachometer' | 'interleaved'; - -/** - * Benchmarks the harness's pages across one or more builds of the workspace with tachometer, and - * writes a combined JSON report. - * - * Cases are the source of truth: the selected `tachometer.json` files determine which pages to - * build, one variant is built per distinct ref, each url is then rewritten to that ref's built page, - * and each case runs as its own tachometer invocation so its result is a clean table for that - * scenario rather than a grid comparing unrelated scenarios against each other. - */ -export async function runTachometer(options: RunTachometerOptions): Promise { - const { - harnessDir, - filters = [], - baseline, - buildCmd = 'pnpm release:build', - install = true, - out, - upload = false, - resolveMode = 'in-place', - engine = 'tachometer', - samples, - warmup, - } = options; - - const repoRoot = await findWorkspaceDir(harnessDir); - if (!repoRoot) { - throw new Error(`Could not find a pnpm workspace root above ${harnessDir}.`); - } - - // Everything a run writes goes under one directory, so a harness has one thing to ignore and - // deleting it is the whole reset story. `packed` holds tarballs — a ref's keyed by commit SHA, - // the working tree's by content hash — and is the one worth caching in CI; `trees` holds the - // install each ref's pages resolve through, cheap to recreate and not portable between - // containers, since its links point into a pnpm store; `builds` holds the pages built per ref; - // `results` the report. - const outputDir = await prepareOutputDir(harnessDir); - const buildsDir = buildsDirOf(harnessDir); - const packedDir = path.join(outputDir, 'packed'); - const treesDir = path.join(outputDir, 'trees'); - - // A run killed outright — Ctrl-C, not a thrown error — never reaches the restore below, leaving - // the repository pinned to tarballs. The copy it left behind is how that is noticed, so put it - // back before doing anything else, whatever mode this run is in. - await restoreWorkspace(repoRoot, outputDir); - - // Each engine runs its own kind of case: tachometer the `tachometer.json` cases, the interleaved - // engine the `*.bench.tsx` files. - const cases = engine === 'tachometer' ? await discoverCases({ harnessDir, filters }) : []; - const benchFiles = - engine === 'interleaved' ? await discoverBenchFiles({ harnessDir, filters }) : []; - if (engine === 'interleaved' && benchFiles.length === 0) { - throw new Error( - filters.length > 0 - ? `No *.bench.tsx file under ${path.join(harnessDir, 'src')} matches ${filters.map((filter) => `"${filter}"`).join(', ')}.` - : `No *.bench.tsx file found under ${path.join(harnessDir, 'src')}.`, - ); - } - // A benchmark file always compares the working tree against the baseline. - const { refs, buildFor } = await resolveBuilds( - benchFiles.length > 0 || cases.some((entry) => entry.comparison === 'baseline'), - baseline, - repoRoot, - ); - - // The commit follows the name, so a revision like `HEAD~1` still says which commit it landed on. - const describeRef = (ref: ResolvedRef) => - ref.sha ? `${refLabel(ref)} (${ref.sha.slice(0, 9)})` : refLabel(ref); - - // Before any build: a driver that cannot open the browser fails the run either way, and finding - // out now costs seconds instead of minutes of packing and installing. - const browserBinary = await resolveBrowserBinary(harnessDir); - // Only tachometer drives the browser through chromedriver. - if (engine === 'tachometer') { - assertDriverMatchesBrowser(harnessDir, browserBinary); - } - // `getuid` is POSIX-only; on Windows nobody is root. - const asRoot = process.getuid?.() === 0; - - // What each case compares, so one that carries no baseline says so up front rather than through - // an empty regression list at the end. - const describeCase = (entry: BenchmarkCase) => - entry.comparison === 'baseline' - ? `${entry.name} (vs baseline)` - : `${entry.name} (${entry.variants.length} variants)`; - - console.log(chalk.cyan(`Engine: ${engine}`)); - if (engine === 'tachometer') { - console.log(chalk.cyan(`Cases: ${cases.map(describeCase).join(', ')}`)); - console.log(chalk.cyan(`Pages: ${pagesOf(cases).join(', ')}`)); - } else { - console.log(chalk.cyan(`Files: ${benchFiles.map((benchFile) => benchFile.file).join(', ')}`)); - } - console.log(chalk.cyan(`Refs: ${refs.map(describeRef).join(', ')}`)); - console.log( - chalk.cyan(`Browser: ${browserBinary}${asRoot ? ' (as root, so without its sandbox)' : ''}`), - ); - - const tmpBase = await mkdtemp(path.join(os.tmpdir(), 'tacho-')); - - try { - // Build one variant per distinct ref, each resolving through its own install so both sides - // of a comparison resolve the library identically. - for (const ref of refs) { - let packages: PackedPackage[]; - if (ref.kind === 'worktree') { - // eslint-disable-next-line no-await-in-loop - packages = await packWorkingTree({ - repoRoot, - outRoot: path.join(packedDir, 'current'), - buildCmd, - }); - } else { - // packRef caches a ref's tarballs by SHA; a hit skips the checkout, install, and build. - // eslint-disable-next-line no-await-in-loop - const packed = await packRef({ - repoRoot, - ref: ref.sha, - outRoot: packedDir, - // An empty install command means "skip"; otherwise packRef's default install runs. - installCmd: install ? undefined : '', - buildCmd, - }); - packages = packed.packages; - } - - // eslint-disable-next-line no-await-in-loop - await buildRefPages({ - harnessDir, - repoRoot, - ref, - packages, - treeDir: path.join(treesDir, ref.id), - outputDir, - outDir: path.join(buildsDir, ref.id), - resolveMode, - }); - } - - const results: CaseRun[] = - engine === 'interleaved' - ? await runInterleaved({ - buildsDir, - benchFiles, - // `resolveBuilds` lists the working tree first, which makes it the reference. - benchRefs: refs, - browserBinary, - asRoot, - samples, - warmup, - }) - : []; - - // Rewrite each leaf url to its ref's built page, then run each case's own config. Tachometer - // resolves relative urls against the config file's directory and these configs are written to a - // temp dir, so the rewritten urls are absolute. - for (const entry of cases) { - for (const leaf of entry.leaves) { - leaf.node.url = `${path.join(buildsDir, buildFor(leaf).id, leaf.page)}${leaf.suffix}`; - } - // Discovery flattened `expand` away, so every benchmark already carries its own effective - // browser and there is no inheritance left to walk. - for (const benchmark of entry.config.benchmarks ?? []) { - benchmark.browser = withBrowserDefaults(benchmark.browser, browserBinary, asRoot); - } - const slug = fileSlugOf(entry.name); - const configPath = path.join(tmpBase, `tachometer-${slug}.json`); - // eslint-disable-next-line no-await-in-loop - await writeFile( - configPath, - // Served from the output directory rather than the harness: tachometer's static server is - // koa-static with its default `hidden: false`, so any dot-prefixed segment in the *served* - // path is refused. Rooting here keeps `.tachometer` out of the url entirely. - `${JSON.stringify({ ...entry.config, root: outputDir }, null, 2)}\n`, - ); - - console.log(chalk.cyan(`\nRunning "${entry.name}"…`)); - const jsonPath = path.join(tmpBase, `result-${slug}.json`); - // eslint-disable-next-line no-await-in-loop - await run( - 'pnpm', - ['exec', 'tachometer', '--config', configPath, '--json-file', jsonPath], - harnessDir, - { verbose: true }, - ); - // Cases run sequentially on purpose — concurrent browser benchmarks would contend for the - // same machine and skew timings — so reading each result in turn is fine. - // eslint-disable-next-line no-await-in-loop - results.push({ entry, json: JSON.parse(await readFile(jsonPath, 'utf8')) }); - } - - const outPath = out ? path.resolve(out) : path.join(outputDir, 'results', 'report.json'); - const report = { - // Consumers render reports from several benchmark axes; the pair identifies which one this - // is and how to read it. - version: 1 as const, - reportType: 'tachometer' as const, - generatedAt: new Date().toISOString(), - head: { - ref: 'HEAD', - sha: execaSync('git', ['rev-parse', 'HEAD'], { cwd: repoRoot }).stdout.trim(), - branch: execaSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { - cwd: repoRoot, - }).stdout.trim(), - }, - browser: browserBinary, - refs, - // Summarising is best-effort per case: a case that produced no usable benchmarks must not - // cost the whole run its report, since `raw` below is the only surviving copy of every other - // case's samples once the temp dir is cleaned up. - cases: results.map((result): CaseResult => { - const { entry } = result; - if ('error' in result) { - return { name: entry.name, comparison: entry.comparison, error: result.error }; - } - try { - return summarizeCase(entry, result.json); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(chalk.yellow(`Could not summarize "${entry.name}": ${message}`)); - return { name: entry.name, comparison: entry.comparison, error: message }; - } - }), - raw: Object.fromEntries( - results.flatMap((result) => ('json' in result ? [[result.entry.name, result.json]] : [])), - ), - }; - await mkdir(path.dirname(outPath), { recursive: true }); - await writeFile(outPath, `${JSON.stringify(report, null, 2)}\n`); - console.log(''); - renderTachometerReport(report); - console.log(chalk.green(`\nWrote JSON report to ${outPath}`)); - - if (upload) { - await publishReport(report); - } - - // Written and uploaded first, so the errors are readable and the comment says what happened — - // then fail, because a harness that measured nothing at all otherwise reports a passing job. - if (report.cases.every((entry) => !isSummarized(entry))) { - throw new Error('No case produced any measurement.'); - } - } finally { - await rm(tmpBase, { recursive: true, force: true }); - // Always, including after a failure: an in-place run leaves the repository resolving the library - // to a tarball until this puts it back. - if (resolveMode === 'in-place') { - await restoreWorkspace(repoRoot, outputDir); - } - } -} - -/** - * Uploads a report and refreshes the pull request comment. - * - * A failure to refresh the comment does not fail the run: the numbers are already measured, written - * and uploaded, and losing them to a comment API hiccup would waste the whole benchmark. - */ -async function publishReport(report: TachometerReport): Promise { - const metadata = await getCiMetadata(); - if (!metadata.repo) { - console.warn( - chalk.yellow('Skipping upload: no repository detected, which usually means this is not CI.'), - ); - return; - } - - await uploadCiReport({ version: 1, reportType: 'tachometer', ...metadata, report }); - - try { - console.log('Syncing PR comment via the dashboard API…'); - const result = await syncPrComment(metadata.repo); - console.log( - result.skipped ? 'No open PR found for this branch, skipping.' : 'PR comment synced.', - ); - } catch (error) { - console.error( - chalk.yellow( - `Failed to sync the PR comment: ${error instanceof Error ? error.message : error}`, - ), - ); - } -} diff --git a/packages/benchmark/src/tachometer/summarizeCase.test.ts b/packages/benchmark/src/tachometer/summarizeCase.test.ts deleted file mode 100644 index 11678777dc..0000000000 --- a/packages/benchmark/src/tachometer/summarizeCase.test.ts +++ /dev/null @@ -1,203 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { summarizeCase, verdictOf } from './summarizeCase'; - -type FakeDifference = { ms: [number, number]; pct: [number, number] } | null; - -/** A tachometer benchmark entry. */ -function benchmark({ - name, - measurement, - differences = [], -}: { - /** Entry name, as tachometer reports it. */ - name: string; - measurement?: string; - /** Differences against every entry, by flat index. */ - differences?: FakeDifference[]; -}): any { - return { - name, - mean: { low: 1, high: 2 }, - samples: [1, 2, 3], - measurement: measurement === undefined ? undefined : { name: measurement }, - differences: differences.map((difference) => - difference - ? { - absolute: { low: difference.ms[0], high: difference.ms[1] }, - percentChange: { low: difference.pct[0], high: difference.pct[1] }, - } - : null, - ), - }; -} - -/** - * A discovered case, as `discoverCases` would return it. `variants` is in order; the first is the - * reference. - */ -function discovered(variants: string[], measurements: string[]): any { - return { - name: 'example', - configPath: '/tmp/tachometer.json', - config: {}, - comparison: 'baseline', - leaves: [], - variants, - measurements, - }; -} - -describe('verdictOf', () => { - it('is slower when both intervals sit above zero', () => { - expect(verdictOf({ absolute: { low: 1, high: 3 }, percentChange: { low: 2, high: 5 } })).toBe( - 'slower', - ); - }); - - it('is faster when both intervals sit below zero', () => { - expect( - verdictOf({ absolute: { low: -3, high: -1 }, percentChange: { low: -5, high: -2 } }), - ).toBe('faster'); - }); - - it('is unsure when the intervals disagree', () => { - // The two are computed separately from the same samples, so near zero one can exclude it while - // the other does not. A verdict resting on only one of them would flip depending which is read. - expect(verdictOf({ absolute: { low: 1, high: 3 }, percentChange: { low: -1, high: 5 } })).toBe( - 'unsure', - ); - }); - - it('is unsure when both intervals straddle zero', () => { - expect(verdictOf({ absolute: { low: -1, high: 3 }, percentChange: { low: -2, high: 5 } })).toBe( - 'unsure', - ); - }); -}); - -describe('summarizeCase', () => { - it('reports every variant of a single-measurement case', () => { - const entry = discovered(['a', 'b'], ['mount']); - const summary = summarizeCase(entry, { - benchmarks: [ - benchmark({ - name: 'a', - measurement: 'mount', - differences: [null, { ms: [-3, -1], pct: [-5, -2] }], - }), - benchmark({ - name: 'b', - measurement: 'mount', - differences: [{ ms: [1, 3], pct: [2, 5] }, null], - }), - ], - }); - - expect(summary.reference).toBe('a'); - expect(summary.measurements).toHaveLength(1); - expect(summary.measurements[0].variants.map((variant) => variant.variant)).toEqual(['a', 'b']); - expect(summary.measurements[0].comparisons).toMatchObject([ - { variant: 'b', verdict: 'faster' }, - ]); - }); - - it('pairs results by measurement rather than by position', () => { - // Tachometer flattens variants × measurements into one list and names entries - // ` []`. Reading the difference matrix positionally would compare `mount` - // against `scroll`. Here the flat order deliberately interleaves the two measurements. - const entry = discovered(['a', 'b'], ['mount', 'scroll']); - const summary = summarizeCase(entry, { - benchmarks: [ - // index 0 - benchmark({ - name: 'a [mount]', - measurement: 'mount', - differences: [null, null, { ms: [-3, -1], pct: [-5, -2] }, null], - }), - // index 1 - benchmark({ - name: 'a [scroll]', - measurement: 'scroll', - differences: [null, null, null, { ms: [1, 3], pct: [2, 5] }], - }), - // index 2 - benchmark({ name: 'b [mount]', measurement: 'mount', differences: [] }), - // index 3 - benchmark({ name: 'b [scroll]', measurement: 'scroll', differences: [] }), - ], - }); - - const [mount, scroll] = summary.measurements; - expect(mount.name).toBe('mount'); - expect(mount.comparisons).toMatchObject([{ variant: 'b', verdict: 'faster' }]); - expect(scroll.name).toBe('scroll'); - expect(scroll.comparisons).toMatchObject([{ variant: 'b', verdict: 'slower' }]); - }); - - it('strips the measurement suffix when pairing variants back', () => { - const entry = discovered(['a', 'b'], ['mount']); - const summary = summarizeCase(entry, { - benchmarks: [ - benchmark({ - name: 'a [mount]', - measurement: 'mount', - differences: [null, { ms: [-3, -1], pct: [-5, -2] }], - }), - benchmark({ name: 'b [mount]', measurement: 'mount', differences: [] }), - ], - }); - - expect(summary.measurements[0].variants.map((variant) => variant.variant)).toEqual(['a', 'b']); - }); - - it('records the reverse comparison, which is not the negation of the forward one', () => { - // Each direction has its own denominator. - const entry = discovered(['a', 'b'], ['mount']); - const summary = summarizeCase(entry, { - benchmarks: [ - benchmark({ - name: 'a', - measurement: 'mount', - differences: [null, { ms: [-3, -1], pct: [-5, -2] }], - }), - benchmark({ - name: 'b', - measurement: 'mount', - differences: [{ ms: [1, 3], pct: [3, 9] }, null], - }), - ], - }); - - expect(summary.measurements[0].comparisons[0]).toMatchObject({ - verdict: 'faster', - versusReference: { verdict: 'slower', percentChange: { low: 3, high: 9 } }, - }); - }); - - it('skips a variant with no result for a measurement instead of failing the case', () => { - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - const entry = discovered(['a', 'b'], ['mount']); - - const summary = summarizeCase(entry, { - benchmarks: [benchmark({ name: 'a', measurement: 'mount', differences: [null] })], - }); - - expect(summary.measurements[0].variants.map((variant) => variant.variant)).toEqual(['a']); - expect(warn).toHaveBeenCalled(); - warn.mockRestore(); - }); - - it('throws when the reference itself has no result for a measurement', () => { - const entry = discovered(['a'], ['mount']); - - expect(() => summarizeCase(entry, { benchmarks: [] })).toThrow( - 'Case "example": no "mount" result for the reference "a".', - ); - }); - - it('throws when the case declares no variants', () => { - expect(() => summarizeCase(discovered([], ['mount']), { benchmarks: [] })).toThrow( - 'Case "example" declares no variants.', - ); - }); -}); diff --git a/packages/benchmark/src/tachometer/summarizeCase.ts b/packages/benchmark/src/tachometer/summarizeCase.ts deleted file mode 100644 index e01bca7b2f..0000000000 --- a/packages/benchmark/src/tachometer/summarizeCase.ts +++ /dev/null @@ -1,195 +0,0 @@ -import chalk from 'chalk'; -import type { BenchmarkCase, CaseComparison } from './discoverCases'; - -/** Reduces tachometer's raw JSON output to a flat, interpretable summary. */ - -export interface ConfidenceInterval { - low: number; - high: number; -} - -export interface TachometerDifference { - /** Absolute difference, in milliseconds. */ - absolute: ConfidenceInterval; - /** Relative difference, as a percentage. */ - percentChange: ConfidenceInterval; -} - -export interface TachometerBenchmark { - /** Variant name, suffixed with ` []` when a page has several. */ - name: string; - /** Mean duration, as a 95% confidence interval. */ - mean: ConfidenceInterval; - samples: number[]; - /** The measurement this entry reports. */ - measurement?: { name?: string }; - /** This entry against every other, by flat index. */ - differences: Array; -} - -export interface TachometerJson { - /** One entry per variant × measurement. */ - benchmarks: TachometerBenchmark[]; -} - -export type Verdict = 'faster' | 'slower' | 'unsure'; - -export interface VariantSummary { - variant: string; - meanMs: ConfidenceInterval; - samples: number; -} - -export interface ComparisonSummary { - variant: string; - verdict: Verdict; - absoluteMs: ConfidenceInterval; - percentChange: ConfidenceInterval; - versusReference?: { - verdict: Verdict; - absoluteMs: ConfidenceInterval; - percentChange: ConfidenceInterval; - }; -} - -export interface MeasurementSummary { - name: string; - variants: VariantSummary[]; - comparisons: ComparisonSummary[]; -} - -export interface CaseSummary { - name: string; - /** What the case compares, which is what says whether a difference is a regression. */ - comparison: CaseComparison; - reference: string; - measurements: MeasurementSummary[]; -} - -/** - * Judges a difference. - * - * A difference only counts as `faster`/`slower` when the millisecond interval and the percent - * interval both exclude zero on the same side. They come from the same samples but are computed - * separately, so near zero one can exclude it while the other does not — and a verdict that rests - * on only one of them flips depending on which is read. - */ -export function verdictOf(difference: TachometerDifference): Verdict { - if (difference.absolute.low > 0 && difference.percentChange.low > 0) { - return 'slower'; - } - if (difference.absolute.high < 0 && difference.percentChange.high < 0) { - return 'faster'; - } - return 'unsure'; -} - -/** - * Indexes tachometer's flat benchmark list by measurement and then by variant, keeping each entry's - * position so the difference matrix can be read back. - */ -function indexByMeasurement( - json: TachometerJson, -): Map> { - const byMeasurement = new Map< - string, - Map - >(); - json.benchmarks.forEach((benchmark, index) => { - const measurement = benchmark.measurement?.name ?? ''; - const suffix = ` [${measurement}]`; - const variant = benchmark.name.endsWith(suffix) - ? benchmark.name.slice(0, -suffix.length) - : benchmark.name; - const forMeasurement = byMeasurement.get(measurement) ?? new Map(); - forMeasurement.set(variant, { benchmark, index }); - byMeasurement.set(measurement, forMeasurement); - }); - return byMeasurement; -} - -/** - * Reduces one case's tachometer output to a per-measurement summary. - * - * Tachometer flattens variants × measurements into one list, naming entries - * ` []` when a page declares more than one, with a difference matrix over the - * whole flat list. Results are paired back by (variant, measurement) name — pairing by position - * would compare unrelated measurements as soon as a case has more than one. - * - * The first declared variant is the reference, because that is the one under test: for the - * auto-expanded regression case it is `[current]`. - */ -/** What a summary needs to know about a case, whichever engine measured it. */ -export type SummarizedCaseShape = Pick< - BenchmarkCase, - 'name' | 'comparison' | 'variants' | 'measurements' ->; - -/** One case as an engine ran it: its raw results, or the error that stopped it. */ -export type CaseRun = - | { entry: SummarizedCaseShape; json: TachometerJson } - | { entry: SummarizedCaseShape; error: string }; - -export function summarizeCase(entry: SummarizedCaseShape, json: TachometerJson): CaseSummary { - const [reference] = entry.variants; - if (!reference) { - throw new Error(`Case "${entry.name}" declares no variants.`); - } - - const byMeasurement = indexByMeasurement(json); - - const measurements = entry.measurements.map((measurement): MeasurementSummary => { - const byVariant = byMeasurement.get(measurement); - const lookup = (variant: string) => byVariant?.get(variant); - - const referenceResult = lookup(reference); - if (!referenceResult) { - throw new Error( - `Case "${entry.name}": no "${measurement}" result for the reference "${reference}".`, - ); - } - - const variants: VariantSummary[] = []; - const comparisons: ComparisonSummary[] = []; - for (const variant of entry.variants) { - const found = lookup(variant); - if (!found) { - console.warn( - chalk.yellow(` ${entry.name}: no "${measurement}" result for "${variant}"; skipping.`), - ); - continue; - } - variants.push({ - variant, - meanMs: found.benchmark.mean, - samples: found.benchmark.samples.length, - }); - if (variant === reference) { - continue; - } - const difference = referenceResult.benchmark.differences[found.index]; - if (!difference) { - continue; - } - // The same pair read from the variant's own matrix row. Each direction has its own - // denominator, so it is not the negation of `difference`. - const reverse = found.benchmark.differences[referenceResult.index]; - comparisons.push({ - variant, - verdict: verdictOf(difference), - absoluteMs: difference.absolute, - percentChange: difference.percentChange, - versusReference: reverse - ? { - verdict: verdictOf(reverse), - absoluteMs: reverse.absolute, - percentChange: reverse.percentChange, - } - : undefined, - }); - } - return { name: measurement, variants, comparisons }; - }); - - return { name: entry.name, comparison: entry.comparison, reference, measurements }; -} diff --git a/packages/benchmark/src/utils/html.ts b/packages/benchmark/src/utils/html.ts new file mode 100644 index 0000000000..6f9145d23d --- /dev/null +++ b/packages/benchmark/src/utils/html.ts @@ -0,0 +1,8 @@ +/** Escapes text for use in HTML content or a double-quoted attribute. */ +export function escapeHtml(text: string): string { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ad88ae0c46..c437218ac6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -188,9 +188,6 @@ importers: baseline-browser-mapping: specifier: 2.11.21 version: 2.11.21 - chromedriver: - specifier: ^151.0.5 - version: 151.0.5(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) eslint-plugin-n: specifier: 18.3.0 version: 18.3.0(@typescript/typescript6@6.0.2)(eslint@10.10.0(jiti@2.7.0)(supports-color@10.2.2)) @@ -1207,18 +1204,11 @@ importers: version: 30.0.1 publishDirectory: build - test/bundle-size: + test/benchmark: dependencies: - '@mui/internal-docs-infra': - specifier: workspace:* - version: link:../../packages/docs-infra/build - devDependencies: - '@mui/internal-bundle-size-checker': + '@mui/internal-test-utils': specifier: workspace:* - version: link:../../packages/bundle-size-checker - - test/performance: - dependencies: + version: link:../../packages/test-utils/build react: specifier: 19.2.8 version: 19.2.8 @@ -1229,21 +1219,31 @@ importers: '@mui/internal-benchmark': specifier: workspace:* version: link:../../packages/benchmark/build + '@playwright/test': + specifier: 1.62.1 + version: 1.62.1 '@types/react': specifier: 19.2.18 version: 19.2.18 - '@vitest/browser-playwright': - specifier: 4.1.11 - version: 4.1.11(playwright@1.63.0)(vite@8.2.2(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0))(vitest@4.1.11) - vitest: - specifier: 4.1.11 - version: 4.1.11(@types/node@22.20.1)(@vitest/browser-playwright@4.1.11)(@vitest/coverage-v8@4.1.11)(jsdom@30.0.1)(vite@8.2.2(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0)) + '@types/react-dom': + specifier: 19.2.7 + version: 19.2.7(@types/react@19.2.18) + vite: + specifier: ^8.2.2 + version: 8.2.2(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0) - test/tachometer: + test/bundle-size: dependencies: - '@mui/internal-test-utils': + '@mui/internal-docs-infra': specifier: workspace:* - version: link:../../packages/test-utils/build + version: link:../../packages/docs-infra/build + devDependencies: + '@mui/internal-bundle-size-checker': + specifier: workspace:* + version: link:../../packages/bundle-size-checker + + test/performance: + dependencies: react: specifier: 19.2.8 version: 19.2.8 @@ -1254,21 +1254,15 @@ importers: '@mui/internal-benchmark': specifier: workspace:* version: link:../../packages/benchmark/build - '@playwright/test': - specifier: 1.62.1 - version: 1.62.1 '@types/react': specifier: 19.2.18 version: 19.2.18 - '@types/react-dom': - specifier: 19.2.7 - version: 19.2.7(@types/react@19.2.18) - tachometer: - specifier: ^0.7.2 - version: 0.7.2(supports-color@10.2.2) - vite: - specifier: ^8.2.2 - version: 8.2.2(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0) + '@vitest/browser-playwright': + specifier: 4.1.11 + version: 4.1.11(playwright@1.63.0)(vite@8.2.2(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0))(vitest@4.1.11) + vitest: + specifier: 4.1.11 + version: 4.1.11(@types/node@22.20.1)(@vitest/browser-playwright@4.1.11)(@vitest/coverage-v8@4.1.11)(jsdom@30.0.1)(vite@8.2.2(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0)) packages: @@ -2008,9 +2002,6 @@ packages: '@types/react': optional: true - '@bazel/runfiles@6.5.0': - resolution: {integrity: sha512-RzahvqTkfpY2jsDxo8YItPX+/iZ6hbiikw1YhE0bA9EKBR5Og8Pa6FHn9PO9M0zaXRVsr0GFQLKbB/0rzy9SzA==} - '@bcoe/v8-coverage@1.0.2': resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} engines: {node: '>=18'} @@ -2461,9 +2452,6 @@ packages: '@floating-ui/utils@0.2.12': resolution: {integrity: sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==} - '@hapi/bourne@3.0.0': - resolution: {integrity: sha512-Waj1cwPXJDucOib4a3bAISsKJVb15MKi9IvmTI/7ssVEm6sywXGjVJDhl6/umt1pK1ZS7PacXU3A1PmFKHEZ2w==} - '@html-validate/stylish@6.0.0': resolution: {integrity: sha512-d1/lI3qIXhGVJF3+MmKEBn1dRX6U4Z+BDaOdDI3E6SXcO+OQ7hC/3mSo6BIjwQlcuV6NdDOKm6QCrzIQL1EezQ==} engines: {node: ^22.16.0 || >= 24.0.0} @@ -3690,10 +3678,6 @@ packages: '@sinclair/typebox@0.34.52': resolution: {integrity: sha512-XiMQh7qqVlxZzcVD+kkGMNGMzcTrDMLWI7S4x7z1MkCkbDPrekpZXEUK0eZqZFMuHQg2a2DZOcDIh9o5v3Gonw==} - '@sindresorhus/is@5.6.0': - resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==} - engines: {node: '>=14.16'} - '@sindresorhus/merge-streams@4.0.0': resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} @@ -3728,10 +3712,6 @@ packages: '@swc/helpers@0.5.23': resolution: {integrity: sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==} - '@szmarczak/http-timer@5.0.1': - resolution: {integrity: sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==} - engines: {node: '>=14.16'} - '@tanstack/query-core@5.102.8': resolution: {integrity: sha512-ZNjkJ33CqvPNec/6lZBnHqLc3EVGPZ9ySLhYahU9TcuRFdmwXewuj0c4hwSWcGHqEUwcSrKeZ+oGcvPBqXcQcg==} @@ -3740,9 +3720,6 @@ packages: peerDependencies: react: ^18 || ^19 - '@testim/chrome-version@1.1.4': - resolution: {integrity: sha512-kIhULpw9TrGYnHp/8VfdcneIcxKnLixmADtukQRtJUmsVlMg0niMkwV0xZmi8hqa57xqilIHjWFA0GKvEjVU5g==} - '@testing-library/dom@10.4.1': resolution: {integrity: sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==} engines: {node: '>=18'} @@ -3870,9 +3847,6 @@ packages: '@types/etag@1.8.4': resolution: {integrity: sha512-f1z/UMth8gQ6636NBqhFmJ3zES7EuDcUnV6K1gl1osHp+85KPKX+VixYWUpqLkw1fftCagyHJjJOZjZkEi2rHw==} - '@types/execa@0.9.0': - resolution: {integrity: sha512-mgfd93RhzjYBUHHV532turHC2j4l/qxsF/PbfDmprHDEUHmNZGlDn1CEsulGK3AfsPdhkWzZQT/S/k0UGhLGsA==} - '@types/find-package-json@1.2.7': resolution: {integrity: sha512-zHPw+sj/9u0cHV8cS99fizuCSEo3K5pk6JxDu6K3DEJAEZgdvjoHnFJz/C1Vzwr75GSPlAcDZR9O30/4CuQo9Q==} @@ -3885,9 +3859,6 @@ packages: '@types/hast@3.0.5': resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==} - '@types/http-cache-semantics@4.2.0': - resolution: {integrity: sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q==} - '@types/is-empty@1.2.3': resolution: {integrity: sha512-4J1l5d79hoIvsrKh5VUKVRA1aIdsOb10Hu5j3J2VfP/msDnfTdGPmNp2E1Wg+vs97Bktzo+MZePFFXSGoykYJw==} @@ -3912,9 +3883,6 @@ packages: '@types/ms@2.1.0': resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} - '@types/node@11.15.54': - resolution: {integrity: sha512-1RWYiq+5UfozGsU6MwJyFX6BtktcT10XRjvcAQmskCtMcW3tPske88lM/nHv7BQG1w9KBXI1zPGuu5PnNCX14g==} - '@types/node@14.18.63': resolution: {integrity: sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==} @@ -3935,9 +3903,6 @@ packages: resolution: {integrity: sha512-EULJ8LApcVEPbrfND0cRQqutIOdiIgJ1Mgrhpy755r14xMohPTEpkV/k28SJvuOs9bHRFW8x+KeDAEPiGQPB9Q==} deprecated: This is a stub types definition. parse-path provides its own type definitions, so you do not need this installed. - '@types/parse5@5.0.3': - resolution: {integrity: sha512-kUNnecmtkunAoQ3CnjmMkzNU/gtxG8guhi+Fk2U/kOpIKjIMKnXGp4IJCgQJrXSgMsWYimYG4TGjz/UzbGEBTw==} - '@types/prop-types@15.7.15': resolution: {integrity: sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==} @@ -4451,10 +4416,6 @@ packages: resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} - accepts@1.3.8: - resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} - engines: {node: '>= 0.6'} - acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -4469,18 +4430,6 @@ packages: engines: {node: '>=0.4.0'} hasBin: true - adm-zip@0.6.0: - resolution: {integrity: sha512-XleryMhbuksdKtofnWZ9Sk+4CUTbms4Mb/EU32SZwToAyZ5RgVos/ki8n+yr0LWHOGKuakbXTuuYNHLQjhddgg==} - engines: {node: '>=14.0'} - - agent-base@6.0.2: - resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} - engines: {node: '>= 6.0.0'} - - agent-base@9.0.0: - resolution: {integrity: sha512-TQf59BsZnytt8GdJKLPfUZ54g/iaUL2OWDSFCCvMOhsHduDQxO8xC4PNeyIkVcA5KwL2phPSv0douC0fgWzmnA==} - engines: {node: '>= 20'} - ajv-formats@2.1.1: resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} peerDependencies: @@ -4506,15 +4455,6 @@ packages: ansi-align@3.0.1: resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} - ansi-escape-sequences@6.2.4: - resolution: {integrity: sha512-2KJQAG1Nk4Iyu0dJENKXQJE9smEASrpu/E0F7LSnR72tQXngKGLqfRkHbkinjNct5vvAQY4BwQNt+4Tvg73nDQ==} - engines: {node: '>=12.17'} - peerDependencies: - '@75lb/nature': latest - peerDependenciesMeta: - '@75lb/nature': - optional: true - ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -4523,10 +4463,6 @@ packages: resolution: {integrity: sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==} engines: {node: '>=12'} - ansi-styles@3.2.1: - resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} - engines: {node: '>=4'} - ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} @@ -4562,18 +4498,6 @@ packages: resolution: {integrity: sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==} engines: {node: '>= 0.4'} - array-back@3.1.0: - resolution: {integrity: sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==} - engines: {node: '>=6'} - - array-back@4.0.2: - resolution: {integrity: sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==} - engines: {node: '>=8'} - - array-back@6.2.3: - resolution: {integrity: sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==} - engines: {node: '>=12.17'} - array-buffer-byte-length@1.0.2: resolution: {integrity: sha512-LHE+8BuR7RYGDKvnrmcuSq3tDcKv9OFEXQt/HpbZhY7V6h0zlUXutnAD82GiFx9rdieCMjkvtcsPqBwgUl1Iiw==} engines: {node: '>= 0.4'} @@ -4619,10 +4543,6 @@ packages: ast-types-flow@0.0.8: resolution: {integrity: sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==} - ast-types@0.13.4: - resolution: {integrity: sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==} - engines: {node: '>=4'} - ast-v8-to-istanbul@1.0.5: resolution: {integrity: sha512-UPAgKJFSEGMWSDr3LX4tqnAb4f7KGT8O40Tyx8wbYmmZ/yn58lNCm8h3svs3eXgiGd5AXxz8NDOvXWvicq+rJA==} @@ -4638,9 +4558,6 @@ packages: resolution: {integrity: sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA==} engines: {node: '>= 0.4'} - asynckit@0.4.0: - resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - autoprefixer@10.5.5: resolution: {integrity: sha512-uiRYvQYe/nNSzBJ7OUnd2/TZVsAdob3blml44teEpee9Cc1f4rGZFewO+JT3Wo8mgFOSzNqes4FHZn/Qz8WOuw==} engines: {node: ^10 || ^12 || >=14} @@ -4660,9 +4577,6 @@ packages: resolution: {integrity: sha512-UzGt8zg7Ny8djbYMhxl2zuEevVa7r2gJjYY5Lwr1xM7+XU2nd6CkIWFTVcCIbAP63vSz71NaVyyuSk9lHKcy0A==} engines: {node: '>=4'} - axios@1.20.0: - resolution: {integrity: sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==} - axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} @@ -4734,10 +4648,6 @@ packages: engines: {node: '>=6.0.0'} hasBin: true - basic-ftp@5.3.1: - resolution: {integrity: sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw==} - engines: {node: '>=10.0.0'} - bcrypt-pbkdf@1.0.2: resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} @@ -4783,9 +4693,6 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true - buffer-equal-constant-time@1.0.1: - resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} - buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -4808,18 +4715,6 @@ packages: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} - cache-content-type@1.0.1: - resolution: {integrity: sha512-IKufZ1o4Ut42YUrZSo8+qnMTrFuKkvyoLXUywKz9GJ5BrhOFGhLdkx9sG4KAnVvbY6kEcSFjLQul+DVmBm2bgA==} - engines: {node: '>= 6.0.0'} - - cacheable-lookup@7.0.0: - resolution: {integrity: sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==} - engines: {node: '>=14.16'} - - cacheable-request@10.2.14: - resolution: {integrity: sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==} - engines: {node: '>=14.16'} - cacheable@2.5.0: resolution: {integrity: sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g==} @@ -4867,10 +4762,6 @@ packages: resolution: {integrity: sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==} engines: {node: '>=12'} - chalk@2.4.2: - resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} - engines: {node: '>=4'} - chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -4907,11 +4798,6 @@ packages: resolution: {integrity: sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ==} engines: {node: '>=6.0'} - chromedriver@151.0.5: - resolution: {integrity: sha512-oPI/VtTeLePbkTOI5o4yvEOlcdpmqEdqQNLZbyu5ha6kpDQYfDRuliisDiph29hBK/UzHSmOyZFKX1mE3efYOQ==} - engines: {node: '>=22'} - hasBin: true - ci-info@4.4.0: resolution: {integrity: sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==} engines: {node: '>=8'} @@ -4954,14 +4840,6 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} - co-body@6.2.0: - resolution: {integrity: sha512-Kbpv2Yd1NdL1V/V4cwLVxraHDV6K8ayohr2rmH0J87Er8+zJjcTa6dAn9QMPC9CRgU8+aNajKbSf1TzDB1yKPA==} - engines: {node: '>=8.0.0'} - - co@4.6.0: - resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} - engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} - codsen-format-diagnostic-value@1.1.0: resolution: {integrity: sha512-Zlza4Webzp1e7/9EaOF6AYZjkdeUnbqMLMCbgJx0Zfw9+aDSDR+xZxExRLYz0zk7YvG5ZHVxSLU0OLzH2PbJEQ==} engines: {node: '>=18.20.8'} @@ -4973,37 +4851,19 @@ packages: collapse-white-space@2.1.0: resolution: {integrity: sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==} - color-convert@1.9.3: - resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} - color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} - color-name@1.1.3: - resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} - color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} colord@2.10.0: resolution: {integrity: sha512-AidJptpBJmjTclAp9BkLwJi0T93fo5epJnbaZslpg6QVzpHjAiveF55mE9AcUJiGMqRHgMDY8soMsQtuNYMHfw==} - combined-stream@1.0.8: - resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} - engines: {node: '>= 0.8'} - comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} - command-line-args@5.2.1: - resolution: {integrity: sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==} - engines: {node: '>=4.0.0'} - - command-line-usage@6.1.3: - resolution: {integrity: sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==} - engines: {node: '>=8.0.0'} - commander@2.20.3: resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} @@ -5022,9 +4882,6 @@ packages: commondir@1.0.1: resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} - compare-versions@6.1.1: - resolution: {integrity: sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg==} - compressible@2.0.18: resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} engines: {node: '>= 0.6'} @@ -5044,10 +4901,6 @@ packages: resolution: {integrity: sha512-kRGRZw3bLlFISDBgwTSA1TMBFN6J6GWDeubmDE3AF+3+yXL8hTWv8r5rkLbqYXY4RjPk/EzHnClI3zQf1cFmHA==} engines: {node: '>= 0.6'} - content-type@1.0.5: - resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} - engines: {node: '>= 0.6'} - content-type@3.0.0: resolution: {integrity: sha512-AIi5H6p0xk5uknXcN3/rmhP8jgp69OfSe/JuKiQAFprJ7UGw7mwj7m4XcmDzlrnJDG+cGpphAINGdU3g3g7kDw==} engines: {node: '>=22'} @@ -5062,13 +4915,6 @@ packages: resolution: {integrity: sha512-JtXpxqDqJ8P0UwEHwhxLzCIXQy97vlYBZR222Sbzb1q1Erex9ASrztJ29SyhWFQjod1AeFBaPzEEC8YvtZMIYg==} engines: {node: '>=6'} - cookies@0.9.1: - resolution: {integrity: sha512-TG2hpqe4ELx54QER/S3HQ9SRVnQnGBtKUz5bLQWtYAQ+o6GpgMs6sYUvaiJjVxb+UXwhRhAEP3m7LbsIZ77Hmw==} - engines: {node: '>= 0.8'} - - copy-to@2.0.1: - resolution: {integrity: sha512-3DdaFaU/Zf1AnpLiFDeNCD4TOWe3Zl2RZaTzUvWiIk5ERzcCodOE20Vqq4fzCbNoHURFHT4/us/Lfq+S2zyY4w==} - core-js-compat@3.50.0: resolution: {integrity: sha512-XGpFGbMLHwSt74YLTKho7Ib242qi6O8MSX+sRokV4oz7iKXvQWGYZthjIhjRGMxjzVkAubBO512dKGYcefmX3Q==} engines: {node: '>=6.4.0'} @@ -5096,10 +4942,6 @@ packages: resolution: {integrity: sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==} engines: {node: '>=10.0.0'} - cross-spawn@6.0.6: - resolution: {integrity: sha512-VqCUuhcd1iB+dsv8gxPttb5iZh/D0iubSP21g36KXdEuf6I5JiioesUVjpCdHV9MZRUfVFlvwtIUyPfxo5trtw==} - engines: {node: '>=4.8'} - cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -5134,9 +4976,6 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} - csv-stringify@6.8.3: - resolution: {integrity: sha512-gIeSCvq5F4VtXV3naV3VAewLhBkiZBz+PPhTOA8H3Y8h/ELa+R1ml0GZck/4/Nzo9ep2lvOluilJ6MJlbZsKMA==} - d3-array@3.2.4: resolution: {integrity: sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==} engines: {node: '>=12'} @@ -5184,10 +5023,6 @@ packages: damerau-levenshtein@1.0.8: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} - data-uri-to-buffer@8.0.0: - resolution: {integrity: sha512-6UHfyCux51b8PTGDgveqtz1tvphBku5DrMKKJbFAZAJOI2zsjDpDoYE1+QGj7FOMS4BdTFNJsJiR3zEB0xH0yQ==} - engines: {node: '>= 20'} - data-urls@7.0.0: resolution: {integrity: sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==} engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0} @@ -5226,15 +5061,6 @@ packages: supports-color: optional: true - debug@4.3.1: - resolution: {integrity: sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -5254,13 +5080,6 @@ packages: decode-named-character-reference@1.3.0: resolution: {integrity: sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==} - decompress-response@6.0.0: - resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} - engines: {node: '>=10'} - - deep-equal@1.0.1: - resolution: {integrity: sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==} - deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} @@ -5276,10 +5095,6 @@ packages: resolution: {integrity: sha512-m1pAzaJgZ/gssEqlOhJkPJp8Xly7QyW6xcrkUa2KKcDeDSEMP7X8xipU3snUcfisTQx0w1AGae+9UtJSfVnXGw==} engines: {node: '>=18'} - defer-to-connect@2.0.1: - resolution: {integrity: sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==} - engines: {node: '>=10'} - define-data-property@1.1.4: resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} engines: {node: '>= 0.4'} @@ -5292,35 +5107,10 @@ packages: resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} engines: {node: '>= 0.4'} - degenerator@7.0.1: - resolution: {integrity: sha512-ABErK0IefDSyHjlPH7WUEenIAX2rPPnrDcDM+TS3z3+zu9TfyKKi07BQM+8rmxpdE2y1v5fjjdoAS/x4D2U60w==} - engines: {node: '>= 20'} - peerDependencies: - quickjs-wasi: ^2.2.0 - - delayed-stream@1.0.0: - resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} - engines: {node: '>=0.4.0'} - - delegates@1.0.0: - resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} - - depd@1.1.2: - resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} - engines: {node: '>= 0.6'} - - depd@2.0.0: - resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} - engines: {node: '>= 0.8'} - dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} - destroy@1.2.0: - resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} - engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} - detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} @@ -5371,12 +5161,6 @@ packages: eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} - ecdsa-sig-formatter@1.0.11: - resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} - - ee-first@1.1.1: - resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} - electron-to-chromium@1.5.422: resolution: {integrity: sha512-UvA/32XqrLDdZSn7Jllo1AYNcWji/G0d5M0GTViE7KoGBiMunw3a34Sb2KO4ZZyrSEhqsxFoVhWWJshdyfKqJA==} @@ -5389,13 +5173,6 @@ packages: emoji-regex@9.2.2: resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} - encodeurl@1.0.2: - resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} - engines: {node: '>= 0.8'} - - end-of-stream@1.4.5: - resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} - enhanced-resolve@5.24.5: resolution: {integrity: sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==} engines: {node: '>=10.13.0'} @@ -5483,13 +5260,6 @@ packages: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} - escape-html@1.0.3: - resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} - - escape-string-regexp@1.0.5: - resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} - engines: {node: '>=0.8.0'} - escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -5498,11 +5268,6 @@ packages: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} - escodegen@2.1.0: - resolution: {integrity: sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==} - engines: {node: '>=6.0'} - hasBin: true - eslint-compat-utils@0.5.1: resolution: {integrity: sha512-3z3vFexKIEnjHE3zCMRo6fn/e44U7T1khUjg+Hp0ZQMCigh28rALD0nPFBcGZuiLC5rLZa2ubQHDRln09JfU2Q==} engines: {node: '>=12'} @@ -5691,11 +5456,6 @@ packages: resolution: {integrity: sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} - esprima@4.0.1: - resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} - engines: {node: '>=4'} - hasBin: true - esquery@1.7.0: resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} engines: {node: '>=0.10'} @@ -5741,10 +5501,6 @@ packages: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} - execa@1.0.0: - resolution: {integrity: sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==} - engines: {node: '>=6'} - execa@10.0.1: resolution: {integrity: sha512-ge98qjkRK4IB7tL7Ju/6qmm5LHoH1eEMt5FNZrz3f4UIYhF28lggX20z3FaX1sgc67msLEn0N0BscOs29iuwyw==} engines: {node: '>=22'} @@ -5849,10 +5605,6 @@ packages: find-package-json@1.2.0: resolution: {integrity: sha512-+SOGcLGYDJHtyqHd87ysBhmaeQ95oWspDKnMXBrnQ9Eq4OkLNqejgoaD8xVWu6GPa0B6roa6KinCMEMcVeqONw==} - find-replace@3.0.0: - resolution: {integrity: sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==} - engines: {node: '>=4.0.0'} - find-root@1.1.0: resolution: {integrity: sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==} @@ -5864,10 +5616,6 @@ packages: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} - find-up@6.3.0: - resolution: {integrity: sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - flat-cache@4.0.1: resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} engines: {node: '>=16'} @@ -5881,15 +5629,6 @@ packages: flatted@3.4.4: resolution: {integrity: sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==} - follow-redirects@1.16.0: - resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - for-each@0.3.5: resolution: {integrity: sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==} engines: {node: '>= 0.4'} @@ -5898,14 +5637,6 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} - form-data-encoder@2.1.4: - resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} - engines: {node: '>= 14.17'} - - form-data@4.0.6: - resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} - engines: {node: '>= 6'} - format-util@1.0.5: resolution: {integrity: sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==} @@ -5916,14 +5647,6 @@ packages: fraction.js@5.3.4: resolution: {integrity: sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==} - fresh@0.5.2: - resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} - engines: {node: '>= 0.6'} - - fs-extra@10.1.0: - resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==} - engines: {node: '>=12'} - fs-readdir-recursive@1.1.0: resolution: {integrity: sha512-GNanXlVr2pf02+sPN40XN8HG+ePaNcvM0q5mZBd668Obwb0yD5GiUbZOFgwn8kGMY6I3mdyDJzieUy3PTYyTRA==} @@ -5981,14 +5704,6 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} - get-stream@4.1.0: - resolution: {integrity: sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==} - engines: {node: '>=6'} - - get-stream@5.2.0: - resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} - engines: {node: '>=8'} - get-stream@6.0.1: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} engines: {node: '>=10'} @@ -6008,10 +5723,6 @@ packages: get-tsconfig@4.14.3: resolution: {integrity: sha512-++QEw4DIY7WGoukz+/+A/8dGYPT9l9yIadnmSgZ8Rjr3YVSVDipQSO9CdnJo9ePqFqUUqh+wk9uIaoiAwsiPkA==} - get-uri@8.0.1: - resolution: {integrity: sha512-/5N/P4Lrh0p/mDwlDRi7Y1+P2o/OyzZI3l6Iz1Ov6XXwwm1y3RlZLuo3gVgML99djrEDtV980bBxSuOeHLk8ww==} - engines: {node: '>= 20'} - git-up@8.1.1: resolution: {integrity: sha512-FDenSF3fVqBYSaJoYy1KSc2wosx0gCvKP+c+PRBht7cAaiCeQlBtfBDX9vgnNOHmdePlSFITVcn4pFfcgNvx3g==} @@ -6076,10 +5787,6 @@ packages: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} - got@12.6.1: - resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==} - engines: {node: '>=14.16'} - graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} @@ -6091,10 +5798,6 @@ packages: resolution: {integrity: sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg==} engines: {node: '>= 0.4'} - has-flag@3.0.0: - resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} - engines: {node: '>=4'} - has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} @@ -6217,41 +5920,6 @@ packages: html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} - http-assert@1.5.0: - resolution: {integrity: sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==} - engines: {node: '>= 0.8'} - - http-cache-semantics@4.2.0: - resolution: {integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==} - - http-errors@1.6.3: - resolution: {integrity: sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==} - engines: {node: '>= 0.6'} - - http-errors@1.8.1: - resolution: {integrity: sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==} - engines: {node: '>= 0.6'} - - http-errors@2.0.1: - resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} - engines: {node: '>= 0.8'} - - http-proxy-agent@9.1.0: - resolution: {integrity: sha512-2NxoveTT58mjYT4n3RPTEfCZGLMbidoO8XEieXfpSYxu+PQJ1qpx4ypwH6N+uF9twBPIvRRgvkvW5HUTYWENig==} - engines: {node: '>= 20'} - - http2-wrapper@2.2.1: - resolution: {integrity: sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==} - engines: {node: '>=10.19.0'} - - https-proxy-agent@5.0.1: - resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} - engines: {node: '>= 6'} - - https-proxy-agent@9.1.0: - resolution: {integrity: sha512-ag87y7cJJ9/3+GxFr8Oy4O5faDsGRGnBGsJj/YjOSsSx/5eadKLYTMPlzuR6obgoCDDm0abAAZitXXQkMOPSpA==} - engines: {node: '>= 20'} - human-signals@2.1.0: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} engines: {node: '>=10.17.0'} @@ -6264,10 +5932,6 @@ packages: resolution: {integrity: sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==} engines: {node: '>=18.18.0'} - iconv-lite@0.4.24: - resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} - engines: {node: '>=0.10.0'} - iconv-lite@0.7.3: resolution: {integrity: sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==} engines: {node: '>=0.10.0'} @@ -6304,17 +5968,10 @@ packages: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} engines: {node: '>=0.8.19'} - inflation@2.1.0: - resolution: {integrity: sha512-t54PPJHG1Pp7VQvxyVCJ9mBbjG3Hqryges9bXoOO6GExCPa+//i/d5GSuFtpx3ALLd7lgIAur6zrIlBQyJuMlQ==} - engines: {node: '>= 0.8.0'} - inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. - inherits@2.0.3: - resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==} - inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} @@ -6339,14 +5996,6 @@ packages: resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==} engines: {node: '>=12'} - ip-address@10.7.0: - resolution: {integrity: sha512-BGFsyJd5mpXp3rK6jIdADLNgpJUK1jnjzvYF8lK+VyDab9JAmqN0YOKDdP17HlgKb2+ehPgDc8EtnRLbGCAMhA==} - engines: {node: '>= 12'} - - ip-regex@2.1.0: - resolution: {integrity: sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw==} - engines: {node: '>=4'} - is-alphabetical@2.0.1: resolution: {integrity: sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==} @@ -6509,10 +6158,6 @@ packages: is-ssh@1.4.1: resolution: {integrity: sha512-JNeu1wQsHjyHgn9NcWTaXq6zWSR6hqE0++zhfZlkFBbScNkyvxCdeV8sRkSBaeLKxmbpR21brail63ACNxJ0Tg==} - is-stream@1.1.0: - resolution: {integrity: sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==} - engines: {node: '>=0.10.0'} - is-stream@2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} @@ -6541,9 +6186,6 @@ packages: resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==} engines: {node: '>=18'} - is-url@1.2.4: - resolution: {integrity: sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==} - is-wayland@0.1.0: resolution: {integrity: sha512-QkbMsWkIfkrzOPxenwye0h56iAXirZYHG9eHVPb22fO9y+wPbaX/CHacOWBa/I++4ohTcByimhM1/nyCsH8KNA==} engines: {node: '>=20'} @@ -6568,10 +6210,6 @@ packages: resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==} engines: {node: '>=16'} - is2@2.0.1: - resolution: {integrity: sha512-+WaJvnaA7aJySz2q/8sLjMb2Mw14KTplHmSwcSpZ/fWJPkUmqw3YTzSWbPJ7OAwRvdYTWF2Wg+yYJ1AdP5Z8CA==} - engines: {node: '>=v0.10.0'} - is64bit@2.0.0: resolution: {integrity: sha512-jv+8jaWCl0g2lSBkNSVXdzfBA0npK1HGC2KtWM9FumFRoGS94g3NbCCLVnCYHLjp4GrW2KZeeSTMo5ddtznmGw==} engines: {node: '>=18'} @@ -6683,19 +6321,6 @@ packages: engines: {node: ^18.0.0 || >=20.0.0} hasBin: true - jsonfile@6.2.1: - resolution: {integrity: sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==} - - jsonschema@1.5.0: - resolution: {integrity: sha512-K+A9hhqbn0f3pJX17Q/7H6yQfD/5OXgdrR5UE12gMXCiN9D5Xq2o5mddV2QEcX/bjla99ASsAAQUyMCCRWAEhw==} - - jsonwebtoken@9.0.3: - resolution: {integrity: sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==} - engines: {node: '>=12', npm: '>=6'} - - jstat@1.9.6: - resolution: {integrity: sha512-rPBkJbK2TnA8pzs93QcDDPlKcrtZWuuCo2dVR0TFLOJSxhqfWOVCSp8aV3/oSbn+4uY4yw1URtLpHQedtmXfug==} - jsx-ast-utils@3.3.5: resolution: {integrity: sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==} engines: {node: '>=4.0'} @@ -6703,19 +6328,9 @@ packages: jszip@3.10.1: resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==} - jwa@2.0.1: - resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} - - jws@4.0.1: - resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} - kebab-case@2.0.2: resolution: {integrity: sha512-NhIP7vecGtqJfNZ85ct0yRQfx//gCJHapJCBZP6/BBcBw/U218Jw7YX7rO6TI5/cCp5dJvlIjN3vbcRTqq7nWA==} - keygrip@1.1.0: - resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} - engines: {node: '>= 0.6'} - keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -6734,36 +6349,6 @@ packages: resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} engines: {node: '>=6'} - koa-bodyparser@4.4.1: - resolution: {integrity: sha512-kBH3IYPMb+iAXnrxIhXnW+gXV8OTzCu8VPDqvcDHW9SQrbkHmqPQtiZwrltNmSq6/lpipHnT7k7PsjlVD7kK0w==} - engines: {node: '>=8.0.0'} - - koa-compose@4.1.0: - resolution: {integrity: sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==} - - koa-convert@2.0.0: - resolution: {integrity: sha512-asOvN6bFlSnxewce2e/DK3p4tltyfC4VM7ZwuTuepI7dEQVcvpyFuBcEARu1+Hxg8DIwytce2n7jrZtRlPrARA==} - engines: {node: '>= 10'} - - koa-mount@4.2.0: - resolution: {integrity: sha512-2iHQc7vbA9qLeVq5gKAYh3m5DOMMlMfIKjW/REPAS18Mf63daCJHHVXY9nbu7ivrnYn5PiPC4CE523Tf5qvjeQ==} - engines: {node: '>= 7.6.0'} - - koa-node-resolve@1.0.0-pre.9: - resolution: {integrity: sha512-WKgqe5TGVD6zuR3NrKnmbb/NNHIbWOCezQVqqnyQLdtLLXWgiothlUQT23S5qQGE0Z623jp6jxpMjvAqyrcZFQ==} - - koa-send@5.0.1: - resolution: {integrity: sha512-tmcyQ/wXXuxpDxyNXv5yNNkdAMdFRqwtegBXUaowiQzUKqJehttS0x2j0eOZDQAyloAth5w6wwBImnFzkUz3pQ==} - engines: {node: '>= 8'} - - koa-static@5.0.0: - resolution: {integrity: sha512-UqyYyH5YEXaJrf9S8E23GoJFQZXkBVJ9zYYMPGz919MSX1KuvAcycIuS0ci150HCoPf4XQVhQ84Qf8xRPWxFaQ==} - engines: {node: '>= 7.6.0'} - - koa@2.16.4: - resolution: {integrity: sha512-3An0GCLDSR34tsCO4H8Tef8Pp2ngtaZDAZnsWJYelqXUK5wyiHvGItgK/xcSkmHLSTn1Jcho1mRQs2ehRzvKKw==} - engines: {node: ^4.8.4 || ^6.10.1 || ^7.10.1 || >= 8.1.4} - language-subtag-registry@0.3.23: resolution: {integrity: sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ==} @@ -6870,13 +6455,6 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} - locate-path@7.2.0: - resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - - lodash.camelcase@4.3.0: - resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} - lodash.clonedeep@4.5.0: resolution: {integrity: sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==} @@ -6889,24 +6467,6 @@ packages: lodash.groupby@4.6.0: resolution: {integrity: sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==} - lodash.includes@4.3.0: - resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} - - lodash.isboolean@3.0.3: - resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} - - lodash.isinteger@4.0.4: - resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==} - - lodash.isnumber@3.0.3: - resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==} - - lodash.isplainobject@4.0.6: - resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} - - lodash.isstring@4.0.1: - resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==} - lodash.memoize@4.1.2: resolution: {integrity: sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==} @@ -6916,9 +6476,6 @@ packages: lodash.mergewith@4.6.2: resolution: {integrity: sha512-GK3g5RPZWTRSeLSpgP8Xhra+pnjBC56q9FZYe1d5RN3TJ35dbkGy3YqBSMbyCrlbi+CM9Z3Jk5yTL7RCsqboyQ==} - lodash.once@4.1.1: - resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==} - lodash.truncate@4.4.2: resolution: {integrity: sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==} @@ -6938,10 +6495,6 @@ packages: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true - lowercase-keys@3.0.0: - resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} @@ -6952,10 +6505,6 @@ packages: lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} - lru-cache@7.18.3: - resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==} - engines: {node: '>=12'} - lru.min@1.1.5: resolution: {integrity: sha512-5J9ysMYUpYIg9RF2vJpy9SinEmSviFSe0GyPpCQ4L5QSkLAgeLXlTAOu2ZwWUU5m+0SBl6gUU1R1ZQB3aKypfA==} engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} @@ -7065,10 +6614,6 @@ packages: mdn-data@2.27.1: resolution: {integrity: sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==} - media-typer@0.3.0: - resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} - engines: {node: '>= 0.6'} - meow@14.1.0: resolution: {integrity: sha512-EDYo6VlmtnumlcBCbh1gLJ//9jvM/ndXHfVXIFrZVr6fGcwTUyCTFNTLCKuY3ffbK8L/+3Mzqnd58RojiZqHVw==} engines: {node: '>=20'} @@ -7196,10 +6741,6 @@ packages: resolution: {integrity: sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==} engines: {node: '>= 0.6'} - mime-db@1.52.0: - resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} - engines: {node: '>= 0.6'} - mime-db@1.54.0: resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} engines: {node: '>= 0.6'} @@ -7208,10 +6749,6 @@ packages: resolution: {integrity: sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==} engines: {node: '>= 0.6'} - mime-types@2.1.35: - resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} - engines: {node: '>= 0.6'} - mime-types@3.0.2: resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} engines: {node: '>=18'} @@ -7224,14 +6761,6 @@ packages: resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==} engines: {node: '>=12'} - mimic-response@3.1.0: - resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} - engines: {node: '>=10'} - - mimic-response@4.0.0: - resolution: {integrity: sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - min-indent@1.0.1: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} engines: {node: '>=4'} @@ -7320,9 +6849,6 @@ packages: ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} - ms@2.1.2: - resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} - ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -7359,10 +6885,6 @@ packages: natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} - negotiator@0.6.3: - resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} - engines: {node: '>= 0.6'} - negotiator@0.6.4: resolution: {integrity: sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==} engines: {node: '>= 0.6'} @@ -7370,10 +6892,6 @@ packages: neo-async@2.6.2: resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} - netmask@2.1.1: - resolution: {integrity: sha512-eonl3sLUha+S1GzTPxychyhnUzKyeQkZ7jLjKrBagJgPla13F+uQ71HgpFefyHgqrjEbCPkDArxYsjY8/+gLKA==} - engines: {node: '>= 0.4.0'} - next@16.3.4: resolution: {integrity: sha512-/Ztf6CeRH+ejEXUrYtqI4gkS66eFIHuSwqi60RgcpWKodxFZx2/dqVCMKBwILfAHXQ+F1b1vAudgj3mnxqtoIA==} engines: {node: '>=20.9.0'} @@ -7395,9 +6913,6 @@ packages: sass: optional: true - nice-try@1.0.5: - resolution: {integrity: sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==} - node-exports-info@1.6.2: resolution: {integrity: sha512-kXs9Go0cah0qHVV2v389IXQLdLCeE1xfFtjOAF+iobu0OIoG1pje8At2vMHyaPMiPMnG/LWP50twML21eMcAag==} engines: {node: '>= 0.4'} @@ -7425,10 +6940,6 @@ packages: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} - normalize-url@8.1.1: - resolution: {integrity: sha512-JYc0DPlpGWB40kH5g07gGTrYuMqV653k3uBKY6uITPWds3M0ov3GaWGp9lbE3Bzngx8+XkfzgvASb9vk9JDFXQ==} - engines: {node: '>=14.16'} - npm-install-checks@6.3.0: resolution: {integrity: sha512-W29RiK/xtpCGqn6f3ixfRYGk+zRyr+Ew9F2E20BfXxT5/euLdA/Nm7fO7OeTGuAmTs30cpgInyJ0cYe708YTZw==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -7445,10 +6956,6 @@ packages: resolution: {integrity: sha512-nkc+3pIIhqHVQr085X9d2JzPzLyjzQS96zbruppqC9aZRm/x8xx6xhI98gHtsfELP2bE+loHq8ZaHFHhe+NauA==} engines: {node: ^16.14.0 || >=18.0.0} - npm-run-path@2.0.2: - resolution: {integrity: sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==} - engines: {node: '>=4'} - npm-run-path@4.0.1: resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==} engines: {node: '>=8'} @@ -7504,10 +7011,6 @@ packages: resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} engines: {node: '>=12.20.0'} - on-finished@2.4.1: - resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} - engines: {node: '>= 0.8'} - on-headers@1.1.0: resolution: {integrity: sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==} engines: {node: '>= 0.8'} @@ -7523,9 +7026,6 @@ packages: resolution: {integrity: sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==} engines: {node: '>=12'} - only@0.0.2: - resolution: {integrity: sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==} - open@11.0.2: resolution: {integrity: sha512-RWqF+pBSkqecEvCKOn8QYhaNdRMJDZRIrlS/7rTDdLHaPcfXGCZ/h8zb413NfvdeAV0MR7T1yJcA34/q+CSm1Q==} engines: {node: '>=20'} @@ -7548,14 +7048,6 @@ packages: resolution: {integrity: sha512-19YVAg7T+WTrxggPukVq7DjTv6+PJ867TmhCvBsYwmbFCsZd344rq2Ld1p0wo8f8Qrrhgp82c6FJRqdXWtSEhg==} engines: {node: '>= 0.4'} - p-cancelable@3.0.0: - resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==} - engines: {node: '>=12.20'} - - p-finally@1.0.0: - resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} - engines: {node: '>=4'} - p-limit@2.3.0: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} engines: {node: '>=6'} @@ -7564,10 +7056,6 @@ packages: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} - p-limit@4.0.0: - resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - p-locate@4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -7576,10 +7064,6 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} - p-locate@6.0.0: - resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - p-retry@8.0.1: resolution: {integrity: sha512-NAigyu8hfvJe7MsS18mnc4is0MZtau3QMdBklaJKK23oBQb6occO3UPM3vHmTckW8KhfyjMZaUOqdTFEYliHgg==} engines: {node: '>=22'} @@ -7588,16 +7072,6 @@ packages: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} - pac-proxy-agent@9.1.0: - resolution: {integrity: sha512-1aU+1mpj3DrQPfo3gh+3Gap3G5x+axnMx1P/y0ZF2ch7kb2meyOCAH8K2k9d27ROsTE7TnAerzxqF9aon2jqnA==} - engines: {node: '>= 20'} - - pac-resolver@9.0.1: - resolution: {integrity: sha512-lJbS008tmkj08VhoM8Hzuv/VE5tK9MS0OIQ/7+s0lIF+BYhiQWFYzkSpML7lXs9iBu2jfmzBTLzhe9n6BX+dYw==} - engines: {node: '>= 20'} - peerDependencies: - quickjs-wasi: ^2.2.0 - package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} @@ -7633,27 +7107,16 @@ packages: resolution: {integrity: sha512-bCgsFI+GeGWPAvAiUv63ZorMeif3/U0zaXABGJbOWt5OH2KCaPHF6S+0ok4aqM9RuIPGyZdx9tR9l13PsW4AYQ==} engines: {node: '>=14.13.0'} - parse5@5.1.1: - resolution: {integrity: sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==} - parse5@7.3.0: resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} parse5@8.0.1: resolution: {integrity: sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==} - parseurl@1.3.3: - resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} - engines: {node: '>= 0.8'} - path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} - path-exists@5.0.0: - resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - path-is-absolute@1.0.1: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} @@ -7661,10 +7124,6 @@ packages: path-is-inside@1.0.2: resolution: {integrity: sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==} - path-key@2.0.1: - resolution: {integrity: sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==} - engines: {node: '>=4'} - path-key@3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} @@ -7720,17 +7179,10 @@ packages: resolution: {integrity: sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==} engines: {node: '>=8'} - pkg-install@1.0.0: - resolution: {integrity: sha512-UGI8bfhrDb1KN01RZ7Bq08GRQc8rmVjxQ2up0g4mUHPCYDTK1FzQ0PMmLOBCHg3yaIijZ2U3Fn9ofLa4N392Ug==} - pkg-pr-new@0.0.88: resolution: {integrity: sha512-Xc6PMJ2gher0WZP+rtjefFk26hIb7V1PTLL30bmy1Z2vsRmSvqiss7Ag1XUdyplTGYzIlFNJp4L3vMNmK44N6g==} hasBin: true - pkg-up@4.0.0: - resolution: {integrity: sha512-N4zdA4sfOe6yCv+ulPCmpnIBQ5I60xfhDr1otdBBhKte9QtEf3bhfrfkW7dTb+IQ0iEx4ZDzas0kc1o5rdWpYg==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - playwright-core@1.62.1: resolution: {integrity: sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==} engines: {node: '>=20'} @@ -7857,10 +7309,6 @@ packages: process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} - progress@2.0.3: - resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} - engines: {node: '>=0.4.0'} - promise-inflight@1.0.1: resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==} peerDependencies: @@ -7889,26 +7337,6 @@ packages: protocols@2.0.2: resolution: {integrity: sha512-hHVTzba3wboROl0/aWRRG9dMytgH6ow//STBZh43l/wQgmMhYhOFi0EHWAPtoCz9IAUymsyP0TSBHkhgMEGNnQ==} - proxy-agent-negotiate@1.1.0: - resolution: {integrity: sha512-N8IBcM3UgCVzz2L2Lqv8DVntDnnC8/hiV4nEDUPkqq72TPUgYWjQc+bdZlBPZK9LzPAvOY//gAt0S0DApoOXWQ==} - engines: {node: '>= 20'} - peerDependencies: - kerberos: ^2.0.0 - peerDependenciesMeta: - kerberos: - optional: true - - proxy-agent@8.0.2: - resolution: {integrity: sha512-idLLRewuemWd7GH/BDJzGiB0dWGfT2SQs3jy6NtZtGWU9uPTTSdeC1/cdbqLwgzhfv027daGFuXX426e2Eg20A==} - engines: {node: '>= 20'} - - proxy-from-env@2.1.0: - resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} - engines: {node: '>=10'} - - pump@3.0.4: - resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} - punycode@1.4.1: resolution: {integrity: sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==} @@ -7932,13 +7360,6 @@ packages: queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} - quick-lru@5.1.1: - resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} - engines: {node: '>=10'} - - quickjs-wasi@2.2.0: - resolution: {integrity: sha512-zQxXmQMrEoD3S+jQdYsloq4qAuaxKFHZj6hHqOYGwB2iQZH+q9e/lf5zQPXCKOk0WJuAjzRFbO4KwHIp2D05Iw==} - quotation@2.0.3: resolution: {integrity: sha512-yEc24TEgCFLXx7D4JHJJkK4JFVtatO8fziwUxY4nB/Jbea9o9CVS3gt22mA0W7rPYAGW2fWzYDSOtD94PwOyqA==} @@ -7965,10 +7386,6 @@ packages: rasterizehtml@1.4.1: resolution: {integrity: sha512-mcE6b5cKhQO0Qka9vduSQW/F/crQ7EUjrTpXDPAdcj9uNtYijBiNmUOxOrIuMMq7GO/X3CR6UckHTgh5ZVqqBA==} - raw-body@2.5.3: - resolution: {integrity: sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==} - engines: {node: '>= 0.8'} - rc@1.2.8: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true @@ -8035,10 +7452,6 @@ packages: recma-stringify@1.0.0: resolution: {integrity: sha512-cjwII1MdIIVloKvC9ErQ+OgAtwHBmcZ0Bg4ciz78FtbT8In39aAYbaA7zvxQ61xVMSPE8WxhLwLbhif4Js2C+g==} - reduce-flatten@2.0.0: - resolution: {integrity: sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w==} - engines: {node: '>=6'} - reflect.getprototypeof@1.0.10: resolution: {integrity: sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==} engines: {node: '>= 0.4'} @@ -8170,17 +7583,10 @@ packages: reselect@5.3.0: resolution: {integrity: sha512-XGoLeRAVzUTcJ1qkxPQhDJyIZ5d6zzZD9nT7AEZOaaU9UbWclhycElmhO+VD5bFeLuzhPBaOV2oXC8uG35ZSpg==} - resolve-alpn@1.2.1: - resolution: {integrity: sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==} - resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} - resolve-path@1.4.0: - resolution: {integrity: sha512-i1xevIst/Qa+nA9olDxLWnLk8YZbi8R/7JPbCMcgyWaFR6bKWaexgJgEB5oc2PKMjYdrHynyz0NY+if+H98t1w==} - engines: {node: '>= 0.8'} - resolve-pkg-maps@1.0.0: resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} @@ -8194,10 +7600,6 @@ packages: engines: {node: '>= 0.4'} hasBin: true - responselike@3.0.0: - resolution: {integrity: sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==} - engines: {node: '>=14.16'} - retry@0.12.0: resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} engines: {node: '>= 4'} @@ -8259,9 +7661,6 @@ packages: sane-domparser-error@0.2.0: resolution: {integrity: sha512-wxjDV5jty95tNv8N/4WA15UNGqqaor/xX7rnNYY961hifN3bheYoKqtXN+V/M6EUgmUAs6pMul3klwUPMEiVXA==} - sanitize-filename@1.6.4: - resolution: {integrity: sha512-9ZyI08PsvdQl2r/bBIGubpVdR3RR9sY6RDiWFPreA21C/EFlQhmgo20UZlNjZMMZNubusLhAQozkA0Od5J21Eg==} - saxes@5.0.1: resolution: {integrity: sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==} engines: {node: '>=10'} @@ -8277,10 +7676,6 @@ packages: resolution: {integrity: sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA==} engines: {node: '>= 10.13.0'} - selenium-webdriver@4.48.0: - resolution: {integrity: sha512-rKM9uXFRWcF9aThrZQDNQH2/9Et/WvMZbg3/x1rnSYWoXiwJuShYeH0IAli8Cuw+c3lEV0UWPfUz88H+fvW9Hg==} - engines: {node: '>= 22.0.0'} - semver@5.7.2: resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} hasBin: true @@ -8323,12 +7718,6 @@ packages: setimmediate@1.0.5: resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==} - setprototypeof@1.1.0: - resolution: {integrity: sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==} - - setprototypeof@1.2.0: - resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} - sharp@0.35.4: resolution: {integrity: sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==} engines: {node: '>=20.9.0'} @@ -8338,18 +7727,10 @@ packages: '@types/node': optional: true - shebang-command@1.2.0: - resolution: {integrity: sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==} - engines: {node: '>=0.10.0'} - shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} - shebang-regex@1.0.0: - resolution: {integrity: sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==} - engines: {node: '>=0.10.0'} - shebang-regex@3.0.0: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} @@ -8403,18 +7784,6 @@ packages: resolution: {integrity: sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==} engines: {node: '>=10'} - smart-buffer@4.2.0: - resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==} - engines: {node: '>= 6.0.0', npm: '>= 3.0.0'} - - socks-proxy-agent@10.1.0: - resolution: {integrity: sha512-WlMj/67cEJ6MDI1OcsnjuYKDNDoyPCCYZ249kuuXPiMDw9F8PXkVaQ7YWu3siTydfQ/4BEZcvGzu+aYvz7dDCQ==} - engines: {node: '>= 20'} - - socks@2.8.10: - resolution: {integrity: sha512-e0VyvkVTwVYViNovRkZ9aodhxVlyoMn7eJhVUPxZ+eK9P/7CBkxvvsBOHqFPEH416726W8tLXXXjKwqgTErrCQ==} - engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} - source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -8468,14 +7837,6 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} - statuses@1.5.0: - resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==} - engines: {node: '>= 0.6'} - - statuses@2.0.2: - resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} - engines: {node: '>= 0.8'} - std-env@4.2.0: resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} @@ -8567,10 +7928,6 @@ packages: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} - strip-eof@1.0.0: - resolution: {integrity: sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==} - engines: {node: '>=0.10.0'} - strip-final-newline@2.0.0: resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} engines: {node: '>=6'} @@ -8647,10 +8004,6 @@ packages: resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} engines: {node: '>=18'} - supports-color@5.5.0: - resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} - engines: {node: '>=4'} - supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} @@ -8685,31 +8038,14 @@ packages: resolution: {integrity: sha512-ulAk51I9UVUyJgxlv9M6lFot2WP3e7t8Kz9+IS6D4rVba1tR9kON+Ey69f+1R4Q8cd45Lod6a4IcJIxnzGc/zA==} engines: {node: '>=18'} - systeminformation@5.33.8: - resolution: {integrity: sha512-v4F6OGYGh7wDvV68YmjOmZwGixV9A/GQ7d2b84t0UF4CaOy9jipNWIJDkHqYDYiTPuiojqlwVQd0hfUKOUN7tQ==} - engines: {node: '>=10.0.0'} - os: [darwin, linux, win32, freebsd, openbsd, netbsd, sunos, android] - hasBin: true - - table-layout@1.0.2: - resolution: {integrity: sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==} - engines: {node: '>=8.0.0'} - table@6.9.0: resolution: {integrity: sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==} engines: {node: '>=10.0.0'} - tachometer@0.7.2: - resolution: {integrity: sha512-e9dtDyWuMFyomQxo16vzti6ymoJd3Dp2yGfKrhAiNzCkb2HiP6A0+SWtqE84hcO6tSKZF8eTFqrxM2uKbliJyQ==} - hasBin: true - tapable@2.3.3: resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} engines: {node: '>=6'} - tcp-port-used@1.0.3: - resolution: {integrity: sha512-4CEQ3qRJYo+mtEbJ+OoQu3dF4TDkwaO3RDVC4UzP5cpAOIUWwuwPjD7sdxDFFqsMUjsXVVYBMlg/boAaloThMA==} - terser@5.51.2: resolution: {integrity: sha512-bWnjSNscmuI+GJze6ZupnHP8G/cTcsJF+bXCeQknk2SHQsgbNJnLrqiH9jZ2W4STPVXH2mDKKRX3iwPhc9Cn/Q==} engines: {node: '>=10'} @@ -8759,10 +8095,6 @@ packages: resolution: {integrity: sha512-m1TdR/rvT7kgGJZhspNtXdsdYk0fddFpJJFlG5s+UkPFo6lkLoZ3YLOaovPYjq1R75NP5JfeTlSHaOsE09peCg==} engines: {node: '>=20'} - toidentifier@1.0.1: - resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} - engines: {node: '>=0.6'} - totalist@3.0.1: resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} engines: {node: '>=6'} @@ -8781,9 +8113,6 @@ packages: trough@2.2.0: resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==} - truncate-utf8-bytes@1.0.2: - resolution: {integrity: sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==} - ts-api-utils@2.5.0: resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} engines: {node: '>=18.12'} @@ -8799,10 +8128,6 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - tsscmp@1.0.6: - resolution: {integrity: sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==} - engines: {node: '>=0.6.x'} - tsx@4.23.13: resolution: {integrity: sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==} engines: {node: '>=18.0.0'} @@ -8835,10 +8160,6 @@ packages: resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} engines: {node: '>=16'} - type-is@1.6.18: - resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} - engines: {node: '>= 0.6'} - typed-array-buffer@1.0.3: resolution: {integrity: sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==} engines: {node: '>= 0.4'} @@ -8879,18 +8200,6 @@ packages: engines: {node: '>=16.20.0'} hasBin: true - typical@4.0.0: - resolution: {integrity: sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==} - engines: {node: '>=8'} - - typical@5.2.0: - resolution: {integrity: sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==} - engines: {node: '>=8'} - - ua-parser-js@1.0.41: - resolution: {integrity: sha512-LbBDqdIC5s8iROCUjMbW1f5dJQTEFB1+KO9ogbvlb3nm9n4YHa5p4KTvFPWvh2Hs8gZMBuiB1/8+pdfe/tDPug==} - hasBin: true - uint8-to-base64@0.2.1: resolution: {integrity: sha512-uO/84GaoDUfiAxpa8EksjVLE77A9Kc7ZTziN4zRpq4de9yLaLcZn3jx1/sVjyupsywcVX6RKWbqLe7gUNyzH+Q==} @@ -8985,14 +8294,6 @@ packages: universal-user-agent@7.0.3: resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==} - universalify@2.0.1: - resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} - engines: {node: '>= 10.0.0'} - - unpipe@1.0.0: - resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} - engines: {node: '>= 0.8'} - unrs-resolver@1.12.2: resolution: {integrity: sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ==} @@ -9017,9 +8318,6 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 - utf8-byte-length@1.0.5: - resolution: {integrity: sha512-Xn0w3MtiQ6zoz2vFyUVruaCL53O/DwUvkEeOvj+uulMm0BkUGYWmBYVyElqZaSLhY6ZD0ulfU3aBra2aVT4xfA==} - util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -9256,10 +8554,6 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} - wordwrapjs@4.0.1: - resolution: {integrity: sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==} - engines: {node: '>=8.0.0'} - wrap-ansi@6.2.0: resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} engines: {node: '>=8'} @@ -9360,18 +8654,10 @@ packages: resolution: {integrity: sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg==} engines: {node: ^20.19.0 || ^22.12.0 || >=23} - ylru@1.4.0: - resolution: {integrity: sha512-2OQsPNEmBCvXuFlIni/a+Rn+R2pHW9INm0BxXJ4hVDA8TirqMj+J/Rp9ItLatT/5pZqWwefVrTQcHpixsxnVlA==} - engines: {node: '>= 4.0.0'} - yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - yocto-queue@1.2.2: - resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} - engines: {node: '>=12.20'} - yoctocolors@2.2.0: resolution: {integrity: sha512-xYqdZFUK/VYazNl/oCDYN+3WloWQwMfZxBoiNt6qNyk+xfOdi598muWE42rNZFp1kNOiqW936q5RhUdnpqElSg==} engines: {node: '>=18'} @@ -10508,8 +9794,6 @@ snapshots: optionalDependencies: '@types/react': 19.2.18 - '@bazel/runfiles@6.5.0': {} - '@bcoe/v8-coverage@1.0.2': {} '@blazediff/core@1.9.1': {} @@ -10915,8 +10199,6 @@ snapshots: '@floating-ui/utils@0.2.12': {} - '@hapi/bourne@3.0.0': {} - '@html-validate/stylish@6.0.0': {} '@humanfs/core@0.19.2': @@ -12031,8 +11313,6 @@ snapshots: '@sinclair/typebox@0.34.52': {} - '@sindresorhus/is@5.6.0': {} - '@sindresorhus/merge-streams@4.0.0': {} '@smithy/core@3.33.3': @@ -12074,10 +11354,6 @@ snapshots: dependencies: tslib: 2.8.1 - '@szmarczak/http-timer@5.0.1': - dependencies: - defer-to-connect: 2.0.1 - '@tanstack/query-core@5.102.8': {} '@tanstack/react-query@5.102.8(react@19.2.8)': @@ -12085,8 +11361,6 @@ snapshots: '@tanstack/query-core': 5.102.8 react: 19.2.8 - '@testim/chrome-version@1.1.4': {} - '@testing-library/dom@10.4.1': dependencies: '@babel/code-frame': 7.29.7 @@ -12234,10 +11508,6 @@ snapshots: dependencies: '@types/node': 22.20.1 - '@types/execa@0.9.0': - dependencies: - '@types/node': 22.20.1 - '@types/find-package-json@1.2.7': dependencies: '@types/node': 22.20.1 @@ -12250,8 +11520,6 @@ snapshots: dependencies: '@types/unist': 3.0.3 - '@types/http-cache-semantics@4.2.0': {} - '@types/is-empty@1.2.3': {} '@types/jsdom@30.0.0': @@ -12277,8 +11545,6 @@ snapshots: '@types/ms@2.1.0': {} - '@types/node@11.15.54': {} - '@types/node@14.18.63': {} '@types/node@18.19.130': @@ -12299,8 +11565,6 @@ snapshots: dependencies: parse-path: 7.1.0 - '@types/parse5@5.0.3': {} - '@types/prop-types@15.7.15': {} '@types/proper-lockfile@4.1.4': @@ -12811,11 +12075,6 @@ snapshots: abbrev@2.0.0: {} - accepts@1.3.8: - dependencies: - mime-types: 2.1.35 - negotiator: 0.6.3 - acorn-jsx@5.3.2(acorn@8.18.0): dependencies: acorn: 8.18.0 @@ -12826,16 +12085,6 @@ snapshots: acorn@8.18.0: {} - adm-zip@0.6.0: {} - - agent-base@6.0.2(supports-color@10.2.2): - dependencies: - debug: 4.4.3(supports-color@10.2.2) - transitivePeerDependencies: - - supports-color - - agent-base@9.0.0: {} - ajv-formats@2.1.1(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -12870,18 +12119,10 @@ snapshots: dependencies: string-width: 4.2.3 - ansi-escape-sequences@6.2.4: - dependencies: - array-back: 6.2.3 - ansi-regex@5.0.1: {} ansi-regex@6.3.0: {} - ansi-styles@3.2.1: - dependencies: - color-convert: 1.9.3 - ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 @@ -12910,12 +12151,6 @@ snapshots: aria-query@5.3.2: {} - array-back@3.1.0: {} - - array-back@4.0.2: {} - - array-back@6.2.3: {} - array-buffer-byte-length@1.0.2: dependencies: call-bound: 1.0.4 @@ -12995,10 +12230,6 @@ snapshots: ast-types-flow@0.0.8: {} - ast-types@0.13.4: - dependencies: - tslib: 2.8.1 - ast-v8-to-istanbul@1.0.5: dependencies: '@jridgewell/trace-mapping': 0.3.31 @@ -13011,8 +12242,6 @@ snapshots: async-function@1.0.0: {} - asynckit@0.4.0: {} - autoprefixer@10.5.5(postcss@8.5.28): dependencies: browserslist: 4.28.9 @@ -13030,16 +12259,6 @@ snapshots: axe-core@4.13.0: {} - axios@1.20.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2): - dependencies: - follow-redirects: 1.16.0(debug@4.4.3(supports-color@10.2.2)) - form-data: 4.0.6 - https-proxy-agent: 5.0.1(supports-color@10.2.2) - proxy-from-env: 2.1.0 - transitivePeerDependencies: - - debug - - supports-color - axobject-query@4.1.0: {} babel-plugin-macros@3.1.0: @@ -13139,8 +12358,6 @@ snapshots: baseline-browser-mapping@2.11.21: {} - basic-ftp@5.3.1: {} - bcrypt-pbkdf@1.0.2: dependencies: tweetnacl: 0.14.5 @@ -13196,8 +12413,6 @@ snapshots: node-releases: 2.0.54 update-browserslist-db: 1.3.2(browserslist@4.28.9) - buffer-equal-constant-time@1.0.1: {} - buffer-from@1.1.2: {} buildcheck@0.0.7: @@ -13215,23 +12430,6 @@ snapshots: bytes@3.1.2: {} - cache-content-type@1.0.1: - dependencies: - mime-types: 2.1.35 - ylru: 1.4.0 - - cacheable-lookup@7.0.0: {} - - cacheable-request@10.2.14: - dependencies: - '@types/http-cache-semantics': 4.2.0 - get-stream: 6.0.1 - http-cache-semantics: 4.2.0 - keyv: 4.5.4 - mimic-response: 4.0.0 - normalize-url: 8.1.1 - responselike: 3.0.0 - cacheable@2.5.0: dependencies: '@cacheable/memory': 2.2.0 @@ -13277,12 +12475,6 @@ snapshots: dependencies: chalk: 4.1.2 - chalk@2.4.2: - dependencies: - ansi-styles: 3.2.1 - escape-string-regexp: 1.0.5 - supports-color: 5.5.0 - chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -13317,20 +12509,6 @@ snapshots: chrome-trace-event@1.0.4: {} - chromedriver@151.0.5(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2): - dependencies: - '@testim/chrome-version': 1.1.4 - adm-zip: 0.6.0 - axios: 1.20.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) - compare-versions: 6.1.1 - proxy-agent: 8.0.2(supports-color@10.2.2) - proxy-from-env: 2.1.0 - tcp-port-used: 1.0.3(supports-color@10.2.2) - transitivePeerDependencies: - - debug - - kerberos - - supports-color - ci-info@4.4.0: {} cli-boxes@3.0.0: {} @@ -13374,16 +12552,6 @@ snapshots: clsx@2.1.1: {} - co-body@6.2.0: - dependencies: - '@hapi/bourne': 3.0.0 - inflation: 2.1.0 - qs: 6.16.0 - raw-body: 2.5.3 - type-is: 1.6.18 - - co@4.6.0: {} - codsen-format-diagnostic-value@1.1.0: {} codsen-utils@1.10.0: @@ -13392,40 +12560,16 @@ snapshots: collapse-white-space@2.1.0: {} - color-convert@1.9.3: - dependencies: - color-name: 1.1.3 - color-convert@2.0.1: dependencies: color-name: 1.1.4 - color-name@1.1.3: {} - color-name@1.1.4: {} colord@2.10.0: {} - combined-stream@1.0.8: - dependencies: - delayed-stream: 1.0.0 - comma-separated-tokens@2.0.3: {} - command-line-args@5.2.1: - dependencies: - array-back: 3.1.0 - find-replace: 3.0.0 - lodash.camelcase: 4.3.0 - typical: 4.0.0 - - command-line-usage@6.1.3: - dependencies: - array-back: 4.0.2 - chalk: 2.4.2 - table-layout: 1.0.2 - typical: 5.2.0 - commander@2.20.3: {} commander@4.1.1: {} @@ -13436,8 +12580,6 @@ snapshots: commondir@1.0.1: {} - compare-versions@6.1.1: {} - compressible@2.0.18: dependencies: mime-db: 1.54.0 @@ -13465,8 +12607,6 @@ snapshots: content-disposition@0.5.2: {} - content-type@1.0.5: {} - content-type@3.0.0: {} convert-source-map@1.9.0: {} @@ -13478,13 +12618,6 @@ snapshots: lodash.clonedeep: 4.5.0 yargs-parser: 20.2.9 - cookies@0.9.1: - dependencies: - depd: 2.0.0 - keygrip: 1.1.0 - - copy-to@2.0.1: {} - core-js-compat@3.50.0: dependencies: browserslist: 4.28.9 @@ -13516,14 +12649,6 @@ snapshots: nan: 2.28.0 optional: true - cross-spawn@6.0.6: - dependencies: - nice-try: 1.0.5 - path-key: 2.0.1 - semver: 5.7.2 - shebang-command: 1.2.0 - which: 1.3.1 - cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -13558,8 +12683,6 @@ snapshots: csstype@3.2.3: {} - csv-stringify@6.8.3: {} - d3-array@3.2.4: dependencies: internmap: 2.0.3 @@ -13602,8 +12725,6 @@ snapshots: damerau-levenshtein@1.0.8: {} - data-uri-to-buffer@8.0.0: {} - data-urls@7.0.0: dependencies: whatwg-mimetype: 5.0.0 @@ -13645,12 +12766,6 @@ snapshots: optionalDependencies: supports-color: 10.2.2 - debug@4.3.1(supports-color@10.2.2): - dependencies: - ms: 2.1.2 - optionalDependencies: - supports-color: 10.2.2 - debug@4.4.3(supports-color@10.2.2): dependencies: ms: 2.1.3 @@ -13671,12 +12786,6 @@ snapshots: dependencies: character-entities: 2.0.2 - decompress-response@6.0.0: - dependencies: - mimic-response: 3.1.0 - - deep-equal@1.0.1: {} - deep-extend@0.6.0: {} deep-is@0.1.4: {} @@ -13688,8 +12797,6 @@ snapshots: bundle-name: 4.1.0 default-browser-id: 5.0.1 - defer-to-connect@2.0.1: {} - define-data-property@1.1.4: dependencies: es-define-property: 1.0.1 @@ -13704,25 +12811,8 @@ snapshots: has-property-descriptors: 1.0.2 object-keys: 1.1.1 - degenerator@7.0.1(quickjs-wasi@2.2.0): - dependencies: - ast-types: 0.13.4 - escodegen: 2.1.0 - esprima: 4.0.1 - quickjs-wasi: 2.2.0 - - delayed-stream@1.0.0: {} - - delegates@1.0.0: {} - - depd@1.1.2: {} - - depd@2.0.0: {} - dequal@2.0.3: {} - destroy@1.2.0: {} - detect-libc@2.1.2: {} devlop@1.1.0: @@ -13774,12 +12864,6 @@ snapshots: eastasianwidth@0.2.0: {} - ecdsa-sig-formatter@1.0.11: - dependencies: - safe-buffer: 5.2.1 - - ee-first@1.1.1: {} - electron-to-chromium@1.5.422: {} emoji-regex@10.6.0: {} @@ -13788,12 +12872,6 @@ snapshots: emoji-regex@9.2.2: {} - encodeurl@1.0.2: {} - - end-of-stream@1.4.5: - dependencies: - once: 1.4.0 - enhanced-resolve@5.24.5: dependencies: graceful-fs: 4.2.11 @@ -13978,22 +13056,10 @@ snapshots: escalade@3.2.0: {} - escape-html@1.0.3: {} - - escape-string-regexp@1.0.5: {} - escape-string-regexp@4.0.0: {} escape-string-regexp@5.0.0: {} - escodegen@2.1.0: - dependencies: - esprima: 4.0.1 - estraverse: 5.3.0 - esutils: 2.0.3 - optionalDependencies: - source-map: 0.6.1 - eslint-compat-utils@0.5.1(eslint@10.10.0(jiti@2.7.0)(supports-color@10.2.2)): dependencies: eslint: 10.10.0(jiti@2.7.0)(supports-color@10.2.2) @@ -14323,8 +13389,6 @@ snapshots: acorn-jsx: 5.3.2(acorn@8.18.0) eslint-visitor-keys: 5.0.1 - esprima@4.0.1: {} - esquery@1.7.0: dependencies: estraverse: 5.3.0 @@ -14374,16 +13438,6 @@ snapshots: events@3.3.0: {} - execa@1.0.0: - dependencies: - cross-spawn: 6.0.6 - get-stream: 4.1.0 - is-stream: 1.1.0 - npm-run-path: 2.0.2 - p-finally: 1.0.0 - signal-exit: 3.0.7 - strip-eof: 1.0.0 - execa@10.0.1: dependencies: '@sindresorhus/merge-streams': 4.0.0 @@ -14523,10 +13577,6 @@ snapshots: find-package-json@1.2.0: {} - find-replace@3.0.0: - dependencies: - array-back: 3.1.0 - find-root@1.1.0: {} find-up@4.1.0: @@ -14539,11 +13589,6 @@ snapshots: locate-path: 6.0.0 path-exists: 4.0.0 - find-up@6.3.0: - dependencies: - locate-path: 7.2.0 - path-exists: 5.0.0 - flat-cache@4.0.1: dependencies: flatted: 3.4.4 @@ -14559,10 +13604,6 @@ snapshots: flatted@3.4.4: {} - follow-redirects@1.16.0(debug@4.4.3(supports-color@10.2.2)): - optionalDependencies: - debug: 4.4.3(supports-color@10.2.2) - for-each@0.3.5: dependencies: is-callable: 1.2.7 @@ -14572,30 +13613,12 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 - form-data-encoder@2.1.4: {} - - form-data@4.0.6: - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - es-set-tostringtag: 2.1.0 - hasown: 2.0.4 - mime-types: 2.1.35 - format-util@1.0.5: {} format@0.2.2: {} fraction.js@5.3.4: {} - fresh@0.5.2: {} - - fs-extra@10.1.0: - dependencies: - graceful-fs: 4.2.11 - jsonfile: 6.2.1 - universalify: 2.0.1 - fs-readdir-recursive@1.1.0: {} fs.realpath@1.0.0: {} @@ -14654,14 +13677,6 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.2 - get-stream@4.1.0: - dependencies: - pump: 3.0.4 - - get-stream@5.2.0: - dependencies: - pump: 3.0.4 - get-stream@6.0.1: {} get-stream@8.0.1: {} @@ -14681,14 +13696,6 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 - get-uri@8.0.1(supports-color@10.2.2): - dependencies: - basic-ftp: 5.3.1 - data-uri-to-buffer: 8.0.0 - debug: 4.4.3(supports-color@10.2.2) - transitivePeerDependencies: - - supports-color - git-up@8.1.1: dependencies: is-ssh: 1.4.1 @@ -14763,20 +13770,6 @@ snapshots: gopd@1.2.0: {} - got@12.6.1: - dependencies: - '@sindresorhus/is': 5.6.0 - '@szmarczak/http-timer': 5.0.1 - cacheable-lookup: 7.0.0 - cacheable-request: 10.2.14 - decompress-response: 6.0.0 - form-data-encoder: 2.1.4 - get-stream: 6.0.1 - http2-wrapper: 2.2.1 - lowercase-keys: 3.0.0 - p-cancelable: 3.0.0 - responselike: 3.0.0 - graceful-fs@4.2.11: {} gzip-size@6.0.0: @@ -14785,8 +13778,6 @@ snapshots: has-bigints@1.1.0: {} - has-flag@3.0.0: {} - has-flag@4.0.0: {} has-flag@5.0.1: {} @@ -14965,76 +13956,12 @@ snapshots: html-void-elements@3.0.0: {} - http-assert@1.5.0: - dependencies: - deep-equal: 1.0.1 - http-errors: 1.8.1 - - http-cache-semantics@4.2.0: {} - - http-errors@1.6.3: - dependencies: - depd: 1.1.2 - inherits: 2.0.3 - setprototypeof: 1.1.0 - statuses: 1.5.0 - - http-errors@1.8.1: - dependencies: - depd: 1.1.2 - inherits: 2.0.4 - setprototypeof: 1.2.0 - statuses: 1.5.0 - toidentifier: 1.0.1 - - http-errors@2.0.1: - dependencies: - depd: 2.0.0 - inherits: 2.0.4 - setprototypeof: 1.2.0 - statuses: 2.0.2 - toidentifier: 1.0.1 - - http-proxy-agent@9.1.0(supports-color@10.2.2): - dependencies: - agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) - proxy-agent-negotiate: 1.1.0 - transitivePeerDependencies: - - kerberos - - supports-color - - http2-wrapper@2.2.1: - dependencies: - quick-lru: 5.1.1 - resolve-alpn: 1.2.1 - - https-proxy-agent@5.0.1(supports-color@10.2.2): - dependencies: - agent-base: 6.0.2(supports-color@10.2.2) - debug: 4.4.3(supports-color@10.2.2) - transitivePeerDependencies: - - supports-color - - https-proxy-agent@9.1.0(supports-color@10.2.2): - dependencies: - agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) - proxy-agent-negotiate: 1.1.0 - transitivePeerDependencies: - - kerberos - - supports-color - human-signals@2.1.0: {} human-signals@5.0.0: {} human-signals@8.0.1: {} - iconv-lite@0.4.24: - dependencies: - safer-buffer: 2.1.2 - iconv-lite@0.7.3: dependencies: safer-buffer: 2.1.2 @@ -15060,15 +13987,11 @@ snapshots: imurmurhash@0.1.4: {} - inflation@2.1.0: {} - inflight@1.0.6: dependencies: once: 1.4.0 wrappy: 1.0.2 - inherits@2.0.3: {} - inherits@2.0.4: {} ini@1.3.8: {} @@ -15091,10 +14014,6 @@ snapshots: internmap@2.0.3: {} - ip-address@10.7.0: {} - - ip-regex@2.1.0: {} - is-alphabetical@2.0.1: {} is-alphanumerical@2.0.1: @@ -15237,8 +14156,6 @@ snapshots: dependencies: protocols: 2.0.2 - is-stream@1.1.0: {} - is-stream@2.0.1: {} is-stream@3.0.0: {} @@ -15262,8 +14179,6 @@ snapshots: is-unicode-supported@2.1.0: {} - is-url@1.2.4: {} - is-wayland@0.1.0: {} is-weakmap@2.0.2: {} @@ -15285,12 +14200,6 @@ snapshots: dependencies: is-inside-container: 1.0.0 - is2@2.0.1: - dependencies: - deep-is: 0.1.4 - ip-regex: 2.1.0 - is-url: 1.2.4 - is64bit@2.0.0: dependencies: system-architecture: 0.1.0 @@ -15403,29 +14312,6 @@ snapshots: dependencies: '@dmsnell/diff-match-patch': 1.1.0 - jsonfile@6.2.1: - dependencies: - universalify: 2.0.1 - optionalDependencies: - graceful-fs: 4.2.11 - - jsonschema@1.5.0: {} - - jsonwebtoken@9.0.3: - dependencies: - jws: 4.0.1 - lodash.includes: 4.3.0 - lodash.isboolean: 3.0.3 - lodash.isinteger: 4.0.4 - lodash.isnumber: 3.0.3 - lodash.isplainobject: 4.0.6 - lodash.isstring: 4.0.1 - lodash.once: 4.1.1 - ms: 2.1.3 - semver: 7.8.5 - - jstat@1.9.6: {} - jsx-ast-utils@3.3.5: dependencies: array-includes: 3.1.9 @@ -15440,23 +14326,8 @@ snapshots: readable-stream: 2.3.8 setimmediate: 1.0.5 - jwa@2.0.1: - dependencies: - buffer-equal-constant-time: 1.0.1 - ecdsa-sig-formatter: 1.0.11 - safe-buffer: 5.2.1 - - jws@4.0.1: - dependencies: - jwa: 2.0.1 - safe-buffer: 5.2.1 - kebab-case@2.0.2: {} - keygrip@1.1.0: - dependencies: - tsscmp: 1.0.6 - keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -15471,82 +14342,6 @@ snapshots: kleur@4.1.5: {} - koa-bodyparser@4.4.1: - dependencies: - co-body: 6.2.0 - copy-to: 2.0.1 - type-is: 1.6.18 - - koa-compose@4.1.0: {} - - koa-convert@2.0.0: - dependencies: - co: 4.6.0 - koa-compose: 4.1.0 - - koa-mount@4.2.0(supports-color@10.2.2): - dependencies: - debug: 4.4.3(supports-color@10.2.2) - koa-compose: 4.1.0 - transitivePeerDependencies: - - supports-color - - koa-node-resolve@1.0.0-pre.9(supports-color@10.2.2): - dependencies: - '@babel/generator': 7.29.8 - '@babel/parser': 7.29.8 - '@babel/traverse': 7.29.8(supports-color@10.2.2) - '@types/babel__generator': 7.27.0 - '@types/parse5': 5.0.3 - get-stream: 5.2.0 - parse5: 5.1.1 - resolve: 1.22.12 - transitivePeerDependencies: - - supports-color - - koa-send@5.0.1(supports-color@10.2.2): - dependencies: - debug: 4.4.3(supports-color@10.2.2) - http-errors: 1.8.1 - resolve-path: 1.4.0 - transitivePeerDependencies: - - supports-color - - koa-static@5.0.0(supports-color@10.2.2): - dependencies: - debug: 3.2.7(supports-color@10.2.2) - koa-send: 5.0.1(supports-color@10.2.2) - transitivePeerDependencies: - - supports-color - - koa@2.16.4(supports-color@10.2.2): - dependencies: - accepts: 1.3.8 - cache-content-type: 1.0.1 - content-disposition: 0.5.2 - content-type: 1.0.5 - cookies: 0.9.1 - debug: 4.4.3(supports-color@10.2.2) - delegates: 1.0.0 - depd: 2.0.0 - destroy: 1.2.0 - encodeurl: 1.0.2 - escape-html: 1.0.3 - fresh: 0.5.2 - http-assert: 1.5.0 - http-errors: 1.8.1 - is-generator-function: 1.1.2 - koa-compose: 4.1.0 - koa-convert: 2.0.0 - on-finished: 2.4.1 - only: 0.0.2 - parseurl: 1.3.3 - statuses: 1.5.0 - type-is: 1.6.18 - vary: 1.1.2 - transitivePeerDependencies: - - supports-color - language-subtag-registry@0.3.23: {} language-tags@1.0.9: @@ -15630,12 +14425,6 @@ snapshots: dependencies: p-locate: 5.0.0 - locate-path@7.2.0: - dependencies: - p-locate: 6.0.0 - - lodash.camelcase@4.3.0: {} - lodash.clonedeep@4.5.0: {} lodash.debounce@4.0.8: {} @@ -15644,26 +14433,12 @@ snapshots: lodash.groupby@4.6.0: {} - lodash.includes@4.3.0: {} - - lodash.isboolean@3.0.3: {} - - lodash.isinteger@4.0.4: {} - - lodash.isnumber@3.0.3: {} - - lodash.isplainobject@4.0.6: {} - - lodash.isstring@4.0.1: {} - lodash.memoize@4.1.2: {} lodash.merge@4.6.2: {} lodash.mergewith@4.6.2: {} - lodash.once@4.1.1: {} - lodash.truncate@4.4.2: {} lodash.uniq@4.5.0: {} @@ -15678,8 +14453,6 @@ snapshots: dependencies: js-tokens: 4.0.0 - lowercase-keys@3.0.0: {} - lru-cache@10.4.3: {} lru-cache@11.5.2: {} @@ -15688,8 +14461,6 @@ snapshots: dependencies: yallist: 3.1.1 - lru-cache@7.18.3: {} - lru.min@1.1.5: {} lucide-react@1.41.0(react@19.2.8): @@ -15920,8 +14691,6 @@ snapshots: mdn-data@2.27.1: {} - media-typer@0.3.0: {} - meow@14.1.0: {} merge-stream@2.0.0: {} @@ -16206,18 +14975,12 @@ snapshots: mime-db@1.33.0: {} - mime-db@1.52.0: {} - mime-db@1.54.0: {} mime-types@2.1.18: dependencies: mime-db: 1.33.0 - mime-types@2.1.35: - dependencies: - mime-db: 1.52.0 - mime-types@3.0.2: dependencies: mime-db: 1.54.0 @@ -16226,10 +14989,6 @@ snapshots: mimic-fn@4.0.0: {} - mimic-response@3.1.0: {} - - mimic-response@4.0.0: {} - min-indent@1.0.1: {} minimatch@10.2.6: @@ -16267,8 +15026,6 @@ snapshots: ms@2.0.0: {} - ms@2.1.2: {} - ms@2.1.3: {} mute-stream@3.0.0: {} @@ -16303,14 +15060,10 @@ snapshots: natural-compare@1.4.0: {} - negotiator@0.6.3: {} - negotiator@0.6.4: {} neo-async@2.6.2: {} - netmask@2.1.1: {} - next@16.3.4(@babel/core@7.29.7(supports-color@10.2.2))(@playwright/test@1.62.1)(@types/node@22.20.1)(babel-plugin-macros@3.1.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8): dependencies: '@next/env': 16.3.4 @@ -16365,8 +15118,6 @@ snapshots: - '@types/node' - babel-plugin-macros - nice-try@1.0.5: {} - node-exports-info@1.6.2: dependencies: array.prototype.flatmap: 1.3.3 @@ -16395,8 +15146,6 @@ snapshots: normalize-path@3.0.0: {} - normalize-url@8.1.1: {} - npm-install-checks@6.3.0: dependencies: semver: 7.8.5 @@ -16417,10 +15166,6 @@ snapshots: npm-package-arg: 11.0.3 semver: 7.8.5 - npm-run-path@2.0.2: - dependencies: - path-key: 2.0.1 - npm-run-path@4.0.1: dependencies: path-key: 3.1.1 @@ -16484,10 +15229,6 @@ snapshots: obug@2.1.4: {} - on-finished@2.4.1: - dependencies: - ee-first: 1.1.1 - on-headers@1.1.0: {} once@1.4.0: @@ -16502,8 +15243,6 @@ snapshots: dependencies: mimic-fn: 4.0.0 - only@0.0.2: {} - open@11.0.2: dependencies: default-browser: 5.5.1 @@ -16537,10 +15276,6 @@ snapshots: object-keys: 1.1.1 safe-push-apply: 1.0.0 - p-cancelable@3.0.0: {} - - p-finally@1.0.0: {} - p-limit@2.3.0: dependencies: p-try: 2.2.0 @@ -16549,10 +15284,6 @@ snapshots: dependencies: yocto-queue: 0.1.0 - p-limit@4.0.0: - dependencies: - yocto-queue: 1.2.2 - p-locate@4.1.0: dependencies: p-limit: 2.3.0 @@ -16561,36 +15292,12 @@ snapshots: dependencies: p-limit: 3.1.0 - p-locate@6.0.0: - dependencies: - p-limit: 4.0.0 - p-retry@8.0.1: dependencies: is-network-error: 1.3.2 p-try@2.2.0: {} - pac-proxy-agent@9.1.0(supports-color@10.2.2): - dependencies: - agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) - get-uri: 8.0.1(supports-color@10.2.2) - http-proxy-agent: 9.1.0(supports-color@10.2.2) - https-proxy-agent: 9.1.0(supports-color@10.2.2) - pac-resolver: 9.0.1(quickjs-wasi@2.2.0) - quickjs-wasi: 2.2.0 - socks-proxy-agent: 10.1.0(supports-color@10.2.2) - transitivePeerDependencies: - - kerberos - - supports-color - - pac-resolver@9.0.1(quickjs-wasi@2.2.0): - dependencies: - degenerator: 7.0.1(quickjs-wasi@2.2.0) - netmask: 2.1.1 - quickjs-wasi: 2.2.0 - package-json-from-dist@1.0.1: {} pako@1.0.11: {} @@ -16637,8 +15344,6 @@ snapshots: '@types/parse-path': 7.1.0 parse-path: 7.1.0 - parse5@5.1.1: {} - parse5@7.3.0: dependencies: entities: 6.0.1 @@ -16647,18 +15352,12 @@ snapshots: dependencies: entities: 8.0.0 - parseurl@1.3.3: {} - path-exists@4.0.0: {} - path-exists@5.0.0: {} - path-is-absolute@1.0.1: {} path-is-inside@1.0.2: {} - path-key@2.0.1: {} - path-key@3.1.1: {} path-key@4.0.0: {} @@ -16696,18 +15395,8 @@ snapshots: dependencies: find-up: 4.1.0 - pkg-install@1.0.0: - dependencies: - '@types/execa': 0.9.0 - '@types/node': 11.15.54 - execa: 1.0.0 - pkg-pr-new@0.0.88: {} - pkg-up@4.0.0: - dependencies: - find-up: 6.3.0 - playwright-core@1.62.1: {} playwright-core@1.63.0: {} @@ -16817,8 +15506,6 @@ snapshots: process-nextick-args@2.0.1: {} - progress@2.0.3: {} - promise-inflight@1.0.1: {} promise-retry@2.0.1: @@ -16847,29 +15534,6 @@ snapshots: protocols@2.0.2: {} - proxy-agent-negotiate@1.1.0: {} - - proxy-agent@8.0.2(supports-color@10.2.2): - dependencies: - agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) - http-proxy-agent: 9.1.0(supports-color@10.2.2) - https-proxy-agent: 9.1.0(supports-color@10.2.2) - lru-cache: 7.18.3 - pac-proxy-agent: 9.1.0(supports-color@10.2.2) - proxy-from-env: 2.1.0 - socks-proxy-agent: 10.1.0(supports-color@10.2.2) - transitivePeerDependencies: - - kerberos - - supports-color - - proxy-from-env@2.1.0: {} - - pump@3.0.4: - dependencies: - end-of-stream: 1.4.5 - once: 1.4.0 - punycode@1.4.1: optional: true @@ -16889,13 +15553,10 @@ snapshots: dependencies: es-define-property: 1.0.1 side-channel: 1.1.1 + optional: true queue-microtask@1.2.3: {} - quick-lru@5.1.1: {} - - quickjs-wasi@2.2.0: {} - quotation@2.0.3: {} range-parser@1.2.0: {} @@ -16928,13 +15589,6 @@ snapshots: xmlserializer: 0.6.1 optional: true - raw-body@2.5.3: - dependencies: - bytes: 3.1.2 - http-errors: 2.0.1 - iconv-lite: 0.4.24 - unpipe: 1.0.0 - rc@1.2.8: dependencies: deep-extend: 0.6.0 @@ -17026,8 +15680,6 @@ snapshots: unified: 11.0.5 vfile: 6.0.3 - reduce-flatten@2.0.0: {} - reflect.getprototypeof@1.0.10: dependencies: call-bind: 1.0.9 @@ -17321,15 +15973,8 @@ snapshots: reselect@5.3.0: {} - resolve-alpn@1.2.1: {} - resolve-from@4.0.0: {} - resolve-path@1.4.0: - dependencies: - http-errors: 1.6.3 - path-is-absolute: 1.0.1 - resolve-pkg-maps@1.0.0: {} resolve@1.22.12: @@ -17348,10 +15993,6 @@ snapshots: path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 - responselike@3.0.0: - dependencies: - lowercase-keys: 3.0.0 - retry@0.12.0: {} reusify@1.1.0: {} @@ -17427,10 +16068,6 @@ snapshots: sane-domparser-error@0.2.0: optional: true - sanitize-filename@1.6.4: - dependencies: - truncate-utf8-bytes: 1.0.2 - saxes@5.0.1: dependencies: xmlchars: 2.2.0 @@ -17448,16 +16085,6 @@ snapshots: ajv-formats: 2.1.1(ajv@8.20.0) ajv-keywords: 5.1.0(ajv@8.20.0) - selenium-webdriver@4.48.0: - dependencies: - '@bazel/runfiles': 6.5.0 - jszip: 3.10.1 - tmp: 0.2.7 - ws: 8.21.3 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - semver@5.7.2: {} semver@6.3.1: {} @@ -17518,10 +16145,6 @@ snapshots: setimmediate@1.0.5: {} - setprototypeof@1.1.0: {} - - setprototypeof@1.2.0: {} - sharp@0.35.4(@types/node@22.20.1): dependencies: '@img/colour': 1.1.0 @@ -17555,16 +16178,10 @@ snapshots: '@img/sharp-win32-x64': 0.35.4 '@types/node': 22.20.1 - shebang-command@1.2.0: - dependencies: - shebang-regex: 1.0.0 - shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 - shebang-regex@1.0.0: {} - shebang-regex@3.0.0: {} side-channel-list@1.0.1: @@ -17625,21 +16242,6 @@ snapshots: astral-regex: 2.0.0 is-fullwidth-code-point: 3.0.0 - smart-buffer@4.2.0: {} - - socks-proxy-agent@10.1.0(supports-color@10.2.2): - dependencies: - agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) - socks: 2.8.10 - transitivePeerDependencies: - - supports-color - - socks@2.8.10: - dependencies: - ip-address: 10.7.0 - smart-buffer: 4.2.0 - source-map-js@1.2.1: {} source-map-support@0.5.21: @@ -17685,10 +16287,6 @@ snapshots: stackback@0.0.2: {} - statuses@1.5.0: {} - - statuses@2.0.2: {} - std-env@4.2.0: {} stop-iteration-iterator@1.1.0: @@ -17823,8 +16421,6 @@ snapshots: strip-bom@3.0.0: {} - strip-eof@1.0.0: {} - strip-final-newline@2.0.0: {} strip-final-newline@3.0.0: {} @@ -17924,10 +16520,6 @@ snapshots: supports-color@10.2.2: {} - supports-color@5.5.0: - dependencies: - has-flag: 3.0.0 - supports-color@7.2.0: dependencies: has-flag: 4.0.0 @@ -17955,15 +16547,6 @@ snapshots: system-architecture@0.1.0: {} - systeminformation@5.33.8: {} - - table-layout@1.0.2: - dependencies: - array-back: 4.0.2 - deep-extend: 0.6.0 - typical: 5.2.0 - wordwrapjs: 4.0.1 - table@6.9.0: dependencies: ajv: 8.20.0 @@ -17972,50 +16555,8 @@ snapshots: string-width: 4.2.3 strip-ansi: 6.0.1 - tachometer@0.7.2(supports-color@10.2.2): - dependencies: - ansi-escape-sequences: 6.2.4 - command-line-args: 5.2.1 - command-line-usage: 6.1.3 - csv-stringify: 6.8.3 - fs-extra: 10.1.0 - get-stream: 6.0.1 - got: 12.6.1 - jsonschema: 1.5.0 - jsonwebtoken: 9.0.3 - jstat: 1.9.6 - koa: 2.16.4(supports-color@10.2.2) - koa-bodyparser: 4.4.1 - koa-mount: 4.2.0(supports-color@10.2.2) - koa-node-resolve: 1.0.0-pre.9(supports-color@10.2.2) - koa-send: 5.0.1(supports-color@10.2.2) - koa-static: 5.0.0(supports-color@10.2.2) - pkg-install: 1.0.0 - pkg-up: 4.0.0 - progress: 2.0.3 - sanitize-filename: 1.6.4 - selenium-webdriver: 4.48.0 - semver: 7.8.5 - source-map-support: 0.5.21 - strip-ansi: 7.2.0 - systeminformation: 5.33.8 - table: 6.9.0 - ua-parser-js: 1.0.41 - transitivePeerDependencies: - - '@75lb/nature' - - bufferutil - - supports-color - - utf-8-validate - tapable@2.3.3: {} - tcp-port-used@1.0.3(supports-color@10.2.2): - dependencies: - debug: 4.3.1(supports-color@10.2.2) - is2: 2.0.1 - transitivePeerDependencies: - - supports-color - terser@5.51.2: dependencies: '@jridgewell/source-map': 0.3.11 @@ -18058,8 +16599,6 @@ snapshots: toad-cache@3.7.4: {} - toidentifier@1.0.1: {} - totalist@3.0.1: {} tough-cookie@6.0.2: @@ -18074,10 +16613,6 @@ snapshots: trough@2.2.0: {} - truncate-utf8-bytes@1.0.2: - dependencies: - utf8-byte-length: 1.0.5 - ts-api-utils@2.5.0(@typescript/typescript6@6.0.2): dependencies: typescript: '@typescript/typescript6@6.0.2' @@ -18093,8 +16628,6 @@ snapshots: tslib@2.8.1: {} - tsscmp@1.0.6: {} - tsx@4.23.13: dependencies: esbuild: 0.28.2 @@ -18117,11 +16650,6 @@ snapshots: type-fest@4.41.0: {} - type-is@1.6.18: - dependencies: - media-typer: 0.3.0 - mime-types: 2.1.35 - typed-array-buffer@1.0.3: dependencies: call-bound: 1.0.4 @@ -18198,12 +16726,6 @@ snapshots: '@typescript/typescript-win32-arm64': 7.0.2 '@typescript/typescript-win32-x64': 7.0.2 - typical@4.0.0: {} - - typical@5.2.0: {} - - ua-parser-js@1.0.41: {} - uint8-to-base64@0.2.1: {} unbox-primitive@1.1.0: @@ -18337,10 +16859,6 @@ snapshots: universal-user-agent@7.0.3: {} - universalify@2.0.1: {} - - unpipe@1.0.0: {} - unrs-resolver@1.12.2: dependencies: napi-postinstall: 0.3.4 @@ -18393,8 +16911,6 @@ snapshots: dependencies: react: 19.2.8 - utf8-byte-length@1.0.5: {} - util-deprecate@1.0.2: {} validate-npm-package-license@3.0.4: @@ -18665,11 +17181,6 @@ snapshots: word-wrap@1.2.5: {} - wordwrapjs@4.0.1: - dependencies: - reduce-flatten: 2.0.0 - typical: 5.2.0 - wrap-ansi@6.2.0: dependencies: ansi-styles: 4.3.0 @@ -18758,12 +17269,8 @@ snapshots: y18n: 5.0.8 yargs-parser: 22.0.0 - ylru@1.4.0: {} - yocto-queue@0.1.0: {} - yocto-queue@1.2.2: {} - yoctocolors@2.2.0: {} zod-validation-error@3.5.4(zod@3.25.76): diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c943e1469c..dce77aded3 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,7 +6,6 @@ packages: engineStrict: true strictDepBuilds: true allowBuilds: - chromedriver: true esbuild: true nx: true sharp: true diff --git a/test/tachometer/README.md b/test/benchmark/README.md similarity index 100% rename from test/tachometer/README.md rename to test/benchmark/README.md diff --git a/test/tachometer/package.json b/test/benchmark/package.json similarity index 71% rename from test/tachometer/package.json rename to test/benchmark/package.json index 57afb031b5..56ac2c7083 100644 --- a/test/tachometer/package.json +++ b/test/benchmark/package.json @@ -1,12 +1,12 @@ { "private": true, - "description": "Tachometer smoke-test harness for the @mui/internal-benchmark tacho tooling", + "description": "Smoke-test harness for `benchmark run` from @mui/internal-benchmark", "scripts": { "dev": "vite", "build": "vite build", "preview": "vite preview", "typescript": "tsc -noEmit", - "test:tacho": "node ./scripts/runBenchmark.mjs tacho run" + "benchmark:run": "node ./scripts/runBenchmark.mjs run" }, "dependencies": { "@mui/internal-test-utils": "workspace:*", @@ -18,7 +18,6 @@ "@playwright/test": "1.62.1", "@types/react": "19.2.18", "@types/react-dom": "19.2.7", - "tachometer": "^0.7.2", "vite": "^8.2.2" } } diff --git a/test/tachometer/scripts/runBenchmark.mjs b/test/benchmark/scripts/runBenchmark.mjs similarity index 100% rename from test/tachometer/scripts/runBenchmark.mjs rename to test/benchmark/scripts/runBenchmark.mjs diff --git a/test/benchmark/src/_shared/ScrollingList.tsx b/test/benchmark/src/_shared/ScrollingList.tsx new file mode 100644 index 0000000000..a2f1ebc946 --- /dev/null +++ b/test/benchmark/src/_shared/ScrollingList.tsx @@ -0,0 +1,21 @@ +import * as React from 'react'; + +/** + * A fixed-size scroller around a list, with a header that re-renders on every scroll event — the + * interaction the scroll benchmarks measure. Shared so every list implementation is scrolled the + * same way. + */ +export function ScrollingList({ children }: { children: React.ReactNode }) { + const [scrollTop, setScrollTop] = React.useState(0); + return ( +
    setScrollTop(event.currentTarget.scrollTop)} + > +

    + Scrolled {Math.round(scrollTop)}px +

    + {children} +
    + ); +} diff --git a/test/tachometer/src/libs/sort/shared.ts b/test/benchmark/src/libs/sort/shared.ts similarity index 100% rename from test/tachometer/src/libs/sort/shared.ts rename to test/benchmark/src/libs/sort/shared.ts diff --git a/test/benchmark/src/libs/sort/sort.alpha.tsx b/test/benchmark/src/libs/sort/sort.alpha.tsx new file mode 100644 index 0000000000..89224ef84f --- /dev/null +++ b/test/benchmark/src/libs/sort/sort.alpha.tsx @@ -0,0 +1,12 @@ +import * as React from 'react'; +import { benchmark } from '@mui/internal-benchmark'; +import { makeValues, report } from './shared'; + +/** Sorts the boxed array in place, paying a comparator call per comparison. */ +function Sorted() { + const values = makeValues(); + values.sort((left, right) => left - right); + return

    {report('alpha', values)}

    ; +} + +benchmark('sort', () => ); diff --git a/test/benchmark/src/libs/sort/sort.bench.tsx b/test/benchmark/src/libs/sort/sort.bench.tsx new file mode 100644 index 0000000000..35edbe9067 --- /dev/null +++ b/test/benchmark/src/libs/sort/sort.bench.tsx @@ -0,0 +1,12 @@ +import { compare } from '@mui/internal-benchmark'; + +/** + * Three sorts compared with each other rather than across builds — the shape a comparison between + * libraries takes. This repository has no competitor libraries to install, so the "libraries" here + * are three sorts with genuinely different constant factors. `ours` is the reference. + */ +compare('libs-sort', { + ours: () => import('./sort.ours'), + alpha: () => import('./sort.alpha'), + beta: () => import('./sort.beta'), +}); diff --git a/test/tachometer/src/libs/sort/beta.ts b/test/benchmark/src/libs/sort/sort.beta.tsx similarity index 80% rename from test/tachometer/src/libs/sort/beta.ts rename to test/benchmark/src/libs/sort/sort.beta.tsx index bd68fad1b3..9accfd3418 100644 --- a/test/tachometer/src/libs/sort/beta.ts +++ b/test/benchmark/src/libs/sort/sort.beta.tsx @@ -1,8 +1,7 @@ -import { measureMount } from '../../_shared/measure'; +import * as React from 'react'; +import { benchmark } from '@mui/internal-benchmark'; import { makeValues, report } from './shared'; -const values = makeValues(); - /** A merge sort written in JavaScript, allocating as it goes — the slow variant of the three. */ function mergeSort(values: number[]): number[] { if (values.length <= 1) { @@ -30,4 +29,8 @@ function mergeSort(values: number[]): number[] { return merged; } -measureMount(() => report('beta', mergeSort(values))); +function Sorted() { + return

    {report('beta', mergeSort(makeValues()))}

    ; +} + +benchmark('sort', () => ); diff --git a/test/benchmark/src/libs/sort/sort.ours.tsx b/test/benchmark/src/libs/sort/sort.ours.tsx new file mode 100644 index 0000000000..1caf8a553c --- /dev/null +++ b/test/benchmark/src/libs/sort/sort.ours.tsx @@ -0,0 +1,12 @@ +import * as React from 'react'; +import { benchmark } from '@mui/internal-benchmark'; +import { makeValues, report } from './shared'; + +/** Copies into a typed array first, so the engine sorts unboxed doubles with no comparator calls. */ +function Sorted() { + const sorted = Float64Array.from(makeValues()); + sorted.sort(); + return

    {report('ours', sorted)}

    ; +} + +benchmark('sort', () => ); diff --git a/test/tachometer/src/list/list.bench.tsx b/test/benchmark/src/list/list.bench.tsx similarity index 59% rename from test/tachometer/src/list/list.bench.tsx rename to test/benchmark/src/list/list.bench.tsx index f85d5c9739..b0e8b33f8c 100644 --- a/test/tachometer/src/list/list.bench.tsx +++ b/test/benchmark/src/list/list.bench.tsx @@ -1,14 +1,15 @@ import * as React from 'react'; +import { ScrollingList } from '../_shared/ScrollingList'; import { benchmark, ScalarMetric } from '@mui/internal-benchmark'; import { reactMajor } from '@mui/internal-test-utils/env'; /** - * `benchmark()` cases, the same API Vitest runs. Under `tacho run --engine interleaved` each case - * runs one iteration at a time in a page that stays open per build, with the builds alternating - * round by round. + * `benchmark()` cases, the same API Vitest runs. Under `benchmark run` each case runs one + * iteration at a time in a page that stays open per build, with the builds alternating round by + * round. * - * Like the `workload` page, it imports a workspace package so both builds resolve it from their own - * packed tarball. + * Like the `workload` benchmark, it imports a workspace package so both builds resolve it from their + * own packed tarball. */ function List({ count }: { count: number }) { @@ -28,27 +29,15 @@ const scrollDuration = new ScalarMetric({ format: { style: 'unit', unit: 'millisecond', maximumFractionDigits: 2 }, }); -function ScrollingList() { - const [scrollTop, setScrollTop] = React.useState(0); - return ( -
    setScrollTop(event.currentTarget.scrollTop)} - > -

    - Scrolled {Math.round(scrollTop)}px -

    - -
    - ); -} - // A trusted wheel-style scroll: the browser generates the whole gesture at frame cadence, so the // list sees the event timing a real mouse wheel would produce. benchmark( 'list scroll', - () => , + () => ( + + + + ), async ({ input, resumeReactRecording, waitForElementTiming }) => { // The scroller has to be painted before a gesture can hit it. await waitForElementTiming('default'); diff --git a/test/tachometer/src/lists/lists.bench.tsx b/test/benchmark/src/lists/lists.bench.tsx similarity index 100% rename from test/tachometer/src/lists/lists.bench.tsx rename to test/benchmark/src/lists/lists.bench.tsx diff --git a/test/tachometer/src/lists/lists.table.tsx b/test/benchmark/src/lists/lists.table.tsx similarity index 62% rename from test/tachometer/src/lists/lists.table.tsx rename to test/benchmark/src/lists/lists.table.tsx index 024fae26c0..230d185de3 100644 --- a/test/tachometer/src/lists/lists.table.tsx +++ b/test/benchmark/src/lists/lists.table.tsx @@ -1,4 +1,5 @@ import * as React from 'react'; +import { ScrollingList } from '../_shared/ScrollingList'; import { benchmark } from '@mui/internal-benchmark'; const ROWS = 1000; @@ -18,26 +19,15 @@ function List() { ); } -function ScrollingList() { - const [scrollTop, setScrollTop] = React.useState(0); - return ( -
    setScrollTop(event.currentTarget.scrollTop)} - > -

    - Scrolled {Math.round(scrollTop)}px -

    - -
    - ); -} - benchmark('mount', () => ); benchmark( 'scroll', - () => , + () => ( + + + + ), async ({ input, resumeReactRecording, waitForElementTiming }) => { // The scroller has to be painted before a gesture can hit it. await waitForElementTiming('default'); diff --git a/test/tachometer/src/lists/lists.ul.tsx b/test/benchmark/src/lists/lists.ul.tsx similarity index 60% rename from test/tachometer/src/lists/lists.ul.tsx rename to test/benchmark/src/lists/lists.ul.tsx index 441d6fb5a7..c174975a47 100644 --- a/test/tachometer/src/lists/lists.ul.tsx +++ b/test/benchmark/src/lists/lists.ul.tsx @@ -1,4 +1,5 @@ import * as React from 'react'; +import { ScrollingList } from '../_shared/ScrollingList'; import { benchmark } from '@mui/internal-benchmark'; const ROWS = 1000; @@ -15,26 +16,15 @@ function List() { ); } -function ScrollingList() { - const [scrollTop, setScrollTop] = React.useState(0); - return ( -
    setScrollTop(event.currentTarget.scrollTop)} - > -

    - Scrolled {Math.round(scrollTop)}px -

    - -
    - ); -} - benchmark('mount', () => ); benchmark( 'scroll', - () => , + () => ( + + + + ), async ({ input, resumeReactRecording, waitForElementTiming }) => { // The scroller has to be painted before a gesture can hit it. await waitForElementTiming('default'); diff --git a/test/benchmark/src/workload/workload.bench.tsx b/test/benchmark/src/workload/workload.bench.tsx new file mode 100644 index 0000000000..6f94ad69e9 --- /dev/null +++ b/test/benchmark/src/workload/workload.bench.tsx @@ -0,0 +1,47 @@ +import * as React from 'react'; +import { benchmark } from '@mui/internal-benchmark'; +import { reactMajor } from '@mui/internal-test-utils/env'; + +/** + * A deterministic CPU workload, sized per case. + * + * The point of this benchmark is not the number it produces — this repository ships build tooling, + * not a browser library, so there is nothing here whose render time is worth tracking. It exists to + * give `benchmark run` something real to drive end to end. + * + * It does import a workspace package (`@mui/internal-test-utils`) and put its value on screen. That + * is the part that matters: under `benchmark run` every ref — the working tree included — resolves + * it from a packed tarball, so both sides of a comparison resolve identically. + */ + +/** A small linear congruential generator, so every sample does identical work. */ +function createRandom(seed: number): () => number { + let state = seed; + return () => { + state = (state * 1664525 + 1013904223) % 0x100000000; + return state / 0x100000000; + }; +} + +function Workload({ size }: { size: number }) { + const random = createRandom(42); + const values = new Float64Array(size); + for (let index = 0; index < size; index += 1) { + values[index] = random(); + } + values.sort(); + + let checksum = 0; + for (let index = 0; index < size; index += 1) { + checksum += values[index] * index; + } + + return ( +

    + size={size} checksum={checksum.toFixed(3)} react={reactMajor} +

    + ); +} + +benchmark('workload', () => ); +benchmark('workload-large', () => ); diff --git a/test/tachometer/tsconfig.json b/test/benchmark/tsconfig.json similarity index 100% rename from test/tachometer/tsconfig.json rename to test/benchmark/tsconfig.json diff --git a/test/benchmark/vite.config.mjs b/test/benchmark/vite.config.mjs new file mode 100644 index 0000000000..b15a239614 --- /dev/null +++ b/test/benchmark/vite.config.mjs @@ -0,0 +1,6 @@ +import { defineConfig } from 'vite'; +import { benchmarkPlugin } from '@mui/internal-benchmark/vitePlugin'; + +// The plugin contributes everything derived from the `*.bench.tsx` files: a page per file as the +// build's entry points, the page runtime they run under, the `src/` root and the index page. +export default defineConfig({ plugins: [benchmarkPlugin()] }); diff --git a/test/tachometer/src/_shared/measure.ts b/test/tachometer/src/_shared/measure.ts deleted file mode 100644 index 00eefa0ffa..0000000000 --- a/test/tachometer/src/_shared/measure.ts +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Shared measurement helper. - * - * Tachometer's `performance` measurement mode polls the page for a `performance.measure()` span of - * a given name, so a page's only obligation is to produce one. - */ - -/** The measure name each case's `tachometer.json` reads. */ -const MEASURE = 'mount'; - -/** - * Runs `work`, writes its result into the page, and closes the `mount` measure once the browser has - * painted it. - * - * The paint is inside the measured window on purpose: a workload whose result is never displayed - * can be optimized away, and the number would then track nothing. - */ -export function measureMount(work: () => string): void { - performance.mark(`${MEASURE}-start`); - const text = work(); - - const root = document.getElementById('root'); - if (!root) { - throw new Error('The page is missing its #root element.'); - } - root.textContent = text; - - // A double rAF closes the measure after the DOM write *and* the paint it triggers; a single one - // would still be inside the frame that performs the paint. - requestAnimationFrame(() => { - requestAnimationFrame(() => { - performance.mark(`${MEASURE}-end`); - performance.measure(MEASURE, `${MEASURE}-start`, `${MEASURE}-end`); - }); - }); -} - -/** - * Reads a positive integer from the page's query string, so one page can back several cases. - */ -export function numberParam(name: string, fallback: number): number { - const raw = new URLSearchParams(window.location.search).get(name); - if (raw === null) { - return fallback; - } - const value = Number(raw); - if (!Number.isInteger(value) || value <= 0) { - throw new Error(`Expected "${name}" to be a positive integer, got "${raw}".`); - } - return value; -} diff --git a/test/tachometer/src/libs/sort/alpha.html b/test/tachometer/src/libs/sort/alpha.html deleted file mode 100644 index 5de8ad125c..0000000000 --- a/test/tachometer/src/libs/sort/alpha.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - libs-sort [alpha] - - -
    - - - diff --git a/test/tachometer/src/libs/sort/alpha.ts b/test/tachometer/src/libs/sort/alpha.ts deleted file mode 100644 index 425c67e246..0000000000 --- a/test/tachometer/src/libs/sort/alpha.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { measureMount } from '../../_shared/measure'; -import { makeValues, report } from './shared'; - -const values = makeValues(); - -/** Sorts the boxed array in place, paying a comparator call per comparison. */ -measureMount(() => { - values.sort((left, right) => left - right); - return report('alpha', values); -}); diff --git a/test/tachometer/src/libs/sort/beta.html b/test/tachometer/src/libs/sort/beta.html deleted file mode 100644 index f0b543d18e..0000000000 --- a/test/tachometer/src/libs/sort/beta.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - libs-sort [beta] - - -
    - - - diff --git a/test/tachometer/src/libs/sort/ours.html b/test/tachometer/src/libs/sort/ours.html deleted file mode 100644 index 68b8517ce4..0000000000 --- a/test/tachometer/src/libs/sort/ours.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - libs-sort [ours] - - -
    - - - diff --git a/test/tachometer/src/libs/sort/ours.ts b/test/tachometer/src/libs/sort/ours.ts deleted file mode 100644 index 15adf702f1..0000000000 --- a/test/tachometer/src/libs/sort/ours.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { measureMount } from '../../_shared/measure'; -import { makeValues, report } from './shared'; - -// Built outside the measured window: every sample is a fresh page load, so the input is still -// generated once per sample — it just no longer counts as part of the sort being measured. -const values = makeValues(); - -/** Copies into a typed array first, so the engine sorts unboxed doubles with no comparator calls. */ -measureMount(() => { - const sorted = Float64Array.from(values); - sorted.sort(); - return report('ours', sorted); -}); diff --git a/test/tachometer/src/libs/sort/tachometer.json b/test/tachometer/src/libs/sort/tachometer.json deleted file mode 100644 index 1f87085259..0000000000 --- a/test/tachometer/src/libs/sort/tachometer.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "$schema": "../../../node_modules/tachometer/config.schema.json", - "sampleSize": 10, - "timeout": 1, - "autoSampleConditions": ["10%"], - "benchmarks": [ - { - "name": "libs-sort", - "measurement": { "mode": "performance", "entryName": "mount" }, - "browser": { - "name": "chrome", - "headless": true, - "windowSize": { "width": 1024, "height": 768 } - }, - "expand": [ - { "name": "libs-sort [ours]", "url": "./ours.html" }, - { "name": "libs-sort [alpha]", "url": "./alpha.html" }, - { "name": "libs-sort [beta]", "url": "./beta.html" } - ] - } - ] -} diff --git a/test/tachometer/src/workload-large/tachometer.json b/test/tachometer/src/workload-large/tachometer.json deleted file mode 100644 index ae102d0e4d..0000000000 --- a/test/tachometer/src/workload-large/tachometer.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "../../node_modules/tachometer/config.schema.json", - "sampleSize": 10, - "timeout": 1, - "autoSampleConditions": ["10%"], - "benchmarks": [ - { - "name": "workload-large", - "url": "../workload/index.html?size=400000", - "measurement": { "mode": "performance", "entryName": "mount" }, - "browser": { - "name": "chrome", - "headless": true, - "windowSize": { "width": 1024, "height": 768 } - } - } - ] -} diff --git a/test/tachometer/src/workload/index.html b/test/tachometer/src/workload/index.html deleted file mode 100644 index 556d47228b..0000000000 --- a/test/tachometer/src/workload/index.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - workload - - -
    - - - diff --git a/test/tachometer/src/workload/index.ts b/test/tachometer/src/workload/index.ts deleted file mode 100644 index e20ed2abfc..0000000000 --- a/test/tachometer/src/workload/index.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { reactMajor } from '@mui/internal-test-utils/env'; -import { measureMount, numberParam } from '../_shared/measure'; - -/** - * A deterministic CPU workload, sized by `?size=`. - * - * The point of this page is not the number it produces — this repository ships build tooling, not a - * browser library, so there is nothing here whose render time is worth tracking. It exists to give - * `benchmark tacho run` something real to drive end to end. - * - * It does import a workspace package (`@mui/internal-test-utils`) and put its value on screen. That - * is the part that matters: under `tacho run` every ref — the working tree included — resolves it - * from a packed tarball in an isolated install, so both sides of a comparison resolve identically. - * A plain `vite dev` or `vite build` resolves it through the workspace link instead. If either path - * breaks, this page fails to build rather than silently measuring nothing. - */ - -/** A small linear congruential generator, so every sample does identical work. */ -function createRandom(seed: number): () => number { - let state = seed; - return () => { - state = (state * 1664525 + 1013904223) % 0x100000000; - return state / 0x100000000; - }; -} - -measureMount(() => { - const size = numberParam('size', 50_000); - const random = createRandom(42); - - const values = new Float64Array(size); - for (let index = 0; index < size; index += 1) { - values[index] = random(); - } - values.sort(); - - let checksum = 0; - for (let index = 0; index < size; index += 1) { - checksum += values[index] * index; - } - - return `size=${size} checksum=${checksum.toFixed(3)} react=${reactMajor}`; -}); diff --git a/test/tachometer/src/workload/tachometer.json b/test/tachometer/src/workload/tachometer.json deleted file mode 100644 index 49d6cecbf3..0000000000 --- a/test/tachometer/src/workload/tachometer.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "../../node_modules/tachometer/config.schema.json", - "sampleSize": 10, - "timeout": 1, - "autoSampleConditions": ["10%"], - "benchmarks": [ - { - "name": "workload", - "url": "./index.html", - "measurement": { "mode": "performance", "entryName": "mount" }, - "browser": { - "name": "chrome", - "headless": true, - "windowSize": { "width": 1024, "height": 768 } - } - } - ] -} diff --git a/test/tachometer/vite.config.mjs b/test/tachometer/vite.config.mjs deleted file mode 100644 index 98416fbdc9..0000000000 --- a/test/tachometer/vite.config.mjs +++ /dev/null @@ -1,6 +0,0 @@ -import { defineConfig } from 'vite'; -import { tachometer } from '@mui/internal-benchmark/tachometerPlugin'; - -// The plugin contributes everything derived from the benchmark cases: the `src/` root, the -// multi-page app type, the build's entry points, and the dev-server case index. -export default defineConfig({ plugins: [tachometer()] }); diff --git a/tsconfig.json b/tsconfig.json index 97d14d0db3..ce12c69e43 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -26,7 +26,7 @@ { "path": "scripts" }, { "path": "docs" }, { "path": "test/performance" }, - { "path": "test/tachometer" } + { "path": "test/benchmark" } ], "files": [] } From e465a3993468010323e87916d0a46ab5c17439ad Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 12:24:56 +0200 Subject: [PATCH 077/173] [code-infra-dashboard] Read benchmark-run reports instead of tachometer The upload route accepts `benchmark-run` in place of `tachometer`, and the tachometer PR-comment section, details view and markdown builder are replaced by ones driven by the shared analyzeRun from @mui/internal-benchmark/runReport: regressions above the fold, a table per benchmark with medians and paired confidence intervals, compare() results as their own tables. Repositories opt in with `prComment.benchmarkRun`. Their benchmarks page gains a master history chart: every master commit was measured against its parent in the same run, so each bar is that commit's own paired change for the selected benchmark and metric. --- .../[owner]/[repo]/page.tsx | 8 +- .../api/ci-reports/sync-pr-comment/route.ts | 18 +- .../app/api/ci-reports/upload/route.ts | 8 +- .../src/components/BenchmarkRunHistory.tsx | 192 ++++++++++ apps/code-infra-dashboard/src/constants.ts | 6 +- .../benchmarkRun/buildMarkdownReport.test.ts | 121 ++++++ .../lib/benchmarkRun/buildMarkdownReport.ts | 120 ++++++ .../src/lib/ciReports/benchmarkRunReport.ts | 33 ++ .../src/lib/ciReports/tachometerReport.ts | 34 -- .../tachometer/buildMarkdownReport.test.ts | 251 ------------- .../src/lib/tachometer/buildMarkdownReport.ts | 179 --------- .../src/utils/fetchCiReport.ts | 4 +- .../src/views/BenchmarkRunDetails.tsx | 192 ++++++++++ .../src/views/RepositoryBenchmarks.tsx | 6 +- .../src/views/TachometerDetails.tsx | 349 ------------------ packages/benchmark/src/page/page.tsx | 2 +- test/benchmark/src/list/list.bench.tsx | 2 +- test/benchmark/src/lists/lists.table.tsx | 2 +- test/benchmark/src/lists/lists.ul.tsx | 2 +- 19 files changed, 689 insertions(+), 840 deletions(-) rename apps/code-infra-dashboard/app/(dashboard)/{tachometer-details => benchmark-run}/[owner]/[repo]/page.tsx (55%) create mode 100644 apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx create mode 100644 apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts create mode 100644 apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts create mode 100644 apps/code-infra-dashboard/src/lib/ciReports/benchmarkRunReport.ts delete mode 100644 apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts delete mode 100644 apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts delete mode 100644 apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts create mode 100644 apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx delete mode 100644 apps/code-infra-dashboard/src/views/TachometerDetails.tsx diff --git a/apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx b/apps/code-infra-dashboard/app/(dashboard)/benchmark-run/[owner]/[repo]/page.tsx similarity index 55% rename from apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx rename to apps/code-infra-dashboard/app/(dashboard)/benchmark-run/[owner]/[repo]/page.tsx index 6d1a42140e..d7da7ee034 100644 --- a/apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx +++ b/apps/code-infra-dashboard/app/(dashboard)/benchmark-run/[owner]/[repo]/page.tsx @@ -1,6 +1,6 @@ import * as React from 'react'; import type { Metadata } from 'next'; -import TachometerDetails from '@/views/TachometerDetails'; +import BenchmarkRunDetails from '@/views/BenchmarkRunDetails'; export async function generateMetadata({ params, @@ -8,9 +8,9 @@ export async function generateMetadata({ params: Promise<{ owner: string; repo: string }>; }): Promise { const { owner, repo } = await params; - return { title: `Tachometer Details - ${owner}/${repo}` }; + return { title: `Benchmark run - ${owner}/${repo}` }; } -export default function TachometerDetailsPage() { - return ; +export default function BenchmarkRunPage() { + return ; } diff --git a/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts b/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts index 0cedc5ddec..8d1fdd8639 100644 --- a/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts +++ b/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts @@ -11,9 +11,9 @@ import { import { generateBenchmarkReport, BENCHMARK_SECTION_TITLE } from '@/lib/ciReports/benchmarkReport'; import { generateDeployPreviewReport } from '@/lib/ciReports/deployPreviewReport'; import { - generateTachometerReport, - TACHOMETER_SECTION_TITLE, -} from '@/lib/ciReports/tachometerReport'; + generateBenchmarkRunReport, + BENCHMARK_RUN_SECTION_TITLE, +} from '@/lib/ciReports/benchmarkRunReport'; import type { ReportResult } from '@/lib/ciReports/types'; import { fetchParentCommits } from '@/utils/fetchCiReportWithFallback'; import { getOctokit } from '@/lib/github'; @@ -122,7 +122,7 @@ export async function POST(request: NextRequest) { // Generate all configured report sections in parallel. // Each generator is wrapped in .catch() so a failure in one doesn't block the others. - const [bundleSizeReport, benchmarkReportResult, tachometerReportResult, deployPreviewReport] = + const [bundleSizeReport, benchmarkReportResult, benchmarkRunReportResult, deployPreviewReport] = await Promise.all([ prCommentConfig?.bundleSize ? generateBundleSizeReport(reportOptions).catch((error): ReportResult => { @@ -140,11 +140,11 @@ export async function POST(request: NextRequest) { }; }) : null, - prCommentConfig?.tachometer - ? generateTachometerReport(reportOptions).catch((error): ReportResult => { - console.error('Failed to generate tachometer report:', error); + prCommentConfig?.benchmarkRun + ? generateBenchmarkRunReport(reportOptions).catch((error): ReportResult => { + console.error('Failed to generate benchmark run report:', error); return { - content: `## ${TACHOMETER_SECTION_TITLE}\n\n:warning: Failed to generate tachometer report.`, + content: `## ${BENCHMARK_RUN_SECTION_TITLE}\n\n:warning: Failed to generate the benchmark report.`, }; }) : null, @@ -164,7 +164,7 @@ export async function POST(request: NextRequest) { deployPreviewReport, bundleSizeReport, benchmarkReportResult, - tachometerReportResult, + benchmarkRunReportResult, ] .filter((report): report is ReportResult => report !== null) .map((report) => report.content) diff --git a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts index e672f85a19..b9591c555d 100644 --- a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts +++ b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts @@ -5,7 +5,7 @@ import { uploadReport } from '@/lib/ciReports/s3'; import { verifyOidcToken } from '@/lib/ciReports/oidcAuth'; import { findAssociatedPr } from '@/lib/ciReports/findAssociatedPr'; -const VALID_REPORT_TYPES = new Set(['size-snapshot', 'benchmark', 'tachometer']); +const VALID_REPORT_TYPES = new Set(['size-snapshot', 'benchmark', 'benchmark-run']); const uploadSchema = z.object({ version: z.number(), @@ -56,9 +56,9 @@ export async function POST(request: NextRequest) { const { commitSha, repo, reportType, branch, report } = parsed.data; // A benchmark report is stored as the whole envelope, every other type as just the inner report. - // Historic, and kept because artifacts written that way are still read back. A tachometer report - // needs none of it: it carries its own `version`, `reportType` and `head`, the repo is in the S3 - // key, and the generator reads the rest from its own route params. + // Historic, and kept because artifacts written that way are still read back. A benchmark-run + // report needs none of it: it carries its own `version` and `head`, the repo is in the S3 key, and + // the generator reads the rest from its own route params. const storedBody = reportType === 'benchmark' ? JSON.stringify(parsed.data) : JSON.stringify(report); diff --git a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx new file mode 100644 index 0000000000..0b458b8144 --- /dev/null +++ b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx @@ -0,0 +1,192 @@ +import * as React from 'react'; +import NextLink from 'next/link'; +import Autocomplete from '@mui/material/Autocomplete'; +import Box from '@mui/material/Box'; +import Button from '@mui/material/Button'; +import Link from '@mui/material/Link'; +import Paper from '@mui/material/Paper'; +import TextField from '@mui/material/TextField'; +import Typography from '@mui/material/Typography'; +import { BarChartPro } from '@mui/x-charts-pro/BarChartPro'; +import { analyzeRun, formatComparison } from '@mui/internal-benchmark/runReport'; +import type { MetricComparison } from '@mui/internal-benchmark/runReport'; +import { useMasterCommits } from '../hooks/useMasterCommits'; +import { useCiReports } from '../hooks/useCiReports'; +import ErrorDisplay from './ErrorDisplay'; + +/** + * Master history of `benchmark run` reports. Every master commit is benchmarked against its parent + * in the same run, so each bar is that commit's own paired change — which commit moved a number is + * read off the chart directly, rather than inferred from a noisy trend. + */ + +type Change = MetricComparison['change']; + +const CHANGE_COLORS: Record = { + worse: 'var(--mui-palette-error-main)', + better: 'var(--mui-palette-success-main)', + unsure: 'var(--mui-palette-grey-500)', +}; + +interface CommitPoint { + sha: string; + date: Date; + /** Per benchmark, per metric: this commit's change against its parent. */ + comparisons: Map>; +} + +interface BenchmarkRunHistoryProps { + repo: string; +} + +export default function BenchmarkRunHistory({ repo }: BenchmarkRunHistoryProps) { + const { commits, isLoading, isFetchingNextPage, hasNextPage, error, fetchNextPage } = + useMasterCommits(repo); + const { reports, isLoading: reportsLoading } = useCiReports(repo, commits, 'benchmark-run.json'); + + const points = React.useMemo( + () => + commits.flatMap(({ timestamp, commit }): CommitPoint[] => { + const report = reports[commit.sha]; + if (!report) { + return []; + } + const comparisons = new Map>(); + for (const { benchmark, metrics } of analyzeRun(report)) { + if (benchmark.kind !== 'baseline') { + continue; + } + comparisons.set( + benchmark.name, + new Map(metrics.map(({ metric, comparisons: [comparison] }) => [metric, comparison])), + ); + } + return [{ sha: commit.sha, date: new Date(timestamp), comparisons }]; + }), + [commits, reports], + ); + + const benchmarkNames = React.useMemo( + () => [...new Set(points.flatMap((point) => [...point.comparisons.keys()]))].sort(), + [points], + ); + const [userBenchmark, setUserBenchmark] = React.useState(null); + const benchmark = userBenchmark ?? benchmarkNames[0] ?? null; + + const metricNames = React.useMemo( + () => + benchmark === null + ? [] + : [ + ...new Set( + points.flatMap((point) => [...(point.comparisons.get(benchmark)?.keys() ?? [])]), + ), + ], + [points, benchmark], + ); + const [userMetric, setUserMetric] = React.useState(null); + const metric = userMetric !== null && metricNames.includes(userMetric) ? userMetric : 'render'; + + const comparisonAt = (point: CommitPoint) => + benchmark === null ? undefined : point.comparisons.get(benchmark)?.get(metric); + + // One series per kind of change, so each bar takes its change's colour. A commit's bar is the + // midpoint of its confidence interval; the tooltip gives the interval itself. + const series = (['worse', 'better', 'unsure'] as const).map((change) => ({ + type: 'bar' as const, + stack: 'change', + label: change, + color: CHANGE_COLORS[change], + data: points.map((point) => { + const comparison = comparisonAt(point); + return comparison?.change === change + ? (comparison.relative.low + comparison.relative.high) / 2 + : null; + }), + valueFormatter: (_value: number | null, { dataIndex }: { dataIndex: number }) => { + const comparison = comparisonAt(points[dataIndex]); + return comparison?.change === change ? formatComparison(comparison) : null; + }, + })); + + const [selectedSha, setSelectedSha] = React.useState(null); + + return ( + + + Benchmark runs + + + {error ? ( + + ) : ( + + + Each master commit against its parent, measured in the same run: the midpoint of the 95% + confidence interval on the paired change. Click a bar to open that run. + + + + setUserBenchmark(value)} + size="small" + sx={{ minWidth: 320 }} + renderInput={(inputParams) => } + /> + setUserMetric(value)} + size="small" + sx={{ minWidth: 200 }} + renderInput={(inputParams) => } + /> + + + point.date), + scaleType: 'band', + valueFormatter: (date: Date, context: { location: string }) => + context.location === 'tick' + ? date.toLocaleDateString() + : `${date.toLocaleString()} (${ + points.find((point) => point.date === date)?.sha.slice(0, 7) ?? '' + })`, + }, + ]} + yAxis={[{ width: 60, valueFormatter: (value: number) => `${value.toFixed(1)}%` }]} + series={series} + onAxisClick={(_event, data) => { + if (data) { + setSelectedSha(points[data.dataIndex]?.sha ?? null); + } + }} + loading={isLoading || reportsLoading} + height={300} + skipAnimation + grid={{ horizontal: true }} + /> + + + + {selectedSha ? ( + + Open the run for {selectedSha.slice(0, 7)} + + ) : null} + + {hasNextPage && ( + + )} + + + )} + + ); +} diff --git a/apps/code-infra-dashboard/src/constants.ts b/apps/code-infra-dashboard/src/constants.ts index 08e8bf2f43..5c6a95e34f 100644 --- a/apps/code-infra-dashboard/src/constants.ts +++ b/apps/code-infra-dashboard/src/constants.ts @@ -25,7 +25,7 @@ export interface Repository { prComment?: { bundleSize?: boolean; benchmark?: boolean; - tachometer?: boolean; + benchmarkRun?: boolean; netlifyDocs?: | boolean | { @@ -167,7 +167,7 @@ export const repositories = new Map( prComment: { bundleSize: true, benchmark: true, - tachometer: true, + benchmarkRun: true, }, }, { @@ -195,7 +195,7 @@ export const repositories = new Map( prComment: { bundleSize: true, benchmark: true, - tachometer: true, + benchmarkRun: true, netlifyDocs: { siteId: 'mui-internal', formatDocPath: (filePath) => { diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts new file mode 100644 index 0000000000..eec60ede31 --- /dev/null +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -0,0 +1,121 @@ +import { describe, expect, it } from 'vitest'; +import type { BenchmarkRunReport, RunBenchmark } from '@mui/internal-benchmark/runReport'; +import { buildBenchmarkRunMarkdownReport } from './buildMarkdownReport'; + +const MS = { style: 'unit', unit: 'millisecond', maximumFractionDigits: 2 } as const; + +function reportOf(benchmarks: RunBenchmark[]): BenchmarkRunReport { + return { + version: 1, + generatedAt: '2026-01-01T00:00:00.000Z', + head: { sha: 'abc' }, + environment: { browser: 'Chromium 151', platform: 'linux', arch: 'x64', launchArgs: [] }, + sampling: { samples: 4, warmup: 2 }, + builds: { current: { label: 'working tree' }, baseline: { sha: 'def', label: 'HEAD~1' } }, + metrics: { + render: { kind: 'scalar', format: MS, alarm: {} }, + 'bench:paint': { kind: 'scalar', format: MS }, + }, + benchmarks, + }; +} + +const unchanged: RunBenchmark = { + name: 'Button mount', + file: 'button.bench.tsx', + kind: 'baseline', + variants: ['current', 'baseline'], + samples: { + current: { render: [10, 11, 10, 11] }, + baseline: { render: [11, 10, 11, 10] }, + }, +}; + +const slower: RunBenchmark = { + name: 'Grid scroll', + file: 'grid.bench.tsx', + kind: 'baseline', + variants: ['current', 'baseline'], + samples: { + current: { render: [22, 24, 23, 25], 'bench:paint': [30, 31, 30, 31] }, + baseline: { render: [20, 22, 21, 23], 'bench:paint': [30, 31, 30, 31] }, + }, +}; + +const libraries: RunBenchmark = { + name: 'libs / mount', + file: 'libs.bench.tsx', + kind: 'compare', + variants: ['ours', 'theirs'], + samples: { ours: { render: [10, 10, 10, 10] }, theirs: { render: [20, 21, 20, 21] } }, +}; + +const broken: RunBenchmark = { + name: 'Chart zoom', + file: 'chart.bench.tsx', + kind: 'baseline', + variants: ['current', 'baseline'], + error: 'Render failed', +}; + +describe('buildBenchmarkRunMarkdownReport', () => { + it('states nothing but the summary when no benchmark regressed', () => { + const markdown = buildBenchmarkRunMarkdownReport(reportOf([unchanged])); + + expect(markdown.split('\n').slice(0, 3)).toEqual([ + '## Benchmarks', + '', + '1 benchmark measured · no regressions', + ]); + }); + + it('lists regressions above the collapsed results, and marks the heading', () => { + expect(buildBenchmarkRunMarkdownReport(reportOf([slower, libraries, broken]))) + .toMatchInlineSnapshot(` + "## Benchmarks ⚠️ + + 🔴 **Grid scroll** · render · \`worse +9.3% – +9.3%\` + + 2 benchmarks measured · 1 with regressions · 1 failed + + ❌ **Chart zoom**: Render failed + +
    + Full results + + **Grid scroll** + + | Metric | current | baseline | Δ vs baseline | + |:----------|----------:|----------:|:----------| + | render | 23.5 ms | 21.5 ms | \`worse +9.3% – +9.3%\` | + | bench:paint | 30.5 ms | 30.5 ms | \`unsure +0.0% – +0.0%\` | + + + **libs / mount** + + | Metric | ours | theirs | theirs vs ours | + |:----------|----------:|----------:|:----------| + | render | 10 ms | 20.5 ms | \`worse +95.8% – +114.2%\` | + + + _Each value is a median. Each Δ is a 95% confidence interval on the paired per-round difference, relative to the variant it is measured against; "unsure" means it straddles zero — the expected result for two equivalent builds._ + +
    " + `); + }); + + it('does not count a difference between libraries as a regression', () => { + const markdown = buildBenchmarkRunMarkdownReport(reportOf([libraries])); + + expect(markdown).toContain('no regressions'); + expect(markdown).not.toContain('🔴'); + }); + + it('links the full run when given where it is', () => { + const markdown = buildBenchmarkRunMarkdownReport(reportOf([unchanged]), { + detailsUrl: 'https://example.com/run', + }); + + expect(markdown).toContain('[See the full run](https://example.com/run)'); + }); +}); diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts new file mode 100644 index 0000000000..9e8f68f87e --- /dev/null +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -0,0 +1,120 @@ +import { formatMarkdownTable } from '@/utils/formatters'; +import { + analyzeRun, + findRegressions, + formatComparison, + formatValue, +} from '@mui/internal-benchmark/runReport'; +import type { BenchmarkAnalysis, BenchmarkRunReport } from '@mui/internal-benchmark/runReport'; + +export const BENCHMARK_RUN_SECTION_TITLE = 'Benchmarks'; + +interface BuildOptions { + /** Where the full results can be inspected. Omitted in tests. */ + detailsUrl?: string; +} + +/** One table per benchmark: every metric, the median of every variant, every comparison. */ +function renderBenchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): string { + const comparisons = metrics[0]?.comparisons ?? []; + const comparisonHeader = (subject: string, against: string) => + benchmark.kind === 'baseline' ? `Δ vs ${against}` : `${subject} vs ${against}`; + + const columns = [ + { field: 'metric', header: 'Metric', align: 'left' as const }, + ...benchmark.variants.map((variant) => ({ + field: `variant:${variant}`, + header: variant, + align: 'right' as const, + })), + ...comparisons.map(({ subject, against }) => ({ + field: `comparison:${subject}:${against}`, + header: comparisonHeader(subject, against), + align: 'left' as const, + })), + ]; + const rows = metrics.map(({ metric, definition, variants, comparisons: metricComparisons }) => ({ + metric, + ...Object.fromEntries( + benchmark.variants.map((variant) => [ + `variant:${variant}`, + formatValue(variants[variant].median, definition), + ]), + ), + ...Object.fromEntries( + metricComparisons.map((comparison) => [ + `comparison:${comparison.subject}:${comparison.against}`, + `\`${formatComparison(comparison)}\``, + ]), + ), + })); + + return `**${benchmark.name}**\n\n${formatMarkdownTable(columns, rows)}`; +} + +/** + * Renders the benchmarks section of the pull request comment. + * + * Regressions are stated above the collapsed block, and mark the heading, because a reader has to + * see them without expanding anything — GitHub renders `
    ` closed, so anything inside it is + * invisible until someone chooses to look. + */ +export function buildBenchmarkRunMarkdownReport( + report: BenchmarkRunReport, + options: BuildOptions = {}, +): string { + const analyses = analyzeRun(report); + const regressions = findRegressions(analyses); + const measured = analyses.filter((analysis) => !analysis.benchmark.error); + const failed = analyses.filter((analysis) => analysis.benchmark.error); + + const lines = [`## ${BENCHMARK_RUN_SECTION_TITLE}${regressions.length > 0 ? ' ⚠️' : ''}`, '']; + + for (const { benchmark, metric, comparison } of regressions) { + const marker = comparison.severity === 'error' ? '🔴' : '🟡'; + lines.push(`${marker} **${benchmark}** · ${metric} · \`${formatComparison(comparison)}\``); + } + if (regressions.length > 0) { + lines.push(''); + } + + const regressedBenchmarks = new Set(regressions.map((regression) => regression.benchmark)); + lines.push( + [ + `${measured.length} benchmark${measured.length === 1 ? '' : 's'} measured`, + regressedBenchmarks.size > 0 + ? `${regressedBenchmarks.size} with regressions` + : 'no regressions', + failed.length > 0 ? `${failed.length} failed` : null, + ] + .filter(Boolean) + .join(' · '), + '', + ); + + for (const { benchmark } of failed) { + lines.push(`❌ **${benchmark.name}**: ${benchmark.error}`); + } + if (failed.length > 0) { + lines.push(''); + } + + if (measured.length > 0) { + lines.push('
    ', 'Full results', ''); + lines.push(measured.map(renderBenchmarkTable).join('\n\n')); + lines.push( + '', + '_Each value is a median. Each Δ is a 95% confidence interval on the paired per-round ' + + 'difference, relative to the variant it is measured against; "unsure" means it straddles ' + + 'zero — the expected result for two equivalent builds._', + '', + '
    ', + ); + } + + if (options.detailsUrl) { + lines.push('', `[See the full run](${options.detailsUrl})`); + } + + return lines.join('\n'); +} diff --git a/apps/code-infra-dashboard/src/lib/ciReports/benchmarkRunReport.ts b/apps/code-infra-dashboard/src/lib/ciReports/benchmarkRunReport.ts new file mode 100644 index 0000000000..00e16bc847 --- /dev/null +++ b/apps/code-infra-dashboard/src/lib/ciReports/benchmarkRunReport.ts @@ -0,0 +1,33 @@ +import { fetchCiReport } from '@/utils/fetchCiReport'; +import { + buildBenchmarkRunMarkdownReport, + BENCHMARK_RUN_SECTION_TITLE, +} from '@/lib/benchmarkRun/buildMarkdownReport'; +import { DASHBOARD_ORIGIN } from '@/constants'; +import type { ReportOptions, ReportResult } from './types'; + +export { BENCHMARK_RUN_SECTION_TITLE }; + +/** + * Generates the benchmarks section of the pull request comment. The run measured its own baseline, + * so there is no base report to fetch: only the head commit's. + */ +export async function generateBenchmarkRunReport( + options: ReportOptions, +): Promise { + const { repo, prNumber, commitSha } = options; + + const report = await fetchCiReport(repo, commitSha, 'benchmark-run.json'); + if (!report) { + return null; + } + + const detailsUrl = new URL(`${DASHBOARD_ORIGIN}/benchmark-run/${repo}`); + detailsUrl.searchParams.set('sha', commitSha); + // The details view reads this to title the page and link back to the PR overview. + detailsUrl.searchParams.set('prNumber', String(prNumber)); + + return { + content: buildBenchmarkRunMarkdownReport(report, { detailsUrl: detailsUrl.href }), + }; +} diff --git a/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts b/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts deleted file mode 100644 index 7224b5c0ee..0000000000 --- a/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { fetchCiReport } from '@/utils/fetchCiReport'; -import { - buildTachometerMarkdownReport, - TACHOMETER_SECTION_TITLE, -} from '@/lib/tachometer/buildMarkdownReport'; -import { DASHBOARD_ORIGIN } from '@/constants'; - -import type { TachometerReport } from '@mui/internal-benchmark/tachometerReport'; -import type { ReportOptions, ReportResult } from './types'; - -export { TACHOMETER_SECTION_TITLE }; - -/** - * Generates the tachometer section of the pull request comment. - */ -export async function generateTachometerReport( - options: ReportOptions, -): Promise { - const { repo, prNumber, commitSha } = options; - - const report: TachometerReport | null = await fetchCiReport(repo, commitSha, 'tachometer.json'); - if (!report?.cases) { - return null; - } - - const detailsUrl = new URL(`${DASHBOARD_ORIGIN}/tachometer-details/${repo}`); - detailsUrl.searchParams.set('sha', commitSha); - // The details view reads this to title the page and link back to the PR overview. - detailsUrl.searchParams.set('prNumber', String(prNumber)); - - return { - content: buildTachometerMarkdownReport(report, { detailsUrl: detailsUrl.href }), - }; -} diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts deleted file mode 100644 index a086e22459..0000000000 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts +++ /dev/null @@ -1,251 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { - CaseResult, - MeasurementResult, - TachometerReport, - Verdict, -} from '@mui/internal-benchmark/tachometerReport'; -import { buildTachometerMarkdownReport, findRegressions } from './buildMarkdownReport'; - -function variant(name: string) { - return { - variant: name, - meanMs: { low: 20, high: 22 }, - samples: 120, - }; -} - -/** The verdict the other way round; an unresolved difference is unresolved from both sides. */ -function mirrorOf(verdict: Verdict): Verdict { - if (verdict === 'slower') { - return 'faster'; - } - return verdict === 'faster' ? 'slower' : verdict; -} - -/** - * A comparison of the reference against `name`, carrying both directions the way a real report - * does. The two are not negations of each other — each has its own denominator — so the numbers - * here differ, which is what lets a test tell which direction was rendered. - */ -function comparison(name: string, verdict: Verdict) { - const sign = verdict === 'slower' ? 1 : -1; - return { - variant: name, - verdict, - absoluteMs: { low: sign * 6.1, high: sign * 15.2 }, - percentChange: { low: sign * 2.7, high: sign * 6.8 }, - versusReference: { - verdict: mirrorOf(verdict), - absoluteMs: { low: sign * -6.1, high: sign * -15.2 }, - percentChange: { low: sign * -2.6, high: sign * -6.4 }, - }, - }; -} - -/** A case measuring the working tree against the baseline, where a difference is a regression. */ -function regressionCase(name: string, verdicts: Record): CaseResult { - const measurements: MeasurementResult[] = Object.entries(verdicts).map( - ([measurement, verdict]) => ({ - name: measurement, - variants: [variant(`${name} [current]`), variant(`${name} [baseline]`)], - comparisons: [comparison(`${name} [baseline]`, verdict)], - }), - ); - return { name, comparison: 'baseline', reference: `${name} [current]`, measurements }; -} - -/** A case comparing libraries with each other, every variant built from the working tree. */ -function libraryCase(name: string, verdict: Verdict): CaseResult { - return { - name, - comparison: 'variants', - reference: `${name} [mosaic]`, - measurements: [ - { - name: 'mount', - variants: [variant(`${name} [mosaic]`), variant(`${name} [ag-grid]`)], - comparisons: [comparison(`${name} [ag-grid]`, verdict)], - }, - ], - }; -} - -function report(cases: CaseResult[]): TachometerReport { - return { - version: 1, - reportType: 'tachometer', - generatedAt: '2026-09-02T00:00:00.000Z', - head: { ref: 'HEAD', sha: 'a'.repeat(40), branch: 'feature' }, - refs: [ - { id: 'current', kind: 'worktree' }, - { id: 'git-abc123456', kind: 'git', requested: 'HEAD~1', sha: 'b'.repeat(40) }, - ], - cases, - raw: {}, - }; -} - -/** Everything a reader sees without expanding anything. */ -function visiblePart(markdown: string): string { - return markdown.split('
    ')[0]; -} - -/** The collapsed table's row for `variant`, rather than any prose that names it too. */ -function tableRowFor(markdown: string, variantName: string): string | undefined { - return markdown - .split('
    ')[1] - ?.split('\n') - .find((line) => line.startsWith('|') && line.includes(variantName)); -} - -describe('findRegressions', () => { - it('finds a slower verdict across two builds', () => { - const found = findRegressions(report([regressionCase('workload', { mount: 'slower' })])); - - expect(found).toMatchObject([{ caseName: 'workload', measurement: 'mount' }]); - }); - - it.each(['unsure', 'faster'])('does not count a %s verdict', (verdict) => { - expect(findRegressions(report([regressionCase('workload', { mount: verdict })]))).toEqual([]); - }); - - it('ignores a slower verdict between variants on the same build', () => { - // A cross-library case runs every variant from one build, so "slower than ag-grid" says nothing - // about what this pull request changed. Flagging it would warn on every comment forever. - expect(findRegressions(report([libraryCase('libs-mount', 'slower')]))).toEqual([]); - }); - - it('reports only the measurement that regressed', () => { - const found = findRegressions( - report([regressionCase('workload', { mount: 'slower', scroll: 'unsure' })]), - ); - - expect(found).toHaveLength(1); - expect(found[0].measurement).toBe('mount'); - }); -}); - -describe('buildTachometerMarkdownReport', () => { - it('states a regression before the collapsed block, where it can be seen', () => { - // GitHub renders
    closed, so a regression inside it is invisible. - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'slower' })]), - ); - - expect(visiblePart(markdown)).toContain('workload'); - expect(visiblePart(markdown)).toContain('mount'); - expect(visiblePart(markdown)).toContain('+2.7% – +6.8%'); - }); - - it('marks the heading when something regressed', () => { - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'slower' })]), - ); - - expect(markdown.split('\n')[0]).toBe('## Tachometer ⚠️'); - }); - - it('leaves the heading unmarked when nothing regressed', () => { - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'unsure' })]), - ); - - expect(markdown.split('\n')[0]).toBe('## Tachometer'); - expect(visiblePart(markdown)).not.toContain('⚠️'); - }); - - it('summarizes a clean run in one line', () => { - const markdown = buildTachometerMarkdownReport( - report([ - regressionCase('workload', { mount: 'unsure' }), - regressionCase('scroll', { mount: 'unsure' }), - ]), - ); - - expect(visiblePart(markdown)).toContain('2 cases measured · 2 unchanged'); - }); - - it('counts a case that regressed in two measurements once', () => { - // The total beside it counts cases, so counting measurements here stopped the line adding up. - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'slower', 'cold-start': 'slower' })]), - ); - - expect(visiblePart(markdown)).toContain('1 case measured · 1 slower'); - }); - - it('does not count beating another library as having got faster', () => { - const markdown = buildTachometerMarkdownReport( - report([ - libraryCase('libs-mount', 'faster'), - regressionCase('workload', { mount: 'unsure' }), - ]), - ); - - expect(visiblePart(markdown)).toContain('2 cases measured · 2 unchanged'); - expect(visiblePart(markdown)).not.toContain('faster'); - }); - - it('counts a faster case separately', () => { - const markdown = buildTachometerMarkdownReport( - report([ - regressionCase('workload', { mount: 'faster' }), - regressionCase('scroll', { mount: 'unsure' }), - ]), - ); - - expect(visiblePart(markdown)).toContain('1 faster'); - }); - - it('reports a case that produced no result', () => { - const markdown = buildTachometerMarkdownReport( - report([{ name: 'broken', comparison: 'baseline', error: 'produced no benchmarks' }]), - ); - - expect(visiblePart(markdown)).toContain('broken'); - expect(visiblePart(markdown)).toContain('produced no benchmarks'); - }); - - it('puts every variant and measurement in the collapsed table', () => { - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'unsure', scroll: 'unsure' })]), - ); - - const details = markdown.split('
    ')[1]; - expect(details).toContain('mount'); - expect(details).toContain('scroll'); - expect(details).toContain('[current]'); - expect(details).toContain('[baseline]'); - }); - - it('states each row as the variant against the reference, not the reference against it', () => { - // The row sits under a "vs reference" heading next to the variant's own mean, so it has to - // read as a statement about that variant. Rendering the direction a regression is stated in - // makes a variant that is slower than the reference look faster than it. - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'slower' })]), - ); - - expect(tableRowFor(markdown, '[baseline]')).toContain('faster `-2.6% – -6.4%`'); - }); - - it('leaves the comparison out when the report has only one direction', () => { - // Older reports, written before both directions were captured. - const oneDirection = report([regressionCase('workload', { mount: 'slower' })]); - delete oneDirection.cases[0].measurements![0].comparisons[0].versusReference; - - const markdown = buildTachometerMarkdownReport(oneDirection); - - expect(tableRowFor(markdown, '[baseline]')).toContain('—'); - }); - - it('links to the full run when given a url', () => { - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'unsure' })]), - { detailsUrl: 'https://example.test/run' }, - ); - - expect(markdown).toContain('[See the full run](https://example.test/run)'); - }); -}); diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts deleted file mode 100644 index cd3ff28bc5..0000000000 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts +++ /dev/null @@ -1,179 +0,0 @@ -import { formatMarkdownTable } from '@/utils/formatters'; -import type { - ConfidenceInterval, - TachometerReport, -} from '@mui/internal-benchmark/tachometerReport'; -import { - formatMean, - formatPercent, - formatSignedMs, - isSummarized, - shortNameOf, -} from '@mui/internal-benchmark/tachometerFormat'; -import type { SummarizedCase } from '@mui/internal-benchmark/tachometerFormat'; - -export const TACHOMETER_SECTION_TITLE = 'Tachometer'; - -interface Regression { - caseName: string; - measurement: string; - variant: string; - percentChange: ConfidenceInterval; - absoluteMs: ConfidenceInterval; -} - -interface BuildOptions { - /** Where the full results can be inspected. Omitted in tests. */ - detailsUrl?: string; -} - -/** - * The comparisons that say something about this pull request. - * - * Only a case measuring the working tree against the baseline does. One comparing pages built from - * the same tree — this library against another — is a standing measurement, and a difference there - * is the point rather than a regression. - */ -function baselineComparisons(entry: SummarizedCase) { - if (entry.comparison !== 'baseline') { - return []; - } - return entry.measurements.flatMap((measurement) => - measurement.comparisons.map((comparison) => ({ measurement, comparison })), - ); -} - -export function findRegressions(report: TachometerReport): Regression[] { - const regressions: Regression[] = []; - - for (const entry of report.cases) { - if (!isSummarized(entry)) { - continue; - } - for (const { measurement, comparison } of baselineComparisons(entry)) { - if (comparison.verdict === 'slower') { - regressions.push({ - caseName: entry.name, - measurement: measurement.name, - variant: shortNameOf(entry.name, comparison.variant), - percentChange: comparison.percentChange, - absoluteMs: comparison.absoluteMs, - }); - } - } - } - - return regressions; -} - -/** One flat table per case: every measurement, every variant, every number the report holds. */ -function renderCaseTable(entry: SummarizedCase): string { - const rows = entry.measurements.flatMap((measurement) => { - const byVariant = new Map( - measurement.comparisons.map((comparison) => [comparison.variant, comparison]), - ); - return measurement.variants.map((variant) => { - const againstReference = byVariant.get(variant.variant)?.versusReference; - let versus = '—'; - if (variant.variant === entry.reference) { - versus = '_reference_'; - } else if (againstReference) { - versus = `${againstReference.verdict} \`${formatPercent(againstReference.percentChange)}\``; - } - return { - measurement: measurement.name, - variant: shortNameOf(entry.name, variant.variant), - mean: formatMean(variant.meanMs), - versus, - samples: variant.samples, - }; - }); - }); - - return `**${entry.name}**\n\n${formatMarkdownTable( - [ - { field: 'measurement', header: 'Measurement', align: 'left' }, - { field: 'variant', header: 'Variant', align: 'left' }, - { field: 'mean', header: 'Mean (95% CI)', align: 'right' }, - { field: 'versus', header: 'vs reference', align: 'left' }, - { field: 'samples', header: 'Samples', align: 'right' }, - ], - rows, - )}`; -} - -/** - * Renders the tachometer section of the pull request comment. - * - * Regressions are stated above the collapsed block, and mark the heading, because a reader has to - * see them without expanding anything — GitHub renders `
    ` closed, so anything inside it is - * invisible until someone chooses to look. - */ -export function buildTachometerMarkdownReport( - report: TachometerReport, - options: BuildOptions = {}, -): string { - const regressions = findRegressions(report); - const summarized = report.cases.filter(isSummarized); - const failed = report.cases.filter((entry) => !isSummarized(entry)); - - const heading = `## ${TACHOMETER_SECTION_TITLE}${regressions.length > 0 ? ' ⚠️' : ''}`; - const lines = [heading, '']; - - if (regressions.length > 0) { - for (const regression of regressions) { - lines.push( - `⚠️ **${regression.caseName}** · ${regression.measurement} · slower than ` + - `${regression.variant} \`${formatPercent(regression.percentChange)}\` ` + - `(\`${formatSignedMs(regression.absoluteMs)}\`)`, - ); - } - lines.push(''); - } - - // Everything that did not regress collapses to a single line, so the regressions are what the eye - // lands on. `unsure` is the expected result for two equivalent builds, not a warning. Counted in - // cases, like the total beside them, and over the same baseline comparisons a regression is read - // from — a case that beats a competing library has not got faster. - const regressedCases = new Set(regressions.map((regression) => regression.caseName)); - const faster = summarized.filter( - (entry) => - !regressedCases.has(entry.name) && - baselineComparisons(entry).some(({ comparison }) => comparison.verdict === 'faster'), - ).length; - const unchanged = summarized.length - regressedCases.size - faster; - const summary = [ - `${summarized.length} case${summarized.length === 1 ? '' : 's'} measured`, - unchanged > 0 ? `${unchanged} unchanged` : null, - faster > 0 ? `${faster} faster` : null, - regressedCases.size > 0 ? `${regressedCases.size} slower` : null, - ] - .filter(Boolean) - .join(' · '); - lines.push(summary, ''); - - for (const entry of failed) { - lines.push(`❌ **${entry.name}**: ${entry.error ?? 'produced no result'}`); - } - if (failed.length > 0) { - lines.push(''); - } - - if (summarized.length > 0) { - lines.push('
    ', 'Full results', ''); - lines.push(summarized.map(renderCaseTable).join('\n\n')); - lines.push( - '', - '_Each cell is a 95% confidence interval for the mean; one sample is one page load._', - '_"unsure" means the interval still straddles zero — the expected result for two equivalent builds._', - '', - '
    ', - ); - } - - if (options.detailsUrl) { - lines.push('', `[See the full run](${options.detailsUrl})`); - } - - return lines.join('\n'); -} diff --git a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts index 075735d832..ba63a7b8c1 100644 --- a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts +++ b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts @@ -1,12 +1,12 @@ import type { SizeSnapshotWithMetadata } from '@/lib/bundleSize/types'; import type { BenchmarkReport, BenchmarkUpload } from '@/lib/benchmark/types'; -import type { TachometerReport } from '@mui/internal-benchmark/tachometerReport'; +import type { BenchmarkRunReport } from '@mui/internal-benchmark/runReport'; import { migrateBenchmarkReport } from '@/lib/benchmark/migrateBenchmarkReport'; export interface CiReportTypes { 'benchmark.json': BenchmarkUpload; 'size-snapshot.json': SizeSnapshotWithMetadata; - 'tachometer.json': TachometerReport; + 'benchmark-run.json': BenchmarkRunReport; } export type CiReportName = keyof CiReportTypes; diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx new file mode 100644 index 0000000000..6af05f0089 --- /dev/null +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -0,0 +1,192 @@ +'use client'; + +import * as React from 'react'; +import { useParams, useSearchParams } from 'next/navigation'; +import { useQuery } from '@tanstack/react-query'; +import Alert from '@mui/material/Alert'; +import Box from '@mui/material/Box'; +import CircularProgress from '@mui/material/CircularProgress'; +import Paper from '@mui/material/Paper'; +import Table from '@mui/material/Table'; +import TableBody from '@mui/material/TableBody'; +import TableCell from '@mui/material/TableCell'; +import TableContainer from '@mui/material/TableContainer'; +import TableHead from '@mui/material/TableHead'; +import TableRow from '@mui/material/TableRow'; +import Typography from '@mui/material/Typography'; +import { fetchCiReport } from '@/utils/fetchCiReport'; +import { analyzeRun, formatComparison, formatValue } from '@mui/internal-benchmark/runReport'; +import type { + BenchmarkAnalysis, + BenchmarkRunReport, + MetricComparison, +} from '@mui/internal-benchmark/runReport'; +import Heading from '../components/Heading'; +import ReportHeader from '../components/ReportHeader'; +import ErrorDisplay from '../components/ErrorDisplay'; + +/** A regression is coloured by severity; `better` is green; `unsure` is the expected result. */ +function comparisonColor(comparison: MetricComparison): string { + if (comparison.severity === 'error') { + return 'error'; + } + if (comparison.severity === 'warning') { + return 'warning.main'; + } + return comparison.change === 'better' ? 'success.main' : 'text.secondary'; +} + +function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { + const { benchmark, metrics } = analysis; + const comparisons = metrics[0]?.comparisons ?? []; + + return ( + + + {benchmark.name}{' '} + + {benchmark.file} + + + + + + Metric + {benchmark.variants.map((variant) => ( + + {variant} + + ))} + {comparisons.map(({ subject, against }) => ( + + {benchmark.kind === 'baseline' ? `Δ vs ${against}` : `${subject} vs ${against}`} + + ))} + Rounds + + + + {metrics.map(({ metric, definition, variants, comparisons: metricComparisons }) => ( + + {metric} + {benchmark.variants.map((variant) => ( + + {formatValue(variants[variant].median, definition)} + + ))} + {metricComparisons.map((comparison) => ( + + + {formatComparison(comparison)} + + + ))} + {variants[benchmark.variants[0]].count} + + ))} + +
    +     + ); +} + +export default function BenchmarkRunDetails() { + const params = useParams<{ owner: string; repo: string }>(); + const searchParams = useSearchParams(); + + if (!params.owner || !params.repo) { + throw new Error('Missing required path parameters'); + } + + const repo = `${params.owner}/${params.repo}`; + const sha = searchParams.get('sha'); + const prNumber = searchParams.get('prNumber'); + + const { + data: report, + isLoading, + error, + } = useQuery({ + queryKey: ['benchmark-run-report', repo, sha], + queryFn: () => fetchCiReport(repo, sha!, 'benchmark-run.json'), + retry: 1, + enabled: Boolean(sha), + }); + + const analyses = React.useMemo(() => (report ? analyzeRun(report) : []), [report]); + + if (!sha) { + return ( + + Benchmark run + + Missing required "sha" query parameter. + + + ); + } + + // The run measured its own baseline, so it is one of the report's builds rather than a separately + // fetched report. + const baseline = report?.builds.baseline; + + return ( + + Benchmark run + + + + + {isLoading && ( + + + Loading benchmark report… + + )} + + {error && } + + {!isLoading && !error && !report && ( + No benchmark report found for this commit. + )} + + {analyses + .filter((analysis) => analysis.benchmark.error) + .map(({ benchmark }) => ( + + {benchmark.name}: {benchmark.error} + + ))} + + {analyses + .filter((analysis) => !analysis.benchmark.error) + .map((analysis) => ( + + ))} + + {report && ( + + + Each value is a median. Each Δ is a 95% confidence interval on the paired per-round + difference, relative to the variant it is measured against; "unsure" means + it straddles zero. + + + {report.sampling.samples} rounds after {report.sampling.warmup} warmup ·{' '} + {report.environment.browser} on {report.environment.platform}/ + {report.environment.arch} + {report.environment.launchArgs.length > 0 && + ` · ${report.environment.launchArgs.join(' ')}`} + + + )} + + + ); +} diff --git a/apps/code-infra-dashboard/src/views/RepositoryBenchmarks.tsx b/apps/code-infra-dashboard/src/views/RepositoryBenchmarks.tsx index 08941c2925..3d88764506 100644 --- a/apps/code-infra-dashboard/src/views/RepositoryBenchmarks.tsx +++ b/apps/code-infra-dashboard/src/views/RepositoryBenchmarks.tsx @@ -6,6 +6,7 @@ import Link from '@mui/material/Link'; import Alert from '@mui/material/Alert'; import Heading from '../components/Heading'; import DailyBenchmarkChart from '../components/DailyBenchmarkChart'; +import BenchmarkRunHistory from '../components/BenchmarkRunHistory'; import { repositories } from '../constants'; export default function RepositoryBenchmarks() { @@ -34,7 +35,10 @@ export default function RepositoryBenchmarks() { API. ) : ( - + + + {repoConfig?.prComment?.benchmarkRun && } + )} ); diff --git a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx deleted file mode 100644 index d2dbdc7e06..0000000000 --- a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx +++ /dev/null @@ -1,349 +0,0 @@ -'use client'; - -import * as React from 'react'; -import { useParams, useSearchParams } from 'next/navigation'; -import { useQuery } from '@tanstack/react-query'; -import Alert from '@mui/material/Alert'; -import Box from '@mui/material/Box'; -import CircularProgress from '@mui/material/CircularProgress'; -import Paper from '@mui/material/Paper'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableContainer from '@mui/material/TableContainer'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Typography from '@mui/material/Typography'; -import { fetchCiReport } from '@/utils/fetchCiReport'; -import type { - ConfidenceInterval, - TachometerReport, - Verdict, -} from '@mui/internal-benchmark/tachometerReport'; -import { - formatMean, - formatPercent, - refLabel, - groupCasesByVariantSet, - isSummarized, - shortNameOf, -} from '@mui/internal-benchmark/tachometerFormat'; -import type { SummarizedCase } from '@mui/internal-benchmark/tachometerFormat'; -import Heading from '../components/Heading'; -import ReportHeader from '../components/ReportHeader'; -import ErrorDisplay from '../components/ErrorDisplay'; - -/** - * The tables the run prints in CI, as HTML: cases grouped by their variant set, a group of two - * variants read down its measurements, a group of more read across its variants. - */ - -/** Only a resolved difference is coloured; `unsure` is the expected result and stays neutral. */ -function verdictColor(verdict: Verdict): 'error' | 'success' | 'text.secondary' { - if (verdict === 'slower') { - return 'error'; - } - if (verdict === 'faster') { - return 'success'; - } - return 'text.secondary'; -} - -/** Auto-sampling stops per case, so the counts are normally equal; when they are not, say both. */ -function formatSamples(samples: number[]): string { - return [...new Set(samples)].join('/'); -} - -function VerdictText({ - verdict, - percentChange, -}: { - verdict: Verdict; - percentChange: ConfidenceInterval; -}) { - return ( - - {verdict} {formatPercent(percentChange)} - - ); -} - -function Muted({ children }: { children: React.ReactNode }) { - return ( - - {children} - - ); -} - -/** - * One case as a table of variants, for a comparison with more than two of them. - */ -function VariantTable({ entry, variants }: { entry: SummarizedCase; variants: string[] }) { - const [reference] = variants; - const samplesOf = (variantName: string) => - entry.measurements.flatMap((measurement) => - measurement.variants - .filter((variant) => shortNameOf(entry.name, variant.variant) === variantName) - .map((variant) => variant.samples), - ); - - return ( - - - - - {entry.name} - {entry.measurements.map((measurement) => ( - - {measurement.name} - vs {reference} - - ))} - Samples - - - - {variants.map((variantName) => { - return ( - - {variantName} - {entry.measurements.map((measurement) => { - const found = measurement.variants.find( - (candidate) => shortNameOf(entry.name, candidate.variant) === variantName, - ); - // The comparison holds the reference relative to this variant; a row about the - // variant needs the direction whose subject is the variant. - const againstReference = measurement.comparisons.find( - (candidate) => shortNameOf(entry.name, candidate.variant) === variantName, - )?.versusReference; - - let versus = —; - if (variantName === reference) { - versus = reference; - } else if (againstReference) { - versus = ( - - ); - } - - return ( - - - {found ? formatMean(found.meanMs) : —} - - {versus} - - ); - })} - {formatSamples(samplesOf(variantName))} - - ); - })} - -
    -     - ); -} - -/** - * A group of cases sharing a variant set, one row per case and measurement. - */ -function CaseGroupTable({ cases, variants }: { cases: SummarizedCase[]; variants: string[] }) { - const [reference, ...others] = variants; - - return ( - - - - - Case - Measurement - {variants.map((variantName) => ( - - {variantName} - - ))} - {others.map((variantName) => ( - Δ vs {variantName} - ))} - Samples - - - - {cases.flatMap((entry, caseIndex) => - entry.measurements.map((measurement, measurementIndex) => { - const byVariant = new Map( - measurement.variants.map((variant) => [ - shortNameOf(entry.name, variant.variant), - variant, - ]), - ); - const byComparison = new Map( - measurement.comparisons.map((comparison) => [ - shortNameOf(entry.name, comparison.variant), - comparison, - ]), - ); - // A rule where the next case starts, since its name only appears on its first row. - const startsCase = measurementIndex === 0 && caseIndex > 0; - - return ( - - {measurementIndex === 0 ? entry.name : ''} - {measurement.name} - {variants.map((variantName) => { - const found = byVariant.get(variantName); - return ( - - {found ? formatMean(found.meanMs) : —} - - ); - })} - {others.map((variantName) => { - const comparison = byComparison.get(variantName); - return ( - - {comparison ? ( - - ) : ( - — - )} - - ); - })} - - {formatSamples(measurement.variants.map((variant) => variant.samples))} - - - ); - }), - )} - -
    - - Δ is tachometer's confidence interval on the difference, {reference} relative to the - other variant{others.length > 1 ? 's' : ''} — negative is faster. - -     - ); -} - -export default function TachometerDetails() { - const params = useParams<{ owner: string; repo: string }>(); - const searchParams = useSearchParams(); - - if (!params.owner || !params.repo) { - throw new Error('Missing required path parameters'); - } - - const repo = `${params.owner}/${params.repo}`; - const sha = searchParams.get('sha'); - const prNumber = searchParams.get('prNumber'); - - const { - data: report, - isLoading, - error, - } = useQuery({ - queryKey: ['tachometer-report', repo, sha], - queryFn: () => fetchCiReport(repo, sha!, 'tachometer.json'), - retry: 1, - enabled: Boolean(sha), - }); - - if (!sha) { - return ( - - Tachometer Details - - Missing required "sha" query parameter. - - - ); - } - - const summarized = (report?.cases ?? []).filter(isSummarized); - const failed = (report?.cases ?? []).filter((entry) => !isSummarized(entry)); - const groups = groupCasesByVariantSet(summarized); - - // A tachometer report carries its own comparison, so the baseline is one of its own refs rather - // than a separately fetched report. - const baselineRef = report?.refs.find((ref) => ref.kind !== 'worktree'); - - return ( - - Tachometer Details - - - - - {isLoading && ( - - - Loading tachometer report… - - )} - - {error && } - - {!isLoading && !error && !report && ( - No tachometer report found for this commit. - )} - - {failed.map((entry) => ( - - {entry.name}: {entry.error ?? 'produced no result'} - - ))} - - {groups.map((group) => - group.variants.length > 2 ? ( - group.cases.map((entry) => ( - - )) - ) : ( - - ), - )} - - {report && ( - - - Each cell is a 95% confidence interval for the mean, in milliseconds; one sample is - one page load. "unsure" means the interval still straddles zero: the - difference did not resolve within the case's sampling budget. - - - Builds: {report.refs.map((ref) => `${ref.id} = ${refLabel(ref)}`).join(' · ')} - - - head: {report.head.sha.slice(0, 9)} ({report.head.branch || '?'}) · measured on the - production bundle, installed from a packed tarball. - - - )} - - - ); -} diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index bbf79769f8..83088cfa9b 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -1,11 +1,11 @@ import type * as React from 'react'; import { createInput } from '../input'; import { measureIteration, PAINT_METRIC_NAME, splitCaseArgs, warnIfNoGc } from '../caseRuntime'; -import type { MeasuredIteration } from '../caseRuntime'; import type { BenchmarkInteraction, BenchmarkOptions, CaseOptions, + MeasuredIteration, VariantLoader, } from '../caseRuntime'; import { seriesName, setMetricRecorder } from '../metricCore'; diff --git a/test/benchmark/src/list/list.bench.tsx b/test/benchmark/src/list/list.bench.tsx index b0e8b33f8c..941fbf386a 100644 --- a/test/benchmark/src/list/list.bench.tsx +++ b/test/benchmark/src/list/list.bench.tsx @@ -1,7 +1,7 @@ import * as React from 'react'; -import { ScrollingList } from '../_shared/ScrollingList'; import { benchmark, ScalarMetric } from '@mui/internal-benchmark'; import { reactMajor } from '@mui/internal-test-utils/env'; +import { ScrollingList } from '../_shared/ScrollingList'; /** * `benchmark()` cases, the same API Vitest runs. Under `benchmark run` each case runs one diff --git a/test/benchmark/src/lists/lists.table.tsx b/test/benchmark/src/lists/lists.table.tsx index 230d185de3..f573a11984 100644 --- a/test/benchmark/src/lists/lists.table.tsx +++ b/test/benchmark/src/lists/lists.table.tsx @@ -1,6 +1,6 @@ import * as React from 'react'; -import { ScrollingList } from '../_shared/ScrollingList'; import { benchmark } from '@mui/internal-benchmark'; +import { ScrollingList } from '../_shared/ScrollingList'; const ROWS = 1000; diff --git a/test/benchmark/src/lists/lists.ul.tsx b/test/benchmark/src/lists/lists.ul.tsx index c174975a47..b77330d8f5 100644 --- a/test/benchmark/src/lists/lists.ul.tsx +++ b/test/benchmark/src/lists/lists.ul.tsx @@ -1,6 +1,6 @@ import * as React from 'react'; -import { ScrollingList } from '../_shared/ScrollingList'; import { benchmark } from '@mui/internal-benchmark'; +import { ScrollingList } from '../_shared/ScrollingList'; const ROWS = 1000; From daaacaa57022af79e25cf66fe0fac46676f98c8a Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 12:29:24 +0200 Subject: [PATCH 078/173] [benchmark] Document benchmark run The READMEs describe benchmark run, its report and baselines in place of the tachometer engine. --- packages/benchmark/README.md | 147 +++++++++++++++-------------------- test/benchmark/README.md | 108 ++++++++++--------------- 2 files changed, 104 insertions(+), 151 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 673dd0ebab..e02a3987a2 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -1,15 +1,14 @@ # Benchmark -Two ways to measure, for two different questions. +Two ways to run the same `benchmark()` files, for two different questions. -**Render durations** — a React component benchmarking tool built on Vitest and Playwright, using -React's profiling build to capture accurate render durations against your source. Everything up to -[Tachometer](#tachometer) covers it. +**Under Vitest** — a React component benchmarking tool built on Vitest and Playwright, using React's +profiling build to capture render durations against your source, tracked commit by commit. Everything +up to [`benchmark run`](#benchmark-run) covers it. -**Wall-clock time** — [Tachometer](#tachometer) below: Google's -[tachometer](https://github.com/google/tachometer) driving a real browser against your **built** -package, comparing two commits of it with interleaved sampling. No profiler and no render counts, -only elapsed milliseconds on what a consumer installs. +**`benchmark run`** — below: the working tree against a baseline commit, both **built** and installed +the way a consumer gets them, sampled alternately in one browser run and compared on paired +differences, so a small change resolves instead of drowning in run-to-run noise. ## Features @@ -119,7 +118,7 @@ benchmark( ); ``` -Under the interleaved engine every variant gets a page of its own that loads only its own module, so +Under `benchmark run` every variant gets a page of its own that loads only its own module, so no variant's code, styles or module state is present while another is measured; all variants come from the working tree. Under Vitest, which has no notion of variants, each variant's cases simply run as tests of their own, named `mount [recharts]`. @@ -361,61 +360,61 @@ BENCHMARK_BASELINE_PATH=/tmp/base-bench.json pnpm test:bench # head run, inlin The feature is opt-in — without `BENCHMARK_BASELINE_PATH` (or the `baselinePath` config option), the dashboard falls back to fetching the base from S3 by merge-base SHA as before. -## Tachometer +## `benchmark run` -`benchmark tacho run` measures a harness package in a real browser, building the workspace once per -commit under comparison and serving the pages from those builds. Run it from the harness. +`benchmark run` measures a harness package's `*.bench.tsx` files — the same `benchmark()` and +`compare()` files Vitest runs — across two builds of the workspace: the working tree and a baseline. +Run it from the harness, whose vite config uses the plugin: -A harness is a package holding a `src/` tree of **case folders**, each with a page and a plain, -unmodified [tachometer config](https://github.com/google/tachometer#config-file). The configs are the -source of truth for which pages exist: the runner reads the selected ones, works out which pages they -reference, builds one variant per distinct commit, and rewrites each url to the built page. Filtering -to one case therefore narrows the build too. +```js +// vite.config.mjs +import { defineConfig } from 'vite'; +import { benchmarkPlugin } from '@mui/internal-benchmark/vitePlugin'; -Sampling — `sampleSize`, `autoSampleConditions`, `timeout` — is set per case in its own config, -because tachometer rejects those as flags once a config file is in play. A case samples until its -difference resolves or the timeout is hit; two builds of equal speed cannot be separated, so such a -case reports `unsure`, which is the expected "nothing changed" outcome rather than a failure. - -### What a case compares - -A case is one of two things, and its `expand` decides which. - -**No `expand`** is the ordinary regression shape. It is expanded for you into `[current]` — the -working tree — against `[baseline]`, both loading the same page: - -```jsonc -{ "benchmarks": [{ "name": "init", "url": "./index.html" }] } +export default defineConfig({ plugins: [benchmarkPlugin()] }); ``` -**Its own `expand`** compares the pages it names with each other, every one of them built from the -working tree. That is how one implementation is measured against another: - -```jsonc -"expand": [ - { "name": "[mosaic]", "url": "./index.html" }, - { "name": "[tanstack]", "url": "./tanstack.html" } -] +```bash +benchmark run --baseline "$(code-infra baseline)" ``` -Such a case reports no regression, by construction: there is no baseline in it to regress against. -Every run says which shape each case has before it starts, so this is visible rather than inferred -from an empty result. To sweep a parameter and still measure it against the baseline, give each -value its own case folder pointing at the shared page — `expand` would turn the case into the other -shape. +### How it measures + +The plugin generates a page per benchmark file under `src/__bench__/`, where +`@mui/internal-benchmark` resolves to a page runtime instead of Vitest; a benchmark file that imports +`vitest` fails the build. The runner drives Chromium through Playwright and gives every variant — the +current and the baseline build, or each variant of a `compare()` — a page of its own, in a browser +context of its own, that stays open for the whole benchmark: every sample is one warm iteration, and +module-scope data is built once. + +Every variant is sampled once per round, in a shuffled order, and a difference is judged on the +per-round differences rather than on two independent sets of samples. Whatever the machine was doing +during a round — thermal throttling, a background process — then affects both sides of it and cancels +out. `--warmup` rounds (10 by default) are discarded once per benchmark, then `--samples` rounds (30 +by default) are measured; the iteration counts in `benchmark()`'s options only apply under Vitest. + +### The report + +The run writes `.benchmark/results/report.json`, prints it as tables, and with `--upload` sends it to +the dashboard, which renders the pull request comment and the repository's history from it. The +report holds **raw samples**, round-aligned across variants, plus what each metric is — its kind, +format and alarm — the builds, and the environment. Conclusions are computed from it, the same way +everywhere, by `analyzeRun` from `@mui/internal-benchmark/runReport`: a 95% confidence interval on the +paired difference per metric, a change (`better`, `worse`, `unsure`) that respects the metric's +direction, and a severity from its alarm. The harness's own metrics are `render` (total render +duration) and `render:count`, which alarm on any resolved change for the worse, and `bench:paint`, +which is informational; custom metrics bring their own alarm. ### Baselines -`--baseline` names the build every `[baseline]` variant loads: a revision — a SHA, a tag, a branch, -`HEAD~1` — on its own or behind `git:`. It is resolved to an immutable SHA before anything is -cached, and defaults to `HEAD~1`, which needs no policy to decide. +`--baseline` names the build the working tree is compared against: a revision — a SHA, a tag, a +branch, `HEAD~1` — on its own or behind `git:`. It is resolved to an immutable SHA before anything is +cached, and defaults to `HEAD~1`, the parent commit. -Which commit a branch should actually be compared against is `code-infra baseline`'s question — it -answers the same way for every job that compares a branch to its base — so pass it in: - -```bash -benchmark tacho run --baseline "$(code-infra baseline)" -``` +Which commit a branch should actually be compared against is `code-infra baseline`'s question: on a +feature branch the fork point, on the base branch the previous commit. On master, then, every commit +is measured against its parent, and the history the dashboard draws is each commit's own paired +change — which commit moved a number reads off the chart rather than out of a noisy trend. `github:/#` and `preview:` are reserved for a baseline that is not a commit in this repository. Neither is implemented, and both are rejected by name rather than handed to git. @@ -423,11 +422,11 @@ this repository. Neither is implemented, and both are rejected by name rather th ### Builds and caching Both sides are built the same way, from packed tarballs installed into an isolated tree, so neither -resolves the library through a workspace link. The benchmark pages themselves stay on the current +resolves the library through a workspace link. The benchmark files themselves stay on the current branch and only the built library changes between refs — so a commit whose public API differs from -today's pages will fail that ref's build, with the error surfaced. +today's benchmarks will fail that ref's build, with the error surfaced. -Packed builds are cached by commit SHA under `.tachometer/packed/`, so repeating a comparison against +Packed builds are cached by commit SHA under `.benchmark/packed/`, so repeating a comparison against the same commit skips the rebuild; the working tree is never cached. In CI, cache that directory keyed on the baseline SHA, and check out with full history so a fork point can be resolved. A run never modifies the checkout it was started from. @@ -436,44 +435,24 @@ By default a run resolves **in place**: it pins the packed build in the reposito `pnpm-workspace.yaml`, installs it there, and resolution is then ordinary. The repository is put back, and reinstalled, when the run ends — including when it fails, and at the start of the next run if one was killed outright. Two things follow from it. A tracked file names tarballs under -`.tachometer/` until the run ends, so do not commit while one is in flight. And the pins are global -to the workspace for that time, because pnpm scopes an override by parent package name and a harness -has none — so nothing else should build against the same checkout meanwhile. +`.benchmark/` until the run ends, so do not commit while one is in flight. And the pins are global to +the workspace for that time, because pnpm scopes an override by parent package name and a harness has +none — so nothing else should build against the same checkout meanwhile. `--resolve-mode isolated` avoids both: it installs each ref into a directory of its own and points resolution there, leaving the repository untouched, at the cost of a second install tree per ref. -`--no-install` skips a ref's install. `benchmark tacho run --help` lists the rest, and -`benchmark tacho report` prints the table again from a saved JSON report. - -### Interleaved engine - -`--engine interleaved` measures the same builds without tachometer, and runs a different kind of -case: the `*.bench.tsx` files under `src/`, written with `benchmark()` exactly as for Vitest. -`tachometer.json` cases stay with the tachometer engine. - -The plugin generates a page per benchmark file under `src/__bench__/`, where -`@mui/internal-benchmark` resolves to a page runtime instead of Vitest. The runner drives Chromium -through Playwright and gives every variant a page of its own, in a browser context of its own, that -stays open for the whole case: every sample is one warm iteration, and module-scope data is built -once. Variants are sampled once per round in a shuffled order, and a difference is judged on the -per-round differences rather than on two independent sets of samples, so whatever the machine was -doing during a round — thermal throttling, a background process — affects both sides of it and -cancels out. Renders, `bench:paint` and custom metrics are reported as measurements; the report, the -upload and the PR comment are the same as tachometer's. - -```bash -benchmark tacho run --engine interleaved --baseline "$(code-infra baseline)" --samples 30 -``` - -The iteration counts in `benchmark()`'s options are ignored: `--warmup` rounds (10 by default) are -discarded once per case, then `--samples` rounds (30 by default) are measured. +`--launch-arg` passes a flag to Chromium, e.g. `--use-angle=metal` for hardware-rendered paint +timings. `--no-install` skips a ref's install. `benchmark run --help` lists the rest, and +`benchmark report` prints the tables again from a saved JSON report. ## API - `benchmark` — define a benchmark test case - `compare` — compare implementations, each a lazily loaded module of `benchmark()` cases - `runCase` — run a single iteration of a case, for custom drivers +- `@mui/internal-benchmark/vitePlugin` — `benchmarkPlugin()`, the harness's vite plugin for `benchmark run` +- `@mui/internal-benchmark/runReport` — the `benchmark run` report schema and `analyzeRun` - `ElementTiming` — invisible marker component for paint timing (renders a `` tracked by the Element Timing API) - `ScalarMetric` — record a continuous custom measurement (with a `console.time`-style timing helper) - `DiscreteMetric` — record a discrete custom count diff --git a/test/benchmark/README.md b/test/benchmark/README.md index f17882ea83..ecc15d21c7 100644 --- a/test/benchmark/README.md +++ b/test/benchmark/README.md @@ -1,74 +1,60 @@ -# Tachometer smoke test +# Benchmark smoke test -A working harness for `benchmark tacho run`, used to exercise that tooling end to end. +A working harness for `benchmark run`, used to exercise that tooling end to end. This is **not** a meaningful benchmark, and it is not trying to be. This repository ships build and test tooling, not a browser library, so there is nothing here whose render time is worth tracking. -The cases run a deterministic CPU workload purely so the pipeline has something real to drive: -discovering cases, resolving the baseline, packing a commit's workspace to tarballs, building each -variant, and running Chrome under tachometer. +The benchmarks run deterministic workloads purely so the pipeline has something real to drive: +discovering benchmark files, resolving the baseline, packing a commit's workspace to tarballs, +building each ref's pages, and sampling them in Chromium. For the real thing, see the consumers this tooling was extracted from — `base-ui-mosaic` and `base-ui-charts` benchmark actual data grids and charts. ## What it covers -The cases are chosen to walk the parts of the pipeline most likely to break: +| Benchmark file | Covers | +| :---------------------------- | :------------------------------------------------------------------------ | +| `workload/workload.bench.tsx` | The ordinary regression shape: current vs baseline, two sizes of workload | +| `list/list.bench.tsx` | A mount, and a trusted scroll gesture with a custom metric around it | +| `lists/lists.bench.tsx` | `compare()` of two implementations, each a module of `benchmark()` cases | +| `libs/sort/sort.bench.tsx` | `compare()` of three variants, standing in for competitor libraries | -| Case | Covers | -| :--------------- | :------------------------------------------------------------------------------- | -| `workload` | The ordinary regression shape: one page, auto-expanded into current vs baseline | -| `workload-large` | A case owning **no page of its own**, parameterising its sibling's with `?size=` | -| `libs-sort` | Three variants from **one build**: the cross-library shape, with no baseline | +`workload` and `list` import `@mui/internal-test-utils` and put its value on screen. That is the +load-bearing part: every ref — the working tree included — resolves it from a packed tarball in that +ref's own tree, so both sides of a comparison consume the library the way a consumer does. If that +path breaks the page fails to build, rather than quietly measuring nothing. -`workload-large` exercises that a case's query parameters survive the url rewrite, that a url -traversing out of its own case folder resolves, and that two cases referencing one page build it -only once. - -`libs-sort` covers the other axis, where nothing is compared across commits. Its three variants are -built from the same commit and differ only in how they sort, standing in for the competitor -libraries this repository has no reason to install. It exercises three things the two-variant cases -cannot: a case with more than two variants, which the report draws as a table of variants rather -than a row per case; a second variant set, which gets a table of its own rather than sharing one -full of blank cells; and a verdict that is genuinely `slower` yet must never be reported as a -regression, because both sides came from the same build. - -`workload` and `workload-large` import `@mui/internal-test-utils` and put its value on screen. That -is the load-bearing part: every ref — the working tree included — resolves it from a packed tarball -in that ref's own tree, so both sides of a comparison consume the library the way a consumer does. -If that path breaks the page fails to build, rather than quietly measuring nothing. +The `compare()` files cover the other axis, where nothing is compared across commits. Their variants +are built from the same commit and differ only in how they render or sort. A difference there is +genuinely `worse` yet must never be reported as a regression, because both sides came from the same +build. ## Running ```bash -pnpm release:build # the harness resolves workspace packages through their build output -pnpm test:tacho # from the repo root +pnpm release:build # the harness resolves workspace packages through their build output +pnpm test:benchmark-run # from the repo root ``` Or from here, with the usual filters and flags: ```bash -pnpm -F ./test/tachometer test:tacho large -pnpm -F ./test/tachometer test:tacho --baseline HEAD +pnpm -F ./test/benchmark benchmark:run workload +pnpm -F ./test/benchmark benchmark:run --baseline HEAD --samples 10 ``` -A filter is a case-insensitive substring of a case's path under `src/`, the way vitest matches test -files. Cases may sit in folders, and since the path is what a filter matches, a folder name selects -everything beneath it — which is how a repository partitions its suite (a quick set for pull -requests, everything for a release) without the runner being taught what any of it means. A case is -named by its benchmark's own `name`, so moving one between folders does not rename it in the report. - -Cases run sequentially, and each auto-samples until its difference resolves or the (deliberately -short) timeout is hit. Because both sides run identical code, the expected verdict is `unsure` — -that is the "no change" outcome, not a failure. +A filter is a case-insensitive substring of a benchmark file's path under `src/`, the way vitest +matches test files. Because both sides run identical code, the expected result is `unsure` — that is +the "no change" outcome, not a failure. -`benchmark tacho report` prints that table again from the saved JSON, so a run kept from earlier — -or a report downloaded from CI — can be read without sampling again. +`benchmark report` prints the tables again from the saved JSON, so a run kept from earlier — or a +report downloaded from CI — can be read without sampling again. -Everything a run writes goes under `.tachometer/`: the report in `results/`, the pages built per ref -in `builds/`, the packed tarballs in `packed/` (the one worth caching in CI, keyed by commit SHA) -and the install each ref resolves through in `trees/`. Deleting that directory resets the harness -completely, and it is the only thing a repository has to ignore. +Everything a run writes goes under `.benchmark/`: the report in `results/`, the pages built per ref in +`builds/`, the packed tarballs in `packed/` (the one worth caching in CI, keyed by commit SHA) and the +install each ref resolves through in `trees/`. Deleting that directory resets the harness completely. +The pages generated per benchmark file live in `src/__bench__/`, which is ignored too. ### Prerequisites @@ -76,40 +62,28 @@ Playwright's Chromium, installed with `pnpm exec playwright install chromium`. R pinned Chrome for Testing rather than whatever Chrome the machine has auto-updated to, so a browser update cannot move the numbers on its own. -A chromedriver only drives its own Chrome major, so the root `chromedriver` devDependency has to -stay aligned with `@playwright/test` — bump the two together. `tacho run` checks the pair before it -builds anything and fails with both versions named, rather than letting you find out minutes later -through `SessionNotCreatedError`. - -One wrinkle: tachometer installs a chromedriver into its own package directory on demand, and that -copy wins over the root pin. If the check reports a driver you did not install, delete -`node_modules/.pnpm/tachometer@*/node_modules/tachometer/node_modules/chromedriver` and re-run. - ### A note on the baseline -`tacho run` does not work out which commit to compare against — `code-infra baseline` does, for +`benchmark run` does not work out which commit to compare against — `code-infra baseline` does, for every job that compares a branch to its base, so bundle size and the benchmarks agree on the answer instead of each deriving it. Pass it in: ```bash -pnpm test:tacho --baseline "$(pnpm code-infra baseline)" +pnpm test:benchmark-run --baseline "$(pnpm code-infra baseline)" ``` -Without `--baseline` the previous commit is used, which needs no policy to decide. +Without `--baseline` the previous commit is used. -A baseline older than the introduction of `benchmark tacho` will fail to build its pages: this +A baseline older than the introduction of `benchmark run` will fail to build its pages: this harness's vite config imports a plugin that did not exist at that commit. `--baseline HEAD` compares the working tree against the current commit while that is still true. -## Working on a case by hand +## Working on a benchmark by hand ```bash -pnpm -F ./test/tachometer dev +pnpm -F ./test/benchmark dev ``` -Every discovered case is listed at `/`, including `workload-large`, which owns no page of its own and -would otherwise be unreachable — its `?size=` query is in the config, not the file tree. The list is -generated from the case configs rather than written by hand, so it cannot go stale. - -`pnpm -F ./test/tachometer build` and `preview` produce and serve the production bundle, which is -what actually gets measured. The index is emitted into that build too, so the same list is there. +Every benchmark file's generated page is listed at `/`. `pnpm -F ./test/benchmark build` and `preview` +produce and serve the production bundle, which is what actually gets measured; the index is emitted +into that build too. From 028a7f9e2de34b8b2dbe1915661ba5235c0173c7 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:38:22 +0200 Subject: [PATCH 079/173] [benchmark] Keep tacho run, upload benchmark run as benchmark report v2 Repositories move to `benchmark run` one at a time, so the paths it replaces stay until the last one has moved: - The tachometer engine, `benchmark tacho run`/`report`, the test/tachometer harness, chromedriver and the dashboard's tachometer section and details view are restored exactly as #1829 has them. - The Vitest reporter keeps uploading version 1 of the `benchmark` report. `benchmark run` now uploads version 2 of that same report type instead of a report type of its own. A repository uploads one version or the other: both are the commit's `benchmark.json`. The dashboard reads `benchmark.json` as a v1 | v2 union: the Performance PR-comment section renders a v2 report with the new paired analysis and links the v2 details page, the v1 comparison, details page and daily charts ignore v2 reports, and the parent-paired history chart draws v2 only, hiding itself for repositories that have none. CI keeps test_tachometer as is and adds test_benchmark_run as a smoke test without upload, since this repository still uploads version 1. --- .circleci/config.yml | 50 +- .gitignore | 5 + .../[owner]/[repo]/page.tsx | 16 + .../api/ci-reports/sync-pr-comment/route.ts | 18 +- .../app/api/ci-reports/upload/route.ts | 8 +- .../src/components/BenchmarkRunHistory.tsx | 14 +- .../src/components/DailyBenchmarkChart.tsx | 18 +- apps/code-infra-dashboard/src/constants.ts | 6 +- .../benchmarkRun/buildMarkdownReport.test.ts | 22 +- .../lib/benchmarkRun/buildMarkdownReport.ts | 10 +- .../src/lib/ciReports/benchmarkReport.ts | 26 +- .../src/lib/ciReports/benchmarkRunReport.ts | 33 - .../src/lib/ciReports/tachometerReport.ts | 34 + .../tachometer/buildMarkdownReport.test.ts | 251 +++ .../src/lib/tachometer/buildMarkdownReport.ts | 179 ++ .../src/utils/fetchCiReport.ts | 29 +- .../src/views/BenchmarkDetails.tsx | 15 +- .../src/views/BenchmarkRunDetails.tsx | 7 +- .../src/views/RepositoryBenchmarks.tsx | 2 +- .../src/views/TachometerDetails.tsx | 349 ++++ package.json | 2 + packages/benchmark/README.md | 111 +- packages/benchmark/package.json | 3 + packages/benchmark/src/cli/index.ts | 10 + packages/benchmark/src/cli/tachoReport.ts | 51 + packages/benchmark/src/cli/tachoRun.ts | 70 + .../src/runReport/analyzeRun.test.ts | 2 +- packages/benchmark/src/runReport/schema.ts | 18 +- .../benchmark/src/runner/runBenchmarks.ts | 2 +- packages/benchmark/src/runner/upload.ts | 19 +- .../benchmark/src/tachometer/browser.test.ts | 172 ++ packages/benchmark/src/tachometer/browser.ts | 160 ++ .../benchmark/src/tachometer/buildPages.ts | 322 ++++ .../benchmark/src/tachometer/ciReport.test.ts | 141 ++ packages/benchmark/src/tachometer/ciReport.ts | 124 ++ .../src/tachometer/discoverCases.test.ts | 577 +++++++ .../benchmark/src/tachometer/discoverCases.ts | 321 ++++ .../benchmark/src/tachometer/format.test.ts | 100 ++ packages/benchmark/src/tachometer/format.ts | 93 + .../benchmark/src/tachometer/outputDir.ts | 36 + .../benchmark/src/tachometer/plugin.test.ts | 186 ++ packages/benchmark/src/tachometer/plugin.ts | 242 +++ .../benchmark/src/tachometer/refs.test.ts | 124 ++ packages/benchmark/src/tachometer/refs.ts | 70 + .../benchmark/src/tachometer/renderReport.ts | 241 +++ .../benchmark/src/tachometer/runTachometer.ts | 322 ++++ .../src/tachometer/summarizeCase.test.ts | 203 +++ .../benchmark/src/tachometer/summarizeCase.ts | 184 ++ pnpm-lock.yaml | 1519 ++++++++++++++++- pnpm-workspace.yaml | 1 + test/tachometer/README.md | 115 ++ test/tachometer/package.json | 22 + test/tachometer/scripts/runBenchmark.mjs | 12 + test/tachometer/src/_shared/measure.ts | 51 + test/tachometer/src/libs/sort/alpha.html | 11 + test/tachometer/src/libs/sort/alpha.ts | 10 + test/tachometer/src/libs/sort/beta.html | 11 + test/tachometer/src/libs/sort/beta.ts | 33 + test/tachometer/src/libs/sort/ours.html | 11 + test/tachometer/src/libs/sort/ours.ts | 13 + test/tachometer/src/libs/sort/shared.ts | 51 + test/tachometer/src/libs/sort/tachometer.json | 22 + .../src/workload-large/tachometer.json | 18 + test/tachometer/src/workload/index.html | 11 + test/tachometer/src/workload/index.ts | 43 + test/tachometer/src/workload/tachometer.json | 18 + test/tachometer/tsconfig.json | 15 + test/tachometer/vite.config.mjs | 6 + tsconfig.json | 3 +- 69 files changed, 6876 insertions(+), 118 deletions(-) create mode 100644 apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx delete mode 100644 apps/code-infra-dashboard/src/lib/ciReports/benchmarkRunReport.ts create mode 100644 apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts create mode 100644 apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts create mode 100644 apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts create mode 100644 apps/code-infra-dashboard/src/views/TachometerDetails.tsx create mode 100644 packages/benchmark/src/cli/tachoReport.ts create mode 100644 packages/benchmark/src/cli/tachoRun.ts create mode 100644 packages/benchmark/src/tachometer/browser.test.ts create mode 100644 packages/benchmark/src/tachometer/browser.ts create mode 100644 packages/benchmark/src/tachometer/buildPages.ts create mode 100644 packages/benchmark/src/tachometer/ciReport.test.ts create mode 100644 packages/benchmark/src/tachometer/ciReport.ts create mode 100644 packages/benchmark/src/tachometer/discoverCases.test.ts create mode 100644 packages/benchmark/src/tachometer/discoverCases.ts create mode 100644 packages/benchmark/src/tachometer/format.test.ts create mode 100644 packages/benchmark/src/tachometer/format.ts create mode 100644 packages/benchmark/src/tachometer/outputDir.ts create mode 100644 packages/benchmark/src/tachometer/plugin.test.ts create mode 100644 packages/benchmark/src/tachometer/plugin.ts create mode 100644 packages/benchmark/src/tachometer/refs.test.ts create mode 100644 packages/benchmark/src/tachometer/refs.ts create mode 100644 packages/benchmark/src/tachometer/renderReport.ts create mode 100644 packages/benchmark/src/tachometer/runTachometer.ts create mode 100644 packages/benchmark/src/tachometer/summarizeCase.test.ts create mode 100644 packages/benchmark/src/tachometer/summarizeCase.ts create mode 100644 test/tachometer/README.md create mode 100644 test/tachometer/package.json create mode 100644 test/tachometer/scripts/runBenchmark.mjs create mode 100644 test/tachometer/src/_shared/measure.ts create mode 100644 test/tachometer/src/libs/sort/alpha.html create mode 100644 test/tachometer/src/libs/sort/alpha.ts create mode 100644 test/tachometer/src/libs/sort/beta.html create mode 100644 test/tachometer/src/libs/sort/beta.ts create mode 100644 test/tachometer/src/libs/sort/ours.html create mode 100644 test/tachometer/src/libs/sort/ours.ts create mode 100644 test/tachometer/src/libs/sort/shared.ts create mode 100644 test/tachometer/src/libs/sort/tachometer.json create mode 100644 test/tachometer/src/workload-large/tachometer.json create mode 100644 test/tachometer/src/workload/index.html create mode 100644 test/tachometer/src/workload/index.ts create mode 100644 test/tachometer/src/workload/tachometer.json create mode 100644 test/tachometer/tsconfig.json create mode 100644 test/tachometer/vite.config.mjs diff --git a/.circleci/config.yml b/.circleci/config.yml index e2a7a21d1c..6ea84a6d7c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -114,7 +114,7 @@ jobs: BENCHMARK_UPLOAD: 'true' command: pnpm -F ./test/performance benchmark - test_benchmark_run: + test_tachometer: <<: *default-job executor: name: code-infra/mui-node-browser @@ -133,20 +133,54 @@ jobs: # is the command's own job, so there is nothing to keep in step with it here. BASE_SHA=$(pnpm code-infra baseline --base-branch master) echo "Baseline commit: $BASE_SHA" - echo "$BASE_SHA" > .benchmark-baseline-sha + echo "$BASE_SHA" > .tachometer-baseline-sha - restore_cache: name: Restore the packed-baseline cache keys: - - benchmark-run-cache-v1-{{ checksum ".benchmark-baseline-sha" }} + - tachometer-cache-v1-{{ checksum ".tachometer-baseline-sha" }} - run: - name: Run the benchmark smoke test + name: Run the tachometer smoke test no_output_timeout: 30m environment: - BENCHMARK_UPLOAD: 'true' - # TEMPORARY: production does not know the benchmark-run report type until this pull + TACHO_UPLOAD: 'true' + # TEMPORARY: production does not know the tachometer report type until this pull # request lands, so it would reject the upload. Points at this pull request's own # dashboard preview until then — remove the line before merging. - CI_REPORT_API_URL: https://code-infra-dashboard-pr-1898.onrender.com + CI_REPORT_API_URL: https://code-infra-dashboard-pr-1829.onrender.com + command: pnpm test:tacho --baseline "$(cat .tachometer-baseline-sha)" + - save_cache: + name: Save the packed-baseline cache + key: tachometer-cache-v1-{{ checksum ".tachometer-baseline-sha" }} + paths: + - test/tachometer/.tachometer/packed + + test_benchmark_run: + <<: *default-job + executor: + name: code-infra/mui-node-browser + playwright-img-version: 'v1.62.1-noble' + steps: + - checkout + - code-infra/install-deps + - run: + name: Build packages + command: pnpm release:build + - run: + name: Resolve the baseline commit + command: | + set -e + BASE_SHA=$(pnpm code-infra baseline --base-branch master) + echo "Baseline commit: $BASE_SHA" + echo "$BASE_SHA" > .benchmark-baseline-sha + - restore_cache: + name: Restore the packed-baseline cache + keys: + - benchmark-run-cache-v1-{{ checksum ".benchmark-baseline-sha" }} + - run: + name: Run the benchmark smoke test + no_output_timeout: 30m + # No upload: this repository still uploads its version 1 benchmark report from + # `test_benchmark`, and both write the commit's one `benchmark.json`. command: pnpm test:benchmark-run --baseline "$(cat .benchmark-baseline-sha)" - save_cache: name: Save the packed-baseline cache @@ -195,6 +229,8 @@ workflows: <<: *default-context - test_benchmark: <<: *default-context + - test_tachometer: + <<: *default-context - test_benchmark_run: <<: *default-context - test_browser: diff --git a/.gitignore b/.gitignore index a5d602a451..e6c210e268 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,8 @@ next-env.d.ts # .gitignore and never reads nested ones. /test/benchmark/.benchmark/ /test/benchmark/src/__bench__/ + +# Everything the tachometer harness writes: built pages per ref, packed workspace tarballs, the +# isolated installs and the run reports. The directory ignores itself from the inside too, but the +# entry is listed here because ESLint is fed from the root .gitignore and never reads nested ones. +/test/tachometer/.tachometer/ diff --git a/apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx b/apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx new file mode 100644 index 0000000000..6d1a42140e --- /dev/null +++ b/apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx @@ -0,0 +1,16 @@ +import * as React from 'react'; +import type { Metadata } from 'next'; +import TachometerDetails from '@/views/TachometerDetails'; + +export async function generateMetadata({ + params, +}: { + params: Promise<{ owner: string; repo: string }>; +}): Promise { + const { owner, repo } = await params; + return { title: `Tachometer Details - ${owner}/${repo}` }; +} + +export default function TachometerDetailsPage() { + return ; +} diff --git a/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts b/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts index 8d1fdd8639..0cedc5ddec 100644 --- a/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts +++ b/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts @@ -11,9 +11,9 @@ import { import { generateBenchmarkReport, BENCHMARK_SECTION_TITLE } from '@/lib/ciReports/benchmarkReport'; import { generateDeployPreviewReport } from '@/lib/ciReports/deployPreviewReport'; import { - generateBenchmarkRunReport, - BENCHMARK_RUN_SECTION_TITLE, -} from '@/lib/ciReports/benchmarkRunReport'; + generateTachometerReport, + TACHOMETER_SECTION_TITLE, +} from '@/lib/ciReports/tachometerReport'; import type { ReportResult } from '@/lib/ciReports/types'; import { fetchParentCommits } from '@/utils/fetchCiReportWithFallback'; import { getOctokit } from '@/lib/github'; @@ -122,7 +122,7 @@ export async function POST(request: NextRequest) { // Generate all configured report sections in parallel. // Each generator is wrapped in .catch() so a failure in one doesn't block the others. - const [bundleSizeReport, benchmarkReportResult, benchmarkRunReportResult, deployPreviewReport] = + const [bundleSizeReport, benchmarkReportResult, tachometerReportResult, deployPreviewReport] = await Promise.all([ prCommentConfig?.bundleSize ? generateBundleSizeReport(reportOptions).catch((error): ReportResult => { @@ -140,11 +140,11 @@ export async function POST(request: NextRequest) { }; }) : null, - prCommentConfig?.benchmarkRun - ? generateBenchmarkRunReport(reportOptions).catch((error): ReportResult => { - console.error('Failed to generate benchmark run report:', error); + prCommentConfig?.tachometer + ? generateTachometerReport(reportOptions).catch((error): ReportResult => { + console.error('Failed to generate tachometer report:', error); return { - content: `## ${BENCHMARK_RUN_SECTION_TITLE}\n\n:warning: Failed to generate the benchmark report.`, + content: `## ${TACHOMETER_SECTION_TITLE}\n\n:warning: Failed to generate tachometer report.`, }; }) : null, @@ -164,7 +164,7 @@ export async function POST(request: NextRequest) { deployPreviewReport, bundleSizeReport, benchmarkReportResult, - benchmarkRunReportResult, + tachometerReportResult, ] .filter((report): report is ReportResult => report !== null) .map((report) => report.content) diff --git a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts index b9591c555d..e672f85a19 100644 --- a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts +++ b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts @@ -5,7 +5,7 @@ import { uploadReport } from '@/lib/ciReports/s3'; import { verifyOidcToken } from '@/lib/ciReports/oidcAuth'; import { findAssociatedPr } from '@/lib/ciReports/findAssociatedPr'; -const VALID_REPORT_TYPES = new Set(['size-snapshot', 'benchmark', 'benchmark-run']); +const VALID_REPORT_TYPES = new Set(['size-snapshot', 'benchmark', 'tachometer']); const uploadSchema = z.object({ version: z.number(), @@ -56,9 +56,9 @@ export async function POST(request: NextRequest) { const { commitSha, repo, reportType, branch, report } = parsed.data; // A benchmark report is stored as the whole envelope, every other type as just the inner report. - // Historic, and kept because artifacts written that way are still read back. A benchmark-run - // report needs none of it: it carries its own `version` and `head`, the repo is in the S3 key, and - // the generator reads the rest from its own route params. + // Historic, and kept because artifacts written that way are still read back. A tachometer report + // needs none of it: it carries its own `version`, `reportType` and `head`, the repo is in the S3 + // key, and the generator reads the rest from its own route params. const storedBody = reportType === 'benchmark' ? JSON.stringify(parsed.data) : JSON.stringify(report); diff --git a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx index 0b458b8144..80ac8817d6 100644 --- a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx +++ b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx @@ -10,6 +10,7 @@ import Typography from '@mui/material/Typography'; import { BarChartPro } from '@mui/x-charts-pro/BarChartPro'; import { analyzeRun, formatComparison } from '@mui/internal-benchmark/runReport'; import type { MetricComparison } from '@mui/internal-benchmark/runReport'; +import { isBenchmarkRunUpload } from '@/utils/fetchCiReport'; import { useMasterCommits } from '../hooks/useMasterCommits'; import { useCiReports } from '../hooks/useCiReports'; import ErrorDisplay from './ErrorDisplay'; @@ -42,15 +43,17 @@ interface BenchmarkRunHistoryProps { export default function BenchmarkRunHistory({ repo }: BenchmarkRunHistoryProps) { const { commits, isLoading, isFetchingNextPage, hasNextPage, error, fetchNextPage } = useMasterCommits(repo); - const { reports, isLoading: reportsLoading } = useCiReports(repo, commits, 'benchmark-run.json'); + const { reports, isLoading: reportsLoading } = useCiReports(repo, commits, 'benchmark.json'); const points = React.useMemo( () => commits.flatMap(({ timestamp, commit }): CommitPoint[] => { - const report = reports[commit.sha]; - if (!report) { + const upload = reports[commit.sha]; + // Version 1 reports are `DailyBenchmarkChart`'s. + if (!upload || !isBenchmarkRunUpload(upload)) { return []; } + const { report } = upload; const comparisons = new Map>(); for (const { benchmark, metrics } of analyzeRun(report)) { if (benchmark.kind !== 'baseline') { @@ -111,6 +114,11 @@ export default function BenchmarkRunHistory({ repo }: BenchmarkRunHistoryProps) const [selectedSha, setSelectedSha] = React.useState(null); + // A repository still on version 1 reports has nothing to draw here. + if (!error && !isLoading && !reportsLoading && points.length === 0) { + return null; + } + return ( diff --git a/apps/code-infra-dashboard/src/components/DailyBenchmarkChart.tsx b/apps/code-infra-dashboard/src/components/DailyBenchmarkChart.tsx index b97d1c3c6a..73f3572558 100644 --- a/apps/code-infra-dashboard/src/components/DailyBenchmarkChart.tsx +++ b/apps/code-infra-dashboard/src/components/DailyBenchmarkChart.tsx @@ -12,6 +12,7 @@ import { BarChartPro } from '@mui/x-charts-pro/BarChartPro'; import { useXScale, useDrawingArea } from '@mui/x-charts-pro/hooks'; import type { BenchmarkReport, MetricDefinition } from '@/lib/benchmark/types'; import { formatMs } from '@/utils/formatters'; +import { isBenchmarkRunUpload } from '@/utils/fetchCiReport'; import { useMasterCommits } from '../hooks/useMasterCommits'; import type { GitHubCommit } from '../hooks/useMasterCommits'; import { useCiReports } from '../hooks/useCiReports'; @@ -124,12 +125,17 @@ export default function DailyBenchmarkChart({ repo }: DailyBenchmarkChartProps) const chartData: CommitReportData[] = React.useMemo( () => - commits.map(({ timestamp, commit }) => ({ - timestamp, - commit, - report: reports[commit.sha]?.report ?? null, - metricDefinitions: reports[commit.sha]?.metricDefinitions, - })), + commits.map(({ timestamp, commit }) => { + // This chart reads version 1 reports; `BenchmarkRunHistory` draws version 2. + const upload = reports[commit.sha]; + const version1 = upload && !isBenchmarkRunUpload(upload) ? upload : undefined; + return { + timestamp, + commit, + report: version1?.report ?? null, + metricDefinitions: version1?.metricDefinitions, + }; + }), [commits, reports], ); diff --git a/apps/code-infra-dashboard/src/constants.ts b/apps/code-infra-dashboard/src/constants.ts index 5c6a95e34f..08e8bf2f43 100644 --- a/apps/code-infra-dashboard/src/constants.ts +++ b/apps/code-infra-dashboard/src/constants.ts @@ -25,7 +25,7 @@ export interface Repository { prComment?: { bundleSize?: boolean; benchmark?: boolean; - benchmarkRun?: boolean; + tachometer?: boolean; netlifyDocs?: | boolean | { @@ -167,7 +167,7 @@ export const repositories = new Map( prComment: { bundleSize: true, benchmark: true, - benchmarkRun: true, + tachometer: true, }, }, { @@ -195,7 +195,7 @@ export const repositories = new Map( prComment: { bundleSize: true, benchmark: true, - benchmarkRun: true, + tachometer: true, netlifyDocs: { siteId: 'mui-internal', formatDocPath: (filePath) => { diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index eec60ede31..8518795d40 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -6,7 +6,7 @@ const MS = { style: 'unit', unit: 'millisecond', maximumFractionDigits: 2 } as c function reportOf(benchmarks: RunBenchmark[]): BenchmarkRunReport { return { - version: 1, + version: 2, generatedAt: '2026-01-01T00:00:00.000Z', head: { sha: 'abc' }, environment: { browser: 'Chromium 151', platform: 'linux', arch: 'x64', launchArgs: [] }, @@ -60,19 +60,24 @@ const broken: RunBenchmark = { describe('buildBenchmarkRunMarkdownReport', () => { it('states nothing but the summary when no benchmark regressed', () => { - const markdown = buildBenchmarkRunMarkdownReport(reportOf([unchanged])); + const markdown = buildBenchmarkRunMarkdownReport(reportOf([unchanged]), { + title: 'Performance', + }); expect(markdown.split('\n').slice(0, 3)).toEqual([ - '## Benchmarks', + '## Performance', '', '1 benchmark measured · no regressions', ]); }); it('lists regressions above the collapsed results, and marks the heading', () => { - expect(buildBenchmarkRunMarkdownReport(reportOf([slower, libraries, broken]))) - .toMatchInlineSnapshot(` - "## Benchmarks ⚠️ + expect( + buildBenchmarkRunMarkdownReport(reportOf([slower, libraries, broken]), { + title: 'Performance', + }), + ).toMatchInlineSnapshot(` + "## Performance ⚠️ 🔴 **Grid scroll** · render · \`worse +9.3% – +9.3%\` @@ -105,7 +110,9 @@ describe('buildBenchmarkRunMarkdownReport', () => { }); it('does not count a difference between libraries as a regression', () => { - const markdown = buildBenchmarkRunMarkdownReport(reportOf([libraries])); + const markdown = buildBenchmarkRunMarkdownReport(reportOf([libraries]), { + title: 'Performance', + }); expect(markdown).toContain('no regressions'); expect(markdown).not.toContain('🔴'); @@ -113,6 +120,7 @@ describe('buildBenchmarkRunMarkdownReport', () => { it('links the full run when given where it is', () => { const markdown = buildBenchmarkRunMarkdownReport(reportOf([unchanged]), { + title: 'Performance', detailsUrl: 'https://example.com/run', }); diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 9e8f68f87e..a526421502 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -7,9 +7,9 @@ import { } from '@mui/internal-benchmark/runReport'; import type { BenchmarkAnalysis, BenchmarkRunReport } from '@mui/internal-benchmark/runReport'; -export const BENCHMARK_RUN_SECTION_TITLE = 'Benchmarks'; - interface BuildOptions { + /** The comment section's heading. */ + title: string; /** Where the full results can be inspected. Omitted in tests. */ detailsUrl?: string; } @@ -53,7 +53,7 @@ function renderBenchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): string } /** - * Renders the benchmarks section of the pull request comment. + * Renders a version 2 benchmark report as its pull request comment section. * * Regressions are stated above the collapsed block, and mark the heading, because a reader has to * see them without expanding anything — GitHub renders `
    ` closed, so anything inside it is @@ -61,14 +61,14 @@ function renderBenchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): string */ export function buildBenchmarkRunMarkdownReport( report: BenchmarkRunReport, - options: BuildOptions = {}, + options: BuildOptions, ): string { const analyses = analyzeRun(report); const regressions = findRegressions(analyses); const measured = analyses.filter((analysis) => !analysis.benchmark.error); const failed = analyses.filter((analysis) => analysis.benchmark.error); - const lines = [`## ${BENCHMARK_RUN_SECTION_TITLE}${regressions.length > 0 ? ' ⚠️' : ''}`, '']; + const lines = [`## ${options.title}${regressions.length > 0 ? ' ⚠️' : ''}`, '']; for (const { benchmark, metric, comparison } of regressions) { const marker = comparison.severity === 'error' ? '🔴' : '🟡'; diff --git a/apps/code-infra-dashboard/src/lib/ciReports/benchmarkReport.ts b/apps/code-infra-dashboard/src/lib/ciReports/benchmarkReport.ts index e7ef61af56..3bda25c4f7 100644 --- a/apps/code-infra-dashboard/src/lib/ciReports/benchmarkReport.ts +++ b/apps/code-infra-dashboard/src/lib/ciReports/benchmarkReport.ts @@ -1,7 +1,8 @@ -import { fetchCiReport } from '@/utils/fetchCiReport'; +import { fetchCiReport, isBenchmarkRunUpload } from '@/utils/fetchCiReport'; import { fetchCiReportWithFallback } from '@/utils/fetchCiReportWithFallback'; import { compareBenchmarkReports } from '@/lib/benchmark/compareBenchmarkReports'; import { buildBenchmarkMarkdownReport } from '@/lib/benchmark/buildMarkdownReport'; +import { buildBenchmarkRunMarkdownReport } from '@/lib/benchmarkRun/buildMarkdownReport'; import { DASHBOARD_ORIGIN } from '@/constants'; import type { ReportOptions, ReportResult } from './types'; @@ -9,8 +10,12 @@ import type { ReportOptions, ReportResult } from './types'; export const BENCHMARK_SECTION_TITLE = 'Performance'; /** - * Generates a complete benchmark report by fetching and comparing benchmark results. + * Generates the benchmark section of the pull request comment from the head commit's report. * Returns null if the head benchmark report is not available. + * + * A version 2 report measured its own baseline in the same run, so it is rendered on its own. A + * version 1 report is compared against the base commit's report, fetched separately — which has to + * be version 1 too, since the two shapes cannot be compared. */ export async function generateBenchmarkReport( options: ReportOptions, @@ -26,8 +31,23 @@ export async function generateBenchmarkReport( return null; } + if (isBenchmarkRunUpload(headReport)) { + const runUrl = new URL(`${DASHBOARD_ORIGIN}/benchmark-run/${repo}`); + runUrl.searchParams.set('sha', commitSha); + // The details view reads this to title the page and link back to the PR overview. + runUrl.searchParams.set('prNumber', String(prNumber)); + return { + content: buildBenchmarkRunMarkdownReport(headReport.report, { + title: BENCHMARK_SECTION_TITLE, + detailsUrl: runUrl.href, + }), + }; + } + const inlinedBase = headReport.base; - const { report: fetchedBaseUpload, actualCommit: actualBaseCommit } = baseResult; + const { actualCommit: actualBaseCommit } = baseResult; + const fetchedBaseUpload = + baseResult.report && !isBenchmarkRunUpload(baseResult.report) ? baseResult.report : null; const fetchedBaseReport = fetchedBaseUpload?.report ?? null; const mergeBaseCommit = baseCandidates[0]; diff --git a/apps/code-infra-dashboard/src/lib/ciReports/benchmarkRunReport.ts b/apps/code-infra-dashboard/src/lib/ciReports/benchmarkRunReport.ts deleted file mode 100644 index 00e16bc847..0000000000 --- a/apps/code-infra-dashboard/src/lib/ciReports/benchmarkRunReport.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { fetchCiReport } from '@/utils/fetchCiReport'; -import { - buildBenchmarkRunMarkdownReport, - BENCHMARK_RUN_SECTION_TITLE, -} from '@/lib/benchmarkRun/buildMarkdownReport'; -import { DASHBOARD_ORIGIN } from '@/constants'; -import type { ReportOptions, ReportResult } from './types'; - -export { BENCHMARK_RUN_SECTION_TITLE }; - -/** - * Generates the benchmarks section of the pull request comment. The run measured its own baseline, - * so there is no base report to fetch: only the head commit's. - */ -export async function generateBenchmarkRunReport( - options: ReportOptions, -): Promise { - const { repo, prNumber, commitSha } = options; - - const report = await fetchCiReport(repo, commitSha, 'benchmark-run.json'); - if (!report) { - return null; - } - - const detailsUrl = new URL(`${DASHBOARD_ORIGIN}/benchmark-run/${repo}`); - detailsUrl.searchParams.set('sha', commitSha); - // The details view reads this to title the page and link back to the PR overview. - detailsUrl.searchParams.set('prNumber', String(prNumber)); - - return { - content: buildBenchmarkRunMarkdownReport(report, { detailsUrl: detailsUrl.href }), - }; -} diff --git a/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts b/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts new file mode 100644 index 0000000000..7224b5c0ee --- /dev/null +++ b/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts @@ -0,0 +1,34 @@ +import { fetchCiReport } from '@/utils/fetchCiReport'; +import { + buildTachometerMarkdownReport, + TACHOMETER_SECTION_TITLE, +} from '@/lib/tachometer/buildMarkdownReport'; +import { DASHBOARD_ORIGIN } from '@/constants'; + +import type { TachometerReport } from '@mui/internal-benchmark/tachometerReport'; +import type { ReportOptions, ReportResult } from './types'; + +export { TACHOMETER_SECTION_TITLE }; + +/** + * Generates the tachometer section of the pull request comment. + */ +export async function generateTachometerReport( + options: ReportOptions, +): Promise { + const { repo, prNumber, commitSha } = options; + + const report: TachometerReport | null = await fetchCiReport(repo, commitSha, 'tachometer.json'); + if (!report?.cases) { + return null; + } + + const detailsUrl = new URL(`${DASHBOARD_ORIGIN}/tachometer-details/${repo}`); + detailsUrl.searchParams.set('sha', commitSha); + // The details view reads this to title the page and link back to the PR overview. + detailsUrl.searchParams.set('prNumber', String(prNumber)); + + return { + content: buildTachometerMarkdownReport(report, { detailsUrl: detailsUrl.href }), + }; +} diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts new file mode 100644 index 0000000000..a086e22459 --- /dev/null +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts @@ -0,0 +1,251 @@ +import { describe, expect, it } from 'vitest'; +import type { + CaseResult, + MeasurementResult, + TachometerReport, + Verdict, +} from '@mui/internal-benchmark/tachometerReport'; +import { buildTachometerMarkdownReport, findRegressions } from './buildMarkdownReport'; + +function variant(name: string) { + return { + variant: name, + meanMs: { low: 20, high: 22 }, + samples: 120, + }; +} + +/** The verdict the other way round; an unresolved difference is unresolved from both sides. */ +function mirrorOf(verdict: Verdict): Verdict { + if (verdict === 'slower') { + return 'faster'; + } + return verdict === 'faster' ? 'slower' : verdict; +} + +/** + * A comparison of the reference against `name`, carrying both directions the way a real report + * does. The two are not negations of each other — each has its own denominator — so the numbers + * here differ, which is what lets a test tell which direction was rendered. + */ +function comparison(name: string, verdict: Verdict) { + const sign = verdict === 'slower' ? 1 : -1; + return { + variant: name, + verdict, + absoluteMs: { low: sign * 6.1, high: sign * 15.2 }, + percentChange: { low: sign * 2.7, high: sign * 6.8 }, + versusReference: { + verdict: mirrorOf(verdict), + absoluteMs: { low: sign * -6.1, high: sign * -15.2 }, + percentChange: { low: sign * -2.6, high: sign * -6.4 }, + }, + }; +} + +/** A case measuring the working tree against the baseline, where a difference is a regression. */ +function regressionCase(name: string, verdicts: Record): CaseResult { + const measurements: MeasurementResult[] = Object.entries(verdicts).map( + ([measurement, verdict]) => ({ + name: measurement, + variants: [variant(`${name} [current]`), variant(`${name} [baseline]`)], + comparisons: [comparison(`${name} [baseline]`, verdict)], + }), + ); + return { name, comparison: 'baseline', reference: `${name} [current]`, measurements }; +} + +/** A case comparing libraries with each other, every variant built from the working tree. */ +function libraryCase(name: string, verdict: Verdict): CaseResult { + return { + name, + comparison: 'variants', + reference: `${name} [mosaic]`, + measurements: [ + { + name: 'mount', + variants: [variant(`${name} [mosaic]`), variant(`${name} [ag-grid]`)], + comparisons: [comparison(`${name} [ag-grid]`, verdict)], + }, + ], + }; +} + +function report(cases: CaseResult[]): TachometerReport { + return { + version: 1, + reportType: 'tachometer', + generatedAt: '2026-09-02T00:00:00.000Z', + head: { ref: 'HEAD', sha: 'a'.repeat(40), branch: 'feature' }, + refs: [ + { id: 'current', kind: 'worktree' }, + { id: 'git-abc123456', kind: 'git', requested: 'HEAD~1', sha: 'b'.repeat(40) }, + ], + cases, + raw: {}, + }; +} + +/** Everything a reader sees without expanding anything. */ +function visiblePart(markdown: string): string { + return markdown.split('
    ')[0]; +} + +/** The collapsed table's row for `variant`, rather than any prose that names it too. */ +function tableRowFor(markdown: string, variantName: string): string | undefined { + return markdown + .split('
    ')[1] + ?.split('\n') + .find((line) => line.startsWith('|') && line.includes(variantName)); +} + +describe('findRegressions', () => { + it('finds a slower verdict across two builds', () => { + const found = findRegressions(report([regressionCase('workload', { mount: 'slower' })])); + + expect(found).toMatchObject([{ caseName: 'workload', measurement: 'mount' }]); + }); + + it.each(['unsure', 'faster'])('does not count a %s verdict', (verdict) => { + expect(findRegressions(report([regressionCase('workload', { mount: verdict })]))).toEqual([]); + }); + + it('ignores a slower verdict between variants on the same build', () => { + // A cross-library case runs every variant from one build, so "slower than ag-grid" says nothing + // about what this pull request changed. Flagging it would warn on every comment forever. + expect(findRegressions(report([libraryCase('libs-mount', 'slower')]))).toEqual([]); + }); + + it('reports only the measurement that regressed', () => { + const found = findRegressions( + report([regressionCase('workload', { mount: 'slower', scroll: 'unsure' })]), + ); + + expect(found).toHaveLength(1); + expect(found[0].measurement).toBe('mount'); + }); +}); + +describe('buildTachometerMarkdownReport', () => { + it('states a regression before the collapsed block, where it can be seen', () => { + // GitHub renders
    closed, so a regression inside it is invisible. + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'slower' })]), + ); + + expect(visiblePart(markdown)).toContain('workload'); + expect(visiblePart(markdown)).toContain('mount'); + expect(visiblePart(markdown)).toContain('+2.7% – +6.8%'); + }); + + it('marks the heading when something regressed', () => { + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'slower' })]), + ); + + expect(markdown.split('\n')[0]).toBe('## Tachometer ⚠️'); + }); + + it('leaves the heading unmarked when nothing regressed', () => { + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'unsure' })]), + ); + + expect(markdown.split('\n')[0]).toBe('## Tachometer'); + expect(visiblePart(markdown)).not.toContain('⚠️'); + }); + + it('summarizes a clean run in one line', () => { + const markdown = buildTachometerMarkdownReport( + report([ + regressionCase('workload', { mount: 'unsure' }), + regressionCase('scroll', { mount: 'unsure' }), + ]), + ); + + expect(visiblePart(markdown)).toContain('2 cases measured · 2 unchanged'); + }); + + it('counts a case that regressed in two measurements once', () => { + // The total beside it counts cases, so counting measurements here stopped the line adding up. + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'slower', 'cold-start': 'slower' })]), + ); + + expect(visiblePart(markdown)).toContain('1 case measured · 1 slower'); + }); + + it('does not count beating another library as having got faster', () => { + const markdown = buildTachometerMarkdownReport( + report([ + libraryCase('libs-mount', 'faster'), + regressionCase('workload', { mount: 'unsure' }), + ]), + ); + + expect(visiblePart(markdown)).toContain('2 cases measured · 2 unchanged'); + expect(visiblePart(markdown)).not.toContain('faster'); + }); + + it('counts a faster case separately', () => { + const markdown = buildTachometerMarkdownReport( + report([ + regressionCase('workload', { mount: 'faster' }), + regressionCase('scroll', { mount: 'unsure' }), + ]), + ); + + expect(visiblePart(markdown)).toContain('1 faster'); + }); + + it('reports a case that produced no result', () => { + const markdown = buildTachometerMarkdownReport( + report([{ name: 'broken', comparison: 'baseline', error: 'produced no benchmarks' }]), + ); + + expect(visiblePart(markdown)).toContain('broken'); + expect(visiblePart(markdown)).toContain('produced no benchmarks'); + }); + + it('puts every variant and measurement in the collapsed table', () => { + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'unsure', scroll: 'unsure' })]), + ); + + const details = markdown.split('
    ')[1]; + expect(details).toContain('mount'); + expect(details).toContain('scroll'); + expect(details).toContain('[current]'); + expect(details).toContain('[baseline]'); + }); + + it('states each row as the variant against the reference, not the reference against it', () => { + // The row sits under a "vs reference" heading next to the variant's own mean, so it has to + // read as a statement about that variant. Rendering the direction a regression is stated in + // makes a variant that is slower than the reference look faster than it. + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'slower' })]), + ); + + expect(tableRowFor(markdown, '[baseline]')).toContain('faster `-2.6% – -6.4%`'); + }); + + it('leaves the comparison out when the report has only one direction', () => { + // Older reports, written before both directions were captured. + const oneDirection = report([regressionCase('workload', { mount: 'slower' })]); + delete oneDirection.cases[0].measurements![0].comparisons[0].versusReference; + + const markdown = buildTachometerMarkdownReport(oneDirection); + + expect(tableRowFor(markdown, '[baseline]')).toContain('—'); + }); + + it('links to the full run when given a url', () => { + const markdown = buildTachometerMarkdownReport( + report([regressionCase('workload', { mount: 'unsure' })]), + { detailsUrl: 'https://example.test/run' }, + ); + + expect(markdown).toContain('[See the full run](https://example.test/run)'); + }); +}); diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts new file mode 100644 index 0000000000..cd3ff28bc5 --- /dev/null +++ b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts @@ -0,0 +1,179 @@ +import { formatMarkdownTable } from '@/utils/formatters'; +import type { + ConfidenceInterval, + TachometerReport, +} from '@mui/internal-benchmark/tachometerReport'; +import { + formatMean, + formatPercent, + formatSignedMs, + isSummarized, + shortNameOf, +} from '@mui/internal-benchmark/tachometerFormat'; +import type { SummarizedCase } from '@mui/internal-benchmark/tachometerFormat'; + +export const TACHOMETER_SECTION_TITLE = 'Tachometer'; + +interface Regression { + caseName: string; + measurement: string; + variant: string; + percentChange: ConfidenceInterval; + absoluteMs: ConfidenceInterval; +} + +interface BuildOptions { + /** Where the full results can be inspected. Omitted in tests. */ + detailsUrl?: string; +} + +/** + * The comparisons that say something about this pull request. + * + * Only a case measuring the working tree against the baseline does. One comparing pages built from + * the same tree — this library against another — is a standing measurement, and a difference there + * is the point rather than a regression. + */ +function baselineComparisons(entry: SummarizedCase) { + if (entry.comparison !== 'baseline') { + return []; + } + return entry.measurements.flatMap((measurement) => + measurement.comparisons.map((comparison) => ({ measurement, comparison })), + ); +} + +export function findRegressions(report: TachometerReport): Regression[] { + const regressions: Regression[] = []; + + for (const entry of report.cases) { + if (!isSummarized(entry)) { + continue; + } + for (const { measurement, comparison } of baselineComparisons(entry)) { + if (comparison.verdict === 'slower') { + regressions.push({ + caseName: entry.name, + measurement: measurement.name, + variant: shortNameOf(entry.name, comparison.variant), + percentChange: comparison.percentChange, + absoluteMs: comparison.absoluteMs, + }); + } + } + } + + return regressions; +} + +/** One flat table per case: every measurement, every variant, every number the report holds. */ +function renderCaseTable(entry: SummarizedCase): string { + const rows = entry.measurements.flatMap((measurement) => { + const byVariant = new Map( + measurement.comparisons.map((comparison) => [comparison.variant, comparison]), + ); + return measurement.variants.map((variant) => { + const againstReference = byVariant.get(variant.variant)?.versusReference; + let versus = '—'; + if (variant.variant === entry.reference) { + versus = '_reference_'; + } else if (againstReference) { + versus = `${againstReference.verdict} \`${formatPercent(againstReference.percentChange)}\``; + } + return { + measurement: measurement.name, + variant: shortNameOf(entry.name, variant.variant), + mean: formatMean(variant.meanMs), + versus, + samples: variant.samples, + }; + }); + }); + + return `**${entry.name}**\n\n${formatMarkdownTable( + [ + { field: 'measurement', header: 'Measurement', align: 'left' }, + { field: 'variant', header: 'Variant', align: 'left' }, + { field: 'mean', header: 'Mean (95% CI)', align: 'right' }, + { field: 'versus', header: 'vs reference', align: 'left' }, + { field: 'samples', header: 'Samples', align: 'right' }, + ], + rows, + )}`; +} + +/** + * Renders the tachometer section of the pull request comment. + * + * Regressions are stated above the collapsed block, and mark the heading, because a reader has to + * see them without expanding anything — GitHub renders `
    ` closed, so anything inside it is + * invisible until someone chooses to look. + */ +export function buildTachometerMarkdownReport( + report: TachometerReport, + options: BuildOptions = {}, +): string { + const regressions = findRegressions(report); + const summarized = report.cases.filter(isSummarized); + const failed = report.cases.filter((entry) => !isSummarized(entry)); + + const heading = `## ${TACHOMETER_SECTION_TITLE}${regressions.length > 0 ? ' ⚠️' : ''}`; + const lines = [heading, '']; + + if (regressions.length > 0) { + for (const regression of regressions) { + lines.push( + `⚠️ **${regression.caseName}** · ${regression.measurement} · slower than ` + + `${regression.variant} \`${formatPercent(regression.percentChange)}\` ` + + `(\`${formatSignedMs(regression.absoluteMs)}\`)`, + ); + } + lines.push(''); + } + + // Everything that did not regress collapses to a single line, so the regressions are what the eye + // lands on. `unsure` is the expected result for two equivalent builds, not a warning. Counted in + // cases, like the total beside them, and over the same baseline comparisons a regression is read + // from — a case that beats a competing library has not got faster. + const regressedCases = new Set(regressions.map((regression) => regression.caseName)); + const faster = summarized.filter( + (entry) => + !regressedCases.has(entry.name) && + baselineComparisons(entry).some(({ comparison }) => comparison.verdict === 'faster'), + ).length; + const unchanged = summarized.length - regressedCases.size - faster; + const summary = [ + `${summarized.length} case${summarized.length === 1 ? '' : 's'} measured`, + unchanged > 0 ? `${unchanged} unchanged` : null, + faster > 0 ? `${faster} faster` : null, + regressedCases.size > 0 ? `${regressedCases.size} slower` : null, + ] + .filter(Boolean) + .join(' · '); + lines.push(summary, ''); + + for (const entry of failed) { + lines.push(`❌ **${entry.name}**: ${entry.error ?? 'produced no result'}`); + } + if (failed.length > 0) { + lines.push(''); + } + + if (summarized.length > 0) { + lines.push('
    ', 'Full results', ''); + lines.push(summarized.map(renderCaseTable).join('\n\n')); + lines.push( + '', + '_Each cell is a 95% confidence interval for the mean; one sample is one page load._', + '_"unsure" means the interval still straddles zero — the expected result for two equivalent builds._', + '', + '
    ', + ); + } + + if (options.detailsUrl) { + lines.push('', `[See the full run](${options.detailsUrl})`); + } + + return lines.join('\n'); +} diff --git a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts index ba63a7b8c1..ea37f3799e 100644 --- a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts +++ b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts @@ -1,25 +1,44 @@ import type { SizeSnapshotWithMetadata } from '@/lib/bundleSize/types'; import type { BenchmarkReport, BenchmarkUpload } from '@/lib/benchmark/types'; -import type { BenchmarkRunReport } from '@mui/internal-benchmark/runReport'; +import type { BenchmarkRunUpload } from '@mui/internal-benchmark/runReport'; +import type { TachometerReport } from '@mui/internal-benchmark/tachometerReport'; import { migrateBenchmarkReport } from '@/lib/benchmark/migrateBenchmarkReport'; export interface CiReportTypes { - 'benchmark.json': BenchmarkUpload; + /** + * Version 1 from the Vitest reporter, or version 2 from `benchmark run`: a repository uploads one or + * the other while it moves over, so every reader narrows on `version`. + */ + 'benchmark.json': BenchmarkUpload | BenchmarkRunUpload; 'size-snapshot.json': SizeSnapshotWithMetadata; - 'benchmark-run.json': BenchmarkRunReport; + 'tachometer.json': TachometerReport; } export type CiReportName = keyof CiReportTypes; +/** Whether a `benchmark.json` is the version 2 report `benchmark run` uploads. */ +export function isBenchmarkRunUpload( + upload: BenchmarkUpload | BenchmarkRunUpload, +): upload is BenchmarkRunUpload { + return upload.version === 2; +} + /** * Legacy artifacts uploaded before the wrapper change store a flat * `Record`. Wrap them so downstream consumers * can read `.report` uniformly. The S3 path already committed us to a * specific sha/repo, so inject those into the returned wrapper — for legacy * artifacts this fills in missing metadata, for new-shape artifacts it - * simply reasserts what the body already contains. + * simply reasserts what the body already contains. A version 2 artifact needs none of it. */ -function normalizeBenchmarkArtifact(raw: unknown, repo: string, sha: string): BenchmarkUpload { +function normalizeBenchmarkArtifact( + raw: unknown, + repo: string, + sha: string, +): BenchmarkUpload | BenchmarkRunUpload { + if (raw && typeof raw === 'object' && 'version' in raw && raw.version === 2) { + return { ...(raw as BenchmarkRunUpload), commitSha: sha, repo }; + } const upload: BenchmarkUpload = raw && typeof raw === 'object' && 'report' in raw ? { ...(raw as BenchmarkUpload), commitSha: sha, repo } diff --git a/apps/code-infra-dashboard/src/views/BenchmarkDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkDetails.tsx index 861d9b2d18..ca222d105b 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkDetails.tsx @@ -9,13 +9,22 @@ import Typography from '@mui/material/Typography'; import CircularProgress from '@mui/material/CircularProgress'; import Box from '@mui/material/Box'; import Link from '@mui/material/Link'; -import { fetchCiReport } from '@/utils/fetchCiReport'; +import { fetchCiReport, isBenchmarkRunUpload } from '@/utils/fetchCiReport'; import Heading from '../components/Heading'; import ReportHeader from '../components/ReportHeader'; import ErrorDisplay from '../components/ErrorDisplay'; import { BenchmarkComparisonReportView } from '../components/BenchmarkComparisonReportView'; import { useBaseSha } from '../hooks/useBaseSha'; +/** + * A commit's version 1 report, or null. This view compares version 1 reports; a version 2 report is + * read by `BenchmarkRunDetails`. + */ +async function fetchVersion1Report(repo: string, sha: string) { + const upload = await fetchCiReport(repo, sha, 'benchmark.json'); + return upload && !isBenchmarkRunUpload(upload) ? upload : null; +} + interface InlinedBaseAlertProps { fetchedBaseSha: string; inlinedBaseSha: string; @@ -59,7 +68,7 @@ export default function BenchmarkDetails() { error, } = useQuery({ queryKey: ['benchmark-report', repo, sha], - queryFn: () => fetchCiReport(repo, sha!, 'benchmark.json'), + queryFn: () => fetchVersion1Report(repo, sha!), retry: 1, enabled: Boolean(sha), }); @@ -70,7 +79,7 @@ export default function BenchmarkDetails() { error: baseError, } = useQuery({ queryKey: ['benchmark-report', repo, baseSha], - queryFn: () => fetchCiReport(repo, baseSha!, 'benchmark.json'), + queryFn: () => fetchVersion1Report(repo, baseSha!), retry: 1, enabled: Boolean(baseSha), }); diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index 6af05f0089..8e161cec85 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -14,7 +14,7 @@ import TableContainer from '@mui/material/TableContainer'; import TableHead from '@mui/material/TableHead'; import TableRow from '@mui/material/TableRow'; import Typography from '@mui/material/Typography'; -import { fetchCiReport } from '@/utils/fetchCiReport'; +import { fetchCiReport, isBenchmarkRunUpload } from '@/utils/fetchCiReport'; import { analyzeRun, formatComparison, formatValue } from '@mui/internal-benchmark/runReport'; import type { BenchmarkAnalysis, @@ -108,7 +108,10 @@ export default function BenchmarkRunDetails() { error, } = useQuery({ queryKey: ['benchmark-run-report', repo, sha], - queryFn: () => fetchCiReport(repo, sha!, 'benchmark-run.json'), + queryFn: async () => { + const upload = await fetchCiReport(repo, sha!, 'benchmark.json'); + return upload && isBenchmarkRunUpload(upload) ? upload.report : null; + }, retry: 1, enabled: Boolean(sha), }); diff --git a/apps/code-infra-dashboard/src/views/RepositoryBenchmarks.tsx b/apps/code-infra-dashboard/src/views/RepositoryBenchmarks.tsx index 3d88764506..7d7d9cbe07 100644 --- a/apps/code-infra-dashboard/src/views/RepositoryBenchmarks.tsx +++ b/apps/code-infra-dashboard/src/views/RepositoryBenchmarks.tsx @@ -37,7 +37,7 @@ export default function RepositoryBenchmarks() { ) : ( - {repoConfig?.prComment?.benchmarkRun && } + )} diff --git a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx new file mode 100644 index 0000000000..d2dbdc7e06 --- /dev/null +++ b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx @@ -0,0 +1,349 @@ +'use client'; + +import * as React from 'react'; +import { useParams, useSearchParams } from 'next/navigation'; +import { useQuery } from '@tanstack/react-query'; +import Alert from '@mui/material/Alert'; +import Box from '@mui/material/Box'; +import CircularProgress from '@mui/material/CircularProgress'; +import Paper from '@mui/material/Paper'; +import Table from '@mui/material/Table'; +import TableBody from '@mui/material/TableBody'; +import TableCell from '@mui/material/TableCell'; +import TableContainer from '@mui/material/TableContainer'; +import TableHead from '@mui/material/TableHead'; +import TableRow from '@mui/material/TableRow'; +import Typography from '@mui/material/Typography'; +import { fetchCiReport } from '@/utils/fetchCiReport'; +import type { + ConfidenceInterval, + TachometerReport, + Verdict, +} from '@mui/internal-benchmark/tachometerReport'; +import { + formatMean, + formatPercent, + refLabel, + groupCasesByVariantSet, + isSummarized, + shortNameOf, +} from '@mui/internal-benchmark/tachometerFormat'; +import type { SummarizedCase } from '@mui/internal-benchmark/tachometerFormat'; +import Heading from '../components/Heading'; +import ReportHeader from '../components/ReportHeader'; +import ErrorDisplay from '../components/ErrorDisplay'; + +/** + * The tables the run prints in CI, as HTML: cases grouped by their variant set, a group of two + * variants read down its measurements, a group of more read across its variants. + */ + +/** Only a resolved difference is coloured; `unsure` is the expected result and stays neutral. */ +function verdictColor(verdict: Verdict): 'error' | 'success' | 'text.secondary' { + if (verdict === 'slower') { + return 'error'; + } + if (verdict === 'faster') { + return 'success'; + } + return 'text.secondary'; +} + +/** Auto-sampling stops per case, so the counts are normally equal; when they are not, say both. */ +function formatSamples(samples: number[]): string { + return [...new Set(samples)].join('/'); +} + +function VerdictText({ + verdict, + percentChange, +}: { + verdict: Verdict; + percentChange: ConfidenceInterval; +}) { + return ( + + {verdict} {formatPercent(percentChange)} + + ); +} + +function Muted({ children }: { children: React.ReactNode }) { + return ( + + {children} + + ); +} + +/** + * One case as a table of variants, for a comparison with more than two of them. + */ +function VariantTable({ entry, variants }: { entry: SummarizedCase; variants: string[] }) { + const [reference] = variants; + const samplesOf = (variantName: string) => + entry.measurements.flatMap((measurement) => + measurement.variants + .filter((variant) => shortNameOf(entry.name, variant.variant) === variantName) + .map((variant) => variant.samples), + ); + + return ( + + + + + {entry.name} + {entry.measurements.map((measurement) => ( + + {measurement.name} + vs {reference} + + ))} + Samples + + + + {variants.map((variantName) => { + return ( + + {variantName} + {entry.measurements.map((measurement) => { + const found = measurement.variants.find( + (candidate) => shortNameOf(entry.name, candidate.variant) === variantName, + ); + // The comparison holds the reference relative to this variant; a row about the + // variant needs the direction whose subject is the variant. + const againstReference = measurement.comparisons.find( + (candidate) => shortNameOf(entry.name, candidate.variant) === variantName, + )?.versusReference; + + let versus = —; + if (variantName === reference) { + versus = reference; + } else if (againstReference) { + versus = ( + + ); + } + + return ( + + + {found ? formatMean(found.meanMs) : —} + + {versus} + + ); + })} + {formatSamples(samplesOf(variantName))} + + ); + })} + +
    +     + ); +} + +/** + * A group of cases sharing a variant set, one row per case and measurement. + */ +function CaseGroupTable({ cases, variants }: { cases: SummarizedCase[]; variants: string[] }) { + const [reference, ...others] = variants; + + return ( + + + + + Case + Measurement + {variants.map((variantName) => ( + + {variantName} + + ))} + {others.map((variantName) => ( + Δ vs {variantName} + ))} + Samples + + + + {cases.flatMap((entry, caseIndex) => + entry.measurements.map((measurement, measurementIndex) => { + const byVariant = new Map( + measurement.variants.map((variant) => [ + shortNameOf(entry.name, variant.variant), + variant, + ]), + ); + const byComparison = new Map( + measurement.comparisons.map((comparison) => [ + shortNameOf(entry.name, comparison.variant), + comparison, + ]), + ); + // A rule where the next case starts, since its name only appears on its first row. + const startsCase = measurementIndex === 0 && caseIndex > 0; + + return ( + + {measurementIndex === 0 ? entry.name : ''} + {measurement.name} + {variants.map((variantName) => { + const found = byVariant.get(variantName); + return ( + + {found ? formatMean(found.meanMs) : —} + + ); + })} + {others.map((variantName) => { + const comparison = byComparison.get(variantName); + return ( + + {comparison ? ( + + ) : ( + — + )} + + ); + })} + + {formatSamples(measurement.variants.map((variant) => variant.samples))} + + + ); + }), + )} + +
    + + Δ is tachometer's confidence interval on the difference, {reference} relative to the + other variant{others.length > 1 ? 's' : ''} — negative is faster. + +     + ); +} + +export default function TachometerDetails() { + const params = useParams<{ owner: string; repo: string }>(); + const searchParams = useSearchParams(); + + if (!params.owner || !params.repo) { + throw new Error('Missing required path parameters'); + } + + const repo = `${params.owner}/${params.repo}`; + const sha = searchParams.get('sha'); + const prNumber = searchParams.get('prNumber'); + + const { + data: report, + isLoading, + error, + } = useQuery({ + queryKey: ['tachometer-report', repo, sha], + queryFn: () => fetchCiReport(repo, sha!, 'tachometer.json'), + retry: 1, + enabled: Boolean(sha), + }); + + if (!sha) { + return ( + + Tachometer Details + + Missing required "sha" query parameter. + + + ); + } + + const summarized = (report?.cases ?? []).filter(isSummarized); + const failed = (report?.cases ?? []).filter((entry) => !isSummarized(entry)); + const groups = groupCasesByVariantSet(summarized); + + // A tachometer report carries its own comparison, so the baseline is one of its own refs rather + // than a separately fetched report. + const baselineRef = report?.refs.find((ref) => ref.kind !== 'worktree'); + + return ( + + Tachometer Details + + + + + {isLoading && ( + + + Loading tachometer report… + + )} + + {error && } + + {!isLoading && !error && !report && ( + No tachometer report found for this commit. + )} + + {failed.map((entry) => ( + + {entry.name}: {entry.error ?? 'produced no result'} + + ))} + + {groups.map((group) => + group.variants.length > 2 ? ( + group.cases.map((entry) => ( + + )) + ) : ( + + ), + )} + + {report && ( + + + Each cell is a 95% confidence interval for the mean, in milliseconds; one sample is + one page load. "unsure" means the interval still straddles zero: the + difference did not resolve within the case's sampling budget. + + + Builds: {report.refs.map((ref) => `${ref.id} = ${refLabel(ref)}`).join(' · ')} + + + head: {report.head.sha.slice(0, 9)} ({report.head.branch || '?'}) · measured on the + production bundle, installed from a packed tarball. + + + )} + + + ); +} diff --git a/package.json b/package.json index 30d813cd59..bb9e4d45a7 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "docs:lib": "pnpm -F \"./packages/docs-infra\" run build", "test:benchmark": "pnpm -F ./test/performance benchmark", "test:benchmark-run": "pnpm -F ./test/benchmark benchmark:run", + "test:tacho": "pnpm -F ./test/tachometer test:tacho", "org-versions": "tsx scripts/orgVersions.mts" }, "dependencies": { @@ -63,6 +64,7 @@ "@vitest/browser-playwright": "4.1.11", "@vitest/coverage-v8": "4.1.11", "baseline-browser-mapping": "2.11.21", + "chromedriver": "^151.0.5", "eslint-plugin-n": "18.3.0", "pkg-pr-new": "0.0.88", "playwright": "1.63.0", diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index e02a3987a2..4153a1d672 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -1,14 +1,18 @@ # Benchmark -Two ways to run the same `benchmark()` files, for two different questions. +Three ways to measure, while repositories move from the first and last to the second. **Under Vitest** — a React component benchmarking tool built on Vitest and Playwright, using React's -profiling build to capture render durations against your source, tracked commit by commit. Everything -up to [`benchmark run`](#benchmark-run) covers it. +profiling build to capture render durations against your source. Everything up to +[`benchmark run`](#benchmark-run) covers it. It uploads version 1 of the `benchmark` report. -**`benchmark run`** — below: the working tree against a baseline commit, both **built** and installed -the way a consumer gets them, sampled alternately in one browser run and compared on paired -differences, so a small change resolves instead of drowning in run-to-run noise. +**`benchmark run`** — the same `benchmark()` files, the working tree against a baseline commit, both +**built** and installed the way a consumer gets them, sampled alternately in one browser run and +compared on paired differences. It uploads version 2 of the `benchmark` report. A repository uploads +one version or the other; the dashboard reads each by its version. + +**[Tachometer](#tachometer)** — Google's [tachometer](https://github.com/google/tachometer) driving +`tachometer.json` pages against your built package, through `benchmark tacho run`. ## Features @@ -396,8 +400,10 @@ by default) are measured; the iteration counts in `benchmark()`'s options only a ### The report The run writes `.benchmark/results/report.json`, prints it as tables, and with `--upload` sends it to -the dashboard, which renders the pull request comment and the repository's history from it. The -report holds **raw samples**, round-aligned across variants, plus what each metric is — its kind, +the dashboard as version 2 of the `benchmark` report — the commit's `benchmark.json`, which the Vitest +reporter writes as version 1, so a repository switches from one to the other rather than running +both. The dashboard renders the pull request comment's Performance section and the repository's +history from it. The report holds **raw samples**, round-aligned across variants, plus what each metric is — its kind, format and alarm — the builds, and the environment. Conclusions are computed from it, the same way everywhere, by `analyzeRun` from `@mui/internal-benchmark/runReport`: a 95% confidence interval on the paired difference per metric, a change (`better`, `worse`, `unsure`) that respects the metric's @@ -405,7 +411,7 @@ direction, and a severity from its alarm. The harness's own metrics are `render` duration) and `render:count`, which alarm on any resolved change for the worse, and `bench:paint`, which is informational; custom metrics bring their own alarm. -### Baselines +### Choosing the baseline `--baseline` names the build the working tree is compared against: a revision — a SHA, a tag, a branch, `HEAD~1` — on its own or behind `git:`. It is resolved to an immutable SHA before anything is @@ -419,7 +425,7 @@ change — which commit moved a number reads off the chart rather than out of a `github:/#` and `preview:` are reserved for a baseline that is not a commit in this repository. Neither is implemented, and both are rejected by name rather than handed to git. -### Builds and caching +### Building and installing each side Both sides are built the same way, from packed tarballs installed into an isolated tree, so neither resolves the library through a workspace link. The benchmark files themselves stay on the current @@ -446,6 +452,91 @@ resolution there, leaving the repository untouched, at the cost of a second inst timings. `--no-install` skips a ref's install. `benchmark run --help` lists the rest, and `benchmark report` prints the tables again from a saved JSON report. +## Tachometer + +`benchmark tacho run` measures a harness package in a real browser, building the workspace once per +commit under comparison and serving the pages from those builds. Run it from the harness. + +A harness is a package holding a `src/` tree of **case folders**, each with a page and a plain, +unmodified [tachometer config](https://github.com/google/tachometer#config-file). The configs are the +source of truth for which pages exist: the runner reads the selected ones, works out which pages they +reference, builds one variant per distinct commit, and rewrites each url to the built page. Filtering +to one case therefore narrows the build too. + +Sampling — `sampleSize`, `autoSampleConditions`, `timeout` — is set per case in its own config, +because tachometer rejects those as flags once a config file is in play. A case samples until its +difference resolves or the timeout is hit; two builds of equal speed cannot be separated, so such a +case reports `unsure`, which is the expected "nothing changed" outcome rather than a failure. + +### What a case compares + +A case is one of two things, and its `expand` decides which. + +**No `expand`** is the ordinary regression shape. It is expanded for you into `[current]` — the +working tree — against `[baseline]`, both loading the same page: + +```jsonc +{ "benchmarks": [{ "name": "init", "url": "./index.html" }] } +``` + +**Its own `expand`** compares the pages it names with each other, every one of them built from the +working tree. That is how one implementation is measured against another: + +```jsonc +"expand": [ + { "name": "[mosaic]", "url": "./index.html" }, + { "name": "[tanstack]", "url": "./tanstack.html" } +] +``` + +Such a case reports no regression, by construction: there is no baseline in it to regress against. +Every run says which shape each case has before it starts, so this is visible rather than inferred +from an empty result. To sweep a parameter and still measure it against the baseline, give each +value its own case folder pointing at the shared page — `expand` would turn the case into the other +shape. + +### Baselines + +`--baseline` names the build every `[baseline]` variant loads: a revision — a SHA, a tag, a branch, +`HEAD~1` — on its own or behind `git:`. It is resolved to an immutable SHA before anything is +cached, and defaults to `HEAD~1`, which needs no policy to decide. + +Which commit a branch should actually be compared against is `code-infra baseline`'s question — it +answers the same way for every job that compares a branch to its base — so pass it in: + +```bash +benchmark tacho run --baseline "$(code-infra baseline)" +``` + +`github:/#` and `preview:` are reserved for a baseline that is not a commit in +this repository. Neither is implemented, and both are rejected by name rather than handed to git. + +### Builds and caching + +Both sides are built the same way, from packed tarballs installed into an isolated tree, so neither +resolves the library through a workspace link. The benchmark pages themselves stay on the current +branch and only the built library changes between refs — so a commit whose public API differs from +today's pages will fail that ref's build, with the error surfaced. + +Packed builds are cached by commit SHA under `.tachometer/packed/`, so repeating a comparison against +the same commit skips the rebuild; the working tree is never cached. In CI, cache that directory +keyed on the baseline SHA, and check out with full history so a fork point can be resolved. A run +never modifies the checkout it was started from. + +By default a run resolves **in place**: it pins the packed build in the repository's own +`pnpm-workspace.yaml`, installs it there, and resolution is then ordinary. The repository is put +back, and reinstalled, when the run ends — including when it fails, and at the start of the next run +if one was killed outright. Two things follow from it. A tracked file names tarballs under +`.tachometer/` until the run ends, so do not commit while one is in flight. And the pins are global +to the workspace for that time, because pnpm scopes an override by parent package name and a harness +has none — so nothing else should build against the same checkout meanwhile. + +`--resolve-mode isolated` avoids both: it installs each ref into a directory of its own and points +resolution there, leaving the repository untouched, at the cost of a second install tree per ref. + +`--no-install` skips a ref's install. `benchmark tacho run --help` lists the rest, and +`benchmark tacho report` prints the table again from a saved JSON report. + ## API - `benchmark` — define a benchmark test case diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index bbe06f6b50..a673c49e1d 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -17,6 +17,9 @@ "./page": "./src/page/page.tsx", "./vitePlugin": "./src/runner/plugin.ts", "./runReport": "./src/runReport/index.ts", + "./tachometerPlugin": "./src/tachometer/plugin.ts", + "./tachometerReport": "./src/tachometer/renderReport.ts", + "./tachometerFormat": "./src/tachometer/format.ts", "./cli": "./src/cli/index.ts" }, "scripts": { diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index 05284741b3..d8261335a0 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -10,6 +10,8 @@ import { hideBin } from 'yargs/helpers'; import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; import report from './report'; import run from './run'; +import tachoReport from './tachoReport'; +import tachoRun from './tachoRun'; let globalArgv: { verbose?: boolean } = {}; @@ -28,6 +30,14 @@ await yargs(hideBin(process.argv)) }, true) .command(run) .command(report) + .command({ + command: 'tacho ', + describe: 'Benchmark a tachometer harness, and read the reports it writes.', + builder: (group) => + group.command(tachoRun).command(tachoReport).demandCommand(1, 'Specify a tacho subcommand.'), + // The group itself does nothing; `demandCommand` above means a subcommand always runs. + handler: () => {}, + }) .fail((msg, err, yargsInstance) => { if (msg) { yargsInstance.showHelp(); diff --git a/packages/benchmark/src/cli/tachoReport.ts b/packages/benchmark/src/cli/tachoReport.ts new file mode 100644 index 0000000000..49b59569ec --- /dev/null +++ b/packages/benchmark/src/cli/tachoReport.ts @@ -0,0 +1,51 @@ +import * as path from 'node:path'; +import { readFile } from 'node:fs/promises'; +import { z } from 'zod/v4'; +import type { CommandModule } from 'yargs'; +import { OUTPUT_DIR } from '../tachometer/outputDir'; + +interface Args { + /** Path to a JSON report written by `tacho run`. */ + file: string; +} + +const DEFAULT_REPORT = path.join(OUTPUT_DIR, 'results', 'report.json'); + +const command: CommandModule<{}, Args> = { + command: 'report [file]', + describe: + 'Print the table for a JSON report written by `tacho run`. Reads the last run of the harness in the current directory when no path is given.', + builder: (yargs) => { + return yargs + .positional('file', { + type: 'string', + default: DEFAULT_REPORT, + describe: 'Path to the JSON report', + }) + .epilogue( + 'The same table `tacho run` prints when it finishes, so a report kept from an earlier run — or downloaded from CI — can be read without sampling again.', + ); + }, + handler: async (argv) => { + const reportPath = path.resolve(argv.file); + const raw = await readFile(reportPath, 'utf8').catch(() => { + throw new Error( + `No report at ${reportPath}. Pass a path, or run \`tacho run\` from a harness first.`, + ); + }); + + const { tachometerReportSchema } = await import('../tachometer/ciReport'); + const parsed = tachometerReportSchema.safeParse(JSON.parse(raw)); + if (!parsed.success) { + throw new Error( + `${reportPath} is not a tachometer report this version can read.\n` + + `${z.prettifyError(parsed.error)}`, + ); + } + + const { renderTachometerReport } = await import('../tachometer/renderReport'); + renderTachometerReport(parsed.data); + }, +}; + +export default command; diff --git a/packages/benchmark/src/cli/tachoRun.ts b/packages/benchmark/src/cli/tachoRun.ts new file mode 100644 index 0000000000..a6c6343e0d --- /dev/null +++ b/packages/benchmark/src/cli/tachoRun.ts @@ -0,0 +1,70 @@ +import type { CommandModule } from 'yargs'; +import type { RunTachometerOptions } from '../tachometer/runTachometer'; + +/** Everything `runTachometer` takes except the directory, which is where the command was run. */ +type Args = Omit; + +const command: CommandModule<{}, Args> = { + command: 'run [filters...]', + describe: + 'Benchmark a tachometer harness across builds of this workspace and write a JSON report. Run from the harness package.', + builder: (yargs) => { + return yargs + .positional('filters', { + type: 'string', + array: true, + describe: 'Only run cases whose path under src contains this substring', + }) + .option('baseline', { + type: 'string', + describe: + 'The build to compare against: a revision, on its own or as git:. Defaults to HEAD~1; `code-infra baseline` resolves a fork point to pass here', + }) + .option('resolve-mode', { + type: 'string', + choices: ['isolated', 'in-place'] as const, + default: 'in-place' as const, + describe: + "How pages find the library: 'isolated' installs each ref beside the repository, 'in-place' pins it in the repository's own pnpm-workspace.yaml for the length of the run", + }) + .option('build-cmd', { + type: 'string', + default: 'pnpm release:build', + describe: + "Command that builds the publishable workspace packages, in each ref's checkout and in the working tree", + }) + .option('install', { + type: 'boolean', + default: true, + describe: "Install dependencies inside a ref's checkout. Use --no-install to skip", + }) + .option('upload', { + type: 'boolean', + default: process.env.TACHO_UPLOAD === 'true', + describe: + 'Upload the report and refresh the PR comment. Defaults to TACHO_UPLOAD=true, so CI can switch it on by environment. Requires CIRCLE_OIDC_TOKEN_V2', + }) + .option('out', { + type: 'string', + describe: 'Write the combined JSON report here. Default: .tachometer/results/report.json', + }) + .epilogue( + 'Sampling (sampleSize, autoSampleConditions, timeout) is configured per case in its own tachometer.json — tachometer rejects those as CLI flags when a config file is used.', + ); + }, + handler: async (argv) => { + const { runTachometer } = await import('../tachometer/runTachometer'); + await runTachometer({ + harnessDir: process.cwd(), + filters: argv.filters ?? [], + baseline: argv.baseline, + resolveMode: argv.resolveMode, + buildCmd: argv.buildCmd, + install: argv.install, + out: argv.out, + upload: argv.upload, + }); + }, +}; + +export default command; diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index 061162fae5..8fbb092d16 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -22,7 +22,7 @@ function drifting(shift: number) { function reportOf(benchmarks: RunBenchmark[], metrics: BenchmarkRunReport['metrics'] = {}) { const report: BenchmarkRunReport = { - version: 1, + version: 2, generatedAt: '2026-01-01T00:00:00.000Z', head: { sha: 'abc' }, environment: { browser: 'Chromium', platform: 'linux', arch: 'x64', launchArgs: [] }, diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts index d5074e5fb5..9c331e67c5 100644 --- a/packages/benchmark/src/runReport/schema.ts +++ b/packages/benchmark/src/runReport/schema.ts @@ -1,7 +1,9 @@ import { z } from 'zod/v4'; /** - * The report `benchmark run` produces, uploads and the dashboard reads. + * The report `benchmark run` produces, uploads and the dashboard reads: version 2 of the `benchmark` + * report, which the Vitest reporter uploads as version 1. A repository uploads one or the other, and + * the dashboard reads each by its version. * * It holds raw samples rather than conclusions: every variant of a benchmark is sampled once per * round, and index `i` of every sample array is round `i`, so any statistic — paired differences @@ -56,7 +58,7 @@ const benchmarkSchema = z.object({ }); export const benchmarkRunReportSchema = z.object({ - version: z.literal(1), + version: z.literal(2), generatedAt: z.string(), head: z.object({ sha: z.string(), branch: z.string().optional() }), environment: z.object({ @@ -74,6 +76,18 @@ export const benchmarkRunReportSchema = z.object({ }); export type BenchmarkRunReport = z.infer; + +/** The version 2 `benchmark.json` as the dashboard stores it: the upload envelope around the report. */ +export interface BenchmarkRunUpload { + version: 2; + timestamp: number; + commitSha: string; + repo: string; + reportType: 'benchmark'; + prNumber?: number; + branch: string; + report: BenchmarkRunReport; +} export type RunMetricDefinition = z.infer; export type RunBenchmark = z.infer; export type RunBuild = z.infer; diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index 4d3282c6c9..13349e8484 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -169,7 +169,7 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise { + it('reads the major from Chrome for Testing output', () => { + expect(majorVersionOf('Google Chrome for Testing 151.0.7922.34')).toBe(151); + }); + + it('reads the major from plain Chrome output', () => { + expect(majorVersionOf('Google Chrome 152.0.6367.60')).toBe(152); + }); + + it('reads the major from a bare package version', () => { + // chromedriver's package.json version, which is what the driver side is read from. + expect(majorVersionOf('151.0.5')).toBe(151); + }); + + it('returns undefined when there is no version to read', () => { + expect(majorVersionOf('')).toBeUndefined(); + expect(majorVersionOf('no version here')).toBeUndefined(); + }); + + it('ignores a leading number that is not part of a version', () => { + // Requires three dot-separated components, so a stray count cannot be mistaken for a version. + expect(majorVersionOf('2 browsers found: Chrome 151.0.7922.34')).toBe(151); + }); +}); + +describe('withBrowserDefaults', () => { + const BINARY = '/path/to/Chrome for Testing'; + + it('fills in the binary for a case that does not pin one', () => { + expect(withBrowserDefaults({ name: 'chrome', headless: true }, BINARY, false)).toEqual({ + name: 'chrome', + headless: true, + binary: BINARY, + }); + }); + + it('leaves a case that pins its own binary alone', () => { + const pinned = withBrowserDefaults({ binary: '/opt/chrome' }, BINARY, false); + + expect(pinned.binary).toBe('/opt/chrome'); + }); + + it('keeps the sandbox when the run is not root', () => { + // The sandbox is a real protection for the pages a benchmark loads; only root cannot have it. + expect(withBrowserDefaults({ name: 'chrome' }, BINARY, false)).not.toHaveProperty( + 'addArguments', + ); + }); + + it('disables the sandbox when the run is root', () => { + expect(withBrowserDefaults({ name: 'chrome' }, BINARY, true).addArguments).toEqual([ + '--no-sandbox', + ]); + }); + + it("keeps a case's own arguments when it disables the sandbox", () => { + // Cases pass flags that hold their measurements still; dropping them would move the numbers. + const merged = withBrowserDefaults( + { addArguments: ['--disable-renderer-backgrounding'] }, + BINARY, + true, + ); + + expect(merged.addArguments).toEqual(['--disable-renderer-backgrounding', '--no-sandbox']); + }); + + it('does not repeat an argument a case already passes', () => { + const merged = withBrowserDefaults({ addArguments: ['--no-sandbox'] }, BINARY, true); + + expect(merged.addArguments).toEqual(['--no-sandbox']); + }); + + it('names the browser for a case that declares none', () => { + // Tachometer defaults an absent `browser` to chrome, but validates the config against a schema + // where every browser variant requires `name` — so injecting the binary alone is rejected + // before a single sample is taken. + expect(withBrowserDefaults(undefined, BINARY, true)).toEqual({ + name: 'chrome', + binary: BINARY, + addArguments: ['--no-sandbox'], + }); + }); + + it('expands the string shorthand instead of spreading it into characters', () => { + expect(withBrowserDefaults('chrome-headless', BINARY, false)).toEqual({ + name: 'chrome', + headless: true, + binary: BINARY, + }); + }); + + it('keeps the remote url out of the browser name', () => { + expect(withBrowserDefaults('firefox@http://localhost:4444', BINARY, false)).toEqual({ + name: 'firefox', + headless: false, + remoteUrl: 'http://localhost:4444', + binary: BINARY, + }); + }); + + it('names the browser for a case whose object leaves it out', () => { + expect(withBrowserDefaults({ headless: true }, BINARY, false).name).toBe('chrome'); + }); +}); + +describe('assertDriverMatchesBrowser', () => { + /** + * A harness with tachometer installed, and optionally a chromedriver where tachometer can reach + * it — one directory up from its own, the way a workspace root is reachable in a real install. + * + * Real directories and real module resolution rather than a stand-in: what is being checked is + * which chromedriver node finds from tachometer, and faking that would assume the answer. + */ + async function makeHarness(driverVersion?: string): Promise { + const harnessDir = await makeTempDir(); + const modules = path.join(harnessDir, 'node_modules'); + + await writeFile(path.join(harnessDir, 'package.json'), JSON.stringify({ name: 'harness' })); + await mkdir(path.join(modules, 'tachometer'), { recursive: true }); + await writeFile( + path.join(modules, 'tachometer', 'package.json'), + JSON.stringify({ name: 'tachometer', version: '0.7.2' }), + ); + + if (driverVersion) { + await mkdir(path.join(modules, 'chromedriver'), { recursive: true }); + await writeFile( + path.join(modules, 'chromedriver', 'package.json'), + JSON.stringify({ name: 'chromedriver', version: driverVersion }), + ); + } + + return harnessDir; + } + + /** An executable standing in for the browser, reporting `version` to `--version`. */ + async function makeBrowser(version: string): Promise { + const dir = await makeTempDir(); + const binary = path.join(dir, 'chrome'); + await writeFile(binary, `#!/bin/sh\necho "Google Chrome for Testing ${version}"\n`); + await chmod(binary, 0o755); + return binary; + } + + it('passes when the driver and the browser share a major', async () => { + const harnessDir = await makeHarness('151.0.5'); + const binary = await makeBrowser('151.0.7922.34'); + + expect(() => assertDriverMatchesBrowser(harnessDir, binary)).not.toThrow(); + }); + + it('fails when the driver is on a different major', async () => { + const harnessDir = await makeHarness('150.0.3'); + const binary = await makeBrowser('151.0.7922.34'); + + expect(() => assertDriverMatchesBrowser(harnessDir, binary)).toThrow(/cannot drive Chrome/); + }); + + it('fails when no driver is reachable, naming the major to pin', async () => { + // Tachometer would install the newest chromedriver here, which does not drive a pinned Chrome. + const harnessDir = await makeHarness(); + const binary = await makeBrowser('151.0.7922.34'); + + expect(() => assertDriverMatchesBrowser(harnessDir, binary)).toThrow(/"chromedriver": "\^151"/); + }); +}); diff --git a/packages/benchmark/src/tachometer/browser.ts b/packages/benchmark/src/tachometer/browser.ts new file mode 100644 index 0000000000..95e539eebc --- /dev/null +++ b/packages/benchmark/src/tachometer/browser.ts @@ -0,0 +1,160 @@ +import * as path from 'node:path'; +import { createRequire } from 'node:module'; +import { execFileSync } from 'node:child_process'; +import chalk from 'chalk'; +import type * as PlaywrightTest from '@playwright/test'; +import { pathExists } from '../utils/path'; + +/** + * Picking the browser to benchmark in, and checking it can actually be driven. + * + * Two separate programs are involved. The browser itself comes from Playwright, which downloads a + * fixed Chrome build. Tachometer needs `chromedriver` to drive that browser, and supplies it itself: + * it uses whichever one it can find next to itself, and downloads the newest if it finds none. + * + * A chromedriver only drives the Chrome version it was built for. Since Playwright's Chrome is + * deliberately a little behind the latest, letting tachometer fetch the newest driver usually gets a + * pair that cannot work together. + * + * So the consumer supplies it instead: a `chromedriver` devDependency at the workspace root, which + * is the only place tachometer resolves one from. Declared beside `@playwright/test`, so one place + * governs both versions and Renovate moves them together. All this module does is compare the + * version tachometer would resolve against the browser's own — which version that is stays the + * consumer's to choose. + */ + +/** + * Resolves the browser every case runs on: Playwright's pinned Chrome for Testing, rather than + * whatever Chrome the machine has auto-updated to. + * + * The path is machine-specific, so callers inject it into each config instead of committing it. + */ +export async function resolveBrowserBinary(harnessDir: string): Promise { + let playwright: typeof PlaywrightTest; + try { + playwright = await import('@playwright/test'); + } catch { + // An optional peer, so it installs cleanly when absent and only fails here. + throw new Error( + `"@playwright/test" is not installed. Add it as a devDependency of ${harnessDir}.`, + ); + } + + const binary = playwright.chromium.executablePath(); + if (!binary || !(await pathExists(binary))) { + throw new Error( + `Playwright's Chromium is not installed at "${binary}". Run \`pnpm exec playwright install chromium\`.`, + ); + } + return binary; +} + +/** + * The major version in output like `Google Chrome for Testing 151.0.7922.34`, or undefined when + * there is no version in it. + * + * At least three components are required, so a stray count in the same line is not read as one. + */ +export function majorVersionOf(versionOutput: string): number | undefined { + const match = /(\d+)\.\d+\.\d+/.exec(versionOutput); + return match ? Number(match[1]) : undefined; +} + +/** + * Throws when the chromedriver tachometer will use cannot drive `binary`. + */ +export function assertDriverMatchesBrowser(harnessDir: string, binary: string): void { + const browserMajor = majorVersionOf( + execFileSync(binary, ['--version'], { encoding: 'utf8' }).trim(), + ); + + // The same lookup tachometer does before launching: it uses whichever chromedriver resolves from + // its own package root, and installs the newest one when none does. + let driverVersion: string | undefined; + try { + const fromHarness = createRequire(path.join(harnessDir, 'package.json')); + const fromTachometer = createRequire(fromHarness.resolve('tachometer/package.json')); + driverVersion = fromTachometer('chromedriver/package.json').version; + } catch { + throw new Error( + `No chromedriver is reachable from tachometer, so it will install the newest one — which ` + + `will not drive Chrome for Testing ${browserMajor}, since Playwright pins a Chrome behind ` + + `the current release. Add "chromedriver": "^${browserMajor}" to the workspace root, where ` + + `tachometer can resolve it.`, + ); + } + + const driverMajor = majorVersionOf(driverVersion ?? ''); + if (browserMajor === undefined || driverMajor === undefined) { + console.warn( + chalk.yellow('Could not read the browser or chromedriver version; skipping the check.'), + ); + return; + } + if (browserMajor !== driverMajor) { + throw new Error( + `chromedriver ${driverVersion} cannot drive Chrome for Testing ${browserMajor}. ` + + `Update the "chromedriver" devDependency to ${browserMajor}.x, or align ` + + `"@playwright/test" with it — bump the two together.`, + ); + } +} + +/** + * A benchmark's `browser` config. Only the two fields a run touches are named; everything else + * tachometer accepts there (`name`, `headless`, `windowSize`, …) passes through untouched. + */ +export type BrowserConfig = { + binary?: string; + addArguments?: string[]; +} & Record; + +/** Chrome cannot enter its sandbox as root, and refusing to start is all it does about it. */ +const NO_SANDBOX = '--no-sandbox'; + +/** Tachometer's default when a benchmark names no browser. */ +const DEFAULT_BROWSER = 'chrome'; + +/** The suffix tachometer's browser shorthand uses to ask for a headless run. */ +const HEADLESS_SUFFIX = '-headless'; + +/** + * Whatever a case put in `browser` as the object form tachometer's schema requires: `name` is + * mandatory there and unknown keys are rejected, so the string shorthand + * `"[-headless][@]"` and an absent `browser` are both expanded here. + */ +function asBrowserObject(browser: string | BrowserConfig | undefined): BrowserConfig { + if (browser === undefined) { + return { name: DEFAULT_BROWSER }; + } + if (typeof browser !== 'string') { + // An object of its own may still leave out `name`, which the schema requires. + return { name: DEFAULT_BROWSER, ...browser }; + } + const at = browser.indexOf('@'); + const named = at === -1 ? browser : browser.slice(0, at); + const headless = named.endsWith(HEADLESS_SUFFIX); + const base: BrowserConfig = { + name: headless ? named.slice(0, -HEADLESS_SUFFIX.length) : named, + headless, + }; + return at === -1 ? base : { ...base, remoteUrl: browser.slice(at + 1) }; +} + +/** + * Merges what the run decides into a case's own `browser` config: the binary, which is + * machine-specific, and the sandbox flag, which depends on the user the run happens under. + */ +export function withBrowserDefaults( + browser: string | BrowserConfig | undefined, + binary: string, + asRoot: boolean, +): BrowserConfig { + const base = asBrowserObject(browser); + const merged: BrowserConfig = { ...base, binary: base.binary ?? binary }; + const addArguments = merged.addArguments ?? []; + if (asRoot && !addArguments.includes(NO_SANDBOX)) { + merged.addArguments = [...addArguments, NO_SANDBOX]; + } + return merged; +} diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts new file mode 100644 index 0000000000..4b910cae47 --- /dev/null +++ b/packages/benchmark/src/tachometer/buildPages.ts @@ -0,0 +1,322 @@ +/* eslint-disable no-console */ + +import * as path from 'node:path'; +import { createRequire } from 'node:module'; +import { readFile, mkdir, rm, writeFile } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; +import chalk from 'chalk'; +import type * as Vite from 'vite'; +import { parse, stringify } from 'yaml'; +// Self-referencing rather than `../../package.json`, which resolves only in the repository — see +// the note in ../cli/index.ts. +import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; +import { run } from '../utils/exec'; +import { tarballFor } from '../utils/packWorkspace'; +import type { PackedPackage } from '../utils/packWorkspace'; +import { installedVersions, readPackageJson, writePackageJson } from '../utils/pnpm'; +import { refLabel } from './format'; +import type { ResolvedRef } from './refs'; +import { pathExists } from '../utils/path'; + +/** + * How a ref's pages find the library under test. + * + * `isolated` installs that ref's packed build into a directory of its own and points resolution + * there, leaving the repository untouched. `in-place` pins the packed build in the repository's own + * `pnpm-workspace.yaml` and installs it, so resolution is ordinary — at the cost of mutating a + * tracked file for as long as the run lasts. + */ +export type ResolveMode = 'isolated' | 'in-place'; + +/** + * Dependencies the run needs but a page never imports, so a ref's tree does without them — + * whichever section of the harness's manifest they sit in, since neither reason below turns on that. + * + * Every one is resolved from the harness rather than from here: the browser and its driver by the + * runner, tachometer by the command that samples, and this package by the harness's own vite config + * — which is loaded from the harness, since the build happens in place. Every one is also expensive + * to install: tachometer pulls a chromedriver, `@playwright/test` a browser, and this package a + * dependency tree many times the size of a harness. + */ +const RUNNER_ONLY_DEPS = ['tachometer', 'chromedriver', '@playwright/test', pkgJson.name]; + +/** + * A dependency map for a ref's tree: every `workspace:` entry points at that ref's packed tarball, + * and everything else is pinned to the version the harness resolved rather than the range it + * declares. + * + * The tree installs without the repository's lockfile, so a range would float to whatever is newest + * at run time — a different library than the repository builds against, measured as if it were + * ours. + */ +function rewriteWorkspaceDeps( + installed: Map, + deps: Partial> | undefined, + packages: PackedPackage[], + omit: string[] = [], +): Record { + const entries = Object.entries(deps ?? {}).filter( + ([name, version]) => version !== undefined && !omit.includes(name), + ) as Array<[string, string]>; + + return Object.fromEntries( + entries.map(([name, version]) => + version.startsWith('workspace:') + ? [name, `file:${tarballFor(packages, name)}`] + : [name, installed.get(name) ?? version], + ), + ); +} + +/** + * Reads the repository's `overrides`, so the ref's tree resolves the same dependency graph the + * repository does. + * + * The tree is its own workspace root and inherits nothing. Without this, a repository that pins a + * transitive dependency through an override would get a *different* graph here than in its real + * install — quietly changing what is being benchmarked — and any policy the override exists to + * satisfy (a blocked resolution, a security pin) would fail here too. + */ +async function readOverrides(repoRoot: string): Promise> { + let raw: string; + try { + raw = await readFile(path.join(repoRoot, 'pnpm-workspace.yaml'), 'utf8'); + } catch (error) { + // A repository without the file simply has no overrides. Anything else — a malformed file, a + // permission error — would otherwise resolve the un-overridden graph and silently benchmark a + // dependency set the repository never ships, which is the failure this function exists to avoid. + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return {}; + } + throw error; + } + return parse(raw)?.overrides ?? {}; +} + +/** + * A vite plugin that resolves every bare import from `treeDir` instead of from the harness. + * + * Bare specifiers only: a relative or absolute id belongs to the harness's own sources and stays + * where it is. Resolution is handed back to vite with the importer rewritten, so vite's own + * conditions (`browser`, `import`) and the target package's `exports` map still decide the answer — + * which is what an alias to a directory would skip. + */ +function resolveFromTree(treeDir: string): Vite.Plugin { + // Resolution needs a base directory, and vite derives one from the importer's path — falling back + // to the project root unless that path exists on disk, which would silently resolve the harness's + // copy of a dependency instead of this ref's. The tree's own manifest is the file that names this + // directory, the same way `createRequire` is pointed at one. + const importer = path.join(treeDir, 'package.json'); + + return { + name: 'mui-benchmark:tachometer-resolve', + enforce: 'pre', + async resolveId(id, _importer, options) { + if (id.startsWith('.') || path.isAbsolute(id) || id.startsWith('\0') || id.includes(':')) { + return null; + } + return this.resolve(id, importer, { ...options, skipSelf: true }); + }, + }; +} + +/** + * The packed pins an in-place run adds to the repository's overrides. + * + * Everything the run itself resolves is left alone, this package above all: pinning it would hand + * the harness's vite config the ref's copy of the very tool doing the building, and a ref old enough + * not to export the plugin then fails to configure at all. + */ +function packedPins(packages: PackedPackage[]): Record { + return Object.fromEntries( + packages + .filter((pkg) => !RUNNER_ONLY_DEPS.includes(pkg.name)) + .map((pkg) => [pkg.name, `file:${pkg.tarball}`]), + ); +} + +/** Where the repository's own manifest is kept while an in-place run has it pinned. */ +function backupPathOf(outputDir: string): string { + return path.join(outputDir, 'pnpm-workspace.yaml.orig'); +} + +/** + * Points the repository's own install at this ref's packed build, through the same overrides the + * isolated mode applies to a workspace of its own. + * + * The manifest is copied aside first and rewritten from that copy each time, so a second ref pins + * over the original rather than over the first ref's pins. A run that dies leaves both the pins and + * the copy: the manifest then names tarballs under the output directory, which fails the next + * install rather than resolving something else and saying nothing. + */ +async function pinPackedPackages( + repoRoot: string, + outputDir: string, + packages: PackedPackage[], +): Promise { + const manifestPath = path.join(repoRoot, 'pnpm-workspace.yaml'); + const backupPath = backupPathOf(outputDir); + if (!(await pathExists(backupPath))) { + await writeFile(backupPath, await readFile(manifestPath, 'utf8')); + } + + const config = parse(await readFile(backupPath, 'utf8')) ?? {}; + await writeFile( + manifestPath, + stringify({ ...config, overrides: { ...config.overrides, ...packedPins(packages) } }), + ); +} + +/** + * Puts the repository's manifest back, and installs from it. + * + * The install is the point: without it the tree on disk still resolves the library to a tarball + * under the output directory, and that surfaces at the next unrelated command instead of here. + * + * `--no-frozen-lockfile` because pnpm turns a frozen install on by itself in CI, and the lockfile + * here still carries the pins this is undoing. + */ +export async function restoreWorkspace(repoRoot: string, outputDir: string): Promise { + const backupPath = backupPathOf(outputDir); + if (!(await pathExists(backupPath))) { + return; + } + console.log(chalk.cyan('\nRestoring the repository install…')); + await writeFile(path.join(repoRoot, 'pnpm-workspace.yaml'), await readFile(backupPath, 'utf8')); + await rm(backupPath, { force: true }); + await run('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], repoRoot); +} + +/** Installs one ref's packed build, with the harness's own dependencies around it. */ +async function installRefTree({ + harnessDir, + repoRoot, + packages, + treeDir, +}: { + /** The harness package directory, whose dependencies are copied. */ + harnessDir: string; + /** The workspace root, whose overrides the tree inherits. */ + repoRoot: string; + /** That ref's packed workspace packages. */ + packages: PackedPackage[]; + /** Where to install. */ + treeDir: string; +}): Promise { + const [harnessPkg, installed] = await Promise.all([ + readPackageJson(harnessDir), + installedVersions(harnessDir), + mkdir(treeDir, { recursive: true }), + ]); + + await writePackageJson(treeDir, { + name: 'tacho-resolve-tree', + private: true, + version: '0.0.0', + dependencies: rewriteWorkspaceDeps( + installed, + harnessPkg.dependencies, + packages, + RUNNER_ONLY_DEPS, + ), + devDependencies: rewriteWorkspaceDeps( + installed, + harnessPkg.devDependencies, + packages, + RUNNER_ONLY_DEPS, + ), + }); + + // Marks this folder as its own workspace root, so pnpm does not walk up into the monorepo. The + // overrides carry the repository's own, then pin every packed workspace package on top — so a + // transitive dependency between them (whose packed `workspace:*` became a concrete, unpublished + // version) resolves to a local build rather than 404ing on the registry. The packed pins win, + // since they are the whole point of this install. + await writeFile( + path.join(treeDir, 'pnpm-workspace.yaml'), + stringify({ + overrides: { + ...(await readOverrides(repoRoot)), + ...Object.fromEntries(packages.map((pkg) => [pkg.name, `file:${pkg.tarball}`])), + }, + }), + ); + + // `--ignore-scripts`: this workspace has no build-script approvals, and pnpm fails an install + // over unapproved ones rather than warning. Nothing installed here needs its scripts either — the + // pages are built from the packages exactly as packed. + await run( + 'pnpm', + ['install', '--prefer-offline', '--ignore-scripts', '--config.engine-strict=false'], + treeDir, + ); +} + +/** + * Builds the benchmark pages against one ref's packed build. + * + * The pages are built **where they live**, from the harness's own directory and its own vite config, + * so a `tacho run` build sees exactly what a plain `vite build` sees — the same tsconfig, the same + * postcss config, the same everything. Only *resolution* differs: a plugin redirects every bare + * import into a small tree installed for this ref, where the library under test is that ref's packed + * build. + * + * That tree holds the harness's own dependencies too, so the pages and the library resolve React + * (and everything else) to one copy. Building one side through a workspace link instead would + * compare two different resolution paths, and that difference would land in the measurement rather + * than in the library. + */ +export async function buildRefPages(options: { + /** The harness package directory, built in place. */ + harnessDir: string; + /** The workspace root, whose overrides the tree inherits. */ + repoRoot: string; + /** The ref being built. */ + ref: ResolvedRef; + /** That ref's packed workspace packages. */ + packages: PackedPackage[]; + /** Persistent directory holding this ref's installed tree. Unused when resolving in place. */ + treeDir: string; + /** The run's output directory, where an in-place run keeps the manifest it replaced. */ + outputDir: string; + /** Absolute output directory for the built pages. */ + outDir: string; + /** How the pages find the library. Defaults to `in-place`. */ + resolveMode?: ResolveMode; +}): Promise { + const { + harnessDir, + repoRoot, + ref, + packages, + treeDir, + outputDir, + outDir, + resolveMode = 'in-place', + } = options; + console.log(chalk.cyan(`\nBuilding benchmark pages for "${refLabel(ref)}"…`)); + + if (resolveMode === 'in-place') { + await pinPackedPackages(repoRoot, outputDir, packages); + // `--no-frozen-lockfile` because changing the overrides is the point, and pnpm turns a frozen + // install on by itself in CI — where it would refuse the very change being made. + await run('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], repoRoot); + } else { + await installRefTree({ harnessDir, repoRoot, packages, treeDir }); + } + + // vite is the harness's own, not this package's: the harness declares the version its pages are + // built with, and the config being run is the harness's. + const requireFromHarness = createRequire(path.join(harnessDir, 'package.json')); + const vite: typeof Vite = await import(pathToFileURL(requireFromHarness.resolve('vite')).href); + + // `root` only points vite's config discovery at the harness; the harness's own plugin then moves + // it to `src/`, the same as for a plain `vite build`. + await vite.build({ + root: harnessDir, + logLevel: 'info', + // Resolving in place needs no plugin: the repository's own install is the one that changed. + plugins: resolveMode === 'in-place' ? [] : [resolveFromTree(treeDir)], + build: { outDir }, + }); +} diff --git a/packages/benchmark/src/tachometer/ciReport.test.ts b/packages/benchmark/src/tachometer/ciReport.test.ts new file mode 100644 index 0000000000..0e10d03b22 --- /dev/null +++ b/packages/benchmark/src/tachometer/ciReport.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from 'vitest'; +import { tachometerUploadSchema } from './ciReport'; + +/** + * A report of the shape `runTachometer` writes, trimmed to one case. + * + * @param {Partial} [overrides] - Fields to replace + * @returns {any} + */ +function report(overrides = {}) { + return { + version: 1, + reportType: 'tachometer', + generatedAt: '2026-09-02T00:00:00.000Z', + head: { ref: 'HEAD', sha: 'a'.repeat(40), branch: 'feature' }, + browser: '/path/to/Chrome for Testing', + refs: [ + { id: 'current', kind: 'worktree' }, + { id: 'git-abc123456', kind: 'git', requested: 'HEAD~1', sha: 'b'.repeat(40) }, + ], + cases: [ + { + name: 'workload', + comparison: 'baseline', + reference: 'workload [current]', + measurements: [ + { + name: 'mount', + variants: [ + { + variant: 'workload [current]', + meanMs: { low: 20.5, high: 22.7 }, + samples: 120, + }, + ], + comparisons: [ + { + variant: 'workload [baseline]', + verdict: 'unsure', + absoluteMs: { low: -1.2, high: 2.1 }, + percentChange: { low: -5.2, high: 9.3 }, + versusReference: { + verdict: 'unsure', + absoluteMs: { low: -2.1, high: 1.2 }, + percentChange: { low: -9.3, high: 5.2 }, + }, + }, + ], + }, + ], + }, + ], + raw: { workload: { benchmarks: [] } }, + ...overrides, + }; +} + +/** + * @param {Partial} [overrides] - Envelope fields to replace + * @returns {any} + */ +function envelope(overrides = {}) { + return { + version: 1, + timestamp: 1_770_000_000_000, + commitSha: 'a'.repeat(40), + repo: 'mui/mui-public', + reportType: 'tachometer', + prNumber: 1829, + branch: 'feature', + report: report(), + ...overrides, + }; +} + +describe('tachometerUploadSchema', () => { + it('accepts a report of the shape the runner writes', () => { + expect(tachometerUploadSchema.safeParse(envelope()).success).toBe(true); + }); + + it('carries the envelope metadata through', () => { + const parsed = tachometerUploadSchema.parse(envelope()); + + expect(parsed).toMatchObject({ + repo: 'mui/mui-public', + prNumber: 1829, + branch: 'feature', + reportType: 'tachometer', + }); + }); + + it('accepts a run with no pull request', () => { + // A push to the base branch uploads too; it just has no PR to comment on. + const { prNumber, ...withoutPr } = envelope(); + + expect(tachometerUploadSchema.safeParse(withoutPr).success).toBe(true); + }); + + it('accepts a case that failed to summarize', () => { + // Kept in the report rather than dropped, so the comment can say the case produced nothing. + const withFailure = envelope({ + report: report({ + cases: [{ name: 'broken', comparison: 'baseline', error: 'produced no benchmarks' }], + }), + }); + + expect(tachometerUploadSchema.safeParse(withFailure).success).toBe(true); + }); + + describe('rejects', () => { + it('a report from another benchmark axis', () => { + // The dashboard routes on reportType, so the wrong one would render as the wrong thing. + const wrongType = envelope({ reportType: 'benchmark' }); + + expect(tachometerUploadSchema.safeParse(wrongType).success).toBe(false); + }); + + it('a commit that is not a full sha', () => { + // The artifact key is built from it, and a short sha would not match what the dashboard + // looks up for the head commit. + expect(tachometerUploadSchema.safeParse(envelope({ commitSha: 'abc1234' })).success).toBe( + false, + ); + }); + + it('a repo that is not owner/name', () => { + expect(tachometerUploadSchema.safeParse(envelope({ repo: 'mui-public' })).success).toBe( + false, + ); + }); + + it('a verdict outside the known set', () => { + const badVerdict = report(); + badVerdict.cases[0].measurements[0].comparisons[0].verdict = 'regressed'; + + expect(tachometerUploadSchema.safeParse(envelope({ report: badVerdict })).success).toBe( + false, + ); + }); + }); +}); diff --git a/packages/benchmark/src/tachometer/ciReport.ts b/packages/benchmark/src/tachometer/ciReport.ts new file mode 100644 index 0000000000..8fe6cb09d4 --- /dev/null +++ b/packages/benchmark/src/tachometer/ciReport.ts @@ -0,0 +1,124 @@ +/* eslint-disable no-console -- progress belongs in the CI log. */ + +import { z } from 'zod/v4'; +import { postToDashboard } from '../ciApi'; +import { ciReportUploadSchema } from '../ciReport'; + +/** + * The tachometer axis's upload envelope. It differs from the Vitest one only in the shape of + * `report`; the transport, the CI metadata and the pull request sync are shared. + */ + +const confidenceIntervalSchema = z.object({ + low: z.number(), + high: z.number(), +}); + +const differenceSchema = z.object({ + verdict: z.enum(['faster', 'slower', 'unsure']), + absoluteMs: confidenceIntervalSchema, + percentChange: confidenceIntervalSchema, +}); + +const variantResultSchema = z.object({ + variant: z.string(), + meanMs: confidenceIntervalSchema, + samples: z.number(), +}); + +const comparisonSchema = differenceSchema.extend({ + variant: z.string(), + versusReference: differenceSchema.optional(), +}); + +const measurementResultSchema = z.object({ + name: z.string(), + variants: z.array(variantResultSchema), + comparisons: z.array(comparisonSchema), +}); + +// A case that failed to summarize carries an `error` and no measurements. It is kept in the report +// rather than dropped, so the comment can say a case produced nothing instead of quietly omitting it. +const caseResultSchema = z.object({ + name: z.string(), + // `baseline` compares the working tree against the baseline build, so a difference there is a + // regression; `variants` compares pages built from the same tree, where a difference is the + // point. Carried per case because that is where the two are told apart. + comparison: z.enum(['baseline', 'variants']), + reference: z.string().optional(), + measurements: z.array(measurementResultSchema).optional(), + error: z.string().optional(), +}); + +/** + * A build the pages were loaded from. The working tree has no commit behind it and the baseline + * always does, so the two carry different fields. + * + * `requested` is the revision as the run was given it — `HEAD~1`, a tag, a SHA — kept instead of a + * ready-made display string so a renderer decides how it reads. + */ +const refSchema = z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('worktree'), id: z.string() }), + z.object({ + kind: z.literal('git'), + id: z.string(), + sha: z.string(), + requested: z.string(), + }), +]); + +const tachometerReportV1Schema = z.object({ + version: z.literal(1), + reportType: z.literal('tachometer'), + generatedAt: z.string(), + head: z.object({ ref: z.string(), sha: z.string(), branch: z.string().optional() }), + browser: z.string().optional(), + refs: z.array(refSchema), + cases: z.array(caseResultSchema), + // Every sample tachometer produced. Not read by the comment, but kept so a richer view can be + // built later without changing what CI uploads. + raw: z.record(z.string(), z.unknown()), +}); + +/** + * Every report shape the store may hold, keyed on `version`. + * + * Reports written months ago by an older version of this tooling are still in S3 and still get + * rendered. A new shape joins this union with its own literal version rather than replacing the old + * one, so a reader narrows on `version` and the compiler makes it handle each. + */ +export const tachometerReportSchema = z.discriminatedUnion('version', [tachometerReportV1Schema]); + +/** + * The upload envelope. Built from the same factory the Vitest axis uses, so both report types reach + * the dashboard's upload route in the shape it expects. + */ +export const tachometerUploadSchema = ciReportUploadSchema('tachometer', 1, tachometerReportSchema); + +export type TachometerUpload = z.infer; +export type TachometerReport = z.infer; +export type ReportRef = z.infer; +export type CaseResult = z.infer; +export type ConfidenceInterval = z.infer; +export type Verdict = z.infer['verdict']; +export type MeasurementResult = z.infer; +export type VariantResult = z.infer; +export type Comparison = z.infer; + +/** + * Uploads a report to the CI report store. + * + * Validates before sending because the server stores `report` without inspecting it — a malformed + * report would be accepted here and only fail much later, when the dashboard tried to render it. + */ +export async function uploadCiReport(upload: TachometerUpload): Promise { + tachometerUploadSchema.parse(upload); + + const responseText = await postToDashboard( + '/api/ci-reports/upload', + upload, + 'The tachometer upload', + ); + const result = JSON.parse(responseText); + console.log(`Tachometer report uploaded. S3 key: ${result.key}`); +} diff --git a/packages/benchmark/src/tachometer/discoverCases.test.ts b/packages/benchmark/src/tachometer/discoverCases.test.ts new file mode 100644 index 0000000000..08d3285563 --- /dev/null +++ b/packages/benchmark/src/tachometer/discoverCases.test.ts @@ -0,0 +1,577 @@ +import * as path from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { describe, expect, it } from 'vitest'; +import { makeTempDir } from '../utils/testUtils'; +import { discoverCases, measurementNameOf, pagesOf } from './discoverCases'; + +/** A case folder to create in a throwaway harness. */ +interface CaseFixture { + config: any; + pages?: string[]; +} + +/** Builds a throwaway harness: `src//tachometer.json` plus whatever pages each case owns. */ +async function makeHarness(cases: Record): Promise { + const harnessDir = await makeTempDir(); + await Promise.all( + Object.entries(cases).map(async ([name, { config, pages = [] }]) => { + const caseDir = path.join(harnessDir, 'src', name); + await mkdir(caseDir, { recursive: true }); + await writeFile(path.join(caseDir, 'tachometer.json'), JSON.stringify(config, null, 2)); + await Promise.all( + pages.map((page) => writeFile(path.join(caseDir, page), '\n')), + ); + }), + ); + return harnessDir; +} + +/** A minimal single-benchmark config. */ +function config(name: string, url: string): any { + return { + $schema: '../../node_modules/tachometer/config.schema.json', + benchmarks: [{ name, url }], + }; +} + +describe('discoverCases', () => { + it('finds every case folder holding a tachometer.json', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + beta: { config: config('beta', './index.html'), pages: ['index.html'] }, + }); + + const cases = await discoverCases({ harnessDir }); + + expect(cases.map((entry) => entry.name)).toEqual(['alpha', 'beta']); + }); + + it('narrows to cases matching a filter', async () => { + const harnessDir = await makeHarness({ + 'grid-init': { config: config('grid-init', './index.html'), pages: ['index.html'] }, + 'grid-scroll': { config: config('grid-scroll', './index.html'), pages: ['index.html'] }, + }); + + const cases = await discoverCases({ harnessDir, filters: ['scroll'] }); + + expect(cases.map((entry) => entry.name)).toEqual(['grid-scroll']); + }); + + it('finds a case nested in a folder', async () => { + const harnessDir = await makeHarness({ + 'libs/mount': { config: config('libs-mount', './index.html'), pages: ['index.html'] }, + }); + + const cases = await discoverCases({ harnessDir }); + + expect(cases.map((entry) => entry.name)).toEqual(['libs-mount']); + }); + + it('takes a case name from its benchmark rather than its folder', async () => { + // The folder is where a case lives; the config is what it calls itself. Keeping them separate + // is what lets a case move between folders without being renamed in every report. + const harnessDir = await makeHarness({ + 'libs/mount': { config: config('libs-mount', './index.html'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.name).toBe('libs-mount'); + }); + + it('selects a whole folder by filtering on the path', async () => { + const harnessDir = await makeHarness({ + 'libs/mount': { config: config('libs-mount', './index.html'), pages: ['index.html'] }, + 'libs/scroll': { config: config('libs-scroll', './index.html'), pages: ['index.html'] }, + regression: { config: config('regression', './index.html'), pages: ['index.html'] }, + }); + + const cases = await discoverCases({ harnessDir, filters: ['libs'] }); + + expect(cases.map((entry) => entry.name)).toEqual(['libs-mount', 'libs-scroll']); + }); + + it('matches a filter case-insensitively', async () => { + const harnessDir = await makeHarness({ + GridScroll: { config: config('grid-scroll', './index.html'), pages: ['index.html'] }, + }); + + const cases = await discoverCases({ harnessDir, filters: ['gridscroll'] }); + + expect(cases.map((entry) => entry.name)).toEqual(['grid-scroll']); + }); + + it('refuses two cases that call themselves the same thing', async () => { + // The name identifies a case in the report and in the file its results are written to, so a + // duplicate would silently overwrite rather than fail. + const harnessDir = await makeHarness({ + 'libs/scroll': { config: config('scroll', './index.html'), pages: ['index.html'] }, + scroll: { config: config('scroll', './index.html'), pages: ['index.html'] }, + }); + + await expect(discoverCases({ harnessDir })).rejects.toThrow( + /Two benchmark cases are both named "scroll"/, + ); + }); + + it('strips $schema, which tachometer rejects in a config it is handed', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.config.$schema).toBeUndefined(); + }); + + it('throws when no case matches', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + await expect(discoverCases({ harnessDir, filters: ['nope'] })).rejects.toThrow( + /No benchmark case .* matches "nope"/, + ); + }); + + describe('auto-expansion', () => { + it('expands a benchmark with no variants into current versus baseline', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.config.benchmarks).toEqual([ + { name: 'alpha [current]', url: './index.html' }, + { name: 'alpha [baseline]', url: './index.html' }, + ]); + }); + + it('points the two variants at the two builds', async () => { + // The temp harness is no repository at all, so anything reaching for git would fail here. + // Which build a variant loads follows from the case's shape, which is what lets a plain + // `vite build` run in a checkout with no history. + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.comparison).toBe('baseline'); + expect(entry.leaves.map((leaf) => leaf.ref)).toEqual(['current', 'baseline']); + }); + + it('hands tachometer benchmarks it cannot expand again', async () => { + // The config is parsed afresh, so a benchmark that still carried `expand` would be expanded a + // second time — and an un-rewritten parent url would point at a page that was never built. + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + for (const benchmark of entry.config.benchmarks) { + expect(benchmark).not.toHaveProperty('expand'); + } + }); + + it('gives both variants the same page, query and fragment', async () => { + // Everything after the page path belongs to the page. A parameterised case that lost it on one + // side would compare the parameters against the defaults and call the difference a regression. + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html?rows=100#state'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + const target = 'alpha/index.html?rows=100#state'; + + expect(entry.leaves.map((leaf) => `${leaf.page}${leaf.suffix}`)).toEqual([target, target]); + }); + + it('merges an expansion over its parent, nearest value winning', async () => { + // Tachometer merges an expansion over its parent before it reads anything, so a field set + // deeper wins and one set only on the parent is inherited. Flattening here has to match, or + // the config handed over would describe different benchmarks than the ones discovered. + const harnessDir = await makeHarness({ + libs: { + config: { + benchmarks: [ + { + name: 'libs', + measurement: { mode: 'performance', entryName: 'mount' }, + expand: [{ name: 'libs [ours]', url: './ours.html' }, { url: './theirs.html' }], + }, + ], + }, + pages: ['ours.html', 'theirs.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.config.benchmarks).toEqual([ + { + name: 'libs [ours]', + url: './ours.html', + measurement: { mode: 'performance', entryName: 'mount' }, + }, + { + name: 'libs', + url: './theirs.html', + measurement: { mode: 'performance', entryName: 'mount' }, + }, + ]); + }); + + it('leaves a benchmark that declares its own variants alone', async () => { + const harnessDir = await makeHarness({ + libs: { + config: { + benchmarks: [ + { + name: 'libs', + expand: [ + { name: 'libs [ours]', url: './ours.html' }, + { name: 'libs [theirs]', url: './theirs.html' }, + ], + }, + ], + }, + pages: ['ours.html', 'theirs.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.leaves.map((leaf) => leaf.page)).toEqual(['libs/ours.html', 'libs/theirs.html']); + // Its own variants, so they are compared with each other and all load the working tree. + expect(entry.comparison).toBe('variants'); + expect(entry.leaves.map((leaf) => leaf.ref)).toEqual(['current', 'current']); + }); + }); + + describe('leaf urls', () => { + it('inherits a parent url through a nested expand', async () => { + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [ + { + name: 'alpha', + url: './index.html', + expand: [ + { name: 'alpha [one]', expand: [{ name: 'alpha [one/a]' }] }, + { name: 'alpha [two]', url: './other.html' }, + ], + }, + ], + }, + pages: ['index.html', 'other.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.leaves.map((leaf) => leaf.page)).toEqual([ + 'alpha/index.html', + 'alpha/other.html', + ]); + }); + + it('carries the query and the fragment through as written', async () => { + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [ + { name: 'alpha', expand: [{ name: 'a', url: './index.html?rows=100&cols=8#x' }] }, + ], + }, + pages: ['index.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.leaves[0].suffix).toBe('?rows=100&cols=8#x'); + }); + + it('resolves a case that owns no page but parameterises a sibling', async () => { + const harnessDir = await makeHarness({ + updates: { config: config('updates', './index.html'), pages: ['index.html'] }, + 'updates-throttled': { + config: { + benchmarks: [ + { + name: 'updates-throttled', + expand: [{ name: 'throttled', url: '../updates/index.html?throttle=16' }], + }, + ], + }, + }, + }); + + const cases = await discoverCases({ harnessDir }); + const throttled = cases.find((entry) => entry.name === 'updates-throttled'); + + expect(throttled?.leaves[0]).toMatchObject({ + page: 'updates/index.html', + suffix: '?throttle=16', + }); + }); + + it('rejects a url resolving outside src', async () => { + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [{ name: 'alpha', expand: [{ name: 'a', url: '../../outside.html' }] }], + }, + }, + }); + + await expect(discoverCases({ harnessDir })).rejects.toThrow(/resolves outside/); + }); + + it('rejects a benchmark that names no url', async () => { + // Named for the benchmark the author wrote rather than for a variant they did not, which is + // what the auto-expanded pair would otherwise be reported as. + const harnessDir = await makeHarness({ + alpha: { config: { benchmarks: [{ name: 'alpha' }] }, pages: ['index.html'] }, + }); + + await expect(discoverCases({ harnessDir })).rejects.toThrow( + /Benchmark "alpha" in .* has no "url"/, + ); + }); + + it('rejects a url pointing at a missing page', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './missing.html') }, + }); + + await expect(discoverCases({ harnessDir })).rejects.toThrow(/missing page/); + }); + }); +}); + +describe('pagesOf', () => { + it('deduplicates the pages a case references from several variants', async () => { + // The auto-expanded [current] and [baseline] variants point at the same page. + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + expect(pagesOf(await discoverCases({ harnessDir }))).toEqual(['alpha/index.html']); + }); + + it('unions the pages of every case, sorted', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + libs: { + config: { + benchmarks: [ + { + name: 'libs', + expand: [ + { name: 'libs [ours]', url: './ours.html' }, + { name: 'libs [theirs]', url: './theirs.html' }, + ], + }, + ], + }, + pages: ['ours.html', 'theirs.html'], + }, + }); + + expect(pagesOf(await discoverCases({ harnessDir }))).toEqual([ + 'alpha/index.html', + 'libs/ours.html', + 'libs/theirs.html', + ]); + }); +}); + +describe('measurementNameOf', () => { + it('uses an explicit name', () => { + expect(measurementNameOf({ mode: 'performance', entryName: 'mount', name: 'paint' })).toBe( + 'paint', + ); + }); + + it('uses the expression for an expression measurement', () => { + expect(measurementNameOf({ mode: 'expression', expression: 'window.total' })).toBe( + 'window.total', + ); + }); + + it('names a callback measurement', () => { + expect(measurementNameOf({ mode: 'callback' })).toBe('callback'); + expect(measurementNameOf('callback')).toBe('callback'); + }); + + it('names the "global" shorthand by the expression tachometer expands it to', () => { + // `global` becomes an expression measurement, and a result is named by the expression — not by + // the shorthand — so taking the shorthand at face value would never match its own result. + expect(measurementNameOf('global')).toBe('window.tachometerResult'); + expect(measurementNameOf('global', 'window.myResult')).toBe('window.myResult'); + }); + + it('falls back to the entry name', () => { + expect(measurementNameOf({ mode: 'performance', entryName: 'mount' })).toBe('mount'); + }); + + it('abbreviates first-contentful-paint the way tachometer does', () => { + expect(measurementNameOf({ mode: 'performance', entryName: 'first-contentful-paint' })).toBe( + 'fcp', + ); + }); +}); + +describe('case variants and measurements', () => { + it('lists the auto-expanded variants in order, the reference first', async () => { + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [ + { + name: 'alpha', + url: './index.html', + measurement: { mode: 'performance', entryName: 'mount' }, + }, + ], + }, + pages: ['index.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.variants).toEqual(['alpha [current]', 'alpha [baseline]']); + expect(entry.measurements).toEqual(['mount']); + }); + + it('falls back to the case name when a benchmark names neither itself nor its variants', async () => { + // The case name is already the fallback for a nameless benchmark, so the variants read + // "alpha [current]" rather than "undefined [current]". + const harnessDir = await makeHarness({ + alpha: { + config: { benchmarks: [{ url: './index.html' }] }, + pages: ['index.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.variants).toEqual(['alpha [current]', 'alpha [baseline]']); + }); + + it('takes a variant name from the nearest node that sets one', async () => { + // Tachometer merges an expansion over its parent, so a name set partway down the tree is what + // it reports — reading only the leaf and the benchmark would look up a name it never used. + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [ + { + name: 'alpha', + expand: [ + { name: 'alpha [ours]', expand: [{ url: './index.html' }] }, + { name: 'alpha [theirs]', expand: [{ url: './other.html' }] }, + ], + }, + ], + }, + pages: ['index.html', 'other.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.variants).toEqual(['alpha [ours]', 'alpha [theirs]']); + }); + + it('collects every measurement a benchmark declares', async () => { + // A page with several measurements is exactly the case whose results must be paired by name. + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [ + { + name: 'alpha', + url: './index.html', + measurement: [ + { mode: 'performance', entryName: 'mount' }, + { mode: 'performance', entryName: 'scroll' }, + ], + }, + ], + }, + pages: ['index.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.measurements).toEqual(['mount', 'scroll']); + }); + + it('collects the measurements an expansion declares for itself', async () => { + // Read from the parent, a case whose children each measure something different reports the + // parent's default while tachometer reports theirs, and pairing the two then finds nothing. + const harnessDir = await makeHarness({ + alpha: { + config: { + benchmarks: [ + { + name: 'alpha', + expand: [ + { + name: 'alpha [one]', + url: './index.html', + measurement: { mode: 'performance', entryName: 'mount' }, + }, + { + name: 'alpha [two]', + url: './other.html', + measurement: { mode: 'performance', entryName: 'paint' }, + }, + ], + }, + ], + }, + pages: ['index.html', 'other.html'], + }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.measurements).toEqual(['mount', 'paint']); + }); + + it('refuses two variants that end up with the same name', async () => { + // An expansion that names itself nothing inherits its parent's name, so this is reachable from + // a config that looks fine. Results are paired back by name, so one would shadow the other. + const harnessDir = await makeHarness({ + libs: { + config: { + benchmarks: [ + { name: 'libs', expand: [{ url: './ours.html' }, { url: './theirs.html' }] }, + ], + }, + pages: ['ours.html', 'theirs.html'], + }, + }); + + await expect(discoverCases({ harnessDir })).rejects.toThrow(/has two variants named "libs"/); + }); + + it('defaults to the callback measurement when none is declared', async () => { + const harnessDir = await makeHarness({ + alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, + }); + + const [entry] = await discoverCases({ harnessDir }); + + expect(entry.measurements).toEqual(['callback']); + }); +}); diff --git a/packages/benchmark/src/tachometer/discoverCases.ts b/packages/benchmark/src/tachometer/discoverCases.ts new file mode 100644 index 0000000000..dbf1b1ce47 --- /dev/null +++ b/packages/benchmark/src/tachometer/discoverCases.ts @@ -0,0 +1,321 @@ +import * as path from 'node:path'; +import { readFile } from 'node:fs/promises'; +import { globby } from 'globby'; +import { pathExists } from '../utils/path'; + +/** + * Case discovery: the `tachometer.json` files under a harness's `src/` are the source of truth for + * which benchmark cases exist and which pages they reference. + * + * Both the runner and the vite plugin read them through here, so there is exactly one notion of + * "what pages exist". Discovery reads only the filesystem — which builds a case loads follows from + * its shape rather than from git, so a plain `vite build` needs no checkout with history. + */ + +/** Which of a run's two builds a variant loads. */ +export type BuildRef = 'current' | 'baseline'; + +/** + * What a case compares. + * + * `baseline` is the working tree against the baseline build: the regression case, and what a case + * gets when it declares no variants of its own. `variants` is the pages the case names compared + * against each other, every one of them built from the working tree. + */ +export type CaseComparison = 'baseline' | 'variants'; + +export interface Leaf { + /** The config node whose `url` this is; rewritten in place once builds exist. */ + node: { url?: string }; + /** Page path relative to `src`, posix, e.g. `data-grid-init/index.html`. */ + page: string; + /** The build this variant loads. */ + ref: BuildRef; + /** Everything after the page path: query and fragment, as written. */ + suffix: string; +} + +export interface BenchmarkCase { + /** The case name, as its benchmark declares it. */ + name: string; + /** Absolute path to its `tachometer.json`. */ + configPath: string; + /** The parsed config, mutated in place as urls are rewritten. */ + config: any; + /** What the case compares. */ + comparison: CaseComparison; + /** Every node that selects a page. */ + leaves: Leaf[]; + /** Variant names in order, as tachometer will report them; the first is the reference. */ + variants: string[]; + /** Measurement names, as tachometer will name them in its output. */ + measurements: string[]; +} + +export interface DiscoverCasesOptions { + /** The harness package directory; cases live under its `src/`. */ + harnessDir: string; + /** + * Only include cases whose path under `src` contains one of these substrings, + * case-insensitively. + */ + filters?: string[]; +} + +/** The file that marks a directory under `src/` as a benchmark case. */ +const CONFIG_NAME = 'tachometer.json'; + +/** Tachometer's expression when a `global` measurement does not name one. */ +const DEFAULT_MEASUREMENT_EXPRESSION = 'window.tachometerResult'; + +/** + * The name tachometer gives a measurement: an explicit `name`, else the expression, else the entry + * name. + * + * With more than one measurement on a page, tachometer appends ` []` to each benchmark's + * name, and results have to be paired back by that name — pairing by position would compare + * unrelated measurements. + */ +export function measurementNameOf(measurement: any, measurementExpression?: string): string { + if (typeof measurement === 'string') { + // Tachometer's string shorthands. `callback` and `fcp` name their results the same way the + // shorthand reads, but `global` expands to an *expression* measurement that is named by the + // expression itself — so taking the shorthand at face value would look for a result that is + // never emitted under that name. + return measurement === 'global' + ? (measurementExpression ?? DEFAULT_MEASUREMENT_EXPRESSION) + : measurement; + } + if (measurement.name) { + return measurement.name; + } + if (measurement.mode === 'expression') { + return measurement.expression; + } + if (measurement.mode === 'callback') { + return 'callback'; + } + return measurement.entryName === 'first-contentful-paint' ? 'fcp' : measurement.entryName; +} + +/** + * Flattens a benchmark's `expand` tree into the list of benchmarks it stands for, the way tachometer + * does before it reads anything else. + * + * `expand` is recursive and an expansion is merged over its parent, so the nearest value of every + * field wins — `url`, `name`, `measurement`, `browser` alike. Doing that merge once, here, is what + * keeps the rest of discovery dealing in whole benchmarks: per-field inheritance is a rule that only + * has value while it agrees with tachometer's exactly, and re-deriving it once per field is how the + * two drift apart. + * + * `expand` itself is dropped. Tachometer leaves the parent's on the merged object, where it is inert + * because expansion has already happened — but the config written from this is parsed afresh, and a + * benchmark still carrying `expand` would be expanded a second time. + */ +function flattenExpansions(benchmark: any): any[] { + const { expand, ...rest } = benchmark; + if (!Array.isArray(expand) || expand.length === 0) { + return [rest]; + } + return expand.flatMap((child) => flattenExpansions(child).map((leaf) => ({ ...rest, ...leaf }))); +} + +/** + * Resolves one variant's url into the source page it references and whatever follows it. + * + * The path is carried through as written rather than through `URL`: these urls are relative to their + * own config's directory and traverse out of it (`../row-updates/index.html?throttle=16`), and `URL` + * would clamp that traversal against whatever base it was given. Everything after the path belongs + * to the page and survives the rewrite verbatim, or a parameterised benchmark would silently run its + * defaults. + */ +async function parseLeafUrl( + url: string, + configDir: string, + srcDir: string, +): Promise<{ page: string; suffix: string }> { + const [pathPart] = url.split(/[?#]/); + const suffix = url.slice(pathPart.length); + const absolute = path.resolve(configDir, decodeURIComponent(pathPart)); + + const relative = path.relative(srcDir, absolute); + if (relative.startsWith('..') || path.isAbsolute(relative)) { + throw new Error(`Benchmark url "${url}" resolves outside ${srcDir}.`); + } + if (!(await pathExists(absolute))) { + throw new Error(`Benchmark url "${url}" points at a missing page (${absolute}).`); + } + // Posix, because a page path is a url as often as it is a path — the case index links to it and + // rollup names a chunk after it — and node takes forward slashes on Windows for the rest. + return { page: relative.split(path.sep).join('/'), suffix }; +} + +/** + * Every directory under `srcDir` that holds a `tachometer.json`, as a posix path relative to it. + * + * Nested rather than flat, so a harness can group its cases in folders — which is the whole of the + * partitioning story: the filter matches these paths, and nothing else has to understand what a + * folder means. + */ +async function findCaseLocations(srcDir: string): Promise { + const configs = await globby(`**/${CONFIG_NAME}`, { cwd: srcDir }); + return ( + configs + // `dirname` gives `.` for a config sitting directly in `src`; that case's location is the root. + .map((config) => (path.posix.dirname(config) === '.' ? '' : path.posix.dirname(config))) + // Sorted so a run's case order — and therefore its report — does not depend on the order the + // filesystem happened to return. + .sort() + ); +} + +/** + * Reads every `tachometer.json` under `src/` (optionally filtered by location) and resolves the page + * each of its variants loads. + * + * `expand` decides what a case compares. A benchmark that declares one owns its variant axis, and + * every variant loads the working tree — a comparison between pages, such as one library against + * another. A benchmark without one is expanded into `[current]` and `[baseline]` over the same page, + * which is the regression case. + */ +export async function discoverCases(options: DiscoverCasesOptions): Promise { + const { harnessDir, filters = [] } = options; + const srcDir = path.join(harnessDir, 'src'); + + const located = await findCaseLocations(srcDir); + // Matched case-insensitively against the location, the way vitest matches its file filters: a + // folder name selects everything under it without the filter having to know what a folder means. + const selected = located.filter( + (location) => + filters.length === 0 || + filters.some((filter) => location.toLowerCase().includes(filter.toLowerCase())), + ); + + if (selected.length === 0) { + throw new Error( + filters.length > 0 + ? `No benchmark case under ${srcDir} matches ${filters.map((filter) => `"${filter}"`).join(', ')}.` + : `No benchmark case with a ${CONFIG_NAME} found under ${srcDir}.`, + ); + } + + const configs = await Promise.all( + selected.map(async (location) => { + const configPath = path.join(srcDir, location, CONFIG_NAME); + const config = JSON.parse(await readFile(configPath, 'utf8')); + // `$schema` is for editors; tachometer rejects it in a config it is handed. + delete config.$schema; + // The benchmark names itself, so a case keeps its identity wherever its folder is moved to, + // and the ` []` prefix that tachometer builds from that name is strippable for + // display by construction rather than by the folder happening to agree with it. + const name: string = config.benchmarks?.[0]?.name ?? path.basename(location); + return { name, configPath, config }; + }), + ); + + const byName = new Map(); + for (const entry of configs) { + const clash = byName.get(entry.name); + if (clash) { + throw new Error( + `Two benchmark cases are both named "${entry.name}": ${clash} and ${entry.configPath}. ` + + `Names identify a case in the report and in its results file, so they have to be unique.`, + ); + } + byName.set(entry.name, entry.configPath); + } + + const cases: BenchmarkCase[] = []; + for (const { name, configPath, config } of configs) { + const configDir = path.dirname(configPath); + + const measurements = new Set(); + const nodes: Array<{ node: any; ref: BuildRef }> = []; + for (const benchmark of config.benchmarks ?? []) { + // A benchmark need not name itself, and the case name is the same fallback its own name gets + // — without it the auto-expanded variants would read "undefined [current]". Set before + // flattening so every benchmark it expands to inherits it. + benchmark.name ??= name; + + const declaresVariants = Array.isArray(benchmark.expand) && benchmark.expand.length > 0; + const expanded = flattenExpansions(benchmark); + + // Read from the flattened nodes, not the parent: an expansion may set its own `measurement`, + // and a parent that names none would otherwise contribute `callback` while tachometer reports + // what the child asked for. `measurement` may also be a single entry or a list, and a page + // with several is exactly the case whose results are paired by name rather than by position. + for (const node of expanded) { + for (const measurement of [node.measurement ?? 'callback'].flat()) { + measurements.add(measurementNameOf(measurement, node.measurementExpression)); + } + } + if (declaresVariants) { + for (const node of expanded) { + nodes.push({ node, ref: 'current' }); + } + } else { + // Without `expand` exactly one benchmark comes back, and the same page is measured twice. + const [base] = expanded; + if (base.url === undefined) { + throw new Error(`Benchmark "${benchmark.name}" in ${configPath} has no "url".`); + } + nodes.push( + { node: { ...base, name: `${benchmark.name} [current]` }, ref: 'current' }, + { node: { ...base, name: `${benchmark.name} [baseline]` }, ref: 'baseline' }, + ); + } + } + + // A declared `expand` tree may bottom out in a variant that names no url and inherits none. + for (const { node } of nodes) { + if (node.url === undefined) { + throw new Error( + `Benchmark "${node.name}" in ${configPath} expands to a variant with no "url".`, + ); + } + } + + // Results are paired back to what was declared by variant name, and an expansion that sets no + // name of its own inherits its parent's — so two variants can end up sharing one, which would + // quietly report the last one's numbers for both. + const named = new Set(); + for (const { node } of nodes) { + if (named.has(node.name)) { + throw new Error( + `Benchmark case "${name}" has two variants named "${node.name}" (${configPath}). ` + + `Names identify a variant in the report and in its results, so they have to be unique.`, + ); + } + named.add(node.name); + } + + // The flattened list is what tachometer is handed: it expands to exactly this, and every later + // step — the url rewrite, the browser defaults — then addresses one whole benchmark at a time. + config.benchmarks = nodes.map((entry) => entry.node); + + // eslint-disable-next-line no-await-in-loop + const leaves: Leaf[] = await Promise.all( + nodes.map(async ({ node, ref }) => ({ + node, + ref, + ...(await parseLeafUrl(node.url, configDir, srcDir)), + })), + ); + + cases.push({ + name, + configPath, + config, + comparison: leaves.some((leaf) => leaf.ref === 'baseline') ? 'baseline' : 'variants', + leaves, + variants: nodes.map((entry) => entry.node.name as string), + measurements: [...measurements], + }); + } + return cases; +} + +/** The distinct pages referenced by a set of cases, as paths relative to `src`, sorted. */ +export function pagesOf(cases: BenchmarkCase[]): string[] { + return [...new Set(cases.flatMap((entry) => entry.leaves.map((leaf) => leaf.page)))].sort(); +} diff --git a/packages/benchmark/src/tachometer/format.test.ts b/packages/benchmark/src/tachometer/format.test.ts new file mode 100644 index 0000000000..19cca7d4a0 --- /dev/null +++ b/packages/benchmark/src/tachometer/format.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from 'vitest'; +import { groupCasesByVariantSet, shortNameOf, variantsOf } from './format'; +import type { SummarizedCase } from './format'; + +function variant(name: string) { + return { + variant: name, + meanMs: { low: 20, high: 22 }, + samples: 120, + }; +} + +function caseWith(name: string, variantNames: string[], measurements = ['mount']): SummarizedCase { + return { + name, + comparison: 'baseline', + reference: `${name} ${variantNames[0]}`, + measurements: measurements.map((measurement) => ({ + name: measurement, + variants: variantNames.map((short) => variant(`${name} ${short}`)), + comparisons: [], + })), + }; +} + +describe('shortNameOf', () => { + it('drops the case prefix tachometer puts on a variant', () => { + expect(shortNameOf('libs-mount', 'libs-mount [ag-grid]')).toBe('[ag-grid]'); + }); + + it('leaves a name that does not carry the prefix alone', () => { + expect(shortNameOf('scroll', 'row-updates [current]')).toBe('row-updates [current]'); + }); +}); + +describe('variantsOf', () => { + it('lists the variants in the order they first appear', () => { + expect(variantsOf(caseWith('libs-mount', ['[mosaic]', '[ag-grid]', '[mui-x]']))).toEqual([ + '[mosaic]', + '[ag-grid]', + '[mui-x]', + ]); + }); + + it('counts a variant once across measurements', () => { + const entry = caseWith('workload', ['[current]', '[baseline]'], ['mount', 'cold-start']); + + expect(variantsOf(entry)).toEqual(['[current]', '[baseline]']); + }); +}); + +describe('groupCasesByVariantSet', () => { + it('puts cases sharing a variant set in one group', () => { + // Auto-expanded cases all carry [current]/[baseline], so they belong in one table. + const groups = groupCasesByVariantSet([ + caseWith('workload', ['[current]', '[baseline]']), + caseWith('scroll', ['[current]', '[baseline]']), + ]); + + expect(groups).toHaveLength(1); + expect(groups[0].cases.map((entry) => entry.name)).toEqual(['workload', 'scroll']); + }); + + it('gives a case with a different variant set its own group', () => { + // A table over the union would leave a blank cell in every row, and could name only one + // reference for both. + const groups = groupCasesByVariantSet([ + caseWith('workload', ['[current]', '[baseline]']), + caseWith('libs-mount', ['[mosaic]', '[ag-grid]']), + ]); + + expect(groups.map((group) => group.variants)).toEqual([ + ['[current]', '[baseline]'], + ['[mosaic]', '[ag-grid]'], + ]); + }); + + it('separates variant sets that differ only in order', () => { + // The first variant is the reference, so the same names in another order is another table. + const groups = groupCasesByVariantSet([ + caseWith('a', ['[current]', '[baseline]']), + caseWith('b', ['[baseline]', '[current]']), + ]); + + expect(groups).toHaveLength(2); + }); + + it('keeps the groups in order of first appearance', () => { + const groups = groupCasesByVariantSet([ + caseWith('libs-mount', ['[mosaic]', '[ag-grid]']), + caseWith('workload', ['[current]', '[baseline]']), + caseWith('libs-scroll', ['[mosaic]', '[ag-grid]']), + ]); + + expect(groups.map((group) => group.cases.map((entry) => entry.name))).toEqual([ + ['libs-mount', 'libs-scroll'], + ['workload'], + ]); + }); +}); diff --git a/packages/benchmark/src/tachometer/format.ts b/packages/benchmark/src/tachometer/format.ts new file mode 100644 index 0000000000..6bdd4f56b8 --- /dev/null +++ b/packages/benchmark/src/tachometer/format.ts @@ -0,0 +1,93 @@ +import type { CaseResult, MeasurementResult, ReportRef, ConfidenceInterval } from './ciReport'; + +/** + * How a report's numbers and names are written, for every renderer of one. + * + * The terminal table and the pull request comment show the same run, so they format it here rather + * than each their own way. Pure, and free of anything node-only, so a browser can import it. + */ + +/** A case that produced results, as opposed to one carrying only the error that stopped it. */ +export type SummarizedCase = CaseResult & { measurements: MeasurementResult[] }; + +export interface VariantGroup { + /** The variant set the group's cases share, reference first. */ + variants: string[]; + cases: SummarizedCase[]; +} + +/** `20.50 – 22.67 ms`, with the unit written once. */ +export function formatMean(interval: ConfidenceInterval): string { + return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; +} + +/** Both bounds signed, so the direction of a difference reads without the verdict beside it. */ +function signedInterval(interval: ConfidenceInterval, unit: string, digits: number): string { + const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(digits)}${unit}`; + return `${signed(interval.low)} – ${signed(interval.high)}`; +} + +/** `+2.7% – +6.8%`. */ +export function formatPercent(interval: ConfidenceInterval): string { + return signedInterval(interval, '%', 1); +} + +/** `+6.10 ms – +15.20 ms`. */ +export function formatSignedMs(interval: ConfidenceInterval): string { + return signedInterval(interval, ' ms', 2); +} + +/** What a ref is called in a report: the working tree, or the revision the run was given. */ +export function refLabel(ref: ReportRef): string { + return ref.kind === 'worktree' ? 'working tree' : ref.requested; +} + +/** + * Tachometer names each variant ` []`, and the case is named by the table itself, so + * the prefix is dropped for display. Dropping it is also what lets cases share a table: every + * auto-expanded case then contributes the same `[current]`/`[baseline]` pair rather than a pair of + * its own. + */ +export function shortNameOf(caseName: string, variant: string): string { + return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; +} + +/** Whether a case produced results, as opposed to carrying only the error that stopped it. */ +export function isSummarized(entry: CaseResult): entry is SummarizedCase { + return entry.measurements !== undefined && entry.measurements.length > 0; +} + +/** The distinct variants a case reports, in order of first appearance. */ +export function variantsOf(entry: SummarizedCase): string[] { + const variants: string[] = []; + for (const measurement of entry.measurements) { + for (const variant of measurement.variants) { + const short = shortNameOf(entry.name, variant.variant); + if (!variants.includes(short)) { + variants.push(short); + } + } + } + return variants; +} + +/** + * Splits cases into the groups that each get their own table. + * + * A regression case carries `[current]`/`[baseline]`, a cross-library case `[ours]`/`[theirs]`, and + * each set has its own reference, the first variant. One table over the union of the columns would + * leave a blank cell in every row, and could name only one reference for all of them. + */ +export function groupCasesByVariantSet(cases: SummarizedCase[]): VariantGroup[] { + const groups = new Map(); + + for (const entry of cases) { + const variants = variantsOf(entry); + const key = JSON.stringify(variants); + const group = groups.get(key) ?? { variants, cases: [] }; + group.cases.push(entry); + groups.set(key, group); + } + + return [...groups.values()]; +} diff --git a/packages/benchmark/src/tachometer/outputDir.ts b/packages/benchmark/src/tachometer/outputDir.ts new file mode 100644 index 0000000000..fee71d0e3e --- /dev/null +++ b/packages/benchmark/src/tachometer/outputDir.ts @@ -0,0 +1,36 @@ +import * as path from 'node:path'; +import { mkdir, writeFile } from 'node:fs/promises'; + +/** + * Everything a run writes — packed tarballs, per-ref installs, built pages, the report — lives + * under this one directory inside the harness, so a repository has a single thing to ignore and + * deleting it is the whole reset story. + */ +export const OUTPUT_DIR = '.tachometer'; + +/** + * Creates the output directory, marking it ignored from the inside. + * + * A `.gitignore` holding `*` ignores the directory's contents and itself, the way pytest marks its + * cache, so a harness that forgets to list it does not end up offering built pages and tarballs for + * commit. It does not replace an entry in the repository's own ignore file: tools that read only + * the root one — this repository's ESLint config among them — never see a nested `.gitignore`. + */ +export async function prepareOutputDir(harnessDir: string): Promise { + const outputDir = path.join(harnessDir, OUTPUT_DIR); + await mkdir(outputDir, { recursive: true }); + await writeFile(path.join(outputDir, '.gitignore'), '*\n'); + return outputDir; +} + +/** + * Where the pages built for `refId` go, or the root of every ref's pages when it is omitted. + * + * Named here rather than composed at each call site because the runner and the vite plugin have to + * agree on it: the runner points tachometer's static server at the output directory and rewrites + * each url under this path, and a plain `vite build` has to land somewhere that does not collide. + */ +export function buildsDirOf(harnessDir: string, refId?: string): string { + const builds = path.join(harnessDir, OUTPUT_DIR, 'builds'); + return refId ? path.join(builds, refId) : builds; +} diff --git a/packages/benchmark/src/tachometer/plugin.test.ts b/packages/benchmark/src/tachometer/plugin.test.ts new file mode 100644 index 0000000000..49175b5582 --- /dev/null +++ b/packages/benchmark/src/tachometer/plugin.test.ts @@ -0,0 +1,186 @@ +import * as path from 'node:path'; +import { mkdir, symlink, writeFile } from 'node:fs/promises'; +import { describe, expect, it } from 'vitest'; +import { makeTempDir } from '../utils/testUtils'; +import { tachometer } from './plugin'; + +/** A case folder to create in a throwaway harness. */ +interface CaseFixture { + config: any; + pages?: string[]; +} + +/** Builds a throwaway harness with a package.json, so the plugin can read its dependencies. */ +async function makeHarness({ + cases, + dependencies = {}, +}: { + cases: Record; + dependencies?: Record; +}): Promise { + const harnessDir = await makeTempDir(); + await writeFile( + path.join(harnessDir, 'package.json'), + JSON.stringify({ name: 'harness', private: true, dependencies }, null, 2), + ); + await Promise.all( + Object.entries(cases).map(async ([name, { config, pages = [] }]) => { + const caseDir = path.join(harnessDir, 'src', name); + await mkdir(caseDir, { recursive: true }); + await writeFile(path.join(caseDir, 'tachometer.json'), JSON.stringify(config, null, 2)); + await Promise.all( + pages.map((page) => writeFile(path.join(caseDir, page), '\n')), + ); + }), + ); + return harnessDir; +} + +/** Invokes the plugin's `config` hook. */ +async function callConfig( + harnessDir: string, + command: 'build' | 'serve', + userConfig: any = {}, +): Promise { + const plugin = tachometer({ harnessDir }); + const hook = plugin.config as any; + return hook(userConfig, { command, mode: command === 'build' ? 'production' : 'development' }); +} + +const oneCase = { + alpha: { + config: { benchmarks: [{ name: 'alpha', url: './index.html' }] }, + pages: ['index.html'], + }, +}; + +describe('tachometer plugin', () => { + describe('config hook', () => { + it('roots the app at src and declares it multi-page', async () => { + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'serve'); + + expect(config.root).toBe(path.join(harnessDir, 'src')); + // Vite defaults to 'spa', whose fallback rewrites unmatched urls toward a root index.html + // that does not exist in this harness. + expect(config.appType).toBe('mpa'); + }); + + it('derives no build input while serving', async () => { + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'serve'); + + expect(config.build?.rollupOptions).toBeUndefined(); + }); + + it('points serving at the output directory too, so preview finds the built pages', async () => { + // `vite preview` runs as `serve` but serves `build.outDir`. Leaving it unset there sends + // preview to `/dist` — inside `src/` — and it exits instead of serving the build. + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'serve'); + + expect(config.build?.outDir).toBe(path.join(harnessDir, '.tachometer', 'builds', 'manual')); + }); + + it('uses relative asset urls for a build', async () => { + // Tachometer serves the build directory and pages live under //, so absolute + // "/assets/…" paths would 404. + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'build'); + + expect(config.base).toBe('./'); + }); + + it('defaults the output directory to the run output folder, outside the source root', async () => { + // Vite would otherwise emit into /dist, i.e. inside src/ next to the case sources. + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'build'); + + expect(config.build.outDir).toBe(path.join(harnessDir, '.tachometer', 'builds', 'manual')); + }); + + it('leaves an output directory the caller chose alone', async () => { + // A plugin's returned config is merged *over* the inline config, so defaulting this + // unconditionally would silently override `vite build --outDir` — which is how `tacho run` + // directs each ref's build into its own directory. + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'build', { + build: { outDir: '/somewhere/builds/current' }, + }); + + expect(config.build.outDir).toBe('/somewhere/builds/current'); + }); + + it('derives the build input from the discovered cases', async () => { + const harnessDir = await makeHarness({ cases: oneCase }); + + const config = await callConfig(harnessDir, 'build'); + + expect(config.build.rollupOptions.input).toEqual({ + 'alpha/index': path.join(harnessDir, 'src', 'alpha', 'index.html'), + }); + }); + + it('keys several pages in one case folder separately', async () => { + // A distinct key per page is what stops a cross-library case's two pages colliding into one + // entry. + const harnessDir = await makeHarness({ + cases: { + libs: { + config: { + benchmarks: [ + { + name: 'libs', + expand: [ + { name: 'libs [ours]', url: './ours.html' }, + { name: 'libs [theirs]', url: './theirs.html' }, + ], + }, + ], + }, + pages: ['ours.html', 'theirs.html'], + }, + }, + }); + + const config = await callConfig(harnessDir, 'build'); + + expect(Object.keys(config.build.rollupOptions.input).sort()).toEqual([ + 'libs/ours', + 'libs/theirs', + ]); + }); + }); + + describe('workspace link check', () => { + it('passes when the harness links to nothing', async () => { + const harnessDir = await makeHarness({ cases: oneCase }); + + await expect(callConfig(harnessDir, 'serve')).resolves.toBeTruthy(); + }); + + it('explains that the library has to be built when its link dangles', async () => { + // A harness resolves the library through its build output, so before the first build the + // link points at a directory that does not exist yet. + const harnessDir = await makeHarness({ + cases: oneCase, + dependencies: { '@scope/library': 'workspace:*' }, + }); + await mkdir(path.join(harnessDir, 'node_modules', '@scope'), { recursive: true }); + await symlink( + path.join(harnessDir, 'packages', 'library', 'build'), + path.join(harnessDir, 'node_modules', '@scope', 'library'), + ); + + await expect(callConfig(harnessDir, 'serve')).rejects.toThrow( + /"@scope\/library" is linked into .* but its target does not exist/, + ); + }); + }); +}); diff --git a/packages/benchmark/src/tachometer/plugin.ts b/packages/benchmark/src/tachometer/plugin.ts new file mode 100644 index 0000000000..0d2a7b72e6 --- /dev/null +++ b/packages/benchmark/src/tachometer/plugin.ts @@ -0,0 +1,242 @@ +import * as path from 'node:path'; +import { readFile, realpath } from 'node:fs/promises'; +import type { Plugin } from 'vite'; +import { discoverCases, pagesOf } from './discoverCases'; +import type { BenchmarkCase } from './discoverCases'; +import { buildsDirOf, prepareOutputDir, OUTPUT_DIR } from './outputDir'; + +/** + * Vite plugin for a tachometer benchmark harness. + * + * The harness keeps its own vite config and runs plain `vite` / `vite build` / `vite preview`; this + * plugin only contributes what is derived from the benchmark cases: + * + * - `root` is the harness's `src/`, and the app is an MPA (vite defaults to `spa`, whose fallback + * would rewrite unmatched urls toward a root `index.html` that does not exist here). + * - The build's entry points come from the `tachometer.json` files, so the configs are the single + * source of truth for which pages exist. + * - The dev server serves a generated index listing every case, including cases that own no page of + * their own and only parameterise a sibling's. + * + * It deliberately never resolves refs, so a plain `vite build` needs no git history. + */ + +export interface TachometerPluginOptions { + /** + * The harness package directory. Defaults to the current working directory, which is where vite + * is run from. + */ + harnessDir?: string; +} + +/** + * Checks that the workspace packages the harness links to have actually been built. + * + * A harness depends on the library under test via `workspace:*`, and `publishConfig.directory` + * makes that link point at the package's build output — which does not exist until the library is + * built. The symlink is then dangling, and vite's failure ("failed to resolve import") does not + * say why. + */ +async function assertWorkspaceDepsBuilt( + harnessDir: string, + deps: Record, +): Promise { + const linked = Object.entries(deps).filter(([, version]) => version.startsWith('workspace:')); + await Promise.all( + linked.map(async ([name]) => { + try { + await realpath(path.join(harnessDir, 'node_modules', ...name.split('/'))); + } catch { + throw new Error( + `"${name}" is linked into ${harnessDir} but its target does not exist. ` + + `Build the workspace packages first (e.g. \`pnpm release:build\`) — the harness ` + + `resolves the library through its build output, not its source.`, + ); + } + }), + ); +} + +/** A case name and a query both come from a config file, so neither is trusted as markup. */ +function escapeHtml(text: string): string { + return text + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +/** + * Renders the case index: one link per case, carrying whatever query the case parameterises its + * page with. + */ +function renderIndex(cases: BenchmarkCase[], note: string): string { + const items = cases + .map((entry) => { + // A case's variants can point at several pages (a cross-library comparison) or at one page + // with different queries (a parameterised case), so dedupe on the pair. + const targets = [...new Set(entry.leaves.map((leaf) => `${leaf.page}${leaf.suffix}`))].sort(); + const links = targets + .map( + (target) => + `
  • ${escapeHtml(target)}
    • `, + ) + .join('\n '); + return `
  • \n ${escapeHtml(entry.name)}\n
      \n ${links}\n
    \n
    • `; + }) + .join('\n'); + + return ` + + + + Tachometer benchmark cases + + + +

    Tachometer benchmark cases

    +

    ${note}

    +
      +${items} +
    + + +`; +} + +/** + * Whether a case's own page occupies the `index.html` the case list would be served at. + * + * A `tachometer.json` directly in `src/` is a supported layout, and such a case may own the root + * page. Asked in one place because the build and the dev server both have to give the same answer: + * otherwise `/` shows the case under one command and the list under the other. + */ +function aCaseOwnsIndex(cases: BenchmarkCase[]): boolean { + return pagesOf(cases).includes('index.html'); +} + +/** Creates the tachometer harness plugin. */ +export function tachometer(options: TachometerPluginOptions = {}): Plugin { + const harnessDir = options.harnessDir ?? process.cwd(); + const srcDir = path.join(harnessDir, 'src'); + let buildCases: BenchmarkCase[] = []; + + return { + name: 'mui-benchmark:tachometer', + + async config(userConfig, env) { + const pkg = JSON.parse(await readFile(path.join(harnessDir, 'package.json'), 'utf8')); + await assertWorkspaceDepsBuilt(harnessDir, { + ...pkg.dependencies, + ...pkg.devDependencies, + }); + + // The run output directory, next to the per-ref directories the runner writes — vite would + // otherwise default to `/dist`, i.e. inside `src/` next to the case sources. + // + // Resolved for every command, not just `build`: `vite preview` runs as `serve` but serves + // `build.outDir`, so leaving it unset there sends preview looking for `/dist` and it + // exits rather than serving the pages that were just built. + // + // Only fill it in when the caller said nothing: a plugin's returned config is merged *over* + // the inline config, so unconditionally setting it here would silently override + // `vite build --outDir`, which is exactly how `tacho run` directs each ref's build. + const outDir = userConfig.build?.outDir ?? buildsDirOf(harnessDir, 'manual'); + + if (env.command !== 'build') { + return { root: srcDir, appType: 'mpa', build: { outDir } }; + } + + // Mark the output directory ignored from the inside before the build writes into it, so a + // harness that does not list `.tachometer` is not offered its built pages for commit. Keyed on + // where the output actually lands rather than on who chose it: `tacho run` passes its own + // `outDir` under the same directory and prepares it itself, and any other caller that writes + // there deserves the marker too. Vite resolves a relative `outDir` against `root`. + if (path.resolve(srcDir, outDir).startsWith(path.join(harnessDir, OUTPUT_DIR))) { + await prepareOutputDir(harnessDir); + } + + const cases = await discoverCases({ harnessDir }); + buildCases = cases; + // Key the input map by the page path without its extension. Vite emits each page at its path + // relative to `root` regardless, so `/index.html` lands at `//index.html`; + // a distinct key per page is what lets one case folder hold several pages — a cross-library + // comparison with `mosaic.html` and `tanstack.html` side by side — without them colliding. + const input = Object.fromEntries( + pagesOf(cases).map((page) => [page.replace(/\.html$/, ''), path.join(srcDir, page)]), + ); + + return { + root: srcDir, + appType: 'mpa', + // Relative asset urls: tachometer serves the build directory and the pages live under + // `//`, so absolute "/assets/…" paths would 404. + base: './', + build: { + outDir, + // The output directory sits outside `root`, which is `src/`; allow vite to clean it anyway. + emptyOutDir: true, + chunkSizeWarningLimit: 9999, + rollupOptions: { input }, + }, + }; + }, + + // The same index the dev server renders, emitted as a real file so the built directory is + // navigable too — under `vite preview`, under any static server, or opened from disk. A page-less + // case is only reachable through this list, so a build without it can only be entered by typing + // urls from memory. + generateBundle() { + // Rollup drops an emit whose name a bundle output already claims, with no error and no + // warning, so the list would simply vanish. Say so instead. + if (aCaseOwnsIndex(buildCases)) { + this.warn( + 'A case owns index.html, so the case list was not emitted. Move the case into a subfolder of src/ to get it back.', + ); + return; + } + + this.emitFile({ + type: 'asset', + fileName: 'index.html', + source: renderIndex( + buildCases, + 'These are production bundles — the same build tachometer measures.', + ), + }); + }, + + configureServer(server) { + server.middlewares.use(async (req, res, next) => { + const url = (req.url ?? '/').split('?')[0]; + if (url !== '/' && url !== '/index.html') { + next(); + return; + } + try { + const cases = await discoverCases({ harnessDir }); + if (aCaseOwnsIndex(cases)) { + // Let vite serve the case's own page, the way the build does. + next(); + return; + } + res.setHeader('Content-Type', 'text/html; charset=utf-8'); + res.end( + renderIndex( + cases, + "Pages are served from the working tree's build of the library. Rebuild it and reload to pick up a source change.", + ), + ); + } catch (error) { + next(error); + } + }); + }, + }; +} diff --git a/packages/benchmark/src/tachometer/refs.test.ts b/packages/benchmark/src/tachometer/refs.test.ts new file mode 100644 index 0000000000..4d55ec205c --- /dev/null +++ b/packages/benchmark/src/tachometer/refs.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from 'vitest'; +import { execa } from 'execa'; +import { makeTempDir } from '../utils/testUtils'; +import { resolveBaselineRef } from './refs'; + +/** + * A repository with three commits on `main`, the first of them carrying an annotated tag. + * + * A real repository rather than a stand-in for git: every behaviour here is a question about what + * git reports — what a tag resolves to, which refnames exist — and a fake would only restate the + * answer the code already assumes. + */ +async function makeRepo(): Promise<{ repoRoot: string; shas: string[] }> { + const repoRoot = await makeTempDir(); + // Identity through the environment rather than `git config`, so the repo needs no extra processes + // and nothing depends on the machine's own git configuration. + const git = (...args: string[]) => + execa('git', args, { + cwd: repoRoot, + env: { + GIT_AUTHOR_NAME: 'Fixture', + GIT_AUTHOR_EMAIL: 'fixture@example.com', + GIT_COMMITTER_NAME: 'Fixture', + GIT_COMMITTER_EMAIL: 'fixture@example.com', + }, + }); + + await git('init', '--initial-branch=main'); + // Empty commits: nothing here reads a file, only the commits' identities matter. + for (const message of ['first', 'second', 'third']) { + // eslint-disable-next-line no-await-in-loop + await git('commit', '--allow-empty', '-m', message); + } + const { stdout } = await git('log', '--format=%H', '--reverse'); + const shas = stdout.trim().split('\n'); + + // Annotated (`-m`), so the tag is its own object with its own SHA. + await git('tag', '-a', 'v1.0.0', '-m', 'release', shas[0]); + + return { repoRoot, shas }; +} + +describe('resolveBaselineRef', () => { + it('takes a bare value as a revision', async () => { + const { repoRoot, shas } = await makeRepo(); + + const ref = await resolveBaselineRef(shas[0], repoRoot); + + expect(ref.sha).toBe(shas[0]); + }); + + it('takes the same revision behind the git scheme', async () => { + const { repoRoot, shas } = await makeRepo(); + + const ref = await resolveBaselineRef(`git:${shas[0]}`, repoRoot); + + expect(ref.sha).toBe(shas[0]); + }); + + it('resolves an annotated tag to the commit it points at', async () => { + const { repoRoot, shas } = await makeRepo(); + + const ref = await resolveBaselineRef('v1.0.0', repoRoot); + + // Without `^{commit}` this is the tag object's own SHA, which is not a commit at all. + expect(ref.sha).toBe(shas[0]); + }); + + it('falls back to the previous commit when nothing names one', async () => { + // Which commit a branch should be compared against is `code-infra baseline`'s question, and the + // harness scripts pass the answer in. Unbound, the previous commit is the only choice here that + // needs no policy of its own. + const { repoRoot, shas } = await makeRepo(); + + const ref = await resolveBaselineRef(undefined, repoRoot); + + expect(ref.sha).toBe(shas[1]); + }); + + it('keeps the revision it was given, without the scheme', async () => { + const { repoRoot } = await makeRepo(); + + // The report shows this, so a tag stays a tag rather than becoming the SHA it points at. + expect((await resolveBaselineRef('v1.0.0', repoRoot)).requested).toBe('v1.0.0'); + expect((await resolveBaselineRef('git:v1.0.0', repoRoot)).requested).toBe('v1.0.0'); + }); + + it('names a revision and a tag the same build, so the commit is built once', async () => { + const { repoRoot, shas } = await makeRepo(); + + const viaTag = await resolveBaselineRef('v1.0.0', repoRoot); + const viaSha = await resolveBaselineRef(shas[0], repoRoot); + + // `id` is the build directory: two ids for one commit means packing and building it twice. + expect(viaTag.id).toBe(viaSha.id); + }); + + it.each(['github:owner/repo#abc1234', 'preview:abc1234'])( + 'reports %s as recognised but not implemented', + async (token) => { + const { repoRoot } = await makeRepo(); + + await expect(() => resolveBaselineRef(token, repoRoot)).rejects.toThrow( + /recognised but not implemented/, + ); + }, + ); + + it('rejects the git scheme with no revision after it', async () => { + const { repoRoot } = await makeRepo(); + + await expect(() => resolveBaselineRef('git:', repoRoot)).rejects.toThrow(/names no revision/); + }); + + it('reports what git said about a revision it cannot resolve', async () => { + const { repoRoot } = await makeRepo(); + + // Git is the authority on what a revision is, so an unknown one surfaces as its own failure + // rather than as a grammar this code would have to keep in step with git's. + await expect(() => resolveBaselineRef('no-such-thing', repoRoot)).rejects.toThrow( + /Could not resolve git ref "no-such-thing"/, + ); + }); +}); diff --git a/packages/benchmark/src/tachometer/refs.ts b/packages/benchmark/src/tachometer/refs.ts new file mode 100644 index 0000000000..ac94c1c6c8 --- /dev/null +++ b/packages/benchmark/src/tachometer/refs.ts @@ -0,0 +1,70 @@ +import { resolveCommit } from '../utils/git'; + +/** + * The builds a run measures: the working tree, and the baseline it is compared against. + * + * Those are the only two. A case either compares the working tree against the baseline or compares + * the pages it names against each other, all built from the working tree — so the only build anyone + * chooses is the baseline, and `--baseline` is where it is named. + */ + +/** + * A build with its identity. The working tree has no commit behind it and a git revision always + * does, so the two carry different fields rather than sharing optional ones a reader has to pair + * with the right `kind` by hand. + * + * `id` is the build directory name — `current`, or `git-230342ee2`. `requested` is the revision as + * it was given, which is what a report shows. + */ +export type ResolvedRef = + | { kind: 'worktree'; id: string; sha?: undefined; requested?: undefined } + | { kind: 'git'; id: string; sha: string; requested: string }; + +/** The working tree, which every run builds. Never cached, having no immutable identity. */ +export const WORKTREE_REF: ResolvedRef = { kind: 'worktree', id: 'current' }; + +/** Schemes a baseline may eventually name, each rejected until it does something. */ +const RESERVED_SCHEMES = ['github', 'preview']; + +/** + * The revision a baseline token names. + * + * A bare value is a revision and `git:` says so explicitly, which leaves room for a baseline that is + * not a commit in this repository — a published preview, another repository — to arrive as its own + * scheme without the flag changing shape. Anything else goes to git, which is the authority on what + * a revision is. + */ +function committishOf(token: string | undefined): string { + if (!token) { + // Which commit a pull request should be measured against is not decided here: `code-infra + // baseline` answers that for every job that compares against a base — bundle size as well as + // benchmarks — and the harness scripts pass the answer in. Without one, the previous commit is + // the only choice that needs no policy. + return 'HEAD~1'; + } + if (token.startsWith('git:')) { + const revision = token.slice('git:'.length); + if (!revision) { + throw new Error('The baseline "git:" names no revision.'); + } + return revision; + } + const reserved = RESERVED_SCHEMES.find((scheme) => token.startsWith(`${scheme}:`)); + if (reserved) { + throw new Error( + `Baseline scheme "${reserved}:" is recognised but not implemented yet (in "${token}"). ` + + `Pass a git revision, on its own or as "git:".`, + ); + } + return token; +} + +/** Resolves the baseline to the commit behind it, so a run stays interpretable after the fact. */ +export async function resolveBaselineRef( + token: string | undefined, + repoRoot: string, +): Promise { + const committish = committishOf(token); + const sha = await resolveCommit(repoRoot, committish); + return { kind: 'git', id: `git-${sha.slice(0, 9)}`, sha, requested: committish }; +} diff --git a/packages/benchmark/src/tachometer/renderReport.ts b/packages/benchmark/src/tachometer/renderReport.ts new file mode 100644 index 0000000000..1ad46cd643 --- /dev/null +++ b/packages/benchmark/src/tachometer/renderReport.ts @@ -0,0 +1,241 @@ +/* eslint-disable no-console -- a reporter: printing the report is its output. */ + +// Column widths have to measure visible characters, so chalk's escapes come off first. +import { stripVTControlCharacters as stripAnsi } from 'node:util'; +import chalk from 'chalk'; +import { + formatMean, + formatPercent, + groupCasesByVariantSet, + isSummarized, + refLabel, + shortNameOf, +} from './format'; +import type { SummarizedCase } from './format'; +import type { TachometerReport, Verdict } from './ciReport'; + +/** The report's types, from the schema that defines it. */ +export type { + TachometerReport, + CaseResult, + MeasurementResult, + VariantResult, + Comparison, + ReportRef, + ConfidenceInterval, + Verdict, +} from './ciReport'; + +/** Prints a table, padding each column to its widest visible cell. */ +function printTable(headers: string[], rows: string[][], leftColumns: number): void { + const widths = headers.map((header, index) => + Math.max(stripAnsi(header).length, ...rows.map((row) => stripAnsi(row[index] ?? '').length)), + ); + const pad = (text: string, width: number, left: boolean) => { + const fill = ' '.repeat(Math.max(0, width - stripAnsi(text).length)); + return left ? text + fill : fill + text; + }; + const line = (cells: string[]) => + cells + .map((cell, index) => pad(cell ?? '', widths[index], index < leftColumns)) + .join(' ') + .trimEnd(); + + console.log(line(headers)); + console.log(widths.map((width) => '─'.repeat(width)).join(' ')); + for (const row of rows) { + console.log(line(row)); + } +} + +function colorVerdict(verdict: Verdict, text: string): string { + if (verdict === 'faster') { + return chalk.green(text); + } + if (verdict === 'slower') { + return chalk.red(text); + } + return chalk.dim(text); +} + +/** + * Renders one case as a table of variants, for a comparison with more than two of them. + * + * Rows are variants, not cases: with several libraries, a column per variant plus a Δ column per + * pair would run off the screen. Each measurement then shows the variant's interval next to its + * difference *relative to the reference* — the direction a row about that library reads in — taken + * from the variant's own row of tachometer's matrix. + */ +function printVariantTable(entry: SummarizedCase, variants: string[]): void { + const [reference] = variants; + const headers = [ + entry.name, + ...entry.measurements.flatMap((measurement) => [measurement.name, `vs ${reference}`]), + 'Samples', + ]; + + const rows = variants.map((variant) => { + // Looked up once per row rather than accumulated while the measurement cells render, so the + // last two columns do not depend on the cell loop having run. + const perMeasurement = entry.measurements.map((measurement) => ({ + found: measurement.variants.find( + (candidate) => shortNameOf(entry.name, candidate.variant) === variant, + ), + comparison: measurement.comparisons.find( + (candidate) => shortNameOf(entry.name, candidate.variant) === variant, + ), + })); + const sampleCounts = new Set( + perMeasurement.flatMap(({ found }) => (found ? [found.samples] : [])), + ); + + const cells = perMeasurement.flatMap(({ found, comparison }) => { + let delta = chalk.dim('—'); + if (variant === reference) { + delta = chalk.dim('reference'); + } else if (comparison?.versusReference) { + const { verdict, percentChange } = comparison.versusReference; + delta = colorVerdict(verdict, `${verdict} ${formatPercent(percentChange)}`); + } + return [found ? formatMean(found.meanMs) : chalk.dim('—'), delta]; + }); + + return [variant, ...cells, [...sampleCounts].join('/')]; + }); + + printTable(headers, rows, 1); + console.log( + chalk.dim( + `vs ${reference} is tachometer's confidence interval on the difference, that variant ` + + `relative to ${reference} — negative is faster than ${reference}.`, + ), + ); +} + +/** Renders a group of cases sharing a variant set, one row per case and measurement. */ +function printCaseTable(cases: SummarizedCase[], variants: string[]): void { + const [reference, ...others] = variants; + const headers = [ + 'Case', + 'Measurement', + ...variants, + ...others.map((variant) => `Δ vs ${variant}`), + 'Samples', + ]; + + const rows: string[][] = []; + for (const [caseIndex, entry] of cases.entries()) { + if (caseIndex > 0) { + rows.push(headers.map(() => '')); + } + let firstRow = true; + for (const measurement of entry.measurements) { + const byVariant = new Map( + measurement.variants.map((variant) => [shortNameOf(entry.name, variant.variant), variant]), + ); + const byComparison = new Map( + measurement.comparisons.map((comparison) => [ + shortNameOf(entry.name, comparison.variant), + comparison, + ]), + ); + // Auto-sampling stops per case, not per variant, so the counts are normally equal; when they + // are not, saying so beats printing one number that describes neither side. + const sampleCounts = [...new Set(measurement.variants.map((variant) => variant.samples))]; + + rows.push([ + firstRow ? entry.name : '', + measurement.name, + ...variants.map((variant) => { + const found = byVariant.get(variant); + return found ? formatMean(found.meanMs) : chalk.dim('—'); + }), + ...others.map((variant) => { + const comparison = byComparison.get(variant); + if (!comparison) { + return chalk.dim('—'); + } + return colorVerdict( + comparison.verdict, + `${comparison.verdict} ${formatPercent(comparison.percentChange)}`, + ); + }), + sampleCounts.join('/'), + ]); + firstRow = false; + } + } + + printTable(headers, rows, 2); + console.log( + chalk.dim( + `Δ is tachometer's confidence interval on the difference, ${reference ?? 'the reference'} ` + + `relative to the other variant${others.length > 1 ? 's' : ''} — negative is faster.`, + ), + ); +} + +/** + * Prints a tachometer report: one table per variant set, with a column per variant holding its 95% + * confidence interval and a Δ column per non-reference variant. + * + * The Δ column is **tachometer's own confidence interval on the difference**, not a delta of the + * two means. That is the whole reason this suite exists next to a profiling one: variants are + * sampled round-robin in a single session and auto-sampled until that interval clears the case's + * `autoSampleConditions`, so the interval — not a point estimate plus a post-hoc test — is the + * result. `unsure` means it never resolved within the sampling budget, which for two builds that + * really are equivalent is the expected outcome rather than a failure. + * + * Signs are the reference relative to the variant, so a faster reference reads negative; the + * verdict word carries the direction so the sign never has to be read twice. + */ +export function renderTachometerReport(report: TachometerReport): void { + // One predicate for both halves: partitioning on two conditions that have to agree left a case + // with an empty `measurements` array rendered nowhere and reported nowhere. + const usable = report.cases.filter(isSummarized); + const failed = report.cases.filter((entry) => !isSummarized(entry)); + + if (usable.length === 0) { + console.log(chalk.dim('No tachometer results to show.')); + for (const entry of failed) { + console.log(chalk.red(` ${entry.name}: ${entry.error ?? 'no result'}`)); + } + return; + } + + for (const [groupIndex, group] of groupCasesByVariantSet(usable).entries()) { + if (groupIndex > 0) { + console.log(''); + } + if (group.variants.length > 2) { + for (const [caseIndex, entry] of group.cases.entries()) { + if (caseIndex > 0) { + console.log(''); + } + printVariantTable(entry, group.variants); + } + continue; + } + printCaseTable(group.cases, group.variants); + } + + const notes = [ + 'Each cell is a 95% confidence interval for the mean, in milliseconds; one sample = one page load.', + '"unsure" means the interval still straddles zero: the difference did not resolve within the case\'s sampling budget.', + ]; + + // No short SHA appended: `id` already ends in one. + const refLabels = report.refs.map((ref) => `${ref.id} = ${refLabel(ref)}`); + if (refLabels.length > 0) { + notes.push(`Builds: ${refLabels.join(' · ')}`); + } + notes.push( + `head: ${report.head.sha.slice(0, 9)} (${report.head.branch || '?'}) · ` + + `measured on the production bundle, installed from a packed tarball.`, + ); + console.log(chalk.dim(`\n${notes.join('\n')}`)); + + for (const entry of failed) { + console.log(chalk.red(`\n${entry.name}: ${entry.error ?? 'no result'}`)); + } +} diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts new file mode 100644 index 0000000000..45ffd3ccc2 --- /dev/null +++ b/packages/benchmark/src/tachometer/runTachometer.ts @@ -0,0 +1,322 @@ +/* eslint-disable no-console */ + +import * as path from 'node:path'; +import * as os from 'node:os'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import chalk from 'chalk'; +import { execaSync } from 'execa'; +import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; +import { packRef, packWorkingTree } from '../utils/packWorkspace'; +import type { PackedPackage } from '../utils/packWorkspace'; +import { resolveBaselineRef, WORKTREE_REF } from './refs'; +import type { ResolvedRef } from './refs'; +import { discoverCases, pagesOf } from './discoverCases'; +import type { BenchmarkCase, Leaf } from './discoverCases'; +import { isSummarized, refLabel } from './format'; +import { buildRefPages, restoreWorkspace } from './buildPages'; +import type { ResolveMode } from './buildPages'; +import { assertDriverMatchesBrowser, resolveBrowserBinary, withBrowserDefaults } from './browser'; +import { summarizeCase } from './summarizeCase'; +import { renderTachometerReport } from './renderReport'; +import { getCiMetadata } from '../ciReport'; +import { syncPrComment } from '../syncPrComment'; +import { uploadCiReport } from './ciReport'; +import type { CaseResult, TachometerReport } from './ciReport'; +import { buildsDirOf, prepareOutputDir } from './outputDir'; +import { run } from '../utils/exec'; + +/** + * A case's name as a filename component. Names come from the benchmark's own `name`, so they are + * prose rather than identifiers and may hold anything — a slash included. + */ +function fileSlugOf(name: string): string { + return name.replace(/[^a-zA-Z0-9._-]+/g, '-'); +} + +/** + * The builds a run measures, and which of them a given variant loads. + * + * The baseline is resolved only when a case compares against it: a harness that only puts pages + * side by side needs no second commit, and asking for one would fail in a clone too shallow to hold + * it. The lookup comes back with the list so that "there is no baseline" is settled once, here, + * rather than being a case every later step has to allow for. + */ +async function resolveBuilds( + cases: BenchmarkCase[], + baseline: string | undefined, + repoRoot: string, +): Promise<{ refs: ResolvedRef[]; buildFor: (leaf: Leaf) => ResolvedRef }> { + if (!cases.some((entry) => entry.comparison === 'baseline')) { + return { refs: [WORKTREE_REF], buildFor: () => WORKTREE_REF }; + } + const baselineRef = await resolveBaselineRef(baseline, repoRoot); + return { + refs: [WORKTREE_REF, baselineRef], + buildFor: (leaf) => (leaf.ref === 'baseline' ? baselineRef : WORKTREE_REF), + }; +} + +export interface RunTachometerOptions { + /** The harness package directory (where the command was run). */ + harnessDir: string; + /** + * Only run cases whose path under `src` contains one of these substrings, case-insensitively. + */ + filters?: string[]; + /** + * The build every `[baseline]` variant loads: a revision, on its own or as `git:`. Defaults + * to `HEAD~1`. + */ + baseline?: string; + /** + * Command that builds the publishable workspace packages. Run for every ref — in each ref's own + * checkout, and in the working tree — so both sides of a comparison are built the same way. + * Defaults to `pnpm release:build`. + */ + buildCmd?: string; + /** Whether to install inside a ref's checkout. Defaults to true. */ + install?: boolean; + /** Where to write the combined JSON report. Defaults to `.tachometer/results/report.json`. */ + out?: string; + /** Upload the report and refresh the pull request comment. */ + upload?: boolean; + /** How a ref's pages find the library under test. Defaults to `in-place`. */ + resolveMode?: ResolveMode; +} + +/** + * Benchmarks the harness's pages across one or more builds of the workspace with tachometer, and + * writes a combined JSON report. + * + * Cases are the source of truth: the selected `tachometer.json` files determine which pages to + * build, one variant is built per distinct ref, each url is then rewritten to that ref's built page, + * and each case runs as its own tachometer invocation so its result is a clean table for that + * scenario rather than a grid comparing unrelated scenarios against each other. + */ +export async function runTachometer(options: RunTachometerOptions): Promise { + const { + harnessDir, + filters = [], + baseline, + buildCmd = 'pnpm release:build', + install = true, + out, + upload = false, + resolveMode = 'in-place', + } = options; + + const repoRoot = await findWorkspaceDir(harnessDir); + if (!repoRoot) { + throw new Error(`Could not find a pnpm workspace root above ${harnessDir}.`); + } + + // Everything a run writes goes under one directory, so a harness has one thing to ignore and + // deleting it is the whole reset story. `packed` holds tarballs — a ref's keyed by commit SHA, + // the working tree's by content hash — and is the one worth caching in CI; `trees` holds the + // install each ref's pages resolve through, cheap to recreate and not portable between + // containers, since its links point into a pnpm store; `builds` holds the pages built per ref; + // `results` the report. + const outputDir = await prepareOutputDir(harnessDir); + const buildsDir = buildsDirOf(harnessDir); + const packedDir = path.join(outputDir, 'packed'); + const treesDir = path.join(outputDir, 'trees'); + + // A run killed outright — Ctrl-C, not a thrown error — never reaches the restore below, leaving + // the repository pinned to tarballs. The copy it left behind is how that is noticed, so put it + // back before doing anything else, whatever mode this run is in. + await restoreWorkspace(repoRoot, outputDir); + + const cases = await discoverCases({ harnessDir, filters }); + const { refs, buildFor } = await resolveBuilds(cases, baseline, repoRoot); + + // The commit follows the name, so a revision like `HEAD~1` still says which commit it landed on. + const describeRef = (ref: ResolvedRef) => + ref.sha ? `${refLabel(ref)} (${ref.sha.slice(0, 9)})` : refLabel(ref); + + // Before any build: a driver that cannot open the browser fails the run either way, and finding + // out now costs seconds instead of minutes of packing and installing. + const browserBinary = await resolveBrowserBinary(harnessDir); + assertDriverMatchesBrowser(harnessDir, browserBinary); + // `getuid` is POSIX-only; on Windows nobody is root. + const asRoot = process.getuid?.() === 0; + + // What each case compares, so one that carries no baseline says so up front rather than through + // an empty regression list at the end. + const describeCase = (entry: BenchmarkCase) => + entry.comparison === 'baseline' + ? `${entry.name} (vs baseline)` + : `${entry.name} (${entry.variants.length} variants)`; + + console.log(chalk.cyan(`Cases: ${cases.map(describeCase).join(', ')}`)); + console.log(chalk.cyan(`Pages: ${pagesOf(cases).join(', ')}`)); + console.log(chalk.cyan(`Refs: ${refs.map(describeRef).join(', ')}`)); + console.log( + chalk.cyan(`Browser: ${browserBinary}${asRoot ? ' (as root, so without its sandbox)' : ''}`), + ); + + const tmpBase = await mkdtemp(path.join(os.tmpdir(), 'tacho-')); + + try { + // Build one variant per distinct ref, each resolving through its own install so both sides + // of a comparison resolve the library identically. + for (const ref of refs) { + let packages: PackedPackage[]; + if (ref.kind === 'worktree') { + // eslint-disable-next-line no-await-in-loop + packages = await packWorkingTree({ + repoRoot, + outRoot: path.join(packedDir, 'current'), + buildCmd, + }); + } else { + // packRef caches a ref's tarballs by SHA; a hit skips the checkout, install, and build. + // eslint-disable-next-line no-await-in-loop + const packed = await packRef({ + repoRoot, + ref: ref.sha, + outRoot: packedDir, + // An empty install command means "skip"; otherwise packRef's default install runs. + installCmd: install ? undefined : '', + buildCmd, + }); + packages = packed.packages; + } + + // eslint-disable-next-line no-await-in-loop + await buildRefPages({ + harnessDir, + repoRoot, + ref, + packages, + treeDir: path.join(treesDir, ref.id), + outputDir, + outDir: path.join(buildsDir, ref.id), + resolveMode, + }); + } + + // Rewrite each leaf url to its ref's built page, then run each case's own config. Tachometer + // resolves relative urls against the config file's directory and these configs are written to a + // temp dir, so the rewritten urls are absolute. + const results: Array<{ entry: BenchmarkCase; json: any }> = []; + for (const entry of cases) { + for (const leaf of entry.leaves) { + leaf.node.url = `${path.join(buildsDir, buildFor(leaf).id, leaf.page)}${leaf.suffix}`; + } + // Discovery flattened `expand` away, so every benchmark already carries its own effective + // browser and there is no inheritance left to walk. + for (const benchmark of entry.config.benchmarks ?? []) { + benchmark.browser = withBrowserDefaults(benchmark.browser, browserBinary, asRoot); + } + const slug = fileSlugOf(entry.name); + const configPath = path.join(tmpBase, `tachometer-${slug}.json`); + // eslint-disable-next-line no-await-in-loop + await writeFile( + configPath, + // Served from the output directory rather than the harness: tachometer's static server is + // koa-static with its default `hidden: false`, so any dot-prefixed segment in the *served* + // path is refused. Rooting here keeps `.tachometer` out of the url entirely. + `${JSON.stringify({ ...entry.config, root: outputDir }, null, 2)}\n`, + ); + + console.log(chalk.cyan(`\nRunning "${entry.name}"…`)); + const jsonPath = path.join(tmpBase, `result-${slug}.json`); + // eslint-disable-next-line no-await-in-loop + await run( + 'pnpm', + ['exec', 'tachometer', '--config', configPath, '--json-file', jsonPath], + harnessDir, + { verbose: true }, + ); + // Cases run sequentially on purpose — concurrent browser benchmarks would contend for the + // same machine and skew timings — so reading each result in turn is fine. + // eslint-disable-next-line no-await-in-loop + results.push({ entry, json: JSON.parse(await readFile(jsonPath, 'utf8')) }); + } + + const outPath = out ? path.resolve(out) : path.join(outputDir, 'results', 'report.json'); + const report = { + // Consumers render reports from several benchmark axes; the pair identifies which one this + // is and how to read it. + version: 1 as const, + reportType: 'tachometer' as const, + generatedAt: new Date().toISOString(), + head: { + ref: 'HEAD', + sha: execaSync('git', ['rev-parse', 'HEAD'], { cwd: repoRoot }).stdout.trim(), + branch: execaSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { + cwd: repoRoot, + }).stdout.trim(), + }, + browser: browserBinary, + refs, + // Summarising is best-effort per case: a case that produced no usable benchmarks must not + // cost the whole run its report, since `raw` below is the only surviving copy of every other + // case's samples once the temp dir is cleaned up. + cases: results.map(({ entry, json }): CaseResult => { + try { + return summarizeCase(entry, json); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + console.error(chalk.yellow(`Could not summarize "${entry.name}": ${message}`)); + return { name: entry.name, comparison: entry.comparison, error: message }; + } + }), + raw: Object.fromEntries(results.map(({ entry, json }) => [entry.name, json])), + }; + await mkdir(path.dirname(outPath), { recursive: true }); + await writeFile(outPath, `${JSON.stringify(report, null, 2)}\n`); + console.log(''); + renderTachometerReport(report); + console.log(chalk.green(`\nWrote JSON report to ${outPath}`)); + + if (upload) { + await publishReport(report); + } + + // Written and uploaded first, so the errors are readable and the comment says what happened — + // then fail, because a harness that measured nothing at all otherwise reports a passing job. + if (report.cases.every((entry) => !isSummarized(entry))) { + throw new Error('No case produced any measurement.'); + } + } finally { + await rm(tmpBase, { recursive: true, force: true }); + // Always, including after a failure: an in-place run leaves the repository resolving the library + // to a tarball until this puts it back. + if (resolveMode === 'in-place') { + await restoreWorkspace(repoRoot, outputDir); + } + } +} + +/** + * Uploads a report and refreshes the pull request comment. + * + * A failure to refresh the comment does not fail the run: the numbers are already measured, written + * and uploaded, and losing them to a comment API hiccup would waste the whole benchmark. + */ +async function publishReport(report: TachometerReport): Promise { + const metadata = await getCiMetadata(); + if (!metadata.repo) { + console.warn( + chalk.yellow('Skipping upload: no repository detected, which usually means this is not CI.'), + ); + return; + } + + await uploadCiReport({ version: 1, reportType: 'tachometer', ...metadata, report }); + + try { + console.log('Syncing PR comment via the dashboard API…'); + const result = await syncPrComment(metadata.repo); + console.log( + result.skipped ? 'No open PR found for this branch, skipping.' : 'PR comment synced.', + ); + } catch (error) { + console.error( + chalk.yellow( + `Failed to sync the PR comment: ${error instanceof Error ? error.message : error}`, + ), + ); + } +} diff --git a/packages/benchmark/src/tachometer/summarizeCase.test.ts b/packages/benchmark/src/tachometer/summarizeCase.test.ts new file mode 100644 index 0000000000..11678777dc --- /dev/null +++ b/packages/benchmark/src/tachometer/summarizeCase.test.ts @@ -0,0 +1,203 @@ +import { describe, expect, it, vi } from 'vitest'; +import { summarizeCase, verdictOf } from './summarizeCase'; + +type FakeDifference = { ms: [number, number]; pct: [number, number] } | null; + +/** A tachometer benchmark entry. */ +function benchmark({ + name, + measurement, + differences = [], +}: { + /** Entry name, as tachometer reports it. */ + name: string; + measurement?: string; + /** Differences against every entry, by flat index. */ + differences?: FakeDifference[]; +}): any { + return { + name, + mean: { low: 1, high: 2 }, + samples: [1, 2, 3], + measurement: measurement === undefined ? undefined : { name: measurement }, + differences: differences.map((difference) => + difference + ? { + absolute: { low: difference.ms[0], high: difference.ms[1] }, + percentChange: { low: difference.pct[0], high: difference.pct[1] }, + } + : null, + ), + }; +} + +/** + * A discovered case, as `discoverCases` would return it. `variants` is in order; the first is the + * reference. + */ +function discovered(variants: string[], measurements: string[]): any { + return { + name: 'example', + configPath: '/tmp/tachometer.json', + config: {}, + comparison: 'baseline', + leaves: [], + variants, + measurements, + }; +} + +describe('verdictOf', () => { + it('is slower when both intervals sit above zero', () => { + expect(verdictOf({ absolute: { low: 1, high: 3 }, percentChange: { low: 2, high: 5 } })).toBe( + 'slower', + ); + }); + + it('is faster when both intervals sit below zero', () => { + expect( + verdictOf({ absolute: { low: -3, high: -1 }, percentChange: { low: -5, high: -2 } }), + ).toBe('faster'); + }); + + it('is unsure when the intervals disagree', () => { + // The two are computed separately from the same samples, so near zero one can exclude it while + // the other does not. A verdict resting on only one of them would flip depending which is read. + expect(verdictOf({ absolute: { low: 1, high: 3 }, percentChange: { low: -1, high: 5 } })).toBe( + 'unsure', + ); + }); + + it('is unsure when both intervals straddle zero', () => { + expect(verdictOf({ absolute: { low: -1, high: 3 }, percentChange: { low: -2, high: 5 } })).toBe( + 'unsure', + ); + }); +}); + +describe('summarizeCase', () => { + it('reports every variant of a single-measurement case', () => { + const entry = discovered(['a', 'b'], ['mount']); + const summary = summarizeCase(entry, { + benchmarks: [ + benchmark({ + name: 'a', + measurement: 'mount', + differences: [null, { ms: [-3, -1], pct: [-5, -2] }], + }), + benchmark({ + name: 'b', + measurement: 'mount', + differences: [{ ms: [1, 3], pct: [2, 5] }, null], + }), + ], + }); + + expect(summary.reference).toBe('a'); + expect(summary.measurements).toHaveLength(1); + expect(summary.measurements[0].variants.map((variant) => variant.variant)).toEqual(['a', 'b']); + expect(summary.measurements[0].comparisons).toMatchObject([ + { variant: 'b', verdict: 'faster' }, + ]); + }); + + it('pairs results by measurement rather than by position', () => { + // Tachometer flattens variants × measurements into one list and names entries + // ` []`. Reading the difference matrix positionally would compare `mount` + // against `scroll`. Here the flat order deliberately interleaves the two measurements. + const entry = discovered(['a', 'b'], ['mount', 'scroll']); + const summary = summarizeCase(entry, { + benchmarks: [ + // index 0 + benchmark({ + name: 'a [mount]', + measurement: 'mount', + differences: [null, null, { ms: [-3, -1], pct: [-5, -2] }, null], + }), + // index 1 + benchmark({ + name: 'a [scroll]', + measurement: 'scroll', + differences: [null, null, null, { ms: [1, 3], pct: [2, 5] }], + }), + // index 2 + benchmark({ name: 'b [mount]', measurement: 'mount', differences: [] }), + // index 3 + benchmark({ name: 'b [scroll]', measurement: 'scroll', differences: [] }), + ], + }); + + const [mount, scroll] = summary.measurements; + expect(mount.name).toBe('mount'); + expect(mount.comparisons).toMatchObject([{ variant: 'b', verdict: 'faster' }]); + expect(scroll.name).toBe('scroll'); + expect(scroll.comparisons).toMatchObject([{ variant: 'b', verdict: 'slower' }]); + }); + + it('strips the measurement suffix when pairing variants back', () => { + const entry = discovered(['a', 'b'], ['mount']); + const summary = summarizeCase(entry, { + benchmarks: [ + benchmark({ + name: 'a [mount]', + measurement: 'mount', + differences: [null, { ms: [-3, -1], pct: [-5, -2] }], + }), + benchmark({ name: 'b [mount]', measurement: 'mount', differences: [] }), + ], + }); + + expect(summary.measurements[0].variants.map((variant) => variant.variant)).toEqual(['a', 'b']); + }); + + it('records the reverse comparison, which is not the negation of the forward one', () => { + // Each direction has its own denominator. + const entry = discovered(['a', 'b'], ['mount']); + const summary = summarizeCase(entry, { + benchmarks: [ + benchmark({ + name: 'a', + measurement: 'mount', + differences: [null, { ms: [-3, -1], pct: [-5, -2] }], + }), + benchmark({ + name: 'b', + measurement: 'mount', + differences: [{ ms: [1, 3], pct: [3, 9] }, null], + }), + ], + }); + + expect(summary.measurements[0].comparisons[0]).toMatchObject({ + verdict: 'faster', + versusReference: { verdict: 'slower', percentChange: { low: 3, high: 9 } }, + }); + }); + + it('skips a variant with no result for a measurement instead of failing the case', () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); + const entry = discovered(['a', 'b'], ['mount']); + + const summary = summarizeCase(entry, { + benchmarks: [benchmark({ name: 'a', measurement: 'mount', differences: [null] })], + }); + + expect(summary.measurements[0].variants.map((variant) => variant.variant)).toEqual(['a']); + expect(warn).toHaveBeenCalled(); + warn.mockRestore(); + }); + + it('throws when the reference itself has no result for a measurement', () => { + const entry = discovered(['a'], ['mount']); + + expect(() => summarizeCase(entry, { benchmarks: [] })).toThrow( + 'Case "example": no "mount" result for the reference "a".', + ); + }); + + it('throws when the case declares no variants', () => { + expect(() => summarizeCase(discovered([], ['mount']), { benchmarks: [] })).toThrow( + 'Case "example" declares no variants.', + ); + }); +}); diff --git a/packages/benchmark/src/tachometer/summarizeCase.ts b/packages/benchmark/src/tachometer/summarizeCase.ts new file mode 100644 index 0000000000..19ab0c9da6 --- /dev/null +++ b/packages/benchmark/src/tachometer/summarizeCase.ts @@ -0,0 +1,184 @@ +import chalk from 'chalk'; +import type { BenchmarkCase, CaseComparison } from './discoverCases'; + +/** Reduces tachometer's raw JSON output to a flat, interpretable summary. */ + +export interface ConfidenceInterval { + low: number; + high: number; +} + +export interface TachometerDifference { + /** Absolute difference, in milliseconds. */ + absolute: ConfidenceInterval; + /** Relative difference, as a percentage. */ + percentChange: ConfidenceInterval; +} + +export interface TachometerBenchmark { + /** Variant name, suffixed with ` []` when a page has several. */ + name: string; + /** Mean duration, as a 95% confidence interval. */ + mean: ConfidenceInterval; + samples: number[]; + /** The measurement this entry reports. */ + measurement?: { name?: string }; + /** This entry against every other, by flat index. */ + differences: Array; +} + +export interface TachometerJson { + /** One entry per variant × measurement. */ + benchmarks: TachometerBenchmark[]; +} + +export type Verdict = 'faster' | 'slower' | 'unsure'; + +export interface VariantSummary { + variant: string; + meanMs: ConfidenceInterval; + samples: number; +} + +export interface ComparisonSummary { + variant: string; + verdict: Verdict; + absoluteMs: ConfidenceInterval; + percentChange: ConfidenceInterval; + versusReference?: { + verdict: Verdict; + absoluteMs: ConfidenceInterval; + percentChange: ConfidenceInterval; + }; +} + +export interface MeasurementSummary { + name: string; + variants: VariantSummary[]; + comparisons: ComparisonSummary[]; +} + +export interface CaseSummary { + name: string; + /** What the case compares, which is what says whether a difference is a regression. */ + comparison: CaseComparison; + reference: string; + measurements: MeasurementSummary[]; +} + +/** + * Judges a difference. + * + * A difference only counts as `faster`/`slower` when the millisecond interval and the percent + * interval both exclude zero on the same side. They come from the same samples but are computed + * separately, so near zero one can exclude it while the other does not — and a verdict that rests + * on only one of them flips depending on which is read. + */ +export function verdictOf(difference: TachometerDifference): Verdict { + if (difference.absolute.low > 0 && difference.percentChange.low > 0) { + return 'slower'; + } + if (difference.absolute.high < 0 && difference.percentChange.high < 0) { + return 'faster'; + } + return 'unsure'; +} + +/** + * Indexes tachometer's flat benchmark list by measurement and then by variant, keeping each entry's + * position so the difference matrix can be read back. + */ +function indexByMeasurement( + json: TachometerJson, +): Map> { + const byMeasurement = new Map< + string, + Map + >(); + json.benchmarks.forEach((benchmark, index) => { + const measurement = benchmark.measurement?.name ?? ''; + const suffix = ` [${measurement}]`; + const variant = benchmark.name.endsWith(suffix) + ? benchmark.name.slice(0, -suffix.length) + : benchmark.name; + const forMeasurement = byMeasurement.get(measurement) ?? new Map(); + forMeasurement.set(variant, { benchmark, index }); + byMeasurement.set(measurement, forMeasurement); + }); + return byMeasurement; +} + +/** + * Reduces one case's tachometer output to a per-measurement summary. + * + * Tachometer flattens variants × measurements into one list, naming entries + * ` []` when a page declares more than one, with a difference matrix over the + * whole flat list. Results are paired back by (variant, measurement) name — pairing by position + * would compare unrelated measurements as soon as a case has more than one. + * + * The first declared variant is the reference, because that is the one under test: for the + * auto-expanded regression case it is `[current]`. + */ +export function summarizeCase(entry: BenchmarkCase, json: TachometerJson): CaseSummary { + const [reference] = entry.variants; + if (!reference) { + throw new Error(`Case "${entry.name}" declares no variants.`); + } + + const byMeasurement = indexByMeasurement(json); + + const measurements = entry.measurements.map((measurement): MeasurementSummary => { + const byVariant = byMeasurement.get(measurement); + const lookup = (variant: string) => byVariant?.get(variant); + + const referenceResult = lookup(reference); + if (!referenceResult) { + throw new Error( + `Case "${entry.name}": no "${measurement}" result for the reference "${reference}".`, + ); + } + + const variants: VariantSummary[] = []; + const comparisons: ComparisonSummary[] = []; + for (const variant of entry.variants) { + const found = lookup(variant); + if (!found) { + console.warn( + chalk.yellow(` ${entry.name}: no "${measurement}" result for "${variant}"; skipping.`), + ); + continue; + } + variants.push({ + variant, + meanMs: found.benchmark.mean, + samples: found.benchmark.samples.length, + }); + if (variant === reference) { + continue; + } + const difference = referenceResult.benchmark.differences[found.index]; + if (!difference) { + continue; + } + // The same pair read from the variant's own matrix row. Each direction has its own + // denominator, so it is not the negation of `difference`. + const reverse = found.benchmark.differences[referenceResult.index]; + comparisons.push({ + variant, + verdict: verdictOf(difference), + absoluteMs: difference.absolute, + percentChange: difference.percentChange, + versusReference: reverse + ? { + verdict: verdictOf(reverse), + absoluteMs: reverse.absolute, + percentChange: reverse.percentChange, + } + : undefined, + }); + } + return { name: measurement, variants, comparisons }; + }); + + return { name: entry.name, comparison: entry.comparison, reference, measurements }; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c437218ac6..901b1ccff5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -188,6 +188,9 @@ importers: baseline-browser-mapping: specifier: 2.11.21 version: 2.11.21 + chromedriver: + specifier: ^151.0.5 + version: 151.0.5(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) eslint-plugin-n: specifier: 18.3.0 version: 18.3.0(@typescript/typescript6@6.0.2)(eslint@10.10.0(jiti@2.7.0)(supports-color@10.2.2)) @@ -1264,6 +1267,31 @@ importers: specifier: 4.1.11 version: 4.1.11(@types/node@22.20.1)(@vitest/browser-playwright@4.1.11)(@vitest/coverage-v8@4.1.11)(jsdom@30.0.1)(vite@8.2.2(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0)) + test/tachometer: + dependencies: + '@mui/internal-test-utils': + specifier: workspace:* + version: link:../../packages/test-utils/build + react: + specifier: 19.2.8 + version: 19.2.8 + devDependencies: + '@mui/internal-benchmark': + specifier: workspace:* + version: link:../../packages/benchmark/build + '@playwright/test': + specifier: 1.62.1 + version: 1.62.1 + '@types/react': + specifier: 19.2.18 + version: 19.2.18 + tachometer: + specifier: ^0.7.2 + version: 0.7.2(supports-color@10.2.2) + vite: + specifier: ^8.2.2 + version: 8.2.2(@types/node@22.20.1)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.0) + packages: '@-xun/debug@2.0.2': @@ -2002,6 +2030,9 @@ packages: '@types/react': optional: true + '@bazel/runfiles@6.5.0': + resolution: {integrity: sha512-RzahvqTkfpY2jsDxo8YItPX+/iZ6hbiikw1YhE0bA9EKBR5Og8Pa6FHn9PO9M0zaXRVsr0GFQLKbB/0rzy9SzA==} + '@bcoe/v8-coverage@1.0.2': resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} engines: {node: '>=18'} @@ -2452,6 +2483,9 @@ packages: '@floating-ui/utils@0.2.12': resolution: {integrity: sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==} + '@hapi/bourne@3.0.0': + resolution: {integrity: sha512-Waj1cwPXJDucOib4a3bAISsKJVb15MKi9IvmTI/7ssVEm6sywXGjVJDhl6/umt1pK1ZS7PacXU3A1PmFKHEZ2w==} + '@html-validate/stylish@6.0.0': resolution: {integrity: sha512-d1/lI3qIXhGVJF3+MmKEBn1dRX6U4Z+BDaOdDI3E6SXcO+OQ7hC/3mSo6BIjwQlcuV6NdDOKm6QCrzIQL1EezQ==} engines: {node: ^22.16.0 || >= 24.0.0} @@ -3678,6 +3712,10 @@ packages: '@sinclair/typebox@0.34.52': resolution: {integrity: sha512-XiMQh7qqVlxZzcVD+kkGMNGMzcTrDMLWI7S4x7z1MkCkbDPrekpZXEUK0eZqZFMuHQg2a2DZOcDIh9o5v3Gonw==} + '@sindresorhus/is@5.6.0': + resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==} + engines: {node: '>=14.16'} + '@sindresorhus/merge-streams@4.0.0': resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} @@ -3712,6 +3750,10 @@ packages: '@swc/helpers@0.5.23': resolution: {integrity: sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==} + '@szmarczak/http-timer@5.0.1': + resolution: {integrity: sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==} + engines: {node: '>=14.16'} + '@tanstack/query-core@5.102.8': resolution: {integrity: sha512-ZNjkJ33CqvPNec/6lZBnHqLc3EVGPZ9ySLhYahU9TcuRFdmwXewuj0c4hwSWcGHqEUwcSrKeZ+oGcvPBqXcQcg==} @@ -3720,6 +3762,9 @@ packages: peerDependencies: react: ^18 || ^19 + '@testim/chrome-version@1.1.4': + resolution: {integrity: sha512-kIhULpw9TrGYnHp/8VfdcneIcxKnLixmADtukQRtJUmsVlMg0niMkwV0xZmi8hqa57xqilIHjWFA0GKvEjVU5g==} + '@testing-library/dom@10.4.1': resolution: {integrity: sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==} engines: {node: '>=18'} @@ -3847,6 +3892,9 @@ packages: '@types/etag@1.8.4': resolution: {integrity: sha512-f1z/UMth8gQ6636NBqhFmJ3zES7EuDcUnV6K1gl1osHp+85KPKX+VixYWUpqLkw1fftCagyHJjJOZjZkEi2rHw==} + '@types/execa@0.9.0': + resolution: {integrity: sha512-mgfd93RhzjYBUHHV532turHC2j4l/qxsF/PbfDmprHDEUHmNZGlDn1CEsulGK3AfsPdhkWzZQT/S/k0UGhLGsA==} + '@types/find-package-json@1.2.7': resolution: {integrity: sha512-zHPw+sj/9u0cHV8cS99fizuCSEo3K5pk6JxDu6K3DEJAEZgdvjoHnFJz/C1Vzwr75GSPlAcDZR9O30/4CuQo9Q==} @@ -3859,6 +3907,9 @@ packages: '@types/hast@3.0.5': resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==} + '@types/http-cache-semantics@4.2.0': + resolution: {integrity: sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q==} + '@types/is-empty@1.2.3': resolution: {integrity: sha512-4J1l5d79hoIvsrKh5VUKVRA1aIdsOb10Hu5j3J2VfP/msDnfTdGPmNp2E1Wg+vs97Bktzo+MZePFFXSGoykYJw==} @@ -3883,6 +3934,9 @@ packages: '@types/ms@2.1.0': resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} + '@types/node@11.15.54': + resolution: {integrity: sha512-1RWYiq+5UfozGsU6MwJyFX6BtktcT10XRjvcAQmskCtMcW3tPske88lM/nHv7BQG1w9KBXI1zPGuu5PnNCX14g==} + '@types/node@14.18.63': resolution: {integrity: sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==} @@ -3903,6 +3957,9 @@ packages: resolution: {integrity: sha512-EULJ8LApcVEPbrfND0cRQqutIOdiIgJ1Mgrhpy755r14xMohPTEpkV/k28SJvuOs9bHRFW8x+KeDAEPiGQPB9Q==} deprecated: This is a stub types definition. parse-path provides its own type definitions, so you do not need this installed. + '@types/parse5@5.0.3': + resolution: {integrity: sha512-kUNnecmtkunAoQ3CnjmMkzNU/gtxG8guhi+Fk2U/kOpIKjIMKnXGp4IJCgQJrXSgMsWYimYG4TGjz/UzbGEBTw==} + '@types/prop-types@15.7.15': resolution: {integrity: sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==} @@ -4416,6 +4473,10 @@ packages: resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} + accepts@1.3.8: + resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} + engines: {node: '>= 0.6'} + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -4430,6 +4491,18 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + adm-zip@0.6.1: + resolution: {integrity: sha512-Xwrja8nx9e5o2N1my4DsKCeKpdrnACyr1wtbPxBDgGzKzKyE9kRtBFA8mWldI+RVlD7CBZNWY/wQ2+ydwOR6kQ==} + engines: {node: '>=14.0'} + + agent-base@6.0.2: + resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} + engines: {node: '>= 6.0.0'} + + agent-base@9.0.0: + resolution: {integrity: sha512-TQf59BsZnytt8GdJKLPfUZ54g/iaUL2OWDSFCCvMOhsHduDQxO8xC4PNeyIkVcA5KwL2phPSv0douC0fgWzmnA==} + engines: {node: '>= 20'} + ajv-formats@2.1.1: resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} peerDependencies: @@ -4455,6 +4528,15 @@ packages: ansi-align@3.0.1: resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} + ansi-escape-sequences@6.2.4: + resolution: {integrity: sha512-2KJQAG1Nk4Iyu0dJENKXQJE9smEASrpu/E0F7LSnR72tQXngKGLqfRkHbkinjNct5vvAQY4BwQNt+4Tvg73nDQ==} + engines: {node: '>=12.17'} + peerDependencies: + '@75lb/nature': latest + peerDependenciesMeta: + '@75lb/nature': + optional: true + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -4463,6 +4545,10 @@ packages: resolution: {integrity: sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==} engines: {node: '>=12'} + ansi-styles@3.2.1: + resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} + engines: {node: '>=4'} + ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} @@ -4498,6 +4584,18 @@ packages: resolution: {integrity: sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==} engines: {node: '>= 0.4'} + array-back@3.1.0: + resolution: {integrity: sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==} + engines: {node: '>=6'} + + array-back@4.0.2: + resolution: {integrity: sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==} + engines: {node: '>=8'} + + array-back@6.2.3: + resolution: {integrity: sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==} + engines: {node: '>=12.17'} + array-buffer-byte-length@1.0.2: resolution: {integrity: sha512-LHE+8BuR7RYGDKvnrmcuSq3tDcKv9OFEXQt/HpbZhY7V6h0zlUXutnAD82GiFx9rdieCMjkvtcsPqBwgUl1Iiw==} engines: {node: '>= 0.4'} @@ -4543,6 +4641,10 @@ packages: ast-types-flow@0.0.8: resolution: {integrity: sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==} + ast-types@0.13.4: + resolution: {integrity: sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==} + engines: {node: '>=4'} + ast-v8-to-istanbul@1.0.5: resolution: {integrity: sha512-UPAgKJFSEGMWSDr3LX4tqnAb4f7KGT8O40Tyx8wbYmmZ/yn58lNCm8h3svs3eXgiGd5AXxz8NDOvXWvicq+rJA==} @@ -4558,6 +4660,9 @@ packages: resolution: {integrity: sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA==} engines: {node: '>= 0.4'} + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + autoprefixer@10.5.5: resolution: {integrity: sha512-uiRYvQYe/nNSzBJ7OUnd2/TZVsAdob3blml44teEpee9Cc1f4rGZFewO+JT3Wo8mgFOSzNqes4FHZn/Qz8WOuw==} engines: {node: ^10 || ^12 || >=14} @@ -4577,6 +4682,9 @@ packages: resolution: {integrity: sha512-UzGt8zg7Ny8djbYMhxl2zuEevVa7r2gJjYY5Lwr1xM7+XU2nd6CkIWFTVcCIbAP63vSz71NaVyyuSk9lHKcy0A==} engines: {node: '>=4'} + axios@1.20.0: + resolution: {integrity: sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==} + axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} @@ -4648,6 +4756,10 @@ packages: engines: {node: '>=6.0.0'} hasBin: true + basic-ftp@5.3.1: + resolution: {integrity: sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw==} + engines: {node: '>=10.0.0'} + bcrypt-pbkdf@1.0.2: resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} @@ -4693,6 +4805,9 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true + buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -4715,6 +4830,18 @@ packages: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} + cache-content-type@1.0.1: + resolution: {integrity: sha512-IKufZ1o4Ut42YUrZSo8+qnMTrFuKkvyoLXUywKz9GJ5BrhOFGhLdkx9sG4KAnVvbY6kEcSFjLQul+DVmBm2bgA==} + engines: {node: '>= 6.0.0'} + + cacheable-lookup@7.0.0: + resolution: {integrity: sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==} + engines: {node: '>=14.16'} + + cacheable-request@10.2.14: + resolution: {integrity: sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==} + engines: {node: '>=14.16'} + cacheable@2.5.0: resolution: {integrity: sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g==} @@ -4762,6 +4889,10 @@ packages: resolution: {integrity: sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==} engines: {node: '>=12'} + chalk@2.4.2: + resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} + engines: {node: '>=4'} + chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -4798,6 +4929,11 @@ packages: resolution: {integrity: sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ==} engines: {node: '>=6.0'} + chromedriver@151.0.5: + resolution: {integrity: sha512-oPI/VtTeLePbkTOI5o4yvEOlcdpmqEdqQNLZbyu5ha6kpDQYfDRuliisDiph29hBK/UzHSmOyZFKX1mE3efYOQ==} + engines: {node: '>=22'} + hasBin: true + ci-info@4.4.0: resolution: {integrity: sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==} engines: {node: '>=8'} @@ -4840,6 +4976,14 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + co-body@6.2.0: + resolution: {integrity: sha512-Kbpv2Yd1NdL1V/V4cwLVxraHDV6K8ayohr2rmH0J87Er8+zJjcTa6dAn9QMPC9CRgU8+aNajKbSf1TzDB1yKPA==} + engines: {node: '>=8.0.0'} + + co@4.6.0: + resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} + engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} + codsen-format-diagnostic-value@1.1.0: resolution: {integrity: sha512-Zlza4Webzp1e7/9EaOF6AYZjkdeUnbqMLMCbgJx0Zfw9+aDSDR+xZxExRLYz0zk7YvG5ZHVxSLU0OLzH2PbJEQ==} engines: {node: '>=18.20.8'} @@ -4851,19 +4995,37 @@ packages: collapse-white-space@2.1.0: resolution: {integrity: sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==} + color-convert@1.9.3: + resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} + color-name@1.1.3: + resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} + color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} colord@2.10.0: resolution: {integrity: sha512-AidJptpBJmjTclAp9BkLwJi0T93fo5epJnbaZslpg6QVzpHjAiveF55mE9AcUJiGMqRHgMDY8soMsQtuNYMHfw==} + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + command-line-args@5.2.1: + resolution: {integrity: sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==} + engines: {node: '>=4.0.0'} + + command-line-usage@6.1.3: + resolution: {integrity: sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==} + engines: {node: '>=8.0.0'} + commander@2.20.3: resolution: {integrity: sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==} @@ -4882,6 +5044,9 @@ packages: commondir@1.0.1: resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} + compare-versions@6.1.1: + resolution: {integrity: sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg==} + compressible@2.0.18: resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} engines: {node: '>= 0.6'} @@ -4901,6 +5066,10 @@ packages: resolution: {integrity: sha512-kRGRZw3bLlFISDBgwTSA1TMBFN6J6GWDeubmDE3AF+3+yXL8hTWv8r5rkLbqYXY4RjPk/EzHnClI3zQf1cFmHA==} engines: {node: '>= 0.6'} + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + content-type@3.0.0: resolution: {integrity: sha512-AIi5H6p0xk5uknXcN3/rmhP8jgp69OfSe/JuKiQAFprJ7UGw7mwj7m4XcmDzlrnJDG+cGpphAINGdU3g3g7kDw==} engines: {node: '>=22'} @@ -4915,6 +5084,13 @@ packages: resolution: {integrity: sha512-JtXpxqDqJ8P0UwEHwhxLzCIXQy97vlYBZR222Sbzb1q1Erex9ASrztJ29SyhWFQjod1AeFBaPzEEC8YvtZMIYg==} engines: {node: '>=6'} + cookies@0.9.2: + resolution: {integrity: sha512-8BIbcC6tPZMv2/PD4PrB3CtKks7LmAWhA7FRZK/6/b/WJ2RUz00KCga7gq50S4/RhZ22kxefRFGgsuts0v3eoA==} + engines: {node: '>= 0.8'} + + copy-to@2.0.1: + resolution: {integrity: sha512-3DdaFaU/Zf1AnpLiFDeNCD4TOWe3Zl2RZaTzUvWiIk5ERzcCodOE20Vqq4fzCbNoHURFHT4/us/Lfq+S2zyY4w==} + core-js-compat@3.50.0: resolution: {integrity: sha512-XGpFGbMLHwSt74YLTKho7Ib242qi6O8MSX+sRokV4oz7iKXvQWGYZthjIhjRGMxjzVkAubBO512dKGYcefmX3Q==} engines: {node: '>=6.4.0'} @@ -4942,6 +5118,10 @@ packages: resolution: {integrity: sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==} engines: {node: '>=10.0.0'} + cross-spawn@6.0.6: + resolution: {integrity: sha512-VqCUuhcd1iB+dsv8gxPttb5iZh/D0iubSP21g36KXdEuf6I5JiioesUVjpCdHV9MZRUfVFlvwtIUyPfxo5trtw==} + engines: {node: '>=4.8'} + cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -4976,6 +5156,9 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + csv-stringify@6.9.0: + resolution: {integrity: sha512-6b3e29CILFfSQ2nDRuZMRROzQOpPuJ+s8zPB5H7eK5ncXBbFOyVuRLU5Sx/zAUmVri3Ly49GyC8NEme4SVSAbA==} + d3-array@3.2.4: resolution: {integrity: sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==} engines: {node: '>=12'} @@ -5023,6 +5206,10 @@ packages: damerau-levenshtein@1.0.8: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} + data-uri-to-buffer@8.0.0: + resolution: {integrity: sha512-6UHfyCux51b8PTGDgveqtz1tvphBku5DrMKKJbFAZAJOI2zsjDpDoYE1+QGj7FOMS4BdTFNJsJiR3zEB0xH0yQ==} + engines: {node: '>= 20'} + data-urls@7.0.0: resolution: {integrity: sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==} engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0} @@ -5061,6 +5248,15 @@ packages: supports-color: optional: true + debug@4.3.1: + resolution: {integrity: sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -5080,6 +5276,13 @@ packages: decode-named-character-reference@1.3.0: resolution: {integrity: sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==} + decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + + deep-equal@1.0.1: + resolution: {integrity: sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==} + deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} @@ -5095,6 +5298,10 @@ packages: resolution: {integrity: sha512-m1pAzaJgZ/gssEqlOhJkPJp8Xly7QyW6xcrkUa2KKcDeDSEMP7X8xipU3snUcfisTQx0w1AGae+9UtJSfVnXGw==} engines: {node: '>=18'} + defer-to-connect@2.0.1: + resolution: {integrity: sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==} + engines: {node: '>=10'} + define-data-property@1.1.4: resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} engines: {node: '>= 0.4'} @@ -5107,10 +5314,35 @@ packages: resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} engines: {node: '>= 0.4'} + degenerator@7.0.1: + resolution: {integrity: sha512-ABErK0IefDSyHjlPH7WUEenIAX2rPPnrDcDM+TS3z3+zu9TfyKKi07BQM+8rmxpdE2y1v5fjjdoAS/x4D2U60w==} + engines: {node: '>= 20'} + peerDependencies: + quickjs-wasi: ^2.2.0 + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + delegates@1.0.0: + resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} + + depd@1.1.2: + resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} + engines: {node: '>= 0.6'} + + depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} + destroy@1.2.0: + resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} + engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} + detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} @@ -5161,6 +5393,12 @@ packages: eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} + ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + electron-to-chromium@1.5.422: resolution: {integrity: sha512-UvA/32XqrLDdZSn7Jllo1AYNcWji/G0d5M0GTViE7KoGBiMunw3a34Sb2KO4ZZyrSEhqsxFoVhWWJshdyfKqJA==} @@ -5173,6 +5411,13 @@ packages: emoji-regex@9.2.2: resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==} + encodeurl@1.0.2: + resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} + engines: {node: '>= 0.8'} + + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + enhanced-resolve@5.24.5: resolution: {integrity: sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==} engines: {node: '>=10.13.0'} @@ -5260,6 +5505,13 @@ packages: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} + escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + + escape-string-regexp@1.0.5: + resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} + engines: {node: '>=0.8.0'} + escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -5268,6 +5520,11 @@ packages: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} + escodegen@2.1.0: + resolution: {integrity: sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==} + engines: {node: '>=6.0'} + hasBin: true + eslint-compat-utils@0.5.1: resolution: {integrity: sha512-3z3vFexKIEnjHE3zCMRo6fn/e44U7T1khUjg+Hp0ZQMCigh28rALD0nPFBcGZuiLC5rLZa2ubQHDRln09JfU2Q==} engines: {node: '>=12'} @@ -5456,6 +5713,11 @@ packages: resolution: {integrity: sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} + esprima@4.0.1: + resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} + engines: {node: '>=4'} + hasBin: true + esquery@1.7.0: resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} engines: {node: '>=0.10'} @@ -5501,6 +5763,10 @@ packages: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} + execa@1.0.0: + resolution: {integrity: sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==} + engines: {node: '>=6'} + execa@10.0.1: resolution: {integrity: sha512-ge98qjkRK4IB7tL7Ju/6qmm5LHoH1eEMt5FNZrz3f4UIYhF28lggX20z3FaX1sgc67msLEn0N0BscOs29iuwyw==} engines: {node: '>=22'} @@ -5605,6 +5871,10 @@ packages: find-package-json@1.2.0: resolution: {integrity: sha512-+SOGcLGYDJHtyqHd87ysBhmaeQ95oWspDKnMXBrnQ9Eq4OkLNqejgoaD8xVWu6GPa0B6roa6KinCMEMcVeqONw==} + find-replace@3.0.0: + resolution: {integrity: sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==} + engines: {node: '>=4.0.0'} + find-root@1.1.0: resolution: {integrity: sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==} @@ -5616,6 +5886,10 @@ packages: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} + find-up@6.3.0: + resolution: {integrity: sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + flat-cache@4.0.1: resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} engines: {node: '>=16'} @@ -5629,6 +5903,15 @@ packages: flatted@3.4.4: resolution: {integrity: sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==} + follow-redirects@1.16.0: + resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + for-each@0.3.5: resolution: {integrity: sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==} engines: {node: '>= 0.4'} @@ -5637,6 +5920,14 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} + form-data-encoder@2.1.4: + resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} + engines: {node: '>= 14.17'} + + form-data@4.0.6: + resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} + engines: {node: '>= 6'} + format-util@1.0.5: resolution: {integrity: sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==} @@ -5647,6 +5938,14 @@ packages: fraction.js@5.3.4: resolution: {integrity: sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==} + fresh@0.5.2: + resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} + engines: {node: '>= 0.6'} + + fs-extra@10.1.0: + resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==} + engines: {node: '>=12'} + fs-readdir-recursive@1.1.0: resolution: {integrity: sha512-GNanXlVr2pf02+sPN40XN8HG+ePaNcvM0q5mZBd668Obwb0yD5GiUbZOFgwn8kGMY6I3mdyDJzieUy3PTYyTRA==} @@ -5704,6 +6003,14 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} + get-stream@4.1.0: + resolution: {integrity: sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==} + engines: {node: '>=6'} + + get-stream@5.2.0: + resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} + engines: {node: '>=8'} + get-stream@6.0.1: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} engines: {node: '>=10'} @@ -5723,6 +6030,10 @@ packages: get-tsconfig@4.14.3: resolution: {integrity: sha512-++QEw4DIY7WGoukz+/+A/8dGYPT9l9yIadnmSgZ8Rjr3YVSVDipQSO9CdnJo9ePqFqUUqh+wk9uIaoiAwsiPkA==} + get-uri@8.0.1: + resolution: {integrity: sha512-/5N/P4Lrh0p/mDwlDRi7Y1+P2o/OyzZI3l6Iz1Ov6XXwwm1y3RlZLuo3gVgML99djrEDtV980bBxSuOeHLk8ww==} + engines: {node: '>= 20'} + git-up@8.1.1: resolution: {integrity: sha512-FDenSF3fVqBYSaJoYy1KSc2wosx0gCvKP+c+PRBht7cAaiCeQlBtfBDX9vgnNOHmdePlSFITVcn4pFfcgNvx3g==} @@ -5787,6 +6098,10 @@ packages: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} + got@12.6.1: + resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==} + engines: {node: '>=14.16'} + graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} @@ -5798,6 +6113,10 @@ packages: resolution: {integrity: sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg==} engines: {node: '>= 0.4'} + has-flag@3.0.0: + resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} + engines: {node: '>=4'} + has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} @@ -5920,6 +6239,41 @@ packages: html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} + http-assert@1.5.0: + resolution: {integrity: sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==} + engines: {node: '>= 0.8'} + + http-cache-semantics@4.2.0: + resolution: {integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==} + + http-errors@1.6.3: + resolution: {integrity: sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==} + engines: {node: '>= 0.6'} + + http-errors@1.8.1: + resolution: {integrity: sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==} + engines: {node: '>= 0.6'} + + http-errors@2.0.1: + resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} + engines: {node: '>= 0.8'} + + http-proxy-agent@9.1.0: + resolution: {integrity: sha512-2NxoveTT58mjYT4n3RPTEfCZGLMbidoO8XEieXfpSYxu+PQJ1qpx4ypwH6N+uF9twBPIvRRgvkvW5HUTYWENig==} + engines: {node: '>= 20'} + + http2-wrapper@2.2.1: + resolution: {integrity: sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==} + engines: {node: '>=10.19.0'} + + https-proxy-agent@5.0.1: + resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} + engines: {node: '>= 6'} + + https-proxy-agent@9.1.0: + resolution: {integrity: sha512-ag87y7cJJ9/3+GxFr8Oy4O5faDsGRGnBGsJj/YjOSsSx/5eadKLYTMPlzuR6obgoCDDm0abAAZitXXQkMOPSpA==} + engines: {node: '>= 20'} + human-signals@2.1.0: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} engines: {node: '>=10.17.0'} @@ -5932,6 +6286,10 @@ packages: resolution: {integrity: sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==} engines: {node: '>=18.18.0'} + iconv-lite@0.4.24: + resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} + engines: {node: '>=0.10.0'} + iconv-lite@0.7.3: resolution: {integrity: sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==} engines: {node: '>=0.10.0'} @@ -5968,10 +6326,17 @@ packages: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} engines: {node: '>=0.8.19'} + inflation@2.1.0: + resolution: {integrity: sha512-t54PPJHG1Pp7VQvxyVCJ9mBbjG3Hqryges9bXoOO6GExCPa+//i/d5GSuFtpx3ALLd7lgIAur6zrIlBQyJuMlQ==} + engines: {node: '>= 0.8.0'} + inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. + inherits@2.0.3: + resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==} + inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} @@ -5996,6 +6361,14 @@ packages: resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==} engines: {node: '>=12'} + ip-address@10.7.2: + resolution: {integrity: sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w==} + engines: {node: '>= 12'} + + ip-regex@2.1.0: + resolution: {integrity: sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw==} + engines: {node: '>=4'} + is-alphabetical@2.0.1: resolution: {integrity: sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==} @@ -6158,6 +6531,10 @@ packages: is-ssh@1.4.1: resolution: {integrity: sha512-JNeu1wQsHjyHgn9NcWTaXq6zWSR6hqE0++zhfZlkFBbScNkyvxCdeV8sRkSBaeLKxmbpR21brail63ACNxJ0Tg==} + is-stream@1.1.0: + resolution: {integrity: sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==} + engines: {node: '>=0.10.0'} + is-stream@2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} @@ -6186,6 +6563,9 @@ packages: resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==} engines: {node: '>=18'} + is-url@1.2.4: + resolution: {integrity: sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==} + is-wayland@0.1.0: resolution: {integrity: sha512-QkbMsWkIfkrzOPxenwye0h56iAXirZYHG9eHVPb22fO9y+wPbaX/CHacOWBa/I++4ohTcByimhM1/nyCsH8KNA==} engines: {node: '>=20'} @@ -6210,6 +6590,10 @@ packages: resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==} engines: {node: '>=16'} + is2@2.0.1: + resolution: {integrity: sha512-+WaJvnaA7aJySz2q/8sLjMb2Mw14KTplHmSwcSpZ/fWJPkUmqw3YTzSWbPJ7OAwRvdYTWF2Wg+yYJ1AdP5Z8CA==} + engines: {node: '>=v0.10.0'} + is64bit@2.0.0: resolution: {integrity: sha512-jv+8jaWCl0g2lSBkNSVXdzfBA0npK1HGC2KtWM9FumFRoGS94g3NbCCLVnCYHLjp4GrW2KZeeSTMo5ddtznmGw==} engines: {node: '>=18'} @@ -6321,6 +6705,19 @@ packages: engines: {node: ^18.0.0 || >=20.0.0} hasBin: true + jsonfile@6.2.1: + resolution: {integrity: sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==} + + jsonschema@1.5.0: + resolution: {integrity: sha512-K+A9hhqbn0f3pJX17Q/7H6yQfD/5OXgdrR5UE12gMXCiN9D5Xq2o5mddV2QEcX/bjla99ASsAAQUyMCCRWAEhw==} + + jsonwebtoken@9.0.3: + resolution: {integrity: sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==} + engines: {node: '>=12', npm: '>=6'} + + jstat@1.9.6: + resolution: {integrity: sha512-rPBkJbK2TnA8pzs93QcDDPlKcrtZWuuCo2dVR0TFLOJSxhqfWOVCSp8aV3/oSbn+4uY4yw1URtLpHQedtmXfug==} + jsx-ast-utils@3.3.5: resolution: {integrity: sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==} engines: {node: '>=4.0'} @@ -6328,9 +6725,19 @@ packages: jszip@3.10.1: resolution: {integrity: sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==} + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + + jws@4.0.1: + resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + kebab-case@2.0.2: resolution: {integrity: sha512-NhIP7vecGtqJfNZ85ct0yRQfx//gCJHapJCBZP6/BBcBw/U218Jw7YX7rO6TI5/cCp5dJvlIjN3vbcRTqq7nWA==} + keygrip@1.1.0: + resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} + engines: {node: '>= 0.6'} + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -6349,6 +6756,36 @@ packages: resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} engines: {node: '>=6'} + koa-bodyparser@4.4.1: + resolution: {integrity: sha512-kBH3IYPMb+iAXnrxIhXnW+gXV8OTzCu8VPDqvcDHW9SQrbkHmqPQtiZwrltNmSq6/lpipHnT7k7PsjlVD7kK0w==} + engines: {node: '>=8.0.0'} + + koa-compose@4.1.0: + resolution: {integrity: sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==} + + koa-convert@2.0.0: + resolution: {integrity: sha512-asOvN6bFlSnxewce2e/DK3p4tltyfC4VM7ZwuTuepI7dEQVcvpyFuBcEARu1+Hxg8DIwytce2n7jrZtRlPrARA==} + engines: {node: '>= 10'} + + koa-mount@4.2.0: + resolution: {integrity: sha512-2iHQc7vbA9qLeVq5gKAYh3m5DOMMlMfIKjW/REPAS18Mf63daCJHHVXY9nbu7ivrnYn5PiPC4CE523Tf5qvjeQ==} + engines: {node: '>= 7.6.0'} + + koa-node-resolve@1.0.0-pre.9: + resolution: {integrity: sha512-WKgqe5TGVD6zuR3NrKnmbb/NNHIbWOCezQVqqnyQLdtLLXWgiothlUQT23S5qQGE0Z623jp6jxpMjvAqyrcZFQ==} + + koa-send@5.0.1: + resolution: {integrity: sha512-tmcyQ/wXXuxpDxyNXv5yNNkdAMdFRqwtegBXUaowiQzUKqJehttS0x2j0eOZDQAyloAth5w6wwBImnFzkUz3pQ==} + engines: {node: '>= 8'} + + koa-static@5.0.0: + resolution: {integrity: sha512-UqyYyH5YEXaJrf9S8E23GoJFQZXkBVJ9zYYMPGz919MSX1KuvAcycIuS0ci150HCoPf4XQVhQ84Qf8xRPWxFaQ==} + engines: {node: '>= 7.6.0'} + + koa@2.16.4: + resolution: {integrity: sha512-3An0GCLDSR34tsCO4H8Tef8Pp2ngtaZDAZnsWJYelqXUK5wyiHvGItgK/xcSkmHLSTn1Jcho1mRQs2ehRzvKKw==} + engines: {node: ^4.8.4 || ^6.10.1 || ^7.10.1 || >= 8.1.4} + language-subtag-registry@0.3.23: resolution: {integrity: sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ==} @@ -6455,6 +6892,13 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} + locate-path@7.2.0: + resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + lodash.camelcase@4.3.0: + resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + lodash.clonedeep@4.5.0: resolution: {integrity: sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==} @@ -6467,6 +6911,24 @@ packages: lodash.groupby@4.6.0: resolution: {integrity: sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==} + lodash.includes@4.3.0: + resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} + + lodash.isboolean@3.0.3: + resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} + + lodash.isinteger@4.0.4: + resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==} + + lodash.isnumber@3.0.3: + resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==} + + lodash.isplainobject@4.0.6: + resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} + + lodash.isstring@4.0.1: + resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==} + lodash.memoize@4.1.2: resolution: {integrity: sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==} @@ -6476,6 +6938,9 @@ packages: lodash.mergewith@4.6.2: resolution: {integrity: sha512-GK3g5RPZWTRSeLSpgP8Xhra+pnjBC56q9FZYe1d5RN3TJ35dbkGy3YqBSMbyCrlbi+CM9Z3Jk5yTL7RCsqboyQ==} + lodash.once@4.1.1: + resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==} + lodash.truncate@4.4.2: resolution: {integrity: sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==} @@ -6495,6 +6960,10 @@ packages: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true + lowercase-keys@3.0.0: + resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} @@ -6505,6 +6974,10 @@ packages: lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} + lru-cache@7.18.3: + resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==} + engines: {node: '>=12'} + lru.min@1.1.5: resolution: {integrity: sha512-5J9ysMYUpYIg9RF2vJpy9SinEmSviFSe0GyPpCQ4L5QSkLAgeLXlTAOu2ZwWUU5m+0SBl6gUU1R1ZQB3aKypfA==} engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} @@ -6614,6 +7087,10 @@ packages: mdn-data@2.27.1: resolution: {integrity: sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==} + media-typer@0.3.0: + resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} + engines: {node: '>= 0.6'} + meow@14.1.0: resolution: {integrity: sha512-EDYo6VlmtnumlcBCbh1gLJ//9jvM/ndXHfVXIFrZVr6fGcwTUyCTFNTLCKuY3ffbK8L/+3Mzqnd58RojiZqHVw==} engines: {node: '>=20'} @@ -6741,6 +7218,10 @@ packages: resolution: {integrity: sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==} engines: {node: '>= 0.6'} + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + mime-db@1.54.0: resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} engines: {node: '>= 0.6'} @@ -6749,6 +7230,10 @@ packages: resolution: {integrity: sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==} engines: {node: '>= 0.6'} + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + mime-types@3.0.2: resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} engines: {node: '>=18'} @@ -6761,6 +7246,14 @@ packages: resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==} engines: {node: '>=12'} + mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + + mimic-response@4.0.0: + resolution: {integrity: sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + min-indent@1.0.1: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} engines: {node: '>=4'} @@ -6849,6 +7342,9 @@ packages: ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + ms@2.1.2: + resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -6885,6 +7381,10 @@ packages: natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} + negotiator@0.6.3: + resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} + engines: {node: '>= 0.6'} + negotiator@0.6.4: resolution: {integrity: sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==} engines: {node: '>= 0.6'} @@ -6892,6 +7392,10 @@ packages: neo-async@2.6.2: resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} + netmask@2.1.1: + resolution: {integrity: sha512-eonl3sLUha+S1GzTPxychyhnUzKyeQkZ7jLjKrBagJgPla13F+uQ71HgpFefyHgqrjEbCPkDArxYsjY8/+gLKA==} + engines: {node: '>= 0.4.0'} + next@16.3.4: resolution: {integrity: sha512-/Ztf6CeRH+ejEXUrYtqI4gkS66eFIHuSwqi60RgcpWKodxFZx2/dqVCMKBwILfAHXQ+F1b1vAudgj3mnxqtoIA==} engines: {node: '>=20.9.0'} @@ -6913,6 +7417,9 @@ packages: sass: optional: true + nice-try@1.0.5: + resolution: {integrity: sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==} + node-exports-info@1.6.2: resolution: {integrity: sha512-kXs9Go0cah0qHVV2v389IXQLdLCeE1xfFtjOAF+iobu0OIoG1pje8At2vMHyaPMiPMnG/LWP50twML21eMcAag==} engines: {node: '>= 0.4'} @@ -6940,6 +7447,10 @@ packages: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} + normalize-url@8.1.1: + resolution: {integrity: sha512-JYc0DPlpGWB40kH5g07gGTrYuMqV653k3uBKY6uITPWds3M0ov3GaWGp9lbE3Bzngx8+XkfzgvASb9vk9JDFXQ==} + engines: {node: '>=14.16'} + npm-install-checks@6.3.0: resolution: {integrity: sha512-W29RiK/xtpCGqn6f3ixfRYGk+zRyr+Ew9F2E20BfXxT5/euLdA/Nm7fO7OeTGuAmTs30cpgInyJ0cYe708YTZw==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -6956,6 +7467,10 @@ packages: resolution: {integrity: sha512-nkc+3pIIhqHVQr085X9d2JzPzLyjzQS96zbruppqC9aZRm/x8xx6xhI98gHtsfELP2bE+loHq8ZaHFHhe+NauA==} engines: {node: ^16.14.0 || >=18.0.0} + npm-run-path@2.0.2: + resolution: {integrity: sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==} + engines: {node: '>=4'} + npm-run-path@4.0.1: resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==} engines: {node: '>=8'} @@ -7011,6 +7526,10 @@ packages: resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} engines: {node: '>=12.20.0'} + on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + on-headers@1.1.0: resolution: {integrity: sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==} engines: {node: '>= 0.8'} @@ -7026,6 +7545,9 @@ packages: resolution: {integrity: sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==} engines: {node: '>=12'} + only@0.0.2: + resolution: {integrity: sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==} + open@11.0.2: resolution: {integrity: sha512-RWqF+pBSkqecEvCKOn8QYhaNdRMJDZRIrlS/7rTDdLHaPcfXGCZ/h8zb413NfvdeAV0MR7T1yJcA34/q+CSm1Q==} engines: {node: '>=20'} @@ -7048,6 +7570,14 @@ packages: resolution: {integrity: sha512-19YVAg7T+WTrxggPukVq7DjTv6+PJ867TmhCvBsYwmbFCsZd344rq2Ld1p0wo8f8Qrrhgp82c6FJRqdXWtSEhg==} engines: {node: '>= 0.4'} + p-cancelable@3.0.0: + resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==} + engines: {node: '>=12.20'} + + p-finally@1.0.0: + resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} + engines: {node: '>=4'} + p-limit@2.3.0: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} engines: {node: '>=6'} @@ -7056,6 +7586,10 @@ packages: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} + p-limit@4.0.0: + resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + p-locate@4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -7064,6 +7598,10 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} + p-locate@6.0.0: + resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + p-retry@8.0.1: resolution: {integrity: sha512-NAigyu8hfvJe7MsS18mnc4is0MZtau3QMdBklaJKK23oBQb6occO3UPM3vHmTckW8KhfyjMZaUOqdTFEYliHgg==} engines: {node: '>=22'} @@ -7072,6 +7610,16 @@ packages: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} + pac-proxy-agent@9.1.0: + resolution: {integrity: sha512-1aU+1mpj3DrQPfo3gh+3Gap3G5x+axnMx1P/y0ZF2ch7kb2meyOCAH8K2k9d27ROsTE7TnAerzxqF9aon2jqnA==} + engines: {node: '>= 20'} + + pac-resolver@9.0.1: + resolution: {integrity: sha512-lJbS008tmkj08VhoM8Hzuv/VE5tK9MS0OIQ/7+s0lIF+BYhiQWFYzkSpML7lXs9iBu2jfmzBTLzhe9n6BX+dYw==} + engines: {node: '>= 20'} + peerDependencies: + quickjs-wasi: ^2.2.0 + package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} @@ -7107,23 +7655,38 @@ packages: resolution: {integrity: sha512-bCgsFI+GeGWPAvAiUv63ZorMeif3/U0zaXABGJbOWt5OH2KCaPHF6S+0ok4aqM9RuIPGyZdx9tR9l13PsW4AYQ==} engines: {node: '>=14.13.0'} + parse5@5.1.1: + resolution: {integrity: sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==} + parse5@7.3.0: resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} parse5@8.0.1: resolution: {integrity: sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==} + parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} - path-is-absolute@1.0.1: + path-exists@5.0.0: + resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + path-is-absolute@1.0.1: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} path-is-inside@1.0.2: resolution: {integrity: sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==} + path-key@2.0.1: + resolution: {integrity: sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==} + engines: {node: '>=4'} + path-key@3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} @@ -7179,10 +7742,17 @@ packages: resolution: {integrity: sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==} engines: {node: '>=8'} + pkg-install@1.0.0: + resolution: {integrity: sha512-UGI8bfhrDb1KN01RZ7Bq08GRQc8rmVjxQ2up0g4mUHPCYDTK1FzQ0PMmLOBCHg3yaIijZ2U3Fn9ofLa4N392Ug==} + pkg-pr-new@0.0.88: resolution: {integrity: sha512-Xc6PMJ2gher0WZP+rtjefFk26hIb7V1PTLL30bmy1Z2vsRmSvqiss7Ag1XUdyplTGYzIlFNJp4L3vMNmK44N6g==} hasBin: true + pkg-up@4.0.0: + resolution: {integrity: sha512-N4zdA4sfOe6yCv+ulPCmpnIBQ5I60xfhDr1otdBBhKte9QtEf3bhfrfkW7dTb+IQ0iEx4ZDzas0kc1o5rdWpYg==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + playwright-core@1.62.1: resolution: {integrity: sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==} engines: {node: '>=20'} @@ -7309,6 +7879,10 @@ packages: process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + progress@2.0.3: + resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} + engines: {node: '>=0.4.0'} + promise-inflight@1.0.1: resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==} peerDependencies: @@ -7337,6 +7911,26 @@ packages: protocols@2.0.2: resolution: {integrity: sha512-hHVTzba3wboROl0/aWRRG9dMytgH6ow//STBZh43l/wQgmMhYhOFi0EHWAPtoCz9IAUymsyP0TSBHkhgMEGNnQ==} + proxy-agent-negotiate@1.1.0: + resolution: {integrity: sha512-N8IBcM3UgCVzz2L2Lqv8DVntDnnC8/hiV4nEDUPkqq72TPUgYWjQc+bdZlBPZK9LzPAvOY//gAt0S0DApoOXWQ==} + engines: {node: '>= 20'} + peerDependencies: + kerberos: ^2.0.0 + peerDependenciesMeta: + kerberos: + optional: true + + proxy-agent@8.0.2: + resolution: {integrity: sha512-idLLRewuemWd7GH/BDJzGiB0dWGfT2SQs3jy6NtZtGWU9uPTTSdeC1/cdbqLwgzhfv027daGFuXX426e2Eg20A==} + engines: {node: '>= 20'} + + proxy-from-env@2.1.0: + resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} + engines: {node: '>=10'} + + pump@3.0.4: + resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} + punycode@1.4.1: resolution: {integrity: sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==} @@ -7360,6 +7954,13 @@ packages: queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + quick-lru@5.1.1: + resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} + engines: {node: '>=10'} + + quickjs-wasi@2.2.0: + resolution: {integrity: sha512-zQxXmQMrEoD3S+jQdYsloq4qAuaxKFHZj6hHqOYGwB2iQZH+q9e/lf5zQPXCKOk0WJuAjzRFbO4KwHIp2D05Iw==} + quotation@2.0.3: resolution: {integrity: sha512-yEc24TEgCFLXx7D4JHJJkK4JFVtatO8fziwUxY4nB/Jbea9o9CVS3gt22mA0W7rPYAGW2fWzYDSOtD94PwOyqA==} @@ -7386,6 +7987,10 @@ packages: rasterizehtml@1.4.1: resolution: {integrity: sha512-mcE6b5cKhQO0Qka9vduSQW/F/crQ7EUjrTpXDPAdcj9uNtYijBiNmUOxOrIuMMq7GO/X3CR6UckHTgh5ZVqqBA==} + raw-body@2.5.3: + resolution: {integrity: sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==} + engines: {node: '>= 0.8'} + rc@1.2.8: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true @@ -7452,6 +8057,10 @@ packages: recma-stringify@1.0.0: resolution: {integrity: sha512-cjwII1MdIIVloKvC9ErQ+OgAtwHBmcZ0Bg4ciz78FtbT8In39aAYbaA7zvxQ61xVMSPE8WxhLwLbhif4Js2C+g==} + reduce-flatten@2.0.0: + resolution: {integrity: sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w==} + engines: {node: '>=6'} + reflect.getprototypeof@1.0.10: resolution: {integrity: sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==} engines: {node: '>= 0.4'} @@ -7583,10 +8192,17 @@ packages: reselect@5.3.0: resolution: {integrity: sha512-XGoLeRAVzUTcJ1qkxPQhDJyIZ5d6zzZD9nT7AEZOaaU9UbWclhycElmhO+VD5bFeLuzhPBaOV2oXC8uG35ZSpg==} + resolve-alpn@1.2.1: + resolution: {integrity: sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==} + resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} + resolve-path@1.4.0: + resolution: {integrity: sha512-i1xevIst/Qa+nA9olDxLWnLk8YZbi8R/7JPbCMcgyWaFR6bKWaexgJgEB5oc2PKMjYdrHynyz0NY+if+H98t1w==} + engines: {node: '>= 0.8'} + resolve-pkg-maps@1.0.0: resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} @@ -7600,6 +8216,10 @@ packages: engines: {node: '>= 0.4'} hasBin: true + responselike@3.0.0: + resolution: {integrity: sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==} + engines: {node: '>=14.16'} + retry@0.12.0: resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} engines: {node: '>= 4'} @@ -7661,6 +8281,9 @@ packages: sane-domparser-error@0.2.0: resolution: {integrity: sha512-wxjDV5jty95tNv8N/4WA15UNGqqaor/xX7rnNYY961hifN3bheYoKqtXN+V/M6EUgmUAs6pMul3klwUPMEiVXA==} + sanitize-filename@1.6.4: + resolution: {integrity: sha512-9ZyI08PsvdQl2r/bBIGubpVdR3RR9sY6RDiWFPreA21C/EFlQhmgo20UZlNjZMMZNubusLhAQozkA0Od5J21Eg==} + saxes@5.0.1: resolution: {integrity: sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==} engines: {node: '>=10'} @@ -7676,6 +8299,10 @@ packages: resolution: {integrity: sha512-eflK8wEtyOE6+hsaRVPxvUKYCpRgzLqDTb8krvAsRIwOGlHoSgYLgBXoubGgLd2fT41/OUYdb48v4k4WWHQurA==} engines: {node: '>= 10.13.0'} + selenium-webdriver@4.49.0: + resolution: {integrity: sha512-16XqeOEMj+4+p+pzLLyvNWmF5jCY05EKIl3e1anZBgv4+zq7PUTtiNjKtQhMpwosZ0wIMKDFAQ2xdyogOHD4DA==} + engines: {node: '>= 22.0.0'} + semver@5.7.2: resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} hasBin: true @@ -7718,6 +8345,12 @@ packages: setimmediate@1.0.5: resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==} + setprototypeof@1.1.0: + resolution: {integrity: sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==} + + setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + sharp@0.35.4: resolution: {integrity: sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==} engines: {node: '>=20.9.0'} @@ -7727,10 +8360,18 @@ packages: '@types/node': optional: true + shebang-command@1.2.0: + resolution: {integrity: sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==} + engines: {node: '>=0.10.0'} + shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} + shebang-regex@1.0.0: + resolution: {integrity: sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==} + engines: {node: '>=0.10.0'} + shebang-regex@3.0.0: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} @@ -7784,6 +8425,18 @@ packages: resolution: {integrity: sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==} engines: {node: '>=10'} + smart-buffer@4.2.0: + resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==} + engines: {node: '>= 6.0.0', npm: '>= 3.0.0'} + + socks-proxy-agent@10.1.0: + resolution: {integrity: sha512-WlMj/67cEJ6MDI1OcsnjuYKDNDoyPCCYZ249kuuXPiMDw9F8PXkVaQ7YWu3siTydfQ/4BEZcvGzu+aYvz7dDCQ==} + engines: {node: '>= 20'} + + socks@2.8.10: + resolution: {integrity: sha512-e0VyvkVTwVYViNovRkZ9aodhxVlyoMn7eJhVUPxZ+eK9P/7CBkxvvsBOHqFPEH416726W8tLXXXjKwqgTErrCQ==} + engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -7837,6 +8490,14 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + statuses@1.5.0: + resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==} + engines: {node: '>= 0.6'} + + statuses@2.0.2: + resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} + engines: {node: '>= 0.8'} + std-env@4.2.0: resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} @@ -7928,6 +8589,10 @@ packages: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} + strip-eof@1.0.0: + resolution: {integrity: sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==} + engines: {node: '>=0.10.0'} + strip-final-newline@2.0.0: resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} engines: {node: '>=6'} @@ -8004,6 +8669,10 @@ packages: resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} engines: {node: '>=18'} + supports-color@5.5.0: + resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} + engines: {node: '>=4'} + supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} @@ -8038,14 +8707,31 @@ packages: resolution: {integrity: sha512-ulAk51I9UVUyJgxlv9M6lFot2WP3e7t8Kz9+IS6D4rVba1tR9kON+Ey69f+1R4Q8cd45Lod6a4IcJIxnzGc/zA==} engines: {node: '>=18'} + systeminformation@5.33.13: + resolution: {integrity: sha512-GtyakkchTQKymM1WeAB652zi3rM1q00RKdc+jZe9KGstuc/mWUwAnQGN/RE5gVYvHIR8xYOUIUD9irVKDtA0wA==} + engines: {node: '>=10.0.0'} + os: [darwin, linux, win32, freebsd, openbsd, netbsd, sunos, android] + hasBin: true + + table-layout@1.0.2: + resolution: {integrity: sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==} + engines: {node: '>=8.0.0'} + table@6.9.0: resolution: {integrity: sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==} engines: {node: '>=10.0.0'} + tachometer@0.7.2: + resolution: {integrity: sha512-e9dtDyWuMFyomQxo16vzti6ymoJd3Dp2yGfKrhAiNzCkb2HiP6A0+SWtqE84hcO6tSKZF8eTFqrxM2uKbliJyQ==} + hasBin: true + tapable@2.3.3: resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} engines: {node: '>=6'} + tcp-port-used@1.0.3: + resolution: {integrity: sha512-4CEQ3qRJYo+mtEbJ+OoQu3dF4TDkwaO3RDVC4UzP5cpAOIUWwuwPjD7sdxDFFqsMUjsXVVYBMlg/boAaloThMA==} + terser@5.51.2: resolution: {integrity: sha512-bWnjSNscmuI+GJze6ZupnHP8G/cTcsJF+bXCeQknk2SHQsgbNJnLrqiH9jZ2W4STPVXH2mDKKRX3iwPhc9Cn/Q==} engines: {node: '>=10'} @@ -8095,6 +8781,10 @@ packages: resolution: {integrity: sha512-m1TdR/rvT7kgGJZhspNtXdsdYk0fddFpJJFlG5s+UkPFo6lkLoZ3YLOaovPYjq1R75NP5JfeTlSHaOsE09peCg==} engines: {node: '>=20'} + toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + totalist@3.0.1: resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} engines: {node: '>=6'} @@ -8113,6 +8803,9 @@ packages: trough@2.2.0: resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==} + truncate-utf8-bytes@1.0.2: + resolution: {integrity: sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==} + ts-api-utils@2.5.0: resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} engines: {node: '>=18.12'} @@ -8128,6 +8821,10 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsscmp@1.0.6: + resolution: {integrity: sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==} + engines: {node: '>=0.6.x'} + tsx@4.23.13: resolution: {integrity: sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==} engines: {node: '>=18.0.0'} @@ -8160,6 +8857,10 @@ packages: resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} engines: {node: '>=16'} + type-is@1.6.18: + resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} + engines: {node: '>= 0.6'} + typed-array-buffer@1.0.3: resolution: {integrity: sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==} engines: {node: '>= 0.4'} @@ -8200,6 +8901,18 @@ packages: engines: {node: '>=16.20.0'} hasBin: true + typical@4.0.0: + resolution: {integrity: sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==} + engines: {node: '>=8'} + + typical@5.2.0: + resolution: {integrity: sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==} + engines: {node: '>=8'} + + ua-parser-js@1.0.41: + resolution: {integrity: sha512-LbBDqdIC5s8iROCUjMbW1f5dJQTEFB1+KO9ogbvlb3nm9n4YHa5p4KTvFPWvh2Hs8gZMBuiB1/8+pdfe/tDPug==} + hasBin: true + uint8-to-base64@0.2.1: resolution: {integrity: sha512-uO/84GaoDUfiAxpa8EksjVLE77A9Kc7ZTziN4zRpq4de9yLaLcZn3jx1/sVjyupsywcVX6RKWbqLe7gUNyzH+Q==} @@ -8294,6 +9007,14 @@ packages: universal-user-agent@7.0.3: resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==} + universalify@2.0.1: + resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} + engines: {node: '>= 10.0.0'} + + unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + unrs-resolver@1.12.2: resolution: {integrity: sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ==} @@ -8318,6 +9039,9 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + utf8-byte-length@1.0.5: + resolution: {integrity: sha512-Xn0w3MtiQ6zoz2vFyUVruaCL53O/DwUvkEeOvj+uulMm0BkUGYWmBYVyElqZaSLhY6ZD0ulfU3aBra2aVT4xfA==} + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -8554,6 +9278,10 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} + wordwrapjs@4.0.1: + resolution: {integrity: sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==} + engines: {node: '>=8.0.0'} + wrap-ansi@6.2.0: resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} engines: {node: '>=8'} @@ -8654,10 +9382,18 @@ packages: resolution: {integrity: sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg==} engines: {node: ^20.19.0 || ^22.12.0 || >=23} + ylru@1.4.0: + resolution: {integrity: sha512-2OQsPNEmBCvXuFlIni/a+Rn+R2pHW9INm0BxXJ4hVDA8TirqMj+J/Rp9ItLatT/5pZqWwefVrTQcHpixsxnVlA==} + engines: {node: '>= 4.0.0'} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + yocto-queue@1.2.2: + resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} + engines: {node: '>=12.20'} + yoctocolors@2.2.0: resolution: {integrity: sha512-xYqdZFUK/VYazNl/oCDYN+3WloWQwMfZxBoiNt6qNyk+xfOdi598muWE42rNZFp1kNOiqW936q5RhUdnpqElSg==} engines: {node: '>=18'} @@ -9794,6 +10530,8 @@ snapshots: optionalDependencies: '@types/react': 19.2.18 + '@bazel/runfiles@6.5.0': {} + '@bcoe/v8-coverage@1.0.2': {} '@blazediff/core@1.9.1': {} @@ -10199,6 +10937,8 @@ snapshots: '@floating-ui/utils@0.2.12': {} + '@hapi/bourne@3.0.0': {} + '@html-validate/stylish@6.0.0': {} '@humanfs/core@0.19.2': @@ -11313,6 +12053,8 @@ snapshots: '@sinclair/typebox@0.34.52': {} + '@sindresorhus/is@5.6.0': {} + '@sindresorhus/merge-streams@4.0.0': {} '@smithy/core@3.33.3': @@ -11354,6 +12096,10 @@ snapshots: dependencies: tslib: 2.8.1 + '@szmarczak/http-timer@5.0.1': + dependencies: + defer-to-connect: 2.0.1 + '@tanstack/query-core@5.102.8': {} '@tanstack/react-query@5.102.8(react@19.2.8)': @@ -11361,6 +12107,8 @@ snapshots: '@tanstack/query-core': 5.102.8 react: 19.2.8 + '@testim/chrome-version@1.1.4': {} + '@testing-library/dom@10.4.1': dependencies: '@babel/code-frame': 7.29.7 @@ -11508,6 +12256,10 @@ snapshots: dependencies: '@types/node': 22.20.1 + '@types/execa@0.9.0': + dependencies: + '@types/node': 22.20.1 + '@types/find-package-json@1.2.7': dependencies: '@types/node': 22.20.1 @@ -11520,6 +12272,8 @@ snapshots: dependencies: '@types/unist': 3.0.3 + '@types/http-cache-semantics@4.2.0': {} + '@types/is-empty@1.2.3': {} '@types/jsdom@30.0.0': @@ -11545,6 +12299,8 @@ snapshots: '@types/ms@2.1.0': {} + '@types/node@11.15.54': {} + '@types/node@14.18.63': {} '@types/node@18.19.130': @@ -11565,6 +12321,8 @@ snapshots: dependencies: parse-path: 7.1.0 + '@types/parse5@5.0.3': {} + '@types/prop-types@15.7.15': {} '@types/proper-lockfile@4.1.4': @@ -12075,6 +12833,11 @@ snapshots: abbrev@2.0.0: {} + accepts@1.3.8: + dependencies: + mime-types: 2.1.35 + negotiator: 0.6.3 + acorn-jsx@5.3.2(acorn@8.18.0): dependencies: acorn: 8.18.0 @@ -12085,6 +12848,16 @@ snapshots: acorn@8.18.0: {} + adm-zip@0.6.1: {} + + agent-base@6.0.2(supports-color@10.2.2): + dependencies: + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + agent-base@9.0.0: {} + ajv-formats@2.1.1(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -12119,10 +12892,18 @@ snapshots: dependencies: string-width: 4.2.3 + ansi-escape-sequences@6.2.4: + dependencies: + array-back: 6.2.3 + ansi-regex@5.0.1: {} ansi-regex@6.3.0: {} + ansi-styles@3.2.1: + dependencies: + color-convert: 1.9.3 + ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 @@ -12151,6 +12932,12 @@ snapshots: aria-query@5.3.2: {} + array-back@3.1.0: {} + + array-back@4.0.2: {} + + array-back@6.2.3: {} + array-buffer-byte-length@1.0.2: dependencies: call-bound: 1.0.4 @@ -12230,6 +13017,10 @@ snapshots: ast-types-flow@0.0.8: {} + ast-types@0.13.4: + dependencies: + tslib: 2.8.1 + ast-v8-to-istanbul@1.0.5: dependencies: '@jridgewell/trace-mapping': 0.3.31 @@ -12242,6 +13033,8 @@ snapshots: async-function@1.0.0: {} + asynckit@0.4.0: {} + autoprefixer@10.5.5(postcss@8.5.28): dependencies: browserslist: 4.28.9 @@ -12259,6 +13052,16 @@ snapshots: axe-core@4.13.0: {} + axios@1.20.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2): + dependencies: + follow-redirects: 1.16.0(debug@4.4.3(supports-color@10.2.2)) + form-data: 4.0.6 + https-proxy-agent: 5.0.1(supports-color@10.2.2) + proxy-from-env: 2.1.0 + transitivePeerDependencies: + - debug + - supports-color + axobject-query@4.1.0: {} babel-plugin-macros@3.1.0: @@ -12358,6 +13161,8 @@ snapshots: baseline-browser-mapping@2.11.21: {} + basic-ftp@5.3.1: {} + bcrypt-pbkdf@1.0.2: dependencies: tweetnacl: 0.14.5 @@ -12413,6 +13218,8 @@ snapshots: node-releases: 2.0.54 update-browserslist-db: 1.3.2(browserslist@4.28.9) + buffer-equal-constant-time@1.0.1: {} + buffer-from@1.1.2: {} buildcheck@0.0.7: @@ -12430,6 +13237,23 @@ snapshots: bytes@3.1.2: {} + cache-content-type@1.0.1: + dependencies: + mime-types: 2.1.35 + ylru: 1.4.0 + + cacheable-lookup@7.0.0: {} + + cacheable-request@10.2.14: + dependencies: + '@types/http-cache-semantics': 4.2.0 + get-stream: 6.0.1 + http-cache-semantics: 4.2.0 + keyv: 4.5.4 + mimic-response: 4.0.0 + normalize-url: 8.1.1 + responselike: 3.0.0 + cacheable@2.5.0: dependencies: '@cacheable/memory': 2.2.0 @@ -12475,6 +13299,12 @@ snapshots: dependencies: chalk: 4.1.2 + chalk@2.4.2: + dependencies: + ansi-styles: 3.2.1 + escape-string-regexp: 1.0.5 + supports-color: 5.5.0 + chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -12509,6 +13339,20 @@ snapshots: chrome-trace-event@1.0.4: {} + chromedriver@151.0.5(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2): + dependencies: + '@testim/chrome-version': 1.1.4 + adm-zip: 0.6.1 + axios: 1.20.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) + compare-versions: 6.1.1 + proxy-agent: 8.0.2(supports-color@10.2.2) + proxy-from-env: 2.1.0 + tcp-port-used: 1.0.3(supports-color@10.2.2) + transitivePeerDependencies: + - debug + - kerberos + - supports-color + ci-info@4.4.0: {} cli-boxes@3.0.0: {} @@ -12552,6 +13396,16 @@ snapshots: clsx@2.1.1: {} + co-body@6.2.0: + dependencies: + '@hapi/bourne': 3.0.0 + inflation: 2.1.0 + qs: 6.16.0 + raw-body: 2.5.3 + type-is: 1.6.18 + + co@4.6.0: {} + codsen-format-diagnostic-value@1.1.0: {} codsen-utils@1.10.0: @@ -12560,16 +13414,40 @@ snapshots: collapse-white-space@2.1.0: {} + color-convert@1.9.3: + dependencies: + color-name: 1.1.3 + color-convert@2.0.1: dependencies: color-name: 1.1.4 + color-name@1.1.3: {} + color-name@1.1.4: {} colord@2.10.0: {} + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + comma-separated-tokens@2.0.3: {} + command-line-args@5.2.1: + dependencies: + array-back: 3.1.0 + find-replace: 3.0.0 + lodash.camelcase: 4.3.0 + typical: 4.0.0 + + command-line-usage@6.1.3: + dependencies: + array-back: 4.0.2 + chalk: 2.4.2 + table-layout: 1.0.2 + typical: 5.2.0 + commander@2.20.3: {} commander@4.1.1: {} @@ -12580,6 +13458,8 @@ snapshots: commondir@1.0.1: {} + compare-versions@6.1.1: {} + compressible@2.0.18: dependencies: mime-db: 1.54.0 @@ -12607,6 +13487,8 @@ snapshots: content-disposition@0.5.2: {} + content-type@1.0.5: {} + content-type@3.0.0: {} convert-source-map@1.9.0: {} @@ -12618,6 +13500,13 @@ snapshots: lodash.clonedeep: 4.5.0 yargs-parser: 20.2.9 + cookies@0.9.2: + dependencies: + depd: 2.0.0 + keygrip: 1.1.0 + + copy-to@2.0.1: {} + core-js-compat@3.50.0: dependencies: browserslist: 4.28.9 @@ -12649,6 +13538,14 @@ snapshots: nan: 2.28.0 optional: true + cross-spawn@6.0.6: + dependencies: + nice-try: 1.0.5 + path-key: 2.0.1 + semver: 5.7.2 + shebang-command: 1.2.0 + which: 1.3.1 + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -12683,6 +13580,8 @@ snapshots: csstype@3.2.3: {} + csv-stringify@6.9.0: {} + d3-array@3.2.4: dependencies: internmap: 2.0.3 @@ -12725,6 +13624,8 @@ snapshots: damerau-levenshtein@1.0.8: {} + data-uri-to-buffer@8.0.0: {} + data-urls@7.0.0: dependencies: whatwg-mimetype: 5.0.0 @@ -12766,6 +13667,12 @@ snapshots: optionalDependencies: supports-color: 10.2.2 + debug@4.3.1(supports-color@10.2.2): + dependencies: + ms: 2.1.2 + optionalDependencies: + supports-color: 10.2.2 + debug@4.4.3(supports-color@10.2.2): dependencies: ms: 2.1.3 @@ -12786,6 +13693,12 @@ snapshots: dependencies: character-entities: 2.0.2 + decompress-response@6.0.0: + dependencies: + mimic-response: 3.1.0 + + deep-equal@1.0.1: {} + deep-extend@0.6.0: {} deep-is@0.1.4: {} @@ -12797,6 +13710,8 @@ snapshots: bundle-name: 4.1.0 default-browser-id: 5.0.1 + defer-to-connect@2.0.1: {} + define-data-property@1.1.4: dependencies: es-define-property: 1.0.1 @@ -12811,8 +13726,25 @@ snapshots: has-property-descriptors: 1.0.2 object-keys: 1.1.1 + degenerator@7.0.1(quickjs-wasi@2.2.0): + dependencies: + ast-types: 0.13.4 + escodegen: 2.1.0 + esprima: 4.0.1 + quickjs-wasi: 2.2.0 + + delayed-stream@1.0.0: {} + + delegates@1.0.0: {} + + depd@1.1.2: {} + + depd@2.0.0: {} + dequal@2.0.3: {} + destroy@1.2.0: {} + detect-libc@2.1.2: {} devlop@1.1.0: @@ -12864,6 +13796,12 @@ snapshots: eastasianwidth@0.2.0: {} + ecdsa-sig-formatter@1.0.11: + dependencies: + safe-buffer: 5.2.1 + + ee-first@1.1.1: {} + electron-to-chromium@1.5.422: {} emoji-regex@10.6.0: {} @@ -12872,6 +13810,12 @@ snapshots: emoji-regex@9.2.2: {} + encodeurl@1.0.2: {} + + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + enhanced-resolve@5.24.5: dependencies: graceful-fs: 4.2.11 @@ -13056,10 +14000,22 @@ snapshots: escalade@3.2.0: {} + escape-html@1.0.3: {} + + escape-string-regexp@1.0.5: {} + escape-string-regexp@4.0.0: {} escape-string-regexp@5.0.0: {} + escodegen@2.1.0: + dependencies: + esprima: 4.0.1 + estraverse: 5.3.0 + esutils: 2.0.3 + optionalDependencies: + source-map: 0.6.1 + eslint-compat-utils@0.5.1(eslint@10.10.0(jiti@2.7.0)(supports-color@10.2.2)): dependencies: eslint: 10.10.0(jiti@2.7.0)(supports-color@10.2.2) @@ -13389,6 +14345,8 @@ snapshots: acorn-jsx: 5.3.2(acorn@8.18.0) eslint-visitor-keys: 5.0.1 + esprima@4.0.1: {} + esquery@1.7.0: dependencies: estraverse: 5.3.0 @@ -13438,6 +14396,16 @@ snapshots: events@3.3.0: {} + execa@1.0.0: + dependencies: + cross-spawn: 6.0.6 + get-stream: 4.1.0 + is-stream: 1.1.0 + npm-run-path: 2.0.2 + p-finally: 1.0.0 + signal-exit: 3.0.7 + strip-eof: 1.0.0 + execa@10.0.1: dependencies: '@sindresorhus/merge-streams': 4.0.0 @@ -13577,6 +14545,10 @@ snapshots: find-package-json@1.2.0: {} + find-replace@3.0.0: + dependencies: + array-back: 3.1.0 + find-root@1.1.0: {} find-up@4.1.0: @@ -13589,6 +14561,11 @@ snapshots: locate-path: 6.0.0 path-exists: 4.0.0 + find-up@6.3.0: + dependencies: + locate-path: 7.2.0 + path-exists: 5.0.0 + flat-cache@4.0.1: dependencies: flatted: 3.4.4 @@ -13604,6 +14581,10 @@ snapshots: flatted@3.4.4: {} + follow-redirects@1.16.0(debug@4.4.3(supports-color@10.2.2)): + optionalDependencies: + debug: 4.4.3(supports-color@10.2.2) + for-each@0.3.5: dependencies: is-callable: 1.2.7 @@ -13613,12 +14594,30 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 + form-data-encoder@2.1.4: {} + + form-data@4.0.6: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.4 + mime-types: 2.1.35 + format-util@1.0.5: {} format@0.2.2: {} fraction.js@5.3.4: {} + fresh@0.5.2: {} + + fs-extra@10.1.0: + dependencies: + graceful-fs: 4.2.11 + jsonfile: 6.2.1 + universalify: 2.0.1 + fs-readdir-recursive@1.1.0: {} fs.realpath@1.0.0: {} @@ -13677,6 +14676,14 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.2 + get-stream@4.1.0: + dependencies: + pump: 3.0.4 + + get-stream@5.2.0: + dependencies: + pump: 3.0.4 + get-stream@6.0.1: {} get-stream@8.0.1: {} @@ -13696,6 +14703,14 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 + get-uri@8.0.1(supports-color@10.2.2): + dependencies: + basic-ftp: 5.3.1 + data-uri-to-buffer: 8.0.0 + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + git-up@8.1.1: dependencies: is-ssh: 1.4.1 @@ -13770,6 +14785,20 @@ snapshots: gopd@1.2.0: {} + got@12.6.1: + dependencies: + '@sindresorhus/is': 5.6.0 + '@szmarczak/http-timer': 5.0.1 + cacheable-lookup: 7.0.0 + cacheable-request: 10.2.14 + decompress-response: 6.0.0 + form-data-encoder: 2.1.4 + get-stream: 6.0.1 + http2-wrapper: 2.2.1 + lowercase-keys: 3.0.0 + p-cancelable: 3.0.0 + responselike: 3.0.0 + graceful-fs@4.2.11: {} gzip-size@6.0.0: @@ -13778,6 +14807,8 @@ snapshots: has-bigints@1.1.0: {} + has-flag@3.0.0: {} + has-flag@4.0.0: {} has-flag@5.0.1: {} @@ -13956,12 +14987,76 @@ snapshots: html-void-elements@3.0.0: {} + http-assert@1.5.0: + dependencies: + deep-equal: 1.0.1 + http-errors: 1.8.1 + + http-cache-semantics@4.2.0: {} + + http-errors@1.6.3: + dependencies: + depd: 1.1.2 + inherits: 2.0.3 + setprototypeof: 1.1.0 + statuses: 1.5.0 + + http-errors@1.8.1: + dependencies: + depd: 1.1.2 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 1.5.0 + toidentifier: 1.0.1 + + http-errors@2.0.1: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.2 + toidentifier: 1.0.1 + + http-proxy-agent@9.1.0(supports-color@10.2.2): + dependencies: + agent-base: 9.0.0 + debug: 4.4.3(supports-color@10.2.2) + proxy-agent-negotiate: 1.1.0 + transitivePeerDependencies: + - kerberos + - supports-color + + http2-wrapper@2.2.1: + dependencies: + quick-lru: 5.1.1 + resolve-alpn: 1.2.1 + + https-proxy-agent@5.0.1(supports-color@10.2.2): + dependencies: + agent-base: 6.0.2(supports-color@10.2.2) + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + https-proxy-agent@9.1.0(supports-color@10.2.2): + dependencies: + agent-base: 9.0.0 + debug: 4.4.3(supports-color@10.2.2) + proxy-agent-negotiate: 1.1.0 + transitivePeerDependencies: + - kerberos + - supports-color + human-signals@2.1.0: {} human-signals@5.0.0: {} human-signals@8.0.1: {} + iconv-lite@0.4.24: + dependencies: + safer-buffer: 2.1.2 + iconv-lite@0.7.3: dependencies: safer-buffer: 2.1.2 @@ -13987,11 +15082,15 @@ snapshots: imurmurhash@0.1.4: {} + inflation@2.1.0: {} + inflight@1.0.6: dependencies: once: 1.4.0 wrappy: 1.0.2 + inherits@2.0.3: {} + inherits@2.0.4: {} ini@1.3.8: {} @@ -14014,6 +15113,10 @@ snapshots: internmap@2.0.3: {} + ip-address@10.7.2: {} + + ip-regex@2.1.0: {} + is-alphabetical@2.0.1: {} is-alphanumerical@2.0.1: @@ -14156,6 +15259,8 @@ snapshots: dependencies: protocols: 2.0.2 + is-stream@1.1.0: {} + is-stream@2.0.1: {} is-stream@3.0.0: {} @@ -14179,6 +15284,8 @@ snapshots: is-unicode-supported@2.1.0: {} + is-url@1.2.4: {} + is-wayland@0.1.0: {} is-weakmap@2.0.2: {} @@ -14200,6 +15307,12 @@ snapshots: dependencies: is-inside-container: 1.0.0 + is2@2.0.1: + dependencies: + deep-is: 0.1.4 + ip-regex: 2.1.0 + is-url: 1.2.4 + is64bit@2.0.0: dependencies: system-architecture: 0.1.0 @@ -14312,6 +15425,29 @@ snapshots: dependencies: '@dmsnell/diff-match-patch': 1.1.0 + jsonfile@6.2.1: + dependencies: + universalify: 2.0.1 + optionalDependencies: + graceful-fs: 4.2.11 + + jsonschema@1.5.0: {} + + jsonwebtoken@9.0.3: + dependencies: + jws: 4.0.1 + lodash.includes: 4.3.0 + lodash.isboolean: 3.0.3 + lodash.isinteger: 4.0.4 + lodash.isnumber: 3.0.3 + lodash.isplainobject: 4.0.6 + lodash.isstring: 4.0.1 + lodash.once: 4.1.1 + ms: 2.1.3 + semver: 7.8.5 + + jstat@1.9.6: {} + jsx-ast-utils@3.3.5: dependencies: array-includes: 3.1.9 @@ -14326,8 +15462,23 @@ snapshots: readable-stream: 2.3.8 setimmediate: 1.0.5 + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + + jws@4.0.1: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.2.1 + kebab-case@2.0.2: {} + keygrip@1.1.0: + dependencies: + tsscmp: 1.0.6 + keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -14342,6 +15493,82 @@ snapshots: kleur@4.1.5: {} + koa-bodyparser@4.4.1: + dependencies: + co-body: 6.2.0 + copy-to: 2.0.1 + type-is: 1.6.18 + + koa-compose@4.1.0: {} + + koa-convert@2.0.0: + dependencies: + co: 4.6.0 + koa-compose: 4.1.0 + + koa-mount@4.2.0(supports-color@10.2.2): + dependencies: + debug: 4.4.3(supports-color@10.2.2) + koa-compose: 4.1.0 + transitivePeerDependencies: + - supports-color + + koa-node-resolve@1.0.0-pre.9(supports-color@10.2.2): + dependencies: + '@babel/generator': 7.29.8 + '@babel/parser': 7.29.8 + '@babel/traverse': 7.29.8(supports-color@10.2.2) + '@types/babel__generator': 7.27.0 + '@types/parse5': 5.0.3 + get-stream: 5.2.0 + parse5: 5.1.1 + resolve: 1.22.12 + transitivePeerDependencies: + - supports-color + + koa-send@5.0.1(supports-color@10.2.2): + dependencies: + debug: 4.4.3(supports-color@10.2.2) + http-errors: 1.8.1 + resolve-path: 1.4.0 + transitivePeerDependencies: + - supports-color + + koa-static@5.0.0(supports-color@10.2.2): + dependencies: + debug: 3.2.7(supports-color@10.2.2) + koa-send: 5.0.1(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + koa@2.16.4(supports-color@10.2.2): + dependencies: + accepts: 1.3.8 + cache-content-type: 1.0.1 + content-disposition: 0.5.2 + content-type: 1.0.5 + cookies: 0.9.2 + debug: 4.4.3(supports-color@10.2.2) + delegates: 1.0.0 + depd: 2.0.0 + destroy: 1.2.0 + encodeurl: 1.0.2 + escape-html: 1.0.3 + fresh: 0.5.2 + http-assert: 1.5.0 + http-errors: 1.8.1 + is-generator-function: 1.1.2 + koa-compose: 4.1.0 + koa-convert: 2.0.0 + on-finished: 2.4.1 + only: 0.0.2 + parseurl: 1.3.3 + statuses: 1.5.0 + type-is: 1.6.18 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + language-subtag-registry@0.3.23: {} language-tags@1.0.9: @@ -14425,6 +15652,12 @@ snapshots: dependencies: p-locate: 5.0.0 + locate-path@7.2.0: + dependencies: + p-locate: 6.0.0 + + lodash.camelcase@4.3.0: {} + lodash.clonedeep@4.5.0: {} lodash.debounce@4.0.8: {} @@ -14433,12 +15666,26 @@ snapshots: lodash.groupby@4.6.0: {} + lodash.includes@4.3.0: {} + + lodash.isboolean@3.0.3: {} + + lodash.isinteger@4.0.4: {} + + lodash.isnumber@3.0.3: {} + + lodash.isplainobject@4.0.6: {} + + lodash.isstring@4.0.1: {} + lodash.memoize@4.1.2: {} lodash.merge@4.6.2: {} lodash.mergewith@4.6.2: {} + lodash.once@4.1.1: {} + lodash.truncate@4.4.2: {} lodash.uniq@4.5.0: {} @@ -14453,6 +15700,8 @@ snapshots: dependencies: js-tokens: 4.0.0 + lowercase-keys@3.0.0: {} + lru-cache@10.4.3: {} lru-cache@11.5.2: {} @@ -14461,6 +15710,8 @@ snapshots: dependencies: yallist: 3.1.1 + lru-cache@7.18.3: {} + lru.min@1.1.5: {} lucide-react@1.41.0(react@19.2.8): @@ -14691,6 +15942,8 @@ snapshots: mdn-data@2.27.1: {} + media-typer@0.3.0: {} + meow@14.1.0: {} merge-stream@2.0.0: {} @@ -14975,12 +16228,18 @@ snapshots: mime-db@1.33.0: {} + mime-db@1.52.0: {} + mime-db@1.54.0: {} mime-types@2.1.18: dependencies: mime-db: 1.33.0 + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + mime-types@3.0.2: dependencies: mime-db: 1.54.0 @@ -14989,6 +16248,10 @@ snapshots: mimic-fn@4.0.0: {} + mimic-response@3.1.0: {} + + mimic-response@4.0.0: {} + min-indent@1.0.1: {} minimatch@10.2.6: @@ -15026,6 +16289,8 @@ snapshots: ms@2.0.0: {} + ms@2.1.2: {} + ms@2.1.3: {} mute-stream@3.0.0: {} @@ -15060,10 +16325,14 @@ snapshots: natural-compare@1.4.0: {} + negotiator@0.6.3: {} + negotiator@0.6.4: {} neo-async@2.6.2: {} + netmask@2.1.1: {} + next@16.3.4(@babel/core@7.29.7(supports-color@10.2.2))(@playwright/test@1.62.1)(@types/node@22.20.1)(babel-plugin-macros@3.1.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.2.8(react@19.2.8))(react@19.2.8): dependencies: '@next/env': 16.3.4 @@ -15118,6 +16387,8 @@ snapshots: - '@types/node' - babel-plugin-macros + nice-try@1.0.5: {} + node-exports-info@1.6.2: dependencies: array.prototype.flatmap: 1.3.3 @@ -15146,6 +16417,8 @@ snapshots: normalize-path@3.0.0: {} + normalize-url@8.1.1: {} + npm-install-checks@6.3.0: dependencies: semver: 7.8.5 @@ -15166,6 +16439,10 @@ snapshots: npm-package-arg: 11.0.3 semver: 7.8.5 + npm-run-path@2.0.2: + dependencies: + path-key: 2.0.1 + npm-run-path@4.0.1: dependencies: path-key: 3.1.1 @@ -15229,6 +16506,10 @@ snapshots: obug@2.1.4: {} + on-finished@2.4.1: + dependencies: + ee-first: 1.1.1 + on-headers@1.1.0: {} once@1.4.0: @@ -15243,6 +16524,8 @@ snapshots: dependencies: mimic-fn: 4.0.0 + only@0.0.2: {} + open@11.0.2: dependencies: default-browser: 5.5.1 @@ -15276,6 +16559,10 @@ snapshots: object-keys: 1.1.1 safe-push-apply: 1.0.0 + p-cancelable@3.0.0: {} + + p-finally@1.0.0: {} + p-limit@2.3.0: dependencies: p-try: 2.2.0 @@ -15284,6 +16571,10 @@ snapshots: dependencies: yocto-queue: 0.1.0 + p-limit@4.0.0: + dependencies: + yocto-queue: 1.2.2 + p-locate@4.1.0: dependencies: p-limit: 2.3.0 @@ -15292,12 +16583,36 @@ snapshots: dependencies: p-limit: 3.1.0 + p-locate@6.0.0: + dependencies: + p-limit: 4.0.0 + p-retry@8.0.1: dependencies: is-network-error: 1.3.2 p-try@2.2.0: {} + pac-proxy-agent@9.1.0(supports-color@10.2.2): + dependencies: + agent-base: 9.0.0 + debug: 4.4.3(supports-color@10.2.2) + get-uri: 8.0.1(supports-color@10.2.2) + http-proxy-agent: 9.1.0(supports-color@10.2.2) + https-proxy-agent: 9.1.0(supports-color@10.2.2) + pac-resolver: 9.0.1(quickjs-wasi@2.2.0) + quickjs-wasi: 2.2.0 + socks-proxy-agent: 10.1.0(supports-color@10.2.2) + transitivePeerDependencies: + - kerberos + - supports-color + + pac-resolver@9.0.1(quickjs-wasi@2.2.0): + dependencies: + degenerator: 7.0.1(quickjs-wasi@2.2.0) + netmask: 2.1.1 + quickjs-wasi: 2.2.0 + package-json-from-dist@1.0.1: {} pako@1.0.11: {} @@ -15344,6 +16659,8 @@ snapshots: '@types/parse-path': 7.1.0 parse-path: 7.1.0 + parse5@5.1.1: {} + parse5@7.3.0: dependencies: entities: 6.0.1 @@ -15352,12 +16669,18 @@ snapshots: dependencies: entities: 8.0.0 + parseurl@1.3.3: {} + path-exists@4.0.0: {} + path-exists@5.0.0: {} + path-is-absolute@1.0.1: {} path-is-inside@1.0.2: {} + path-key@2.0.1: {} + path-key@3.1.1: {} path-key@4.0.0: {} @@ -15395,8 +16718,18 @@ snapshots: dependencies: find-up: 4.1.0 + pkg-install@1.0.0: + dependencies: + '@types/execa': 0.9.0 + '@types/node': 11.15.54 + execa: 1.0.0 + pkg-pr-new@0.0.88: {} + pkg-up@4.0.0: + dependencies: + find-up: 6.3.0 + playwright-core@1.62.1: {} playwright-core@1.63.0: {} @@ -15506,6 +16839,8 @@ snapshots: process-nextick-args@2.0.1: {} + progress@2.0.3: {} + promise-inflight@1.0.1: {} promise-retry@2.0.1: @@ -15534,6 +16869,29 @@ snapshots: protocols@2.0.2: {} + proxy-agent-negotiate@1.1.0: {} + + proxy-agent@8.0.2(supports-color@10.2.2): + dependencies: + agent-base: 9.0.0 + debug: 4.4.3(supports-color@10.2.2) + http-proxy-agent: 9.1.0(supports-color@10.2.2) + https-proxy-agent: 9.1.0(supports-color@10.2.2) + lru-cache: 7.18.3 + pac-proxy-agent: 9.1.0(supports-color@10.2.2) + proxy-from-env: 2.1.0 + socks-proxy-agent: 10.1.0(supports-color@10.2.2) + transitivePeerDependencies: + - kerberos + - supports-color + + proxy-from-env@2.1.0: {} + + pump@3.0.4: + dependencies: + end-of-stream: 1.4.5 + once: 1.4.0 + punycode@1.4.1: optional: true @@ -15553,10 +16911,13 @@ snapshots: dependencies: es-define-property: 1.0.1 side-channel: 1.1.1 - optional: true queue-microtask@1.2.3: {} + quick-lru@5.1.1: {} + + quickjs-wasi@2.2.0: {} + quotation@2.0.3: {} range-parser@1.2.0: {} @@ -15589,6 +16950,13 @@ snapshots: xmlserializer: 0.6.1 optional: true + raw-body@2.5.3: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.1 + iconv-lite: 0.4.24 + unpipe: 1.0.0 + rc@1.2.8: dependencies: deep-extend: 0.6.0 @@ -15680,6 +17048,8 @@ snapshots: unified: 11.0.5 vfile: 6.0.3 + reduce-flatten@2.0.0: {} + reflect.getprototypeof@1.0.10: dependencies: call-bind: 1.0.9 @@ -15973,8 +17343,15 @@ snapshots: reselect@5.3.0: {} + resolve-alpn@1.2.1: {} + resolve-from@4.0.0: {} + resolve-path@1.4.0: + dependencies: + http-errors: 1.6.3 + path-is-absolute: 1.0.1 + resolve-pkg-maps@1.0.0: {} resolve@1.22.12: @@ -15993,6 +17370,10 @@ snapshots: path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 + responselike@3.0.0: + dependencies: + lowercase-keys: 3.0.0 + retry@0.12.0: {} reusify@1.1.0: {} @@ -16068,6 +17449,10 @@ snapshots: sane-domparser-error@0.2.0: optional: true + sanitize-filename@1.6.4: + dependencies: + truncate-utf8-bytes: 1.0.2 + saxes@5.0.1: dependencies: xmlchars: 2.2.0 @@ -16085,6 +17470,16 @@ snapshots: ajv-formats: 2.1.1(ajv@8.20.0) ajv-keywords: 5.1.0(ajv@8.20.0) + selenium-webdriver@4.49.0: + dependencies: + '@bazel/runfiles': 6.5.0 + jszip: 3.10.1 + tmp: 0.2.7 + ws: 8.21.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + semver@5.7.2: {} semver@6.3.1: {} @@ -16145,6 +17540,10 @@ snapshots: setimmediate@1.0.5: {} + setprototypeof@1.1.0: {} + + setprototypeof@1.2.0: {} + sharp@0.35.4(@types/node@22.20.1): dependencies: '@img/colour': 1.1.0 @@ -16178,10 +17577,16 @@ snapshots: '@img/sharp-win32-x64': 0.35.4 '@types/node': 22.20.1 + shebang-command@1.2.0: + dependencies: + shebang-regex: 1.0.0 + shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 + shebang-regex@1.0.0: {} + shebang-regex@3.0.0: {} side-channel-list@1.0.1: @@ -16242,6 +17647,21 @@ snapshots: astral-regex: 2.0.0 is-fullwidth-code-point: 3.0.0 + smart-buffer@4.2.0: {} + + socks-proxy-agent@10.1.0(supports-color@10.2.2): + dependencies: + agent-base: 9.0.0 + debug: 4.4.3(supports-color@10.2.2) + socks: 2.8.10 + transitivePeerDependencies: + - supports-color + + socks@2.8.10: + dependencies: + ip-address: 10.7.2 + smart-buffer: 4.2.0 + source-map-js@1.2.1: {} source-map-support@0.5.21: @@ -16287,6 +17707,10 @@ snapshots: stackback@0.0.2: {} + statuses@1.5.0: {} + + statuses@2.0.2: {} + std-env@4.2.0: {} stop-iteration-iterator@1.1.0: @@ -16421,6 +17845,8 @@ snapshots: strip-bom@3.0.0: {} + strip-eof@1.0.0: {} + strip-final-newline@2.0.0: {} strip-final-newline@3.0.0: {} @@ -16520,6 +17946,10 @@ snapshots: supports-color@10.2.2: {} + supports-color@5.5.0: + dependencies: + has-flag: 3.0.0 + supports-color@7.2.0: dependencies: has-flag: 4.0.0 @@ -16547,6 +17977,15 @@ snapshots: system-architecture@0.1.0: {} + systeminformation@5.33.13: {} + + table-layout@1.0.2: + dependencies: + array-back: 4.0.2 + deep-extend: 0.6.0 + typical: 5.2.0 + wordwrapjs: 4.0.1 + table@6.9.0: dependencies: ajv: 8.20.0 @@ -16555,8 +17994,50 @@ snapshots: string-width: 4.2.3 strip-ansi: 6.0.1 + tachometer@0.7.2(supports-color@10.2.2): + dependencies: + ansi-escape-sequences: 6.2.4 + command-line-args: 5.2.1 + command-line-usage: 6.1.3 + csv-stringify: 6.9.0 + fs-extra: 10.1.0 + get-stream: 6.0.1 + got: 12.6.1 + jsonschema: 1.5.0 + jsonwebtoken: 9.0.3 + jstat: 1.9.6 + koa: 2.16.4(supports-color@10.2.2) + koa-bodyparser: 4.4.1 + koa-mount: 4.2.0(supports-color@10.2.2) + koa-node-resolve: 1.0.0-pre.9(supports-color@10.2.2) + koa-send: 5.0.1(supports-color@10.2.2) + koa-static: 5.0.0(supports-color@10.2.2) + pkg-install: 1.0.0 + pkg-up: 4.0.0 + progress: 2.0.3 + sanitize-filename: 1.6.4 + selenium-webdriver: 4.49.0 + semver: 7.8.5 + source-map-support: 0.5.21 + strip-ansi: 7.2.0 + systeminformation: 5.33.13 + table: 6.9.0 + ua-parser-js: 1.0.41 + transitivePeerDependencies: + - '@75lb/nature' + - bufferutil + - supports-color + - utf-8-validate + tapable@2.3.3: {} + tcp-port-used@1.0.3(supports-color@10.2.2): + dependencies: + debug: 4.3.1(supports-color@10.2.2) + is2: 2.0.1 + transitivePeerDependencies: + - supports-color + terser@5.51.2: dependencies: '@jridgewell/source-map': 0.3.11 @@ -16599,6 +18080,8 @@ snapshots: toad-cache@3.7.4: {} + toidentifier@1.0.1: {} + totalist@3.0.1: {} tough-cookie@6.0.2: @@ -16613,6 +18096,10 @@ snapshots: trough@2.2.0: {} + truncate-utf8-bytes@1.0.2: + dependencies: + utf8-byte-length: 1.0.5 + ts-api-utils@2.5.0(@typescript/typescript6@6.0.2): dependencies: typescript: '@typescript/typescript6@6.0.2' @@ -16628,6 +18115,8 @@ snapshots: tslib@2.8.1: {} + tsscmp@1.0.6: {} + tsx@4.23.13: dependencies: esbuild: 0.28.2 @@ -16650,6 +18139,11 @@ snapshots: type-fest@4.41.0: {} + type-is@1.6.18: + dependencies: + media-typer: 0.3.0 + mime-types: 2.1.35 + typed-array-buffer@1.0.3: dependencies: call-bound: 1.0.4 @@ -16726,6 +18220,12 @@ snapshots: '@typescript/typescript-win32-arm64': 7.0.2 '@typescript/typescript-win32-x64': 7.0.2 + typical@4.0.0: {} + + typical@5.2.0: {} + + ua-parser-js@1.0.41: {} + uint8-to-base64@0.2.1: {} unbox-primitive@1.1.0: @@ -16859,6 +18359,10 @@ snapshots: universal-user-agent@7.0.3: {} + universalify@2.0.1: {} + + unpipe@1.0.0: {} + unrs-resolver@1.12.2: dependencies: napi-postinstall: 0.3.4 @@ -16911,6 +18415,8 @@ snapshots: dependencies: react: 19.2.8 + utf8-byte-length@1.0.5: {} + util-deprecate@1.0.2: {} validate-npm-package-license@3.0.4: @@ -17181,6 +18687,11 @@ snapshots: word-wrap@1.2.5: {} + wordwrapjs@4.0.1: + dependencies: + reduce-flatten: 2.0.0 + typical: 5.2.0 + wrap-ansi@6.2.0: dependencies: ansi-styles: 4.3.0 @@ -17269,8 +18780,12 @@ snapshots: y18n: 5.0.8 yargs-parser: 22.0.0 + ylru@1.4.0: {} + yocto-queue@0.1.0: {} + yocto-queue@1.2.2: {} + yoctocolors@2.2.0: {} zod-validation-error@3.5.4(zod@3.25.76): diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index dce77aded3..c943e1469c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,6 +6,7 @@ packages: engineStrict: true strictDepBuilds: true allowBuilds: + chromedriver: true esbuild: true nx: true sharp: true diff --git a/test/tachometer/README.md b/test/tachometer/README.md new file mode 100644 index 0000000000..f17882ea83 --- /dev/null +++ b/test/tachometer/README.md @@ -0,0 +1,115 @@ +# Tachometer smoke test + +A working harness for `benchmark tacho run`, used to exercise that tooling end to end. + +This is **not** a meaningful benchmark, and it is not trying to be. This repository ships build and +test tooling, not a browser library, so there is nothing here whose render time is worth tracking. +The cases run a deterministic CPU workload purely so the pipeline has something real to drive: +discovering cases, resolving the baseline, packing a commit's workspace to tarballs, building each +variant, and running Chrome under tachometer. + +For the real thing, see the consumers this tooling was extracted from — `base-ui-mosaic` and +`base-ui-charts` benchmark actual data grids and charts. + +## What it covers + +The cases are chosen to walk the parts of the pipeline most likely to break: + +| Case | Covers | +| :--------------- | :------------------------------------------------------------------------------- | +| `workload` | The ordinary regression shape: one page, auto-expanded into current vs baseline | +| `workload-large` | A case owning **no page of its own**, parameterising its sibling's with `?size=` | +| `libs-sort` | Three variants from **one build**: the cross-library shape, with no baseline | + +`workload-large` exercises that a case's query parameters survive the url rewrite, that a url +traversing out of its own case folder resolves, and that two cases referencing one page build it +only once. + +`libs-sort` covers the other axis, where nothing is compared across commits. Its three variants are +built from the same commit and differ only in how they sort, standing in for the competitor +libraries this repository has no reason to install. It exercises three things the two-variant cases +cannot: a case with more than two variants, which the report draws as a table of variants rather +than a row per case; a second variant set, which gets a table of its own rather than sharing one +full of blank cells; and a verdict that is genuinely `slower` yet must never be reported as a +regression, because both sides came from the same build. + +`workload` and `workload-large` import `@mui/internal-test-utils` and put its value on screen. That +is the load-bearing part: every ref — the working tree included — resolves it from a packed tarball +in that ref's own tree, so both sides of a comparison consume the library the way a consumer does. +If that path breaks the page fails to build, rather than quietly measuring nothing. + +## Running + +```bash +pnpm release:build # the harness resolves workspace packages through their build output +pnpm test:tacho # from the repo root +``` + +Or from here, with the usual filters and flags: + +```bash +pnpm -F ./test/tachometer test:tacho large +pnpm -F ./test/tachometer test:tacho --baseline HEAD +``` + +A filter is a case-insensitive substring of a case's path under `src/`, the way vitest matches test +files. Cases may sit in folders, and since the path is what a filter matches, a folder name selects +everything beneath it — which is how a repository partitions its suite (a quick set for pull +requests, everything for a release) without the runner being taught what any of it means. A case is +named by its benchmark's own `name`, so moving one between folders does not rename it in the report. + +Cases run sequentially, and each auto-samples until its difference resolves or the (deliberately +short) timeout is hit. Because both sides run identical code, the expected verdict is `unsure` — +that is the "no change" outcome, not a failure. + +`benchmark tacho report` prints that table again from the saved JSON, so a run kept from earlier — +or a report downloaded from CI — can be read without sampling again. + +Everything a run writes goes under `.tachometer/`: the report in `results/`, the pages built per ref +in `builds/`, the packed tarballs in `packed/` (the one worth caching in CI, keyed by commit SHA) +and the install each ref resolves through in `trees/`. Deleting that directory resets the harness +completely, and it is the only thing a repository has to ignore. + +### Prerequisites + +Playwright's Chromium, installed with `pnpm exec playwright install chromium`. Runs use Playwright's +pinned Chrome for Testing rather than whatever Chrome the machine has auto-updated to, so a browser +update cannot move the numbers on its own. + +A chromedriver only drives its own Chrome major, so the root `chromedriver` devDependency has to +stay aligned with `@playwright/test` — bump the two together. `tacho run` checks the pair before it +builds anything and fails with both versions named, rather than letting you find out minutes later +through `SessionNotCreatedError`. + +One wrinkle: tachometer installs a chromedriver into its own package directory on demand, and that +copy wins over the root pin. If the check reports a driver you did not install, delete +`node_modules/.pnpm/tachometer@*/node_modules/tachometer/node_modules/chromedriver` and re-run. + +### A note on the baseline + +`tacho run` does not work out which commit to compare against — `code-infra baseline` does, for +every job that compares a branch to its base, so bundle size and the benchmarks agree on the answer +instead of each deriving it. Pass it in: + +```bash +pnpm test:tacho --baseline "$(pnpm code-infra baseline)" +``` + +Without `--baseline` the previous commit is used, which needs no policy to decide. + +A baseline older than the introduction of `benchmark tacho` will fail to build its pages: this +harness's vite config imports a plugin that did not exist at that commit. `--baseline HEAD` compares +the working tree against the current commit while that is still true. + +## Working on a case by hand + +```bash +pnpm -F ./test/tachometer dev +``` + +Every discovered case is listed at `/`, including `workload-large`, which owns no page of its own and +would otherwise be unreachable — its `?size=` query is in the config, not the file tree. The list is +generated from the case configs rather than written by hand, so it cannot go stale. + +`pnpm -F ./test/tachometer build` and `preview` produce and serve the production bundle, which is +what actually gets measured. The index is emitted into that build too, so the same list is there. diff --git a/test/tachometer/package.json b/test/tachometer/package.json new file mode 100644 index 0000000000..cca8712442 --- /dev/null +++ b/test/tachometer/package.json @@ -0,0 +1,22 @@ +{ + "private": true, + "description": "Tachometer smoke-test harness for the @mui/internal-benchmark tacho tooling", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview", + "typescript": "tsc -noEmit", + "test:tacho": "node ./scripts/runBenchmark.mjs tacho run" + }, + "dependencies": { + "@mui/internal-test-utils": "workspace:*", + "react": "19.2.8" + }, + "devDependencies": { + "@mui/internal-benchmark": "workspace:*", + "@playwright/test": "1.62.1", + "@types/react": "19.2.18", + "tachometer": "^0.7.2", + "vite": "^8.2.2" + } +} diff --git a/test/tachometer/scripts/runBenchmark.mjs b/test/tachometer/scripts/runBenchmark.mjs new file mode 100644 index 0000000000..63da240419 --- /dev/null +++ b/test/tachometer/scripts/runBenchmark.mjs @@ -0,0 +1,12 @@ +// Thin wrapper that runs the `@mui/internal-benchmark` CLI from the workspace package's build +// output. Importing the package by name (rather than relying on its `benchmark` bin) avoids +// depending on pnpm linking the bin shim: the package publishes from `build/`, so the workspace +// link points at a directory that does not exist until the first build — and a later build cannot +// create a shim that install already skipped. The same wrapper exists in `docs/scripts` for the +// same reason. +// +// A consumer installing the published package gets a self-contained directory and can use the bin +// directly; only this repository, where the dependency is a workspace link, needs this. +// +// The CLI parses `process.argv` itself on import. +import '@mui/internal-benchmark/cli'; diff --git a/test/tachometer/src/_shared/measure.ts b/test/tachometer/src/_shared/measure.ts new file mode 100644 index 0000000000..00eefa0ffa --- /dev/null +++ b/test/tachometer/src/_shared/measure.ts @@ -0,0 +1,51 @@ +/** + * Shared measurement helper. + * + * Tachometer's `performance` measurement mode polls the page for a `performance.measure()` span of + * a given name, so a page's only obligation is to produce one. + */ + +/** The measure name each case's `tachometer.json` reads. */ +const MEASURE = 'mount'; + +/** + * Runs `work`, writes its result into the page, and closes the `mount` measure once the browser has + * painted it. + * + * The paint is inside the measured window on purpose: a workload whose result is never displayed + * can be optimized away, and the number would then track nothing. + */ +export function measureMount(work: () => string): void { + performance.mark(`${MEASURE}-start`); + const text = work(); + + const root = document.getElementById('root'); + if (!root) { + throw new Error('The page is missing its #root element.'); + } + root.textContent = text; + + // A double rAF closes the measure after the DOM write *and* the paint it triggers; a single one + // would still be inside the frame that performs the paint. + requestAnimationFrame(() => { + requestAnimationFrame(() => { + performance.mark(`${MEASURE}-end`); + performance.measure(MEASURE, `${MEASURE}-start`, `${MEASURE}-end`); + }); + }); +} + +/** + * Reads a positive integer from the page's query string, so one page can back several cases. + */ +export function numberParam(name: string, fallback: number): number { + const raw = new URLSearchParams(window.location.search).get(name); + if (raw === null) { + return fallback; + } + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`Expected "${name}" to be a positive integer, got "${raw}".`); + } + return value; +} diff --git a/test/tachometer/src/libs/sort/alpha.html b/test/tachometer/src/libs/sort/alpha.html new file mode 100644 index 0000000000..5de8ad125c --- /dev/null +++ b/test/tachometer/src/libs/sort/alpha.html @@ -0,0 +1,11 @@ + + + + + libs-sort [alpha] + + +
    + + + diff --git a/test/tachometer/src/libs/sort/alpha.ts b/test/tachometer/src/libs/sort/alpha.ts new file mode 100644 index 0000000000..425c67e246 --- /dev/null +++ b/test/tachometer/src/libs/sort/alpha.ts @@ -0,0 +1,10 @@ +import { measureMount } from '../../_shared/measure'; +import { makeValues, report } from './shared'; + +const values = makeValues(); + +/** Sorts the boxed array in place, paying a comparator call per comparison. */ +measureMount(() => { + values.sort((left, right) => left - right); + return report('alpha', values); +}); diff --git a/test/tachometer/src/libs/sort/beta.html b/test/tachometer/src/libs/sort/beta.html new file mode 100644 index 0000000000..f0b543d18e --- /dev/null +++ b/test/tachometer/src/libs/sort/beta.html @@ -0,0 +1,11 @@ + + + + + libs-sort [beta] + + +
    + + + diff --git a/test/tachometer/src/libs/sort/beta.ts b/test/tachometer/src/libs/sort/beta.ts new file mode 100644 index 0000000000..bd68fad1b3 --- /dev/null +++ b/test/tachometer/src/libs/sort/beta.ts @@ -0,0 +1,33 @@ +import { measureMount } from '../../_shared/measure'; +import { makeValues, report } from './shared'; + +const values = makeValues(); + +/** A merge sort written in JavaScript, allocating as it goes — the slow variant of the three. */ +function mergeSort(values: number[]): number[] { + if (values.length <= 1) { + return values; + } + const middle = Math.floor(values.length / 2); + const left = mergeSort(values.slice(0, middle)); + const right = mergeSort(values.slice(middle)); + + const merged: number[] = new Array(values.length); + let leftIndex = 0; + let rightIndex = 0; + for (let index = 0; index < merged.length; index += 1) { + if ( + leftIndex < left.length && + (rightIndex >= right.length || left[leftIndex] <= right[rightIndex]) + ) { + merged[index] = left[leftIndex]; + leftIndex += 1; + } else { + merged[index] = right[rightIndex]; + rightIndex += 1; + } + } + return merged; +} + +measureMount(() => report('beta', mergeSort(values))); diff --git a/test/tachometer/src/libs/sort/ours.html b/test/tachometer/src/libs/sort/ours.html new file mode 100644 index 0000000000..68b8517ce4 --- /dev/null +++ b/test/tachometer/src/libs/sort/ours.html @@ -0,0 +1,11 @@ + + + + + libs-sort [ours] + + +
    + + + diff --git a/test/tachometer/src/libs/sort/ours.ts b/test/tachometer/src/libs/sort/ours.ts new file mode 100644 index 0000000000..15adf702f1 --- /dev/null +++ b/test/tachometer/src/libs/sort/ours.ts @@ -0,0 +1,13 @@ +import { measureMount } from '../../_shared/measure'; +import { makeValues, report } from './shared'; + +// Built outside the measured window: every sample is a fresh page load, so the input is still +// generated once per sample — it just no longer counts as part of the sort being measured. +const values = makeValues(); + +/** Copies into a typed array first, so the engine sorts unboxed doubles with no comparator calls. */ +measureMount(() => { + const sorted = Float64Array.from(values); + sorted.sort(); + return report('ours', sorted); +}); diff --git a/test/tachometer/src/libs/sort/shared.ts b/test/tachometer/src/libs/sort/shared.ts new file mode 100644 index 0000000000..56e7dc3ba8 --- /dev/null +++ b/test/tachometer/src/libs/sort/shared.ts @@ -0,0 +1,51 @@ +/** + * The workload the three variants share, so they differ only in how they sort. + * + * This case stands in for a cross-library comparison: the shape where every variant is built from + * the *same* commit, and the difference worth knowing is between implementations rather than + * between two builds of ours. This repository has no competitor libraries to install, so the + * "libraries" here are three sorts with genuinely different constant factors. + */ + +/** Values per sample. Large enough that the sort, not the page load, dominates the measurement. */ +export const SIZE = 100_000; + +/** A small linear congruential generator, so every sample sorts identical input. */ +function createRandom(seed: number): () => number { + let state = seed; + return () => { + state = (state * 1664525 + 1013904223) % 0x100000000; + return state / 0x100000000; + }; +} + +/** A fresh unsorted array. Fresh per sample: re-sorting a sorted array measures something else. */ +export function makeValues(): number[] { + const random = createRandom(42); + const values: number[] = new Array(SIZE); + for (let index = 0; index < SIZE; index += 1) { + values[index] = random(); + } + return values; +} + +/** + * A checksum that depends on the order, so a variant that returned an unsorted or truncated array + * could not quietly report the same number as one that sorted correctly. + * + * Takes an `ArrayLike` so a typed array can be checked without being copied back into a boxed one — + * that copy would have landed inside the measured window of the very variant whose point is to + * avoid boxing. + */ +export function checksum(values: ArrayLike): number { + let total = 0; + for (let index = 0; index < values.length; index += 1) { + total += values[index] * index; + } + return total; +} + +/** The line each variant puts on screen, so the three differ only in how they sort. */ +export function report(variant: string, sorted: ArrayLike): string { + return `libs-sort [${variant}] size=${SIZE} checksum=${checksum(sorted).toFixed(3)}`; +} diff --git a/test/tachometer/src/libs/sort/tachometer.json b/test/tachometer/src/libs/sort/tachometer.json new file mode 100644 index 0000000000..1f87085259 --- /dev/null +++ b/test/tachometer/src/libs/sort/tachometer.json @@ -0,0 +1,22 @@ +{ + "$schema": "../../../node_modules/tachometer/config.schema.json", + "sampleSize": 10, + "timeout": 1, + "autoSampleConditions": ["10%"], + "benchmarks": [ + { + "name": "libs-sort", + "measurement": { "mode": "performance", "entryName": "mount" }, + "browser": { + "name": "chrome", + "headless": true, + "windowSize": { "width": 1024, "height": 768 } + }, + "expand": [ + { "name": "libs-sort [ours]", "url": "./ours.html" }, + { "name": "libs-sort [alpha]", "url": "./alpha.html" }, + { "name": "libs-sort [beta]", "url": "./beta.html" } + ] + } + ] +} diff --git a/test/tachometer/src/workload-large/tachometer.json b/test/tachometer/src/workload-large/tachometer.json new file mode 100644 index 0000000000..ae102d0e4d --- /dev/null +++ b/test/tachometer/src/workload-large/tachometer.json @@ -0,0 +1,18 @@ +{ + "$schema": "../../node_modules/tachometer/config.schema.json", + "sampleSize": 10, + "timeout": 1, + "autoSampleConditions": ["10%"], + "benchmarks": [ + { + "name": "workload-large", + "url": "../workload/index.html?size=400000", + "measurement": { "mode": "performance", "entryName": "mount" }, + "browser": { + "name": "chrome", + "headless": true, + "windowSize": { "width": 1024, "height": 768 } + } + } + ] +} diff --git a/test/tachometer/src/workload/index.html b/test/tachometer/src/workload/index.html new file mode 100644 index 0000000000..556d47228b --- /dev/null +++ b/test/tachometer/src/workload/index.html @@ -0,0 +1,11 @@ + + + + + workload + + +
    + + + diff --git a/test/tachometer/src/workload/index.ts b/test/tachometer/src/workload/index.ts new file mode 100644 index 0000000000..e20ed2abfc --- /dev/null +++ b/test/tachometer/src/workload/index.ts @@ -0,0 +1,43 @@ +import { reactMajor } from '@mui/internal-test-utils/env'; +import { measureMount, numberParam } from '../_shared/measure'; + +/** + * A deterministic CPU workload, sized by `?size=`. + * + * The point of this page is not the number it produces — this repository ships build tooling, not a + * browser library, so there is nothing here whose render time is worth tracking. It exists to give + * `benchmark tacho run` something real to drive end to end. + * + * It does import a workspace package (`@mui/internal-test-utils`) and put its value on screen. That + * is the part that matters: under `tacho run` every ref — the working tree included — resolves it + * from a packed tarball in an isolated install, so both sides of a comparison resolve identically. + * A plain `vite dev` or `vite build` resolves it through the workspace link instead. If either path + * breaks, this page fails to build rather than silently measuring nothing. + */ + +/** A small linear congruential generator, so every sample does identical work. */ +function createRandom(seed: number): () => number { + let state = seed; + return () => { + state = (state * 1664525 + 1013904223) % 0x100000000; + return state / 0x100000000; + }; +} + +measureMount(() => { + const size = numberParam('size', 50_000); + const random = createRandom(42); + + const values = new Float64Array(size); + for (let index = 0; index < size; index += 1) { + values[index] = random(); + } + values.sort(); + + let checksum = 0; + for (let index = 0; index < size; index += 1) { + checksum += values[index] * index; + } + + return `size=${size} checksum=${checksum.toFixed(3)} react=${reactMajor}`; +}); diff --git a/test/tachometer/src/workload/tachometer.json b/test/tachometer/src/workload/tachometer.json new file mode 100644 index 0000000000..49d6cecbf3 --- /dev/null +++ b/test/tachometer/src/workload/tachometer.json @@ -0,0 +1,18 @@ +{ + "$schema": "../../node_modules/tachometer/config.schema.json", + "sampleSize": 10, + "timeout": 1, + "autoSampleConditions": ["10%"], + "benchmarks": [ + { + "name": "workload", + "url": "./index.html", + "measurement": { "mode": "performance", "entryName": "mount" }, + "browser": { + "name": "chrome", + "headless": true, + "windowSize": { "width": 1024, "height": 768 } + } + } + ] +} diff --git a/test/tachometer/tsconfig.json b/test/tachometer/tsconfig.json new file mode 100644 index 0000000000..cc3ed604af --- /dev/null +++ b/test/tachometer/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "bundler", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "types": ["vite/client"], + "strict": true, + "noEmit": true, + "isolatedModules": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true + }, + "include": ["src", "vite.config.mjs"] +} diff --git a/test/tachometer/vite.config.mjs b/test/tachometer/vite.config.mjs new file mode 100644 index 0000000000..98416fbdc9 --- /dev/null +++ b/test/tachometer/vite.config.mjs @@ -0,0 +1,6 @@ +import { defineConfig } from 'vite'; +import { tachometer } from '@mui/internal-benchmark/tachometerPlugin'; + +// The plugin contributes everything derived from the benchmark cases: the `src/` root, the +// multi-page app type, the build's entry points, and the dev-server case index. +export default defineConfig({ plugins: [tachometer()] }); diff --git a/tsconfig.json b/tsconfig.json index ce12c69e43..4ce11c9516 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -26,7 +26,8 @@ { "path": "scripts" }, { "path": "docs" }, { "path": "test/performance" }, - { "path": "test/benchmark" } + { "path": "test/benchmark" }, + { "path": "test/tachometer" } ], "files": [] } From 42ba08d7edf345f76a9084ef67683aa93aef7c1e Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 13:49:55 +0200 Subject: [PATCH 080/173] [benchmark] Simplify benchmark run - The v2 upload envelope is one zod schema in runReport/schema.ts, used by the runner to validate and by the dashboard for its type; the report's metric definitions are tied to MetricDefinition with `satisfies`. - The three renderers share formatComparisonLabel, formatRunSummary and RUN_REPORT_FOOTNOTE, so the terminal and the PR comment count regressions the same way. - index.tsx and page.tsx export the same public API from publicApi.ts and share the empty-recording message and millisecond format. - runInterleaved returns metric definitions instead of threading an accumulator, reading them from the reference page. - Both sides are packed in parallel; page builds stay sequential. - The PR comment checks the head report's version before waiting on the base lookup a v2 report doesn't need. - The history chart analyses each report once and memoizes its series. - variantOf and resultsPathOf name what was spelled out twice; the head SHA comes from resolveCommit; the median reuses quantile; CLI defaults live only in runBenchmarks; internal analysis helpers are no longer exported. --- .../src/components/BenchmarkRunHistory.tsx | 89 +++++++++++-------- .../lib/benchmarkRun/buildMarkdownReport.ts | 34 ++----- .../src/lib/ciReports/benchmarkReport.ts | 40 +++++---- .../src/views/BenchmarkRunDetails.tsx | 18 ++-- packages/benchmark/README.md | 11 ++- packages/benchmark/src/caseRuntime.tsx | 12 +++ packages/benchmark/src/cli/report.ts | 4 +- packages/benchmark/src/cli/run.ts | 17 ++-- packages/benchmark/src/index.tsx | 25 ++---- packages/benchmark/src/metricCore.ts | 8 +- packages/benchmark/src/page/page.tsx | 48 +++------- packages/benchmark/src/publicApi.ts | 18 ++++ .../benchmark/src/runReport/analyzeRun.ts | 20 ++--- packages/benchmark/src/runReport/format.ts | 32 ++++++- packages/benchmark/src/runReport/schema.ts | 35 +++++--- packages/benchmark/src/runner/outputDir.ts | 5 ++ packages/benchmark/src/runner/printReport.ts | 37 ++++---- packages/benchmark/src/runner/refs.ts | 8 ++ .../benchmark/src/runner/runBenchmarks.ts | 52 +++++------ .../benchmark/src/runner/runInterleaved.ts | 87 +++++++++--------- packages/benchmark/src/runner/upload.ts | 13 +-- 21 files changed, 324 insertions(+), 289 deletions(-) create mode 100644 packages/benchmark/src/publicApi.ts diff --git a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx index 80ac8817d6..6f3c0e58e6 100644 --- a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx +++ b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx @@ -9,7 +9,11 @@ import TextField from '@mui/material/TextField'; import Typography from '@mui/material/Typography'; import { BarChartPro } from '@mui/x-charts-pro/BarChartPro'; import { analyzeRun, formatComparison } from '@mui/internal-benchmark/runReport'; -import type { MetricComparison } from '@mui/internal-benchmark/runReport'; +import type { + BenchmarkRunReport, + Change, + MetricComparison, +} from '@mui/internal-benchmark/runReport'; import { isBenchmarkRunUpload } from '@/utils/fetchCiReport'; import { useMasterCommits } from '../hooks/useMasterCommits'; import { useCiReports } from '../hooks/useCiReports'; @@ -21,19 +25,40 @@ import ErrorDisplay from './ErrorDisplay'; * read off the chart directly, rather than inferred from a noisy trend. */ -type Change = MetricComparison['change']; - const CHANGE_COLORS: Record = { worse: 'var(--mui-palette-error-main)', better: 'var(--mui-palette-success-main)', unsure: 'var(--mui-palette-grey-500)', }; +/** Per benchmark, per metric: a commit's change against its parent. */ +type CommitComparisons = Map>; + interface CommitPoint { sha: string; date: Date; - /** Per benchmark, per metric: this commit's change against its parent. */ - comparisons: Map>; + comparisons: CommitComparisons; +} + +// A report is analysed once, however often the list of loaded reports changes around it. Keyed by +// the report object, which the query cache keeps stable for as long as it holds the report. +const comparisonsByReport = new WeakMap(); + +function comparisonsOf(report: BenchmarkRunReport): CommitComparisons { + let comparisons = comparisonsByReport.get(report); + if (!comparisons) { + comparisons = new Map(); + for (const { benchmark, metrics } of analyzeRun(report)) { + if (benchmark.kind === 'baseline') { + comparisons.set( + benchmark.name, + new Map(metrics.map(({ metric, comparisons: [comparison] }) => [metric, comparison])), + ); + } + } + comparisonsByReport.set(report, comparisons); + } + return comparisons; } interface BenchmarkRunHistoryProps { @@ -53,18 +78,9 @@ export default function BenchmarkRunHistory({ repo }: BenchmarkRunHistoryProps) if (!upload || !isBenchmarkRunUpload(upload)) { return []; } - const { report } = upload; - const comparisons = new Map>(); - for (const { benchmark, metrics } of analyzeRun(report)) { - if (benchmark.kind !== 'baseline') { - continue; - } - comparisons.set( - benchmark.name, - new Map(metrics.map(({ metric, comparisons: [comparison] }) => [metric, comparison])), - ); - } - return [{ sha: commit.sha, date: new Date(timestamp), comparisons }]; + return [ + { sha: commit.sha, date: new Date(timestamp), comparisons: comparisonsOf(upload.report) }, + ]; }), [commits, reports], ); @@ -90,27 +106,28 @@ export default function BenchmarkRunHistory({ repo }: BenchmarkRunHistoryProps) const [userMetric, setUserMetric] = React.useState(null); const metric = userMetric !== null && metricNames.includes(userMetric) ? userMetric : 'render'; - const comparisonAt = (point: CommitPoint) => - benchmark === null ? undefined : point.comparisons.get(benchmark)?.get(metric); - // One series per kind of change, so each bar takes its change's colour. A commit's bar is the // midpoint of its confidence interval; the tooltip gives the interval itself. - const series = (['worse', 'better', 'unsure'] as const).map((change) => ({ - type: 'bar' as const, - stack: 'change', - label: change, - color: CHANGE_COLORS[change], - data: points.map((point) => { - const comparison = comparisonAt(point); - return comparison?.change === change - ? (comparison.relative.low + comparison.relative.high) / 2 - : null; - }), - valueFormatter: (_value: number | null, { dataIndex }: { dataIndex: number }) => { - const comparison = comparisonAt(points[dataIndex]); - return comparison?.change === change ? formatComparison(comparison) : null; - }, - })); + const series = React.useMemo(() => { + const selected = points.map((point) => + benchmark === null ? undefined : point.comparisons.get(benchmark)?.get(metric), + ); + return (['worse', 'better', 'unsure'] as const satisfies Change[]).map((change) => ({ + type: 'bar' as const, + stack: 'change', + label: change, + color: CHANGE_COLORS[change], + data: selected.map((comparison) => + comparison?.change === change + ? (comparison.relative.low + comparison.relative.high) / 2 + : null, + ), + valueFormatter: (_value: number | null, { dataIndex }: { dataIndex: number }) => { + const comparison = selected[dataIndex]; + return comparison?.change === change ? formatComparison(comparison) : null; + }, + })); + }, [points, benchmark, metric]); const [selectedSha, setSelectedSha] = React.useState(null); diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index a526421502..a93a5929d2 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -3,7 +3,10 @@ import { analyzeRun, findRegressions, formatComparison, + formatComparisonLabel, + formatRunSummary, formatValue, + RUN_REPORT_FOOTNOTE, } from '@mui/internal-benchmark/runReport'; import type { BenchmarkAnalysis, BenchmarkRunReport } from '@mui/internal-benchmark/runReport'; @@ -17,8 +20,6 @@ interface BuildOptions { /** One table per benchmark: every metric, the median of every variant, every comparison. */ function renderBenchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): string { const comparisons = metrics[0]?.comparisons ?? []; - const comparisonHeader = (subject: string, against: string) => - benchmark.kind === 'baseline' ? `Δ vs ${against}` : `${subject} vs ${against}`; const columns = [ { field: 'metric', header: 'Metric', align: 'left' as const }, @@ -27,9 +28,9 @@ function renderBenchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): string header: variant, align: 'right' as const, })), - ...comparisons.map(({ subject, against }) => ({ - field: `comparison:${subject}:${against}`, - header: comparisonHeader(subject, against), + ...comparisons.map((comparison) => ({ + field: `comparison:${comparison.subject}:${comparison.against}`, + header: formatComparisonLabel(benchmark, comparison), align: 'left' as const, })), ]; @@ -78,19 +79,7 @@ export function buildBenchmarkRunMarkdownReport( lines.push(''); } - const regressedBenchmarks = new Set(regressions.map((regression) => regression.benchmark)); - lines.push( - [ - `${measured.length} benchmark${measured.length === 1 ? '' : 's'} measured`, - regressedBenchmarks.size > 0 - ? `${regressedBenchmarks.size} with regressions` - : 'no regressions', - failed.length > 0 ? `${failed.length} failed` : null, - ] - .filter(Boolean) - .join(' · '), - '', - ); + lines.push(formatRunSummary(analyses, regressions), ''); for (const { benchmark } of failed) { lines.push(`❌ **${benchmark.name}**: ${benchmark.error}`); @@ -102,14 +91,7 @@ export function buildBenchmarkRunMarkdownReport( if (measured.length > 0) { lines.push('
    ', 'Full results', ''); lines.push(measured.map(renderBenchmarkTable).join('\n\n')); - lines.push( - '', - '_Each value is a median. Each Δ is a 95% confidence interval on the paired per-round ' + - 'difference, relative to the variant it is measured against; "unsure" means it straddles ' + - 'zero — the expected result for two equivalent builds._', - '', - '
    ', - ); + lines.push('', `_${RUN_REPORT_FOOTNOTE}_`, '', '
    '); } if (options.detailsUrl) { diff --git a/apps/code-infra-dashboard/src/lib/ciReports/benchmarkReport.ts b/apps/code-infra-dashboard/src/lib/ciReports/benchmarkReport.ts index 3bda25c4f7..7901b17cdb 100644 --- a/apps/code-infra-dashboard/src/lib/ciReports/benchmarkReport.ts +++ b/apps/code-infra-dashboard/src/lib/ciReports/benchmarkReport.ts @@ -5,10 +5,28 @@ import { buildBenchmarkMarkdownReport } from '@/lib/benchmark/buildMarkdownRepor import { buildBenchmarkRunMarkdownReport } from '@/lib/benchmarkRun/buildMarkdownReport'; import { DASHBOARD_ORIGIN } from '@/constants'; +import type { BenchmarkRunReport } from '@mui/internal-benchmark/runReport'; import type { ReportOptions, ReportResult } from './types'; export const BENCHMARK_SECTION_TITLE = 'Performance'; +/** A version 2 report measured its own baseline in the same run, so it is rendered on its own. */ +function generateBenchmarkRunSection( + { repo, prNumber, commitSha }: ReportOptions, + report: BenchmarkRunReport, +): ReportResult { + const runUrl = new URL(`${DASHBOARD_ORIGIN}/benchmark-run/${repo}`); + runUrl.searchParams.set('sha', commitSha); + // The details view reads this to title the page and link back to the PR overview. + runUrl.searchParams.set('prNumber', String(prNumber)); + return { + content: buildBenchmarkRunMarkdownReport(report, { + title: BENCHMARK_SECTION_TITLE, + detailsUrl: runUrl.href, + }), + }; +} + /** * Generates the benchmark section of the pull request comment from the head commit's report. * Returns null if the head benchmark report is not available. @@ -22,28 +40,20 @@ export async function generateBenchmarkReport( ): Promise { const { repo, prNumber, commitSha, pr, baseCandidates } = options; - const [baseResult, headReport] = await Promise.all([ - fetchCiReportWithFallback(repo, baseCandidates, 'benchmark.json'), - fetchCiReport(repo, commitSha, 'benchmark.json'), - ]); + // Started before the head is known so a version 1 report does not wait for it twice; a version 2 + // report does not need it, and returns without waiting for it at all. + const baseResultPromise = fetchCiReportWithFallback(repo, baseCandidates, 'benchmark.json'); + baseResultPromise.catch(() => {}); + const headReport = await fetchCiReport(repo, commitSha, 'benchmark.json'); if (!headReport) { return null; } - if (isBenchmarkRunUpload(headReport)) { - const runUrl = new URL(`${DASHBOARD_ORIGIN}/benchmark-run/${repo}`); - runUrl.searchParams.set('sha', commitSha); - // The details view reads this to title the page and link back to the PR overview. - runUrl.searchParams.set('prNumber', String(prNumber)); - return { - content: buildBenchmarkRunMarkdownReport(headReport.report, { - title: BENCHMARK_SECTION_TITLE, - detailsUrl: runUrl.href, - }), - }; + return generateBenchmarkRunSection(options, headReport.report); } + const baseResult = await baseResultPromise; const inlinedBase = headReport.base; const { actualCommit: actualBaseCommit } = baseResult; const fetchedBaseUpload = diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index 8e161cec85..bf33b91e60 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -15,7 +15,13 @@ import TableHead from '@mui/material/TableHead'; import TableRow from '@mui/material/TableRow'; import Typography from '@mui/material/Typography'; import { fetchCiReport, isBenchmarkRunUpload } from '@/utils/fetchCiReport'; -import { analyzeRun, formatComparison, formatValue } from '@mui/internal-benchmark/runReport'; +import { + analyzeRun, + formatComparison, + formatComparisonLabel, + formatValue, + RUN_REPORT_FOOTNOTE, +} from '@mui/internal-benchmark/runReport'; import type { BenchmarkAnalysis, BenchmarkRunReport, @@ -57,9 +63,9 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { {variant} ))} - {comparisons.map(({ subject, against }) => ( - - {benchmark.kind === 'baseline' ? `Δ vs ${against}` : `${subject} vs ${against}`} + {comparisons.map((comparison) => ( + + {formatComparisonLabel(benchmark, comparison)} ))} Rounds @@ -176,9 +182,7 @@ export default function BenchmarkRunDetails() { {report && ( - Each value is a median. Each Δ is a 95% confidence interval on the paired per-round - difference, relative to the variant it is measured against; "unsure" means - it straddles zero. + {RUN_REPORT_FOOTNOTE} {report.sampling.samples} rounds after {report.sampling.warmup} warmup ·{' '} diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 4153a1d672..4392113eec 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -427,15 +427,14 @@ this repository. Neither is implemented, and both are rejected by name rather th ### Building and installing each side -Both sides are built the same way, from packed tarballs installed into an isolated tree, so neither -resolves the library through a workspace link. The benchmark files themselves stay on the current -branch and only the built library changes between refs — so a commit whose public API differs from -today's benchmarks will fail that ref's build, with the error surfaced. +Both sides are built the same way: packed to tarballs, in parallel, and installed the way a consumer +gets them, so neither resolves the library through a workspace link. The benchmark files themselves +stay on the current branch and only the built library changes between refs — so a commit whose public +API differs from today's benchmarks will fail that ref's build, with the error surfaced. Packed builds are cached by commit SHA under `.benchmark/packed/`, so repeating a comparison against the same commit skips the rebuild; the working tree is never cached. In CI, cache that directory -keyed on the baseline SHA, and check out with full history so a fork point can be resolved. A run -never modifies the checkout it was started from. +keyed on the baseline SHA, and check out with full history so a fork point can be resolved. By default a run resolves **in place**: it pins the packed build in the repository's own `pnpm-workspace.yaml`, installs it there, and resolution is then ordinary. The repository is put diff --git a/packages/benchmark/src/caseRuntime.tsx b/packages/benchmark/src/caseRuntime.tsx index acc59b8d4e..8c9eb31c3a 100644 --- a/packages/benchmark/src/caseRuntime.tsx +++ b/packages/benchmark/src/caseRuntime.tsx @@ -29,6 +29,18 @@ export type BenchmarkInteraction = (ctx: InteractionContext) => Promise | */ export const PAINT_METRIC_NAME = 'bench:paint'; +/** How the harness's time metrics are formatted. */ +export const MILLISECONDS: Intl.NumberFormatOptions = { + style: 'unit', + unit: 'millisecond', + maximumFractionDigits: 2, +}; + +/** Why a case failed when React recording was active yet captured no renders. */ +export const EMPTY_RECORDING_MESSAGE = + 'React recording was active but captured no renders. If you only measure imperative DOM ' + + 'updates or custom metrics, keep recording paused (reactRecordingPaused) instead of resuming.'; + export interface CaseOptions { afterEach?: () => Promise | void; /** diff --git a/packages/benchmark/src/cli/report.ts b/packages/benchmark/src/cli/report.ts index c7ac7f7327..aeab2980a0 100644 --- a/packages/benchmark/src/cli/report.ts +++ b/packages/benchmark/src/cli/report.ts @@ -2,14 +2,14 @@ import * as path from 'node:path'; import { readFile } from 'node:fs/promises'; import { z } from 'zod/v4'; import type { CommandModule } from 'yargs'; -import { OUTPUT_DIR } from '../runner/outputDir'; +import { resultsPathOf } from '../runner/outputDir'; interface Args { /** Path to a JSON report written by `benchmark run`. */ file: string; } -const DEFAULT_REPORT = path.join(OUTPUT_DIR, 'results', 'report.json'); +const DEFAULT_REPORT = path.relative(process.cwd(), resultsPathOf(process.cwd())); const command: CommandModule<{}, Args> = { command: 'report [file]', diff --git a/packages/benchmark/src/cli/run.ts b/packages/benchmark/src/cli/run.ts index e439ceec00..a083066406 100644 --- a/packages/benchmark/src/cli/run.ts +++ b/packages/benchmark/src/cli/run.ts @@ -22,13 +22,12 @@ const command: CommandModule<{}, Args> = { }) .option('samples', { type: 'number', - default: 30, - describe: 'Measured rounds per benchmark; each round samples every variant once', + describe: + 'Measured rounds per benchmark; each round samples every variant once. Default: 30', }) .option('warmup', { type: 'number', - default: 10, - describe: 'Discarded rounds before measuring, once per benchmark', + describe: 'Discarded rounds before measuring, once per benchmark. Default: 10', }) .option('launch-arg', { type: 'string', @@ -39,20 +38,18 @@ const command: CommandModule<{}, Args> = { .option('resolve-mode', { type: 'string', choices: ['isolated', 'in-place'] as const, - default: 'in-place' as const, describe: - "How pages find the library: 'isolated' installs each ref beside the repository, 'in-place' pins it in the repository's own pnpm-workspace.yaml for the length of the run", + "How pages find the library: 'isolated' installs each ref beside the repository, 'in-place' pins it in the repository's own pnpm-workspace.yaml for the length of the run. Default: in-place", }) .option('build-cmd', { type: 'string', - default: 'pnpm release:build', describe: - "Command that builds the publishable workspace packages, in each ref's checkout and in the working tree", + "Command that builds the publishable workspace packages, in each ref's checkout and in the working tree. Default: pnpm release:build", }) .option('install', { type: 'boolean', - default: true, - describe: "Install dependencies inside a ref's checkout. Use --no-install to skip", + describe: + "Install dependencies inside a ref's checkout (the default). Use --no-install to skip", }) .option('upload', { type: 'boolean', diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index 823c31ba01..5e3e6c51a4 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -10,7 +10,9 @@ import { createInput } from './input'; import { createCaseRuntime, createElementTimingWaiter, + EMPTY_RECORDING_MESSAGE, measureIteration, + MILLISECONDS, PAINT_METRIC_NAME, splitCaseArgs, warnIfNoGc, @@ -26,20 +28,7 @@ import './Metric'; // Import for TaskMeta augmentation side effect import './taskMetaAugmentation'; -export type { RenderEvent, IterationData, InteractionContext } from './types'; -export type { - MetricKind, - MetricDirection, - MetricAlarm, - MetricConfig, - MetricDefinition, -} from './types'; -export type { BenchmarkInput } from './input'; -export type { BenchmarkInteraction, BenchmarkOptions, VariantLoader } from './caseRuntime'; -export { ElementTiming } from './ElementTiming'; -export { Metric, type MetricRecordOptions } from './Metric'; -export { ScalarMetric }; -export { DiscreteMetric } from './DiscreteMetric'; +export * from './publicApi'; // When true, `benchmark()` opens an interactive profiling session in a headed // browser instead of running the automated measurement loop. Enabled by @@ -78,7 +67,7 @@ export interface RunCaseResult { // Duration regression signal and floods the report on any broadly-regressed run. const paint = new ScalarMetric({ name: PAINT_METRIC_NAME, - format: { style: 'unit', unit: 'millisecond', maximumFractionDigits: 2 }, + format: MILLISECONDS, }); /** @@ -221,11 +210,7 @@ export function benchmark( // Every active recording window must capture at least one render. Windows where recording was // never running (e.g. a fully-paused, metric-only benchmark) are not checked. - expect( - sawEmptyActiveWindow, - 'React recording was active but captured no renders. If you only measure imperative DOM ' + - 'updates or custom metrics, keep recording paused (reactRecordingPaused) instead of resuming.', - ).toBe(false); + expect(sawEmptyActiveWindow, EMPTY_RECORDING_MESSAGE).toBe(false); // Validate all iterations produced the same render events (count + order). // This runs after meta is set so the reporter can still display results on failure. diff --git a/packages/benchmark/src/metricCore.ts b/packages/benchmark/src/metricCore.ts index 2549844964..b9ebe1546b 100644 --- a/packages/benchmark/src/metricCore.ts +++ b/packages/benchmark/src/metricCore.ts @@ -5,15 +5,15 @@ export interface MetricRecordOptions { id?: string; } -/** - * Where recorded values go. The runtime a benchmark runs under installs one: the Vitest harness - * accumulates them per running test, the A/B page emits them as `performance.measure` entries. - */ /** The name a sub-series is reported under: `name#id`, or `name` for the base series. */ export function seriesName(name: string, id: string | undefined): string { return id === undefined ? name : `${name}#${id}`; } +/** + * Where recorded values go. The runtime a benchmark runs under installs one: the Vitest harness + * accumulates them per running test, the page runtime emits them as `performance.measure` entries. + */ export type MetricRecorder = ( metric: Metric, value: number, diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 83088cfa9b..09ace5a1b1 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -1,6 +1,13 @@ import type * as React from 'react'; import { createInput } from '../input'; -import { measureIteration, PAINT_METRIC_NAME, splitCaseArgs, warnIfNoGc } from '../caseRuntime'; +import { + EMPTY_RECORDING_MESSAGE, + measureIteration, + MILLISECONDS, + PAINT_METRIC_NAME, + splitCaseArgs, + warnIfNoGc, +} from '../caseRuntime'; import type { BenchmarkInteraction, BenchmarkOptions, @@ -9,27 +16,14 @@ import type { VariantLoader, } from '../caseRuntime'; import { seriesName, setMetricRecorder } from '../metricCore'; -import type { MetricConfig, MetricKind } from '../types'; +import type { MetricDefinition } from '../types'; // What `@mui/internal-benchmark` resolves to when benchmark files are built into a page for // `benchmark run`. Same authoring API, different driver: `benchmark()` registers a case, and the // runner calls `window.benchmarkPage.sample()` to run one iteration of it at a time. Every result // leaves the page as a `performance.measure` entry, so it also shows up in a DevTools trace. -export type { RenderEvent, IterationData, InteractionContext } from '../types'; -export type { - MetricKind, - MetricDirection, - MetricAlarm, - MetricConfig, - MetricDefinition, -} from '../types'; -export type { BenchmarkInput } from '../input'; -export type { BenchmarkInteraction, BenchmarkOptions, VariantLoader } from '../caseRuntime'; -export { ElementTiming } from '../ElementTiming'; -export { Metric, type MetricRecordOptions } from '../metricCore'; -export { ScalarMetric } from '../ScalarMetric'; -export { DiscreteMetric } from '../DiscreteMetric'; +export * from '../publicApi'; interface PageCase { renderFn: () => React.ReactElement; @@ -107,22 +101,9 @@ interface RecordedValue { // where recorded values are dropped. let sampleValues: RecordedValue[] | null = null; -/** How the runner is told to read a metric: what its report carries per metric name. */ -export interface PageMetricDefinition { - kind: MetricKind; - format?: Intl.NumberFormatOptions; - alarm?: MetricConfig['alarm']; -} - -const MILLISECONDS: Intl.NumberFormatOptions = { - style: 'unit', - unit: 'millisecond', - maximumFractionDigits: 2, -}; - // The harness's own metrics. Render time and render count alarm on any resolved change for the // worse; paint dominates each case's duration and duplicates that signal, so it is informational. -const metricDefinitions = new Map([ +const metricDefinitions = new Map([ ['render', { kind: 'scalar', format: MILLISECONDS, alarm: {} }], ['render:count', { kind: 'discrete', alarm: {} }], ['render:mount', { kind: 'scalar', format: MILLISECONDS }], @@ -147,7 +128,7 @@ export interface BenchPage { /** The file's `compare()` calls, with their variant keys in order. */ comparisons: () => Array<{ name: string; variants: string[] }>; /** Every metric reported so far, by name: the harness's own and those the cases recorded. */ - metricDefinitions: () => Record; + metricDefinitions: () => Record; /** * Runs one iteration of a case. A measured sample leaves its results as `performance.measure` * entries for the runner to collect; a warmup sample leaves none. @@ -233,10 +214,7 @@ async function sample(name: string, { warmup }: { warmup: boolean }): Promise left - right); - const middle = Math.floor(sorted.length / 2); - return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle]; + return quantile( + [...values].sort((left, right) => left - right), + 0.5, + ); } -export function summarizeSamples(values: number[]): SampleSummary { +function summarizeSamples(values: number[]): SampleSummary { return { count: values.length, median: median(values), mean: meanInterval(values) }; } const DEFAULT_DEFINITION: RunMetricDefinition = { kind: 'scalar' }; /** A metric's definition; a `name#id` sub-series is defined by its base metric. */ -export function definitionOf(report: BenchmarkRunReport, metric: string): RunMetricDefinition { +function definitionOf(report: BenchmarkRunReport, metric: string): RunMetricDefinition { const [base] = metric.split('#'); return report.metrics[metric] ?? report.metrics[base] ?? DEFAULT_DEFINITION; } @@ -118,7 +119,7 @@ function worseningAtLeast( return (lowerIsBetter ? comparison.relative.low : -comparison.relative.high) / 100; } -export function severityOf( +function severityOf( comparison: Pick, definition: RunMetricDefinition, ): Severity { @@ -175,10 +176,7 @@ function pairsOf(benchmark: RunBenchmark): Array<{ subject: string; against: str : others.map((other) => ({ subject: other, against: reference })); } -export function analyzeBenchmark( - report: BenchmarkRunReport, - benchmark: RunBenchmark, -): BenchmarkAnalysis { +function analyzeBenchmark(report: BenchmarkRunReport, benchmark: RunBenchmark): BenchmarkAnalysis { const { samples } = benchmark; if (!samples) { return { benchmark, metrics: [] }; diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index d469a168f7..04a6e79f18 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -1,5 +1,5 @@ -import type { Interval, MetricComparison } from './analyzeRun'; -import type { RunMetricDefinition } from './schema'; +import type { BenchmarkAnalysis, Interval, MetricComparison, Regression } from './analyzeRun'; +import type { RunBenchmark, RunMetricDefinition } from './schema'; /** * How a run report's numbers read, for every renderer of one: the terminal table, the pull request @@ -27,3 +27,31 @@ export function formatPercent(interval: Interval): string { export function formatComparison(comparison: MetricComparison): string { return `${comparison.change} ${formatPercent(comparison.relative)}`; } + +/** What a comparison column is headed: `Δ vs baseline` for a build, `theirs vs ours` for a variant. */ +export function formatComparisonLabel( + benchmark: RunBenchmark, + { subject, against }: Pick, +): string { + return benchmark.kind === 'baseline' ? `Δ vs ${against}` : `${subject} vs ${against}`; +} + +/** The line a run is summed up in: how many benchmarks measured, regressed and failed. */ +export function formatRunSummary(analyses: BenchmarkAnalysis[], regressions: Regression[]): string { + const measured = analyses.filter((analysis) => !analysis.benchmark.error).length; + const failed = analyses.length - measured; + const regressed = new Set(regressions.map((regression) => regression.benchmark)).size; + return [ + `${measured} benchmark${measured === 1 ? '' : 's'} measured`, + regressed > 0 ? `${regressed} with regressions` : 'no regressions', + failed > 0 ? `${failed} failed` : null, + ] + .filter(Boolean) + .join(' · '); +} + +/** How to read the numbers, under every rendering of a run. */ +export const RUN_REPORT_FOOTNOTE = + 'Each value is a median. Each Δ is a 95% confidence interval on the paired per-round ' + + 'difference, relative to the variant it is measured against; "unsure" means it straddles ' + + 'zero — the expected result for two equivalent builds.'; diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts index 9c331e67c5..8a5905e727 100644 --- a/packages/benchmark/src/runReport/schema.ts +++ b/packages/benchmark/src/runReport/schema.ts @@ -1,4 +1,5 @@ import { z } from 'zod/v4'; +import type { MetricDefinition } from '../types'; /** * The report `benchmark run` produces, uploads and the dashboard reads: version 2 of the `benchmark` @@ -24,12 +25,14 @@ const alarmSchema = z.object({ error: z.number().min(0).optional(), }); +// The same shape a page reports and a metric's config declares, so a field added there has to be +// added here too — `satisfies` fails the build otherwise, rather than zod dropping it at upload. const metricDefinitionSchema = z.object({ kind: z.enum(['scalar', 'discrete']), format: z.custom().optional(), /** Present when a change in this metric is a regression; without it the metric is informational. */ alarm: alarmSchema.optional(), -}); +}) satisfies z.ZodType; /** A build the benchmark pages were loaded from. */ const buildSchema = z.object({ @@ -75,19 +78,23 @@ export const benchmarkRunReportSchema = z.object({ benchmarks: z.array(benchmarkSchema), }); -export type BenchmarkRunReport = z.infer; +/** + * The version 2 `benchmark` upload, which the dashboard stores as the commit's `benchmark.json`: the + * same envelope as every CI report around this report. Declared here, browser-safe, so the runner + * validates with it and the dashboard reads with its type. + */ +export const benchmarkRunUploadSchema = z.object({ + version: z.literal(2), + timestamp: z.number(), + commitSha: z.string().regex(/^[0-9a-f]{40}$/, 'Must be a 40-character hex string'), + repo: z.string().includes('/', 'Must be in owner/repo format'), + reportType: z.literal('benchmark'), + prNumber: z.number().int().positive().optional(), + branch: z.string(), + report: benchmarkRunReportSchema, +}); -/** The version 2 `benchmark.json` as the dashboard stores it: the upload envelope around the report. */ -export interface BenchmarkRunUpload { - version: 2; - timestamp: number; - commitSha: string; - repo: string; - reportType: 'benchmark'; - prNumber?: number; - branch: string; - report: BenchmarkRunReport; -} +export type BenchmarkRunReport = z.infer; +export type BenchmarkRunUpload = z.infer; export type RunMetricDefinition = z.infer; export type RunBenchmark = z.infer; -export type RunBuild = z.infer; diff --git a/packages/benchmark/src/runner/outputDir.ts b/packages/benchmark/src/runner/outputDir.ts index b6c0b9c203..9251a60a9f 100644 --- a/packages/benchmark/src/runner/outputDir.ts +++ b/packages/benchmark/src/runner/outputDir.ts @@ -30,6 +30,11 @@ export async function prepareOutputDir(harnessDir: string): Promise { * agree on it: the runner serves every ref's pages from under this path, and a plain `vite build` * has to land somewhere that does not collide. */ +/** Where a run writes its JSON report, unless told otherwise. */ +export function resultsPathOf(harnessDir: string): string { + return path.join(harnessDir, OUTPUT_DIR, 'results', 'report.json'); +} + export function buildsDirOf(harnessDir: string, refId?: string): string { const builds = path.join(harnessDir, OUTPUT_DIR, 'builds'); return refId ? path.join(builds, refId) : builds; diff --git a/packages/benchmark/src/runner/printReport.ts b/packages/benchmark/src/runner/printReport.ts index 31ab8770f5..89a85da149 100644 --- a/packages/benchmark/src/runner/printReport.ts +++ b/packages/benchmark/src/runner/printReport.ts @@ -1,18 +1,26 @@ /* eslint-disable no-console */ -import { dim, green, printTable, red, yellow } from '../format'; -import { analyzeRun, findRegressions, formatComparison, formatValue } from '../runReport'; +import { dim, printTable, red, yellow } from '../format'; +import { + analyzeRun, + findRegressions, + formatComparison, + formatComparisonLabel, + formatRunSummary, + formatValue, + RUN_REPORT_FOOTNOTE, +} from '../runReport'; import type { BenchmarkAnalysis, BenchmarkRunReport } from '../runReport'; function printBenchmark({ benchmark, metrics }: BenchmarkAnalysis): void { const [reference] = benchmark.variants; - const comparedTo = (subject: string, against: string) => - benchmark.kind === 'baseline' ? `Δ vs ${against}` : `${subject} vs ${against}`; const headers = [ 'Metric', ...benchmark.variants.map((variant) => `${variant} (median)`), - ...(metrics[0]?.comparisons ?? []).map(({ subject, against }) => comparedTo(subject, against)), + ...(metrics[0]?.comparisons ?? []).map((comparison) => + formatComparisonLabel(benchmark, comparison), + ), 'Rounds', ]; const rows = metrics.map(({ metric, definition, variants, comparisons }) => [ @@ -54,21 +62,6 @@ export function printRunReport(report: BenchmarkRunReport): void { const color = comparison.severity === 'error' ? red : yellow; console.log(color(`⚠ ${benchmark} · ${metric} · ${formatComparison(comparison)}`)); } - const measured = analyses.filter((analysis) => !analysis.benchmark.error).length; - const failed = analyses.length - measured; - console.log( - [ - `${measured} benchmark${measured === 1 ? '' : 's'} measured`, - regressions.length > 0 ? red(`${regressions.length} regression(s)`) : green('no regressions'), - failed > 0 ? red(`${failed} failed`) : null, - ] - .filter(Boolean) - .join(' · '), - ); - console.log( - dim( - 'Each Δ is a 95% confidence interval on the paired per-round difference, relative to the ' + - 'variant it is measured against; "unsure" means it straddles zero.', - ), - ); + console.log(formatRunSummary(analyses, regressions)); + console.log(dim(RUN_REPORT_FOOTNOTE)); } diff --git a/packages/benchmark/src/runner/refs.ts b/packages/benchmark/src/runner/refs.ts index b563008718..c8fc29df6f 100644 --- a/packages/benchmark/src/runner/refs.ts +++ b/packages/benchmark/src/runner/refs.ts @@ -25,6 +25,14 @@ export function refLabel(ref: ResolvedRef): string { return ref.kind === 'worktree' ? 'working tree' : ref.requested; } +/** + * The variant name a ref's pages are reported under: `current` for the working tree, `baseline` for + * the revision it is compared against. The report's `builds` are keyed by the same names. + */ +export function variantOf(ref: ResolvedRef): 'current' | 'baseline' { + return ref.kind === 'worktree' ? 'current' : 'baseline'; +} + /** The working tree, which every run builds. Never cached, having no immutable identity. */ export const WORKTREE_REF: ResolvedRef = { kind: 'worktree', id: 'current' }; diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index 13349e8484..2813c13b30 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -3,12 +3,12 @@ import * as path from 'node:path'; import { mkdir, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; -import { execaSync } from 'execa'; +import { execa } from 'execa'; import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; import { packRef, packWorkingTree } from '../utils/packWorkspace'; -import type { PackedPackage } from '../utils/packWorkspace'; +import { resolveCommit } from '../utils/git'; import type { BenchmarkRunReport } from '../runReport'; -import { refLabel, resolveBaselineRef, WORKTREE_REF } from './refs'; +import { refLabel, resolveBaselineRef, variantOf, WORKTREE_REF } from './refs'; import type { ResolvedRef } from './refs'; import { discoverBenchFiles } from './benchFiles'; import { runInterleaved } from './runInterleaved'; @@ -17,7 +17,7 @@ import type { ResolveMode } from './buildPages'; import { resolveBrowserBinary } from './browser'; import { printRunReport } from './printReport'; import { publishRunReport } from './upload'; -import { buildsDirOf, prepareOutputDir } from './outputDir'; +import { buildsDirOf, prepareOutputDir, resultsPathOf } from './outputDir'; export interface RunBenchmarksOptions { /** The harness package directory (where the command was run). */ @@ -120,20 +120,13 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise { + if (ref.kind === 'worktree') { + return packWorkingTree({ repoRoot, outRoot: path.join(packedDir, 'current'), buildCmd }); + } // packRef caches a ref's tarballs by SHA; a hit skips the checkout, install, and build. - // eslint-disable-next-line no-await-in-loop const packed = await packRef({ repoRoot, ref: ref.sha, @@ -142,15 +135,20 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise [ - ref.kind === 'worktree' ? 'current' : 'baseline', - { sha: ref.sha, label: refLabel(ref) }, - ]), + refs.map((ref) => [variantOf(ref), { sha: ref.sha, label: refLabel(ref) }]), ), metrics: results.metrics, benchmarks: results.benchmarks, }; - const outPath = out ? path.resolve(out) : path.join(outputDir, 'results', 'report.json'); + const outPath = out ? path.resolve(out) : resultsPathOf(harnessDir); await mkdir(path.dirname(outPath), { recursive: true }); await writeFile(outPath, `${JSON.stringify(report, null, 2)}\n`); console.log(''); diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index a495502f2a..635a5099e3 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -5,6 +5,7 @@ import type { Browser, BrowserContext, CDPSession, Page } from '@playwright/test import { BENCHMARK_LAUNCH_ARGS } from '../launchArgs'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; import type { BenchFile } from './benchFiles'; +import { variantOf } from './refs'; import type { ResolvedRef } from './refs'; import { serveDirectory } from './serveDirectory'; // Types for `window.benchmarkPage`, which the evaluated functions below call into. @@ -207,15 +208,9 @@ async function measureBenchCase( } rounds.push(round); } - // Every page defines the same harness metrics, and a custom metric in whichever case records it. - const metrics = Object.assign( - {}, - ...(await Promise.all( - targets.map((target) => - target.page.evaluate(() => window.benchmarkPage!.metricDefinitions()), - ), - )), - ); + // Read from the reference page: a metric whose config the change under test altered is + // described the way the change describes it. + const metrics = await targets[0].page.evaluate(() => window.benchmarkPage!.metricDefinitions()); return { rounds, metrics }; } finally { await closeBenchPages(targets); @@ -235,6 +230,12 @@ function samplesOf(slots: PageSlot[], rounds: Round[]): RunBenchmark['samples'] ); } +/** A benchmark as it goes into the report, with the definitions of the metrics it reported. */ +interface CaseResult { + benchmark: RunBenchmark; + metrics: Record; +} + interface CaseGroup { kind: RunBenchmark['kind']; file: string; @@ -251,20 +252,22 @@ async function runListedCases( listed: OpenedPage[], group: CaseGroup, options: RunInterleavedOptions, - metrics: Record, -): Promise { +): Promise { const slots = listed.map((target) => target.slot); const variants = slots.map((slot) => slot.variant); const [reference, ...others] = listed; - const benchmarks: RunBenchmark[] = []; + const results: CaseResult[] = []; for (const caseName of reference.caseNames) { const entry = { name: group.nameOf(caseName), file: group.file, kind: group.kind, variants }; const missing = others.find((target) => !target.caseNames.includes(caseName)); if (missing) { // A case added by the change under test, or one library lacks, has nothing to compare with. - benchmarks.push({ - ...entry, - error: `"${caseName}" does not exist in [${missing.slot.variant}].`, + results.push({ + benchmark: { + ...entry, + error: `"${caseName}" does not exist in [${missing.slot.variant}].`, + }, + metrics: {}, }); continue; } @@ -273,14 +276,16 @@ async function runListedCases( try { // eslint-disable-next-line no-await-in-loop const measured = await measureBenchCase(browser, slots, caseName, options); - Object.assign(metrics, measured.metrics); - benchmarks.push({ ...entry, samples: samplesOf(slots, measured.rounds) }); + results.push({ + benchmark: { ...entry, samples: samplesOf(slots, measured.rounds) }, + metrics: measured.metrics, + }); } catch (error) { console.error(chalk.red(` ${errorMessage(error)}`)); - benchmarks.push({ ...entry, error: errorMessage(error) }); + results.push({ benchmark: { ...entry, error: errorMessage(error) }, metrics: {} }); } } - return benchmarks; + return results; } /** Opens a set of pages just to learn what they define, then runs their cases. */ @@ -289,13 +294,12 @@ async function listAndRun( slots: PageSlot[], group: CaseGroup, options: RunInterleavedOptions, - metrics: Record, -): Promise<{ benchmarks: RunBenchmark[]; comparisons: OpenedPage['comparisons'] }> { +): Promise<{ results: CaseResult[]; comparisons: OpenedPage['comparisons'] }> { // Every case then measures in fresh pages of its own. const listed = await openBenchPages(browser, slots); await closeBenchPages(listed); return { - benchmarks: await runListedCases(browser, listed, group, options, metrics), + results: await runListedCases(browser, listed, group, options), comparisons: listed[0].comparisons, }; } @@ -313,18 +317,16 @@ async function runBenchFile( origin: string, benchFile: BenchFile, options: RunInterleavedOptions, - metrics: Record, -): Promise { +): Promise { const buildSlots = options.benchRefs.map((ref): PageSlot => ({ - variant: ref.kind === 'worktree' ? 'current' : 'baseline', + variant: variantOf(ref), url: benchPageUrl(origin, ref, benchFile).href, })); - const { benchmarks, comparisons } = await listAndRun( + const { results, comparisons } = await listAndRun( browser, buildSlots, { kind: 'baseline', file: benchFile.file, nameOf: (caseName) => caseName }, options, - metrics, ); const [worktreeRef] = options.benchRefs; @@ -346,21 +348,23 @@ async function runBenchFile( nameOf: (caseName) => `${comparison} / ${caseName}`, }, options, - metrics, ); - benchmarks.push(...compared.benchmarks); + results.push(...compared.results); } catch (error) { console.error(chalk.red(` ${errorMessage(error)}`)); - benchmarks.push({ - name: comparison, - file: benchFile.file, - kind: 'compare', - variants, - error: errorMessage(error), + results.push({ + benchmark: { + name: comparison, + file: benchFile.file, + kind: 'compare', + variants, + error: errorMessage(error), + }, + metrics: {}, }); } } - return benchmarks; + return results; } export async function runInterleaved(options: RunInterleavedOptions): Promise { @@ -375,14 +379,17 @@ export async function runInterleaved(options: RunInterleavedOptions): Promise = {}; + const results: CaseResult[] = []; // Sequential on purpose: concurrent cases would contend for the same machine. for (const benchFile of options.benchFiles) { // eslint-disable-next-line no-await-in-loop - benchmarks.push(...(await runBenchFile(browser, server.origin, benchFile, options, metrics))); + results.push(...(await runBenchFile(browser, server.origin, benchFile, options))); } - return { benchmarks, metrics, browserVersion: browser.version() }; + return { + benchmarks: results.map((result) => result.benchmark), + metrics: Object.assign({}, ...results.map((result) => result.metrics)), + browserVersion: browser.version(), + }; } finally { await browser.close(); await server.close(); diff --git a/packages/benchmark/src/runner/upload.ts b/packages/benchmark/src/runner/upload.ts index 1bb15a43d8..ea40965be2 100644 --- a/packages/benchmark/src/runner/upload.ts +++ b/packages/benchmark/src/runner/upload.ts @@ -2,18 +2,11 @@ import chalk from 'chalk'; import { postToDashboard } from '../ciApi'; -import { ciReportUploadSchema, getCiMetadata } from '../ciReport'; +import { getCiMetadata } from '../ciReport'; import { syncPrComment } from '../syncPrComment'; -import { benchmarkRunReportSchema } from '../runReport'; +import { benchmarkRunUploadSchema } from '../runReport'; import type { BenchmarkRunReport } from '../runReport'; -/** - * Version 2 of the `benchmark` upload. It shares the report type with the Vitest reporter's version - * 1 — and so the commit's one `benchmark.json` — so a repository uploads one or the other; the - * dashboard reads each by its version. - */ -const benchmarkUploadV2Schema = ciReportUploadSchema('benchmark', 2, benchmarkRunReportSchema); - /** * Uploads a report and refreshes the pull request comment. * @@ -30,7 +23,7 @@ export async function publishRunReport(report: BenchmarkRunReport): Promise Date: Tue, 29 Sep 2026 13:57:14 +0200 Subject: [PATCH 081/173] [benchmark] Upload the benchmark run report to the PR preview in CI --- .circleci/config.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6ea84a6d7c..31c5ea6812 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -110,8 +110,9 @@ jobs: command: pnpm release:build - run: name: Run benchmarks - environment: - BENCHMARK_UPLOAD: 'true' + # TEMPORARY: `test_benchmark_run` uploads this commit's `benchmark.json` as version 2 + # instead, so the pull request comment shows the new report. Restore + # `BENCHMARK_UPLOAD: 'true'` here before merging. command: pnpm -F ./test/performance benchmark test_tachometer: @@ -179,8 +180,12 @@ jobs: - run: name: Run the benchmark smoke test no_output_timeout: 30m - # No upload: this repository still uploads its version 1 benchmark report from - # `test_benchmark`, and both write the commit's one `benchmark.json`. + environment: + # TEMPORARY: uploads in place of `test_benchmark`, since both write the commit's one + # `benchmark.json`, and to this pull request's dashboard preview, since production does + # not read version 2 reports yet. Remove both lines before merging. + BENCHMARK_UPLOAD: 'true' + CI_REPORT_API_URL: https://code-infra-dashboard-pr-1898.onrender.com command: pnpm test:benchmark-run --baseline "$(cat .benchmark-baseline-sha)" - save_cache: name: Save the packed-baseline cache From b5a582cf02e3ad098682914e74b0d2db367a942a Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 14:04:39 +0200 Subject: [PATCH 082/173] [benchmark] Remove the benchmark report command `benchmark run` already prints the tables when it finishes, and a saved or uploaded report is readable in the dashboard. --- packages/benchmark/README.md | 3 +- packages/benchmark/src/cli/index.ts | 2 -- packages/benchmark/src/cli/report.ts | 47 ---------------------------- test/benchmark/README.md | 3 -- 4 files changed, 1 insertion(+), 54 deletions(-) delete mode 100644 packages/benchmark/src/cli/report.ts diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 4392113eec..37a508bc85 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -448,8 +448,7 @@ none — so nothing else should build against the same checkout meanwhile. resolution there, leaving the repository untouched, at the cost of a second install tree per ref. `--launch-arg` passes a flag to Chromium, e.g. `--use-angle=metal` for hardware-rendered paint -timings. `--no-install` skips a ref's install. `benchmark run --help` lists the rest, and -`benchmark report` prints the tables again from a saved JSON report. +timings. `--no-install` skips a ref's install. `benchmark run --help` lists the rest. ## Tachometer diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index d8261335a0..1ee03a37b4 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -8,7 +8,6 @@ import { hideBin } from 'yargs/helpers'; // repository than in the published package, which is built from `build/`, so a relative path // cannot reach the manifest from both — and the repository's workspace symlink hides the break. import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; -import report from './report'; import run from './run'; import tachoReport from './tachoReport'; import tachoRun from './tachoRun'; @@ -29,7 +28,6 @@ await yargs(hideBin(process.argv)) globalArgv = argv; }, true) .command(run) - .command(report) .command({ command: 'tacho ', describe: 'Benchmark a tachometer harness, and read the reports it writes.', diff --git a/packages/benchmark/src/cli/report.ts b/packages/benchmark/src/cli/report.ts deleted file mode 100644 index aeab2980a0..0000000000 --- a/packages/benchmark/src/cli/report.ts +++ /dev/null @@ -1,47 +0,0 @@ -import * as path from 'node:path'; -import { readFile } from 'node:fs/promises'; -import { z } from 'zod/v4'; -import type { CommandModule } from 'yargs'; -import { resultsPathOf } from '../runner/outputDir'; - -interface Args { - /** Path to a JSON report written by `benchmark run`. */ - file: string; -} - -const DEFAULT_REPORT = path.relative(process.cwd(), resultsPathOf(process.cwd())); - -const command: CommandModule<{}, Args> = { - command: 'report [file]', - describe: - 'Print the tables for a JSON report written by `benchmark run`. Reads the last run of the harness in the current directory when no path is given.', - builder: (yargs) => { - return yargs.positional('file', { - type: 'string', - default: DEFAULT_REPORT, - describe: 'Path to the JSON report', - }); - }, - handler: async (argv) => { - const reportPath = path.resolve(argv.file); - const raw = await readFile(reportPath, 'utf8').catch(() => { - throw new Error( - `No report at ${reportPath}. Pass a path, or run \`benchmark run\` from a harness first.`, - ); - }); - - const { benchmarkRunReportSchema } = await import('../runReport'); - const parsed = benchmarkRunReportSchema.safeParse(JSON.parse(raw)); - if (!parsed.success) { - throw new Error( - `${reportPath} is not a benchmark report this version can read.\n` + - `${z.prettifyError(parsed.error)}`, - ); - } - - const { printRunReport } = await import('../runner/printReport'); - printRunReport(parsed.data); - }, -}; - -export default command; diff --git a/test/benchmark/README.md b/test/benchmark/README.md index ecc15d21c7..01b336bf31 100644 --- a/test/benchmark/README.md +++ b/test/benchmark/README.md @@ -48,9 +48,6 @@ A filter is a case-insensitive substring of a benchmark file's path under `src/` matches test files. Because both sides run identical code, the expected result is `unsure` — that is the "no change" outcome, not a failure. -`benchmark report` prints the tables again from the saved JSON, so a run kept from earlier — or a -report downloaded from CI — can be read without sampling again. - Everything a run writes goes under `.benchmark/`: the report in `results/`, the pages built per ref in `builds/`, the packed tarballs in `packed/` (the one worth caching in CI, keyed by commit SHA) and the install each ref resolves through in `trees/`. Deleting that directory resets the harness completely. From 49304c09da08db0585d353b6a085b0c98fb0c5e5 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 14:08:56 +0200 Subject: [PATCH 083/173] [benchmark] Trim benchmark run options Pages always resolve in place, and a ref's checkout is always installed before it is built. Samples, warmup and extra launch arguments are fixed for now. --- packages/benchmark/README.md | 12 +- packages/benchmark/src/cli/run.ts | 33 +-- packages/benchmark/src/runner/buildPages.ts | 218 ++---------------- packages/benchmark/src/runner/outputDir.ts | 2 +- .../benchmark/src/runner/runBenchmarks.ts | 69 ++---- test/benchmark/README.md | 6 +- 6 files changed, 48 insertions(+), 292 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 37a508bc85..e8a1737baf 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -394,8 +394,8 @@ module-scope data is built once. Every variant is sampled once per round, in a shuffled order, and a difference is judged on the per-round differences rather than on two independent sets of samples. Whatever the machine was doing during a round — thermal throttling, a background process — then affects both sides of it and cancels -out. `--warmup` rounds (10 by default) are discarded once per benchmark, then `--samples` rounds (30 -by default) are measured; the iteration counts in `benchmark()`'s options only apply under Vitest. +out. 10 warmup rounds are discarded once per benchmark, then 30 rounds are measured; the iteration +counts in `benchmark()`'s options only apply under Vitest. ### The report @@ -436,7 +436,7 @@ Packed builds are cached by commit SHA under `.benchmark/packed/`, so repeating the same commit skips the rebuild; the working tree is never cached. In CI, cache that directory keyed on the baseline SHA, and check out with full history so a fork point can be resolved. -By default a run resolves **in place**: it pins the packed build in the repository's own +A run resolves **in place**: it pins the packed build in the repository's own `pnpm-workspace.yaml`, installs it there, and resolution is then ordinary. The repository is put back, and reinstalled, when the run ends — including when it fails, and at the start of the next run if one was killed outright. Two things follow from it. A tracked file names tarballs under @@ -444,11 +444,7 @@ if one was killed outright. Two things follow from it. A tracked file names tarb the workspace for that time, because pnpm scopes an override by parent package name and a harness has none — so nothing else should build against the same checkout meanwhile. -`--resolve-mode isolated` avoids both: it installs each ref into a directory of its own and points -resolution there, leaving the repository untouched, at the cost of a second install tree per ref. - -`--launch-arg` passes a flag to Chromium, e.g. `--use-angle=metal` for hardware-rendered paint -timings. `--no-install` skips a ref's install. `benchmark run --help` lists the rest. +`benchmark run --help` lists the remaining options. ## Tachometer diff --git a/packages/benchmark/src/cli/run.ts b/packages/benchmark/src/cli/run.ts index a083066406..f74a1e79b1 100644 --- a/packages/benchmark/src/cli/run.ts +++ b/packages/benchmark/src/cli/run.ts @@ -2,7 +2,7 @@ import type { CommandModule } from 'yargs'; import type { RunBenchmarksOptions } from '../runner/runBenchmarks'; /** Everything `runBenchmarks` takes except the directory, which is where the command was run. */ -type Args = Omit & { launchArg?: string[] }; +type Args = Omit; const command: CommandModule<{}, Args> = { command: 'run [filters...]', @@ -20,37 +20,11 @@ const command: CommandModule<{}, Args> = { describe: 'The build to compare against: a revision, on its own or as git:. Defaults to HEAD~1, the parent commit; `code-infra baseline` resolves a fork point to pass here', }) - .option('samples', { - type: 'number', - describe: - 'Measured rounds per benchmark; each round samples every variant once. Default: 30', - }) - .option('warmup', { - type: 'number', - describe: 'Discarded rounds before measuring, once per benchmark. Default: 10', - }) - .option('launch-arg', { - type: 'string', - array: true, - describe: - 'Extra Chromium launch argument, repeatable — e.g. --use-angle=metal for hardware-rendered paint timings', - }) - .option('resolve-mode', { - type: 'string', - choices: ['isolated', 'in-place'] as const, - describe: - "How pages find the library: 'isolated' installs each ref beside the repository, 'in-place' pins it in the repository's own pnpm-workspace.yaml for the length of the run. Default: in-place", - }) .option('build-cmd', { type: 'string', describe: "Command that builds the publishable workspace packages, in each ref's checkout and in the working tree. Default: pnpm release:build", }) - .option('install', { - type: 'boolean', - describe: - "Install dependencies inside a ref's checkout (the default). Use --no-install to skip", - }) .option('upload', { type: 'boolean', default: process.env.BENCHMARK_UPLOAD === 'true', @@ -68,12 +42,7 @@ const command: CommandModule<{}, Args> = { harnessDir: process.cwd(), filters: argv.filters ?? [], baseline: argv.baseline, - samples: argv.samples, - warmup: argv.warmup, - launchArgs: argv.launchArg ?? [], - resolveMode: argv.resolveMode, buildCmd: argv.buildCmd, - install: argv.install, out: argv.out, upload: argv.upload, }); diff --git a/packages/benchmark/src/runner/buildPages.ts b/packages/benchmark/src/runner/buildPages.ts index 64890e8782..9096331946 100644 --- a/packages/benchmark/src/runner/buildPages.ts +++ b/packages/benchmark/src/runner/buildPages.ts @@ -2,7 +2,7 @@ import * as path from 'node:path'; import { createRequire } from 'node:module'; -import { readFile, mkdir, rm, writeFile } from 'node:fs/promises'; +import { readFile, rm, writeFile } from 'node:fs/promises'; import { pathToFileURL } from 'node:url'; import chalk from 'chalk'; import type * as Vite from 'vite'; @@ -11,116 +11,19 @@ import { parse, stringify } from 'yaml'; // the note in ../cli/index.ts. import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; import { run } from '../utils/exec'; -import { tarballFor } from '../utils/packWorkspace'; import type { PackedPackage } from '../utils/packWorkspace'; -import { installedVersions, readPackageJson, writePackageJson } from '../utils/pnpm'; import { refLabel } from './refs'; import type { ResolvedRef } from './refs'; import { pathExists } from '../utils/path'; /** - * How a ref's pages find the library under test. - * - * `isolated` installs that ref's packed build into a directory of its own and points resolution - * there, leaving the repository untouched. `in-place` pins the packed build in the repository's own - * `pnpm-workspace.yaml` and installs it, so resolution is ordinary — at the cost of mutating a - * tracked file for as long as the run lasts. - */ -export type ResolveMode = 'isolated' | 'in-place'; - -/** - * Dependencies the run needs but a page never imports, so a ref's tree does without them — - * whichever section of the harness's manifest they sit in, since neither reason below turns on that. - * - * Both are resolved from the harness rather than from here: the browser by the runner, and this - * package by the harness's own vite config — which is loaded from the harness, since the build - * happens in place. Both are also expensive to install: `@playwright/test` pulls a browser, and this - * package a dependency tree many times the size of a harness. + * Packages the run itself resolves from the harness, so they are never pinned to a ref's build: the + * browser by the runner, and this package by the harness's own vite config. */ const RUNNER_ONLY_DEPS = ['@playwright/test', pkgJson.name]; /** - * A dependency map for a ref's tree: every `workspace:` entry points at that ref's packed tarball, - * and everything else is pinned to the version the harness resolved rather than the range it - * declares. - * - * The tree installs without the repository's lockfile, so a range would float to whatever is newest - * at run time — a different library than the repository builds against, measured as if it were - * ours. - */ -function rewriteWorkspaceDeps( - installed: Map, - deps: Partial> | undefined, - packages: PackedPackage[], - omit: string[] = [], -): Record { - const entries = Object.entries(deps ?? {}).filter( - ([name, version]) => version !== undefined && !omit.includes(name), - ) as Array<[string, string]>; - - return Object.fromEntries( - entries.map(([name, version]) => - version.startsWith('workspace:') - ? [name, `file:${tarballFor(packages, name)}`] - : [name, installed.get(name) ?? version], - ), - ); -} - -/** - * Reads the repository's `overrides`, so the ref's tree resolves the same dependency graph the - * repository does. - * - * The tree is its own workspace root and inherits nothing. Without this, a repository that pins a - * transitive dependency through an override would get a *different* graph here than in its real - * install — quietly changing what is being benchmarked — and any policy the override exists to - * satisfy (a blocked resolution, a security pin) would fail here too. - */ -async function readOverrides(repoRoot: string): Promise> { - let raw: string; - try { - raw = await readFile(path.join(repoRoot, 'pnpm-workspace.yaml'), 'utf8'); - } catch (error) { - // A repository without the file simply has no overrides. Anything else — a malformed file, a - // permission error — would otherwise resolve the un-overridden graph and silently benchmark a - // dependency set the repository never ships, which is the failure this function exists to avoid. - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return {}; - } - throw error; - } - return parse(raw)?.overrides ?? {}; -} - -/** - * A vite plugin that resolves every bare import from `treeDir` instead of from the harness. - * - * Bare specifiers only: a relative or absolute id belongs to the harness's own sources and stays - * where it is. Resolution is handed back to vite with the importer rewritten, so vite's own - * conditions (`browser`, `import`) and the target package's `exports` map still decide the answer — - * which is what an alias to a directory would skip. - */ -function resolveFromTree(treeDir: string): Vite.Plugin { - // Resolution needs a base directory, and vite derives one from the importer's path — falling back - // to the project root unless that path exists on disk, which would silently resolve the harness's - // copy of a dependency instead of this ref's. The tree's own manifest is the file that names this - // directory, the same way `createRequire` is pointed at one. - const importer = path.join(treeDir, 'package.json'); - - return { - name: 'mui-benchmark:resolve-from-tree', - enforce: 'pre', - async resolveId(id, _importer, options) { - if (id.startsWith('.') || path.isAbsolute(id) || id.startsWith('\0') || id.includes(':')) { - return null; - } - return this.resolve(id, importer, { ...options, skipSelf: true }); - }, - }; -} - -/** - * The packed pins an in-place run adds to the repository's overrides. + * The packed pins a run adds to the repository's overrides. * * Everything the run itself resolves is left alone, this package above all: pinning it would hand * the harness's vite config the ref's copy of the very tool doing the building, and a ref old enough @@ -134,14 +37,13 @@ function packedPins(packages: PackedPackage[]): Record { ); } -/** Where the repository's own manifest is kept while an in-place run has it pinned. */ +/** Where the repository's own manifest is kept while a run has it pinned. */ function backupPathOf(outputDir: string): string { return path.join(outputDir, 'pnpm-workspace.yaml.orig'); } /** - * Points the repository's own install at this ref's packed build, through the same overrides the - * isolated mode applies to a workspace of its own. + * Points the repository's own install at this ref's packed build, through its overrides. * * The manifest is copied aside first and rewritten from that copy each time, so a second ref pins * over the original rather than over the first ref's pins. A run that dies leaves both the pins and @@ -186,84 +88,17 @@ export async function restoreWorkspace(repoRoot: string, outputDir: string): Pro await run('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], repoRoot); } -/** Installs one ref's packed build, with the harness's own dependencies around it. */ -async function installRefTree({ - harnessDir, - repoRoot, - packages, - treeDir, -}: { - /** The harness package directory, whose dependencies are copied. */ - harnessDir: string; - /** The workspace root, whose overrides the tree inherits. */ - repoRoot: string; - /** That ref's packed workspace packages. */ - packages: PackedPackage[]; - /** Where to install. */ - treeDir: string; -}): Promise { - const [harnessPkg, installed] = await Promise.all([ - readPackageJson(harnessDir), - installedVersions(harnessDir), - mkdir(treeDir, { recursive: true }), - ]); - - await writePackageJson(treeDir, { - name: 'mui-benchmark:resolve-tree', - private: true, - version: '0.0.0', - dependencies: rewriteWorkspaceDeps( - installed, - harnessPkg.dependencies, - packages, - RUNNER_ONLY_DEPS, - ), - devDependencies: rewriteWorkspaceDeps( - installed, - harnessPkg.devDependencies, - packages, - RUNNER_ONLY_DEPS, - ), - }); - - // Marks this folder as its own workspace root, so pnpm does not walk up into the monorepo. The - // overrides carry the repository's own, then pin every packed workspace package on top — so a - // transitive dependency between them (whose packed `workspace:*` became a concrete, unpublished - // version) resolves to a local build rather than 404ing on the registry. The packed pins win, - // since they are the whole point of this install. - await writeFile( - path.join(treeDir, 'pnpm-workspace.yaml'), - stringify({ - overrides: { - ...(await readOverrides(repoRoot)), - ...Object.fromEntries(packages.map((pkg) => [pkg.name, `file:${pkg.tarball}`])), - }, - }), - ); - - // `--ignore-scripts`: this workspace has no build-script approvals, and pnpm fails an install - // over unapproved ones rather than warning. Nothing installed here needs its scripts either — the - // pages are built from the packages exactly as packed. - await run( - 'pnpm', - ['install', '--prefer-offline', '--ignore-scripts', '--config.engine-strict=false'], - treeDir, - ); -} - /** * Builds the benchmark pages against one ref's packed build. * * The pages are built **where they live**, from the harness's own directory and its own vite config, * so a `benchmark run` build sees exactly what a plain `vite build` sees — the same tsconfig, the same - * postcss config, the same everything. Only *resolution* differs: a plugin redirects every bare - * import into a small tree installed for this ref, where the library under test is that ref's packed - * build. + * postcss config, the same everything. Only the library differs: the repository's install is pinned + * to that ref's packed build first, so both sides resolve it the same way. Building one side through a + * workspace link instead would compare two different resolution paths, and that difference would land + * in the measurement rather than in the library. * - * That tree holds the harness's own dependencies too, so the pages and the library resolve React - * (and everything else) to one copy. Building one side through a workspace link instead would - * compare two different resolution paths, and that difference would land in the measurement rather - * than in the library. + * The pins stay until `restoreWorkspace` puts the repository back. */ export async function buildRefPages(options: { /** The harness package directory, built in place. */ @@ -274,35 +109,18 @@ export async function buildRefPages(options: { ref: ResolvedRef; /** That ref's packed workspace packages. */ packages: PackedPackage[]; - /** Persistent directory holding this ref's installed tree. Unused when resolving in place. */ - treeDir: string; - /** The run's output directory, where an in-place run keeps the manifest it replaced. */ + /** The run's output directory, where the manifest the run replaced is kept. */ outputDir: string; /** Absolute output directory for the built pages. */ outDir: string; - /** How the pages find the library. Defaults to `in-place`. */ - resolveMode?: ResolveMode; }): Promise { - const { - harnessDir, - repoRoot, - ref, - packages, - treeDir, - outputDir, - outDir, - resolveMode = 'in-place', - } = options; + const { harnessDir, repoRoot, ref, packages, outputDir, outDir } = options; console.log(chalk.cyan(`\nBuilding benchmark pages for "${refLabel(ref)}"…`)); - if (resolveMode === 'in-place') { - await pinPackedPackages(repoRoot, outputDir, packages); - // `--no-frozen-lockfile` because changing the overrides is the point, and pnpm turns a frozen - // install on by itself in CI — where it would refuse the very change being made. - await run('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], repoRoot); - } else { - await installRefTree({ harnessDir, repoRoot, packages, treeDir }); - } + await pinPackedPackages(repoRoot, outputDir, packages); + // `--no-frozen-lockfile` because changing the overrides is the point, and pnpm turns a frozen + // install on by itself in CI — where it would refuse the very change being made. + await run('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], repoRoot); // vite is the harness's own, not this package's: the harness declares the version its pages are // built with, and the config being run is the harness's. @@ -314,8 +132,6 @@ export async function buildRefPages(options: { await vite.build({ root: harnessDir, logLevel: 'info', - // Resolving in place needs no plugin: the repository's own install is the one that changed. - plugins: resolveMode === 'in-place' ? [] : [resolveFromTree(treeDir)], build: { outDir }, }); } diff --git a/packages/benchmark/src/runner/outputDir.ts b/packages/benchmark/src/runner/outputDir.ts index 9251a60a9f..67cec3362c 100644 --- a/packages/benchmark/src/runner/outputDir.ts +++ b/packages/benchmark/src/runner/outputDir.ts @@ -2,7 +2,7 @@ import * as path from 'node:path'; import { mkdir, writeFile } from 'node:fs/promises'; /** - * Everything a run writes — packed tarballs, per-ref installs, built pages, the report — lives + * Everything a run writes — packed tarballs, built pages, the report — lives * under this one directory inside the harness, so a repository has a single thing to ignore and * deleting it is the whole reset story. */ diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index 2813c13b30..6d9d8430e4 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -13,7 +13,6 @@ import type { ResolvedRef } from './refs'; import { discoverBenchFiles } from './benchFiles'; import { runInterleaved } from './runInterleaved'; import { buildRefPages, restoreWorkspace } from './buildPages'; -import type { ResolveMode } from './buildPages'; import { resolveBrowserBinary } from './browser'; import { printRunReport } from './printReport'; import { publishRunReport } from './upload'; @@ -35,22 +34,17 @@ export interface RunBenchmarksOptions { * Defaults to `pnpm release:build`. */ buildCmd?: string; - /** Whether to install inside a ref's checkout. Defaults to true. */ - install?: boolean; /** Where to write the JSON report. Defaults to `.benchmark/results/report.json`. */ out?: string; /** Upload the report and refresh the pull request comment. */ upload?: boolean; - /** How a ref's pages find the library under test. Defaults to `in-place`. */ - resolveMode?: ResolveMode; - /** Measured rounds per case. Defaults to 30. */ - samples?: number; - /** Discarded rounds before measuring, once per case. Defaults to 10. */ - warmup?: number; - /** Extra Chromium launch arguments, e.g. to select a hardware GPU backend for paint timings. */ - launchArgs?: string[]; } +/** Measured rounds per benchmark; each round samples every variant once. */ +const SAMPLES = 30; +/** Discarded rounds before measuring, once per benchmark. */ +const WARMUP = 10; + /** * Benchmarks a harness's `*.bench.tsx` files across two builds of the workspace — the working tree * and a baseline — and writes, prints and optionally uploads the run report. @@ -64,13 +58,8 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise ref.sha ? `${refLabel(ref)} (${ref.sha.slice(0, 9)})` : refLabel(ref); console.log(chalk.cyan(`Files: ${benchFiles.map((benchFile) => benchFile.file).join(', ')}`)); console.log(chalk.cyan(`Refs: ${refs.map(describeRef).join(', ')}`)); - console.log(chalk.cyan(`Browser: ${browserBinary} ${allLaunchArgs.join(' ')}`)); + console.log(chalk.cyan(`Browser: ${browserBinary} ${launchArgs.join(' ')}`)); try { // Pack both sides at once: each builds in its own checkout, and nothing is measured yet. @@ -127,21 +113,14 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise [variantOf(ref), { sha: ref.sha, label: refLabel(ref) }]), ), @@ -206,10 +183,8 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise Date: Tue, 29 Sep 2026 14:24:41 +0200 Subject: [PATCH 084/173] [benchmark] Split the page runtime into benchmark and reactBenchmark benchmark() in @mui/internal-benchmark/page is framework-free: it runs a function once per sample and reports whatever that records through a Metric. reactBenchmark() wraps it, recording render, render:count, render: and bench:paint as ordinary metrics. Benchmark files import @mui/internal-benchmark/page directly instead of having the plugin redirect the root import, so they type-check against the runtime that runs them. --- packages/benchmark/README.md | 58 +++++-- packages/benchmark/package.json | 2 +- packages/benchmark/src/caseRuntime.tsx | 7 +- packages/benchmark/src/page/index.ts | 7 + packages/benchmark/src/page/page.tsx | 146 ++++++------------ .../benchmark/src/page/reactBenchmark.tsx | 84 ++++++++++ packages/benchmark/src/publicApi.ts | 5 +- packages/benchmark/src/runner/benchFiles.ts | 6 +- packages/benchmark/src/runner/plugin.ts | 20 +-- test/benchmark/src/json/json.bench.tsx | 26 ++++ test/benchmark/src/libs/sort/sort.alpha.tsx | 4 +- test/benchmark/src/libs/sort/sort.bench.tsx | 2 +- test/benchmark/src/libs/sort/sort.beta.tsx | 4 +- test/benchmark/src/libs/sort/sort.ours.tsx | 4 +- test/benchmark/src/list/list.bench.tsx | 11 +- test/benchmark/src/lists/lists.bench.tsx | 4 +- test/benchmark/src/lists/lists.table.tsx | 6 +- test/benchmark/src/lists/lists.ul.tsx | 6 +- .../benchmark/src/workload/workload.bench.tsx | 6 +- test/benchmark/vite.config.mjs | 2 +- 20 files changed, 255 insertions(+), 155 deletions(-) create mode 100644 packages/benchmark/src/page/index.ts create mode 100644 packages/benchmark/src/page/reactBenchmark.tsx create mode 100644 test/benchmark/src/json/json.bench.tsx diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index e8a1737baf..1fcecb2450 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -366,9 +366,8 @@ The feature is opt-in — without `BENCHMARK_BASELINE_PATH` (or the `baselinePat ## `benchmark run` -`benchmark run` measures a harness package's `*.bench.tsx` files — the same `benchmark()` and -`compare()` files Vitest runs — across two builds of the workspace: the working tree and a baseline. -Run it from the harness, whose vite config uses the plugin: +`benchmark run` measures a harness package's `*.bench.tsx` files across two builds of the workspace: +the working tree and a baseline. Run it from the harness, whose vite config uses the plugin: ```js // vite.config.mjs @@ -382,11 +381,49 @@ export default defineConfig({ plugins: [benchmarkPlugin()] }); benchmark run --baseline "$(code-infra baseline)" ``` +### Defining cases + +Benchmark files import `@mui/internal-benchmark/page`. `benchmark()` registers a case: the runner +calls its function once per sample, and the case records what it measures through a `ScalarMetric` +or `DiscreteMetric`. Nothing is measured implicitly, and a sample that records nothing fails. + +```tsx +import { benchmark, ScalarMetric } from '@mui/internal-benchmark/page'; + +const parseTime = new ScalarMetric({ name: 'json:parse', alarm: {} }); + +benchmark('parse', () => { + parseTime.time(); + JSON.parse(payload); + parseTime.timeEnd(); +}); +``` + +`reactBenchmark()` wraps it for React: every sample mounts the element, runs the interaction, waits +for the paint and unmounts, and records `render` (total render duration), `render:count` and +`bench:paint`, plus a `render:` split when there is more than one phase. It takes the same +arguments as the Vitest `benchmark()` — so moving a file over is a rename and a new import. + +```tsx +import { reactBenchmark } from '@mui/internal-benchmark/page'; + +reactBenchmark('mount', () => ); +reactBenchmark( + 'scroll', + () => , + async ({ input }) => { + await input.scroll({ x: 100, y: 100, deltaY: 2000 }); + }, +); +``` + +`compare()` pairs cases across implementations, each a module of `benchmark()` or `reactBenchmark()` +cases. + ### How it measures -The plugin generates a page per benchmark file under `src/__bench__/`, where -`@mui/internal-benchmark` resolves to a page runtime instead of Vitest; a benchmark file that imports -`vitest` fails the build. The runner drives Chromium through Playwright and gives every variant — the +The plugin generates a page per benchmark file under `src/__bench__/`. A benchmark file that imports +`vitest`, or the Vitest entry `@mui/internal-benchmark`, fails the build. The runner drives Chromium through Playwright and gives every variant — the current and the baseline build, or each variant of a `compare()` — a page of its own, in a browser context of its own, that stays open for the whole benchmark: every sample is one warm iteration, and module-scope data is built once. @@ -394,8 +431,7 @@ module-scope data is built once. Every variant is sampled once per round, in a shuffled order, and a difference is judged on the per-round differences rather than on two independent sets of samples. Whatever the machine was doing during a round — thermal throttling, a background process — then affects both sides of it and cancels -out. 10 warmup rounds are discarded once per benchmark, then 30 rounds are measured; the iteration -counts in `benchmark()`'s options only apply under Vitest. +out. 10 warmup rounds are discarded once per benchmark, then 30 rounds are measured. ### The report @@ -407,9 +443,9 @@ history from it. The report holds **raw samples**, round-aligned across variants format and alarm — the builds, and the environment. Conclusions are computed from it, the same way everywhere, by `analyzeRun` from `@mui/internal-benchmark/runReport`: a 95% confidence interval on the paired difference per metric, a change (`better`, `worse`, `unsure`) that respects the metric's -direction, and a severity from its alarm. The harness's own metrics are `render` (total render -duration) and `render:count`, which alarm on any resolved change for the worse, and `bench:paint`, -which is informational; custom metrics bring their own alarm. +direction, and a severity from its alarm. `reactBenchmark()`'s `render` and `render:count` alarm on +any resolved change for the worse, and `bench:paint` is informational; every other metric brings its +own alarm. ### Choosing the baseline diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index a673c49e1d..95e347e14c 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -14,7 +14,7 @@ "./ciReport": "./src/ciReport.ts", "./reporter": "./src/reporter.ts", "./vitest": "./src/vitest.ts", - "./page": "./src/page/page.tsx", + "./page": "./src/page/index.ts", "./vitePlugin": "./src/runner/plugin.ts", "./runReport": "./src/runReport/index.ts", "./tachometerPlugin": "./src/tachometer/plugin.ts", diff --git a/packages/benchmark/src/caseRuntime.tsx b/packages/benchmark/src/caseRuntime.tsx index 8c9eb31c3a..c5fe6ea021 100644 --- a/packages/benchmark/src/caseRuntime.tsx +++ b/packages/benchmark/src/caseRuntime.tsx @@ -8,8 +8,9 @@ import { ElementTiming } from './ElementTiming'; import { createReactRecordingControls } from './reactRecording'; import type { ReactRecordingControls } from './reactRecording'; -// Everything here runs a case without knowing who drives it: the Vitest harness (`index.tsx`) and -// the A/B page (`page/page.tsx`) both build on it, so nothing in this module may import Vitest. +// Everything here runs a React case without knowing who drives it: the Vitest harness (`index.tsx`) +// and the page runtime's `reactBenchmark()` both build on it, so nothing in this module may import +// Vitest. interface PerformanceElementTiming extends PerformanceEntry { readonly entryType: 'element'; @@ -51,7 +52,7 @@ export interface CaseOptions { reactRecordingPaused?: boolean; } -/** `benchmark()`'s options. The page runtime ignores the iteration counts: the runner decides them. */ +/** The Vitest `benchmark()`'s options. */ export interface BenchmarkOptions extends CaseOptions { runs?: number; warmupRuns?: number; diff --git a/packages/benchmark/src/page/index.ts b/packages/benchmark/src/page/index.ts new file mode 100644 index 0000000000..38751ef244 --- /dev/null +++ b/packages/benchmark/src/page/index.ts @@ -0,0 +1,7 @@ +// `@mui/internal-benchmark/page`: what a benchmark file for `benchmark run` imports. + +export * from '../publicApi'; +export { benchmark, compare, markPageReady } from './page'; +export type { BenchmarkContext, BenchmarkRun, BenchPage } from './page'; +export { reactBenchmark } from './reactBenchmark'; +export type { ReactBenchmarkOptions } from './reactBenchmark'; diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 09ace5a1b1..313d11bde9 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -1,55 +1,43 @@ -import type * as React from 'react'; import { createInput } from '../input'; -import { - EMPTY_RECORDING_MESSAGE, - measureIteration, - MILLISECONDS, - PAINT_METRIC_NAME, - splitCaseArgs, - warnIfNoGc, -} from '../caseRuntime'; -import type { - BenchmarkInteraction, - BenchmarkOptions, - CaseOptions, - MeasuredIteration, - VariantLoader, -} from '../caseRuntime'; +import type { BenchmarkInput } from '../input'; +import type { VariantLoader } from '../caseRuntime'; import { seriesName, setMetricRecorder } from '../metricCore'; import type { MetricDefinition } from '../types'; -// What `@mui/internal-benchmark` resolves to when benchmark files are built into a page for -// `benchmark run`. Same authoring API, different driver: `benchmark()` registers a case, and the -// runner calls `window.benchmarkPage.sample()` to run one iteration of it at a time. Every result -// leaves the page as a `performance.measure` entry, so it also shows up in a DevTools trace. +// The page runtime `benchmark run` measures. `benchmark()` registers a case, and the runner calls +// `window.benchmarkPage.sample()` to run one iteration of it at a time. What a case measures is +// whatever it records through a `Metric`; every recorded value leaves the page as a +// `performance.measure` entry, so it also shows up in a DevTools trace. -export * from '../publicApi'; - -interface PageCase { - renderFn: () => React.ReactElement; - interaction: BenchmarkInteraction | undefined; - options: CaseOptions | undefined; +/** What a case's `run` is handed on every iteration. */ +export interface BenchmarkContext { + /** Trusted input — scroll, pinch, tap — dispatched by the browser rather than from script. */ + input: BenchmarkInput; } +/** + * One iteration of a case. It records its results through `Metric`s — `ScalarMetric`, + * `DiscreteMetric` — and must record at least one value. + */ +export type BenchmarkRun = (context: BenchmarkContext) => Promise | void; + /** The file's own `benchmark()` cases. */ -const fileCases = new Map(); +const fileCases = new Map(); // Where `benchmark()` registers: the file's own cases, except while a `compare()` variant module is // being loaded, whose cases belong to that variant. let registering = fileCases; // The cases this page runs: the file's own, or the variant the url selects. let pageCases = fileCases; -export function benchmark( - name: string, - renderFn: () => React.ReactElement, - interactionOrOptions?: BenchmarkInteraction | BenchmarkOptions, - maybeOptions?: BenchmarkOptions, -): void { +/** + * Registers a benchmark case. The runner calls `run` once per sample, warmup included, and decides + * how many samples to take. + */ +export function benchmark(name: string, run: BenchmarkRun): void { if (registering.has(name)) { throw new Error(`Two benchmarks share the name "${name}". Benchmark names must be unique.`); } - // The iteration counts in `options` are ignored here: the runner decides them. - registering.set(name, { renderFn, ...splitCaseArgs(interactionOrOptions, maybeOptions) }); + registering.set(name, run); } const comparisons = new Map>(); @@ -82,7 +70,7 @@ async function selectVariant(): Promise { if (!load) { throw new Error(`No variant "${variant}" in comparison "${comparison}".`); } - const variantCases = new Map(); + const variantCases = new Map(); registering = variantCases; try { await load(); @@ -97,20 +85,12 @@ interface RecordedValue { value: number; } -// Values custom metrics record during a measured sample. `null` outside one (module scope, warmup), -// where recorded values are dropped. +// Values recorded during a measured sample. `null` outside one (module scope, warmup), where +// recorded values are dropped. let sampleValues: RecordedValue[] | null = null; -// The harness's own metrics. Render time and render count alarm on any resolved change for the -// worse; paint dominates each case's duration and duplicates that signal, so it is informational. -const metricDefinitions = new Map([ - ['render', { kind: 'scalar', format: MILLISECONDS, alarm: {} }], - ['render:count', { kind: 'discrete', alarm: {} }], - ['render:mount', { kind: 'scalar', format: MILLISECONDS }], - ['render:update', { kind: 'scalar', format: MILLISECONDS }], - ['render:nested-update', { kind: 'scalar', format: MILLISECONDS }], - [PAINT_METRIC_NAME, { kind: 'scalar', format: MILLISECONDS }], -]); +// Every metric recorded so far, by name, as the report describes it. +const metricDefinitions = new Map(); setMetricRecorder((metric, value, options) => { if (!metricDefinitions.has(metric.name)) { @@ -127,7 +107,7 @@ export interface BenchPage { caseNames: () => string[]; /** The file's `compare()` calls, with their variant keys in order. */ comparisons: () => Array<{ name: string; variants: string[] }>; - /** Every metric reported so far, by name: the harness's own and those the cases recorded. */ + /** Every metric recorded so far, by name. */ metricDefinitions: () => Record; /** * Runs one iteration of a case. A measured sample leaves its results as `performance.measure` @@ -154,70 +134,37 @@ const input = createInput((method, params) => { return window.benchmarkCdp(method, params); }); -function emitSample(result: MeasuredIteration, values: RecordedValue[]): void { - // Render time is reported as totals rather than per render: an interaction's render count follows - // whatever the browser coalesced, so per-render entries would not line up across iterations. The - // per-phase split only adds information when there is more than one phase. - const byPhase = new Map(); - let total = 0; - for (const render of result.renders) { - byPhase.set(render.phase, (byPhase.get(render.phase) ?? 0) + render.actualDuration); - total += render.actualDuration; - } - if (result.renders.length > 0) { - const start = result.renders[0].startTime; - performance.measure('render', { start, duration: total, detail: { value: total } }); - performance.measure('render:count', { - start, - duration: 0, - detail: { value: result.renders.length }, - }); - if (byPhase.size > 1) { - for (const [phase, duration] of byPhase) { - performance.measure(`render:${phase}`, { start, duration, detail: { value: duration } }); - } - } - } - for (const { id, start, end } of result.paints) { - performance.measure(seriesName(PAINT_METRIC_NAME, id), { start, end }); - } - // Custom metrics aren't necessarily durations, so the value travels in `detail`. - const now = performance.now(); - for (const { name, value } of values) { - performance.measure(name, { start: now, duration: 0, detail: { value } }); - } -} - async function sample(name: string, { warmup }: { warmup: boolean }): Promise { - const benchCase = pageCases.get(name); - if (!benchCase) { + const run = pageCases.get(name); + if (!run) { throw new Error(`No benchmark named "${name}". Known: ${[...pageCases.keys()].join(', ')}`); } performance.clearMarks(); performance.clearMeasures(); const values: RecordedValue[] = []; - sampleValues = warmup ? null : values; - let result: MeasuredIteration; + // Recorded even during warmup, so a case that records nothing fails before a single sample is + // measured; only a measured sample's values leave the page. + sampleValues = values; try { - result = await measureIteration( - benchCase.renderFn, - benchCase.interaction, - benchCase.options, - input, - ); + await run({ input }); } finally { sampleValues = null; } - if (result.renderError) { - throw result.renderError; + if (values.length === 0) { + throw new Error( + `Benchmark "${name}" recorded no value. Record its results through a ScalarMetric or DiscreteMetric.`, + ); } - if (result.hadEmptyActiveWindow) { - throw new Error(EMPTY_RECORDING_MESSAGE); + if (warmup) { + return; } - if (!warmup) { - emitSample(result, values); + // The runner collects measures by name, and a value isn't necessarily a duration, so it travels + // in `detail`. + const now = performance.now(); + for (const { name: series, value } of values) { + performance.measure(series, { start: now, duration: 0, detail: { value } }); } } @@ -226,7 +173,6 @@ async function sample(name: string, { warmup }: { warmup: boolean }): Promise { - warnIfNoGc(); try { await selectVariant(); } catch (error) { diff --git a/packages/benchmark/src/page/reactBenchmark.tsx b/packages/benchmark/src/page/reactBenchmark.tsx new file mode 100644 index 0000000000..7f73a37971 --- /dev/null +++ b/packages/benchmark/src/page/reactBenchmark.tsx @@ -0,0 +1,84 @@ +import type * as React from 'react'; +import { + EMPTY_RECORDING_MESSAGE, + measureIteration, + MILLISECONDS, + PAINT_METRIC_NAME, + splitCaseArgs, + warnIfNoGc, +} from '../caseRuntime'; +import type { BenchmarkInteraction, CaseOptions } from '../caseRuntime'; +import { DiscreteMetric } from '../DiscreteMetric'; +import { ScalarMetric } from '../ScalarMetric'; +import { benchmark } from './page'; + +export type ReactBenchmarkOptions = CaseOptions; + +// Render time and render count alarm on any resolved change for the worse; paint dominates each +// case's duration and duplicates that signal, so it is informational, as is the per-phase split. +const renderMetric = new ScalarMetric({ name: 'render', format: MILLISECONDS, alarm: {} }); +const renderCountMetric = new DiscreteMetric({ name: 'render:count', alarm: {} }); +const paintMetric = new ScalarMetric({ name: PAINT_METRIC_NAME, format: MILLISECONDS }); +const phaseMetrics = new Map(); + +function phaseMetricOf(phase: string): ScalarMetric { + let metric = phaseMetrics.get(phase); + if (!metric) { + metric = new ScalarMetric({ name: `render:${phase}`, format: MILLISECONDS }); + phaseMetrics.set(phase, metric); + } + return metric; +} + +// Every iteration forces a GC before mounting, which needs `--expose-gc`; warned about once. +let checkedGc = false; + +/** + * Registers a React benchmark case: every sample mounts `renderFn()`, runs the interaction, waits + * for the paint and unmounts. It records the renders React's profiler captured and the time to + * paint; custom metrics recorded along the way are reported with them. + */ +export function reactBenchmark( + name: string, + renderFn: () => React.ReactElement, + interactionOrOptions?: BenchmarkInteraction | ReactBenchmarkOptions, + maybeOptions?: ReactBenchmarkOptions, +): void { + if (!checkedGc) { + checkedGc = true; + warnIfNoGc(); + } + const { interaction, options } = splitCaseArgs(interactionOrOptions, maybeOptions); + + benchmark(name, async ({ input }) => { + const result = await measureIteration(renderFn, interaction, options, input); + if (result.renderError) { + throw result.renderError; + } + if (result.hadEmptyActiveWindow) { + throw new Error(EMPTY_RECORDING_MESSAGE); + } + + // Render time is reported as totals rather than per render: an interaction's render count + // follows whatever the browser coalesced, so per-render values would not line up across + // iterations. The per-phase split only adds information when there is more than one phase. + if (result.renders.length > 0) { + const byPhase = new Map(); + let total = 0; + for (const render of result.renders) { + byPhase.set(render.phase, (byPhase.get(render.phase) ?? 0) + render.actualDuration); + total += render.actualDuration; + } + renderMetric.record(total); + renderCountMetric.record(result.renders.length); + if (byPhase.size > 1) { + for (const [phase, duration] of byPhase) { + phaseMetricOf(phase).record(duration); + } + } + } + for (const { id, start, end } of result.paints) { + paintMetric.record(end - start, id === undefined ? undefined : { id }); + } + }); +} diff --git a/packages/benchmark/src/publicApi.ts b/packages/benchmark/src/publicApi.ts index 86b80069ef..ac0312e86d 100644 --- a/packages/benchmark/src/publicApi.ts +++ b/packages/benchmark/src/publicApi.ts @@ -1,6 +1,5 @@ -// What `@mui/internal-benchmark` exports whichever runtime it resolves to — the Vitest harness -// (`index.tsx`) or the page runtime (`page/page.tsx`) — so a benchmark file type-checks and runs the -// same against both. Each entry adds its own `benchmark`, `compare` and, for Vitest, `runCase`. +// What both runtimes export: the Vitest harness (`@mui/internal-benchmark`) and the page runtime +// `benchmark run` measures (`@mui/internal-benchmark/page`). Each entry adds its own case API. export type { RenderEvent, IterationData, InteractionContext } from './types'; export type { diff --git a/packages/benchmark/src/runner/benchFiles.ts b/packages/benchmark/src/runner/benchFiles.ts index 0a67d0d3cb..23b8608097 100644 --- a/packages/benchmark/src/runner/benchFiles.ts +++ b/packages/benchmark/src/runner/benchFiles.ts @@ -4,9 +4,9 @@ import { globby } from 'globby'; import { escapeHtml } from '../utils/html'; /** - * `*.bench.tsx` files: `benchmark()` cases, the same files Vitest runs, measured by the interleaved - * engine instead. Each file gets a generated page that loads it with `@mui/internal-benchmark` - * swapped for the page runtime, so the file itself needs no page, config or entry of its own. + * `*.bench.tsx` files: cases defined with `@mui/internal-benchmark/page`, measured by the interleaved + * engine. Each file gets a generated page that loads it, so the file itself needs no page, config or + * entry of its own. */ /** Where the generated pages live, under `src/`. Ignored from the inside, like `.benchmark/`. */ diff --git a/packages/benchmark/src/runner/plugin.ts b/packages/benchmark/src/runner/plugin.ts index 14b71e3a26..d676e2834c 100644 --- a/packages/benchmark/src/runner/plugin.ts +++ b/packages/benchmark/src/runner/plugin.ts @@ -14,8 +14,8 @@ import { buildsDirOf, prepareOutputDir, OUTPUT_DIR } from './outputDir'; * * - A generated page per benchmark file, under `src/__bench__/`, and those pages as the build's * entry points. - * - `@mui/internal-benchmark` resolved to the page runtime, which runs the file's `benchmark()` - * cases one iteration at a time for `benchmark run`. + * - A clear error when a page imports Vitest, or the Vitest entry of this package instead of + * `@mui/internal-benchmark/page`. * - `root` at the harness's `src/`, as an MPA (vite defaults to `spa`, whose fallback would rewrite * unmatched urls toward a root `index.html` that does not exist here), and an index page listing * the benchmark files. @@ -99,21 +99,23 @@ export function benchmarkPlugin(options: BenchmarkPluginOptions = {}): Plugin { return { name: 'mui-benchmark:pages', - // Ahead of vite's own resolver, so the redirect below wins over the package's `exports`. + // Ahead of vite's own resolver, so the errors below win over the package's `exports`. enforce: 'pre', - // Benchmark files are written against `@mui/internal-benchmark`; in a page that is the page - // runtime, which the runner drives one iteration at a time. Tests are not benchmarks, so a page - // has no Vitest to import. + // Tests are not benchmarks, so a page has no Vitest to import — nor this package's Vitest entry, + // which imports it. async resolveId(source, importer) { if (source === '@mui/internal-benchmark') { - return this.resolve('@mui/internal-benchmark/page', importer, { skipSelf: true }); + this.error( + `${importer ?? 'A benchmark page'} imports "@mui/internal-benchmark", the Vitest ` + + 'harness. Benchmark files for `benchmark run` import "@mui/internal-benchmark/page".', + ); } if (source === 'vitest' || source.startsWith('vitest/')) { this.error( `${importer ?? 'A benchmark page'} imports "${source}". Benchmark files run outside ` + - 'Vitest: define cases with benchmark() or compare(), and use the interaction context ' + - '(input, waitForElementTiming, …) instead of Vitest APIs.', + 'Vitest: define cases with benchmark(), reactBenchmark() or compare(), and use the ' + + 'interaction context (input, waitForElementTiming, …) instead of Vitest APIs.', ); } return null; diff --git a/test/benchmark/src/json/json.bench.tsx b/test/benchmark/src/json/json.bench.tsx new file mode 100644 index 0000000000..670ca28321 --- /dev/null +++ b/test/benchmark/src/json/json.bench.tsx @@ -0,0 +1,26 @@ +import { benchmark, ScalarMetric } from '@mui/internal-benchmark/page'; + +/** + * A `benchmark()` case without React: every sample runs the function once, and what it measures is + * whatever it records. + */ + +const parseTime = new ScalarMetric({ + name: 'json:parse', + format: { style: 'unit', unit: 'millisecond', maximumFractionDigits: 2 }, + alarm: {}, +}); + +const payload = JSON.stringify( + Array.from({ length: 20_000 }, (_, index) => ({ + id: index, + label: `Item ${index}`, + done: false, + })), +); + +benchmark('parse', () => { + parseTime.time(); + JSON.parse(payload); + parseTime.timeEnd(); +}); diff --git a/test/benchmark/src/libs/sort/sort.alpha.tsx b/test/benchmark/src/libs/sort/sort.alpha.tsx index 89224ef84f..9a5cdf9642 100644 --- a/test/benchmark/src/libs/sort/sort.alpha.tsx +++ b/test/benchmark/src/libs/sort/sort.alpha.tsx @@ -1,5 +1,5 @@ import * as React from 'react'; -import { benchmark } from '@mui/internal-benchmark'; +import { reactBenchmark } from '@mui/internal-benchmark/page'; import { makeValues, report } from './shared'; /** Sorts the boxed array in place, paying a comparator call per comparison. */ @@ -9,4 +9,4 @@ function Sorted() { return

    {report('alpha', values)}

    ; } -benchmark('sort', () => ); +reactBenchmark('sort', () => ); diff --git a/test/benchmark/src/libs/sort/sort.bench.tsx b/test/benchmark/src/libs/sort/sort.bench.tsx index 35edbe9067..c75f71da1c 100644 --- a/test/benchmark/src/libs/sort/sort.bench.tsx +++ b/test/benchmark/src/libs/sort/sort.bench.tsx @@ -1,4 +1,4 @@ -import { compare } from '@mui/internal-benchmark'; +import { compare } from '@mui/internal-benchmark/page'; /** * Three sorts compared with each other rather than across builds — the shape a comparison between diff --git a/test/benchmark/src/libs/sort/sort.beta.tsx b/test/benchmark/src/libs/sort/sort.beta.tsx index 9accfd3418..c4a3d9dfad 100644 --- a/test/benchmark/src/libs/sort/sort.beta.tsx +++ b/test/benchmark/src/libs/sort/sort.beta.tsx @@ -1,5 +1,5 @@ import * as React from 'react'; -import { benchmark } from '@mui/internal-benchmark'; +import { reactBenchmark } from '@mui/internal-benchmark/page'; import { makeValues, report } from './shared'; /** A merge sort written in JavaScript, allocating as it goes — the slow variant of the three. */ @@ -33,4 +33,4 @@ function Sorted() { return

    {report('beta', mergeSort(makeValues()))}

    ; } -benchmark('sort', () => ); +reactBenchmark('sort', () => ); diff --git a/test/benchmark/src/libs/sort/sort.ours.tsx b/test/benchmark/src/libs/sort/sort.ours.tsx index 1caf8a553c..dd7cf0a21c 100644 --- a/test/benchmark/src/libs/sort/sort.ours.tsx +++ b/test/benchmark/src/libs/sort/sort.ours.tsx @@ -1,5 +1,5 @@ import * as React from 'react'; -import { benchmark } from '@mui/internal-benchmark'; +import { reactBenchmark } from '@mui/internal-benchmark/page'; import { makeValues, report } from './shared'; /** Copies into a typed array first, so the engine sorts unboxed doubles with no comparator calls. */ @@ -9,4 +9,4 @@ function Sorted() { return

    {report('ours', sorted)}

    ; } -benchmark('sort', () => ); +reactBenchmark('sort', () => ); diff --git a/test/benchmark/src/list/list.bench.tsx b/test/benchmark/src/list/list.bench.tsx index 941fbf386a..87931b5bd0 100644 --- a/test/benchmark/src/list/list.bench.tsx +++ b/test/benchmark/src/list/list.bench.tsx @@ -1,12 +1,11 @@ import * as React from 'react'; -import { benchmark, ScalarMetric } from '@mui/internal-benchmark'; +import { reactBenchmark, ScalarMetric } from '@mui/internal-benchmark/page'; import { reactMajor } from '@mui/internal-test-utils/env'; import { ScrollingList } from '../_shared/ScrollingList'; /** - * `benchmark()` cases, the same API Vitest runs. Under `benchmark run` each case runs one - * iteration at a time in a page that stays open per build, with the builds alternating round by - * round. + * `reactBenchmark()` cases. Under `benchmark run` each case runs one iteration at a time in a page + * that stays open per build, with the builds alternating round by round. * * Like the `workload` benchmark, it imports a workspace package so both builds resolve it from their * own packed tarball. @@ -22,7 +21,7 @@ function List({ count }: { count: number }) { ); } -benchmark('list mount', () => ); +reactBenchmark('list mount', () => ); const scrollDuration = new ScalarMetric({ name: 'scroll_gesture', @@ -31,7 +30,7 @@ const scrollDuration = new ScalarMetric({ // A trusted wheel-style scroll: the browser generates the whole gesture at frame cadence, so the // list sees the event timing a real mouse wheel would produce. -benchmark( +reactBenchmark( 'list scroll', () => ( diff --git a/test/benchmark/src/lists/lists.bench.tsx b/test/benchmark/src/lists/lists.bench.tsx index 7a66d17e64..09fe0947de 100644 --- a/test/benchmark/src/lists/lists.bench.tsx +++ b/test/benchmark/src/lists/lists.bench.tsx @@ -1,8 +1,8 @@ -import { compare } from '@mui/internal-benchmark'; +import { compare } from '@mui/internal-benchmark/page'; /** * Two implementations of the same list, compared with each other rather than across builds — the - * shape a comparison between libraries takes. Each variant is a module of plain `benchmark()` cases; + * shape a comparison between libraries takes. Each variant is a module of plain `reactBenchmark()` cases; * cases with the same name are paired, and the first variant is the reference. * * Every variant gets a page of its own that loads only its own module, so neither implementation's diff --git a/test/benchmark/src/lists/lists.table.tsx b/test/benchmark/src/lists/lists.table.tsx index f573a11984..aa54c9230b 100644 --- a/test/benchmark/src/lists/lists.table.tsx +++ b/test/benchmark/src/lists/lists.table.tsx @@ -1,5 +1,5 @@ import * as React from 'react'; -import { benchmark } from '@mui/internal-benchmark'; +import { reactBenchmark } from '@mui/internal-benchmark/page'; import { ScrollingList } from '../_shared/ScrollingList'; const ROWS = 1000; @@ -19,9 +19,9 @@ function List() { ); } -benchmark('mount', () => ); +reactBenchmark('mount', () => ); -benchmark( +reactBenchmark( 'scroll', () => ( diff --git a/test/benchmark/src/lists/lists.ul.tsx b/test/benchmark/src/lists/lists.ul.tsx index b77330d8f5..62ad2a5521 100644 --- a/test/benchmark/src/lists/lists.ul.tsx +++ b/test/benchmark/src/lists/lists.ul.tsx @@ -1,5 +1,5 @@ import * as React from 'react'; -import { benchmark } from '@mui/internal-benchmark'; +import { reactBenchmark } from '@mui/internal-benchmark/page'; import { ScrollingList } from '../_shared/ScrollingList'; const ROWS = 1000; @@ -16,9 +16,9 @@ function List() { ); } -benchmark('mount', () => ); +reactBenchmark('mount', () => ); -benchmark( +reactBenchmark( 'scroll', () => ( diff --git a/test/benchmark/src/workload/workload.bench.tsx b/test/benchmark/src/workload/workload.bench.tsx index 6f94ad69e9..4c01263150 100644 --- a/test/benchmark/src/workload/workload.bench.tsx +++ b/test/benchmark/src/workload/workload.bench.tsx @@ -1,5 +1,5 @@ import * as React from 'react'; -import { benchmark } from '@mui/internal-benchmark'; +import { reactBenchmark } from '@mui/internal-benchmark/page'; import { reactMajor } from '@mui/internal-test-utils/env'; /** @@ -43,5 +43,5 @@ function Workload({ size }: { size: number }) { ); } -benchmark('workload', () => ); -benchmark('workload-large', () => ); +reactBenchmark('workload', () => ); +reactBenchmark('workload-large', () => ); diff --git a/test/benchmark/vite.config.mjs b/test/benchmark/vite.config.mjs index b15a239614..fcf7c9bafd 100644 --- a/test/benchmark/vite.config.mjs +++ b/test/benchmark/vite.config.mjs @@ -2,5 +2,5 @@ import { defineConfig } from 'vite'; import { benchmarkPlugin } from '@mui/internal-benchmark/vitePlugin'; // The plugin contributes everything derived from the `*.bench.tsx` files: a page per file as the -// build's entry points, the page runtime they run under, the `src/` root and the index page. +// build's entry points, the `src/` root and the index page. export default defineConfig({ plugins: [benchmarkPlugin()] }); From 95285ae6713580986aacc33cae1465b1e34e9034 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 14:54:18 +0200 Subject: [PATCH 085/173] [benchmark] Compare cases directly instead of variant modules benchmark() and reactBenchmark() return their case, and compare(name, cases) measures those cases against each other on the working tree's build. A compared case is no longer run against the baseline on its own. Every case still gets a page of its own; the page loads the whole file, so a case that must keep a library out of the others' pages imports it inside the case. This drops the variant modules, the ?compare=&variant= page selection and the per-variant case registry. --- packages/benchmark/README.md | 20 +- packages/benchmark/src/index.tsx | 1 + packages/benchmark/src/page/index.ts | 2 +- packages/benchmark/src/page/page.tsx | 104 ++++---- .../benchmark/src/page/reactBenchmark.tsx | 5 +- packages/benchmark/src/publicApi.ts | 2 +- .../benchmark/src/runner/benchFiles.test.ts | 2 +- packages/benchmark/src/runner/benchFiles.ts | 2 +- .../benchmark/src/runner/runInterleaved.ts | 233 +++++++----------- test/benchmark/README.md | 6 +- test/benchmark/src/libs/sort/sort.alpha.tsx | 5 +- test/benchmark/src/libs/sort/sort.bench.tsx | 16 +- test/benchmark/src/libs/sort/sort.beta.tsx | 5 +- test/benchmark/src/libs/sort/sort.ours.tsx | 5 +- test/benchmark/src/lists/lists.bench.tsx | 57 ++++- test/benchmark/src/lists/lists.table.tsx | 22 +- test/benchmark/src/lists/lists.ul.tsx | 22 +- 17 files changed, 228 insertions(+), 281 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 1fcecb2450..b0efa0e6ad 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -417,14 +417,28 @@ reactBenchmark( ); ``` -`compare()` pairs cases across implementations, each a module of `benchmark()` or `reactBenchmark()` -cases. +`compare()` measures cases against each other — one library's implementation against another's — +on the working tree's build, instead of each against its baseline. The first case is the reference. + +```tsx +compare('scatter', [ + reactBenchmark('ours', () => ), + benchmark('other', async () => { + const { renderScatter } = await import('other-charts'); + // … + }), +]); +``` + +Every case is measured in a page of its own, but a page loads the whole benchmark file, so a static +import reaches every case's page. Import what only one case needs inside that case: the first +iteration is a warmup, so the `await import()` is never measured. ### How it measures The plugin generates a page per benchmark file under `src/__bench__/`. A benchmark file that imports `vitest`, or the Vitest entry `@mui/internal-benchmark`, fails the build. The runner drives Chromium through Playwright and gives every variant — the -current and the baseline build, or each variant of a `compare()` — a page of its own, in a browser +current and the baseline build, or each case of a `compare()` — a page of its own, in a browser context of its own, that stays open for the whole benchmark: every sample is one warm iteration, and module-scope data is built once. diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index 5e3e6c51a4..36c0d2bf6e 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -29,6 +29,7 @@ import './Metric'; import './taskMetaAugmentation'; export * from './publicApi'; +export type { VariantLoader }; // When true, `benchmark()` opens an interactive profiling session in a headed // browser instead of running the automated measurement loop. Enabled by diff --git a/packages/benchmark/src/page/index.ts b/packages/benchmark/src/page/index.ts index 38751ef244..5a7ce5c0a5 100644 --- a/packages/benchmark/src/page/index.ts +++ b/packages/benchmark/src/page/index.ts @@ -2,6 +2,6 @@ export * from '../publicApi'; export { benchmark, compare, markPageReady } from './page'; -export type { BenchmarkContext, BenchmarkRun, BenchPage } from './page'; +export type { BenchmarkCase, BenchmarkContext, BenchmarkRun, BenchPage } from './page'; export { reactBenchmark } from './reactBenchmark'; export type { ReactBenchmarkOptions } from './reactBenchmark'; diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 313d11bde9..984262f6c1 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -1,6 +1,5 @@ import { createInput } from '../input'; import type { BenchmarkInput } from '../input'; -import type { VariantLoader } from '../caseRuntime'; import { seriesName, setMetricRecorder } from '../metricCore'; import type { MetricDefinition } from '../types'; @@ -21,63 +20,55 @@ export interface BenchmarkContext { */ export type BenchmarkRun = (context: BenchmarkContext) => Promise | void; -/** The file's own `benchmark()` cases. */ -const fileCases = new Map(); -// Where `benchmark()` registers: the file's own cases, except while a `compare()` variant module is -// being loaded, whose cases belong to that variant. -let registering = fileCases; -// The cases this page runs: the file's own, or the variant the url selects. -let pageCases = fileCases; +/** A registered case, as `benchmark()` returns it for `compare()` to group. */ +export interface BenchmarkCase { + readonly name: string; +} + +/** Every case the file registered, by name. */ +const cases = new Map(); /** * Registers a benchmark case. The runner calls `run` once per sample, warmup included, and decides - * how many samples to take. + * how many samples to take. A case on its own is measured across the builds; one passed to + * `compare()` is measured against the other cases there instead. */ -export function benchmark(name: string, run: BenchmarkRun): void { - if (registering.has(name)) { +export function benchmark(name: string, run: BenchmarkRun): BenchmarkCase { + if (cases.has(name)) { throw new Error(`Two benchmarks share the name "${name}". Benchmark names must be unique.`); } - registering.set(name, run); + cases.set(name, run); + return { name }; } -const comparisons = new Map>(); +/** Each `compare()`'s cases, by name, the reference first. */ +const comparisons = new Map(); /** - * Compares implementations against each other — one library against another, say. Each variant is - * a module of ordinary `benchmark()` calls; cases are paired across variants by name, and the first - * variant is the reference. A variant's page loads only its own module, so no variant's code, - * styles or module state is present while another is measured. + * Compares cases with each other — one library's implementation against another's, say — on the + * working tree's build, rather than each against its baseline. The first case is the reference. + * + * Every case is measured in a page of its own, but a page loads the whole file. Import what only one + * case needs inside that case (`await import()` resolves during warmup) to keep it out of the others' + * pages. */ -export function compare(name: string, variants: Record): void { +export function compare(name: string, compared: BenchmarkCase[]): void { if (comparisons.has(name)) { throw new Error(`Two comparisons share the name "${name}". Comparison names must be unique.`); } - if (Object.keys(variants).length < 2) { - throw new Error(`Comparison "${name}" needs at least two variants.`); - } - comparisons.set(name, variants); -} - -/** Loads the variant `?compare=&variant=` selects, if any, and runs its cases instead. */ -async function selectVariant(): Promise { - const params = new URLSearchParams(window.location.search); - const comparison = params.get('compare'); - const variant = params.get('variant'); - if (comparison === null || variant === null) { - return; - } - const load = comparisons.get(comparison)?.[variant]; - if (!load) { - throw new Error(`No variant "${variant}" in comparison "${comparison}".`); + if (compared.length < 2) { + throw new Error(`Comparison "${name}" needs at least two cases.`); } - const variantCases = new Map(); - registering = variantCases; - try { - await load(); - } finally { - registering = fileCases; + for (const benchCase of compared) { + const owner = [...comparisons].find(([, names]) => names.includes(benchCase.name)); + if (owner) { + throw new Error(`"${benchCase.name}" is already compared in "${owner[0]}".`); + } } - pageCases = variantCases; + comparisons.set( + name, + compared.map((benchCase) => benchCase.name), + ); } interface RecordedValue { @@ -104,9 +95,10 @@ setMetricRecorder((metric, value, options) => { }); export interface BenchPage { + /** The cases measured across the builds: every case no `compare()` took. */ caseNames: () => string[]; - /** The file's `compare()` calls, with their variant keys in order. */ - comparisons: () => Array<{ name: string; variants: string[] }>; + /** The file's `compare()` calls, with their cases' names in order. */ + comparisons: () => Array<{ name: string; cases: string[] }>; /** Every metric recorded so far, by name. */ metricDefinitions: () => Record; /** @@ -120,8 +112,6 @@ declare global { interface Window { /** Set by `markPageReady()` once the benchmark file has registered its cases. */ benchmarkPage?: BenchPage; - /** Set instead of `benchmarkPage` when the page could not get ready. */ - benchmarkPageError?: string; /** Exposed by the runner: forwards a CDP command to this page's session. */ benchmarkCdp?: (method: string, params?: Record) => Promise; } @@ -135,9 +125,9 @@ const input = createInput((method, params) => { }); async function sample(name: string, { warmup }: { warmup: boolean }): Promise { - const run = pageCases.get(name); + const run = cases.get(name); if (!run) { - throw new Error(`No benchmark named "${name}". Known: ${[...pageCases.keys()].join(', ')}`); + throw new Error(`No benchmark named "${name}". Known: ${[...cases.keys()].join(', ')}`); } performance.clearMarks(); performance.clearMeasures(); @@ -169,21 +159,15 @@ async function sample(name: string, { warmup }: { warmup: boolean }): Promise { - try { - await selectVariant(); - } catch (error) { - window.benchmarkPageError = - error instanceof Error ? (error.stack ?? error.message) : String(error); - return; - } +export function markPageReady(): void { + const compared = new Set([...comparisons.values()].flat()); window.benchmarkPage = { - caseNames: () => [...pageCases.keys()], + caseNames: () => [...cases.keys()].filter((name) => !compared.has(name)), comparisons: () => - [...comparisons].map(([name, variants]) => ({ name, variants: Object.keys(variants) })), + [...comparisons].map(([name, comparedNames]) => ({ name, cases: comparedNames })), metricDefinitions: () => Object.fromEntries(metricDefinitions), sample, }; diff --git a/packages/benchmark/src/page/reactBenchmark.tsx b/packages/benchmark/src/page/reactBenchmark.tsx index 7f73a37971..d4f1bc9b63 100644 --- a/packages/benchmark/src/page/reactBenchmark.tsx +++ b/packages/benchmark/src/page/reactBenchmark.tsx @@ -11,6 +11,7 @@ import type { BenchmarkInteraction, CaseOptions } from '../caseRuntime'; import { DiscreteMetric } from '../DiscreteMetric'; import { ScalarMetric } from '../ScalarMetric'; import { benchmark } from './page'; +import type { BenchmarkCase } from './page'; export type ReactBenchmarkOptions = CaseOptions; @@ -43,14 +44,14 @@ export function reactBenchmark( renderFn: () => React.ReactElement, interactionOrOptions?: BenchmarkInteraction | ReactBenchmarkOptions, maybeOptions?: ReactBenchmarkOptions, -): void { +): BenchmarkCase { if (!checkedGc) { checkedGc = true; warnIfNoGc(); } const { interaction, options } = splitCaseArgs(interactionOrOptions, maybeOptions); - benchmark(name, async ({ input }) => { + return benchmark(name, async ({ input }) => { const result = await measureIteration(renderFn, interaction, options, input); if (result.renderError) { throw result.renderError; diff --git a/packages/benchmark/src/publicApi.ts b/packages/benchmark/src/publicApi.ts index ac0312e86d..8b40bb8e50 100644 --- a/packages/benchmark/src/publicApi.ts +++ b/packages/benchmark/src/publicApi.ts @@ -10,7 +10,7 @@ export type { MetricDefinition, } from './types'; export type { BenchmarkInput } from './input'; -export type { BenchmarkInteraction, BenchmarkOptions, VariantLoader } from './caseRuntime'; +export type { BenchmarkInteraction, BenchmarkOptions } from './caseRuntime'; export { ElementTiming } from './ElementTiming'; export { Metric, type MetricRecordOptions } from './metricCore'; export { ScalarMetric } from './ScalarMetric'; diff --git a/packages/benchmark/src/runner/benchFiles.test.ts b/packages/benchmark/src/runner/benchFiles.test.ts index c896cebf94..b2880abcad 100644 --- a/packages/benchmark/src/runner/benchFiles.test.ts +++ b/packages/benchmark/src/runner/benchFiles.test.ts @@ -53,7 +53,7 @@ describe('writeBenchPages', () => { "import '../button/button.bench'; import { markPageReady } from '@mui/internal-benchmark/page'; - void markPageReady(); + markPageReady(); " `); }); diff --git a/packages/benchmark/src/runner/benchFiles.ts b/packages/benchmark/src/runner/benchFiles.ts index 23b8608097..b598113791 100644 --- a/packages/benchmark/src/runner/benchFiles.ts +++ b/packages/benchmark/src/runner/benchFiles.ts @@ -79,7 +79,7 @@ function pageEntry(benchFile: BenchFile): string { return `import '${specifier}'; import { markPageReady } from '@mui/internal-benchmark/page'; -void markPageReady(); +markPageReady(); `; } diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index 635a5099e3..fbe721990d 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -12,8 +12,8 @@ import { serveDirectory } from './serveDirectory'; import type {} from '../page/page'; /** - * Runs the `benchmark()` cases of `*.bench.tsx` files in Playwright. Every variant gets a page of - * its own, in a browser context of its own, that stays open for the whole case: every sample is one + * Runs the cases of `*.bench.tsx` files in Playwright. Every variant gets a page of its own, in a + * browser context of its own, that stays open for the whole benchmark: every sample is one * iteration, so module-scope data and the JIT stay warm, and interactions get trusted input through * a CDP session the page is bridged to. Variants are sampled once per round in a shuffled order, so * the samples are round-aligned and can be compared on paired differences. @@ -73,25 +73,21 @@ function stableMetrics(rounds: Round[]): string[] { ); } -/** - * A page a benchmark file's cases are measured in: one build of the file, or one variant of a - * `compare()` in it. - */ -interface PageSlot { - /** How the slot is named in the report: `current`, `baseline`, or the variant's key. */ - variant: string; - url: string; +/** What a benchmark file's page defines. */ +interface PageListing { + /** The cases measured across the builds. */ + caseNames: string[]; + /** The cases measured against each other, on the working tree's build. */ + comparisons: Array<{ name: string; cases: string[] }>; } interface OpenedPage { - slot: PageSlot; context: BrowserContext; page: Page; - caseNames: string[]; - comparisons: Array<{ name: string; variants: string[] }>; + listing: PageListing; } -async function openBenchPage(browser: Browser, slot: PageSlot): Promise { +async function openBenchPage(browser: Browser, url: string): Promise { const context = await browser.newContext({ viewport: VIEWPORT }); const page = await context.newPage(); page.setDefaultTimeout(SAMPLE_TIMEOUT_MS); @@ -104,11 +100,8 @@ async function openBenchPage(browser: Browser, slot: PageSlot): Promise) => session.send(method as CdpMethod, params), ); - await page.goto(slot.url); + await page.goto(url); const ready = await page.waitForFunction(() => { - if (window.benchmarkPageError !== undefined) { - return { error: window.benchmarkPageError }; - } if (window.benchmarkPage === undefined) { return false; } @@ -120,24 +113,21 @@ async function openBenchPage(browser: Browser, slot: PageSlot): Promise { await Promise.all(targets.map((target) => target.context.close())); } -async function openBenchPages(browser: Browser, slots: PageSlot[]): Promise { - const settled = await Promise.allSettled(slots.map((slot) => openBenchPage(browser, slot))); +async function openBenchPages(browser: Browser, urls: string[]): Promise { + const settled = await Promise.allSettled(urls.map((url) => openBenchPage(browser, url))); const opened = settled.flatMap((result) => (result.status === 'fulfilled' ? [result.value] : [])); const failed = settled.find((result) => result.status === 'rejected'); if (failed) { @@ -147,16 +137,23 @@ async function openBenchPages(browser: Browser, slots: PageSlot[]): Promise { + const opened = await openBenchPages(browser, urls); + await closeBenchPages(opened); + return opened.map((target) => target.listing); +} + /** * Runs one iteration of a case. A measured one comes back as its `performance.measure` values by * name, read in the same round trip. */ function sampleBenchCase( - target: OpenedPage, + page: Page, name: string, warmup: boolean, ): Promise | null> { - return target.page.evaluate( + return page.evaluate( async (args) => { await window.benchmarkPage!.sample(args.name, { warmup: args.warmup }); if (args.warmup) { @@ -175,28 +172,43 @@ function sampleBenchCase( ); } +/** + * A variant of a benchmark: the page it is sampled in and the case it runs there. One build of a + * benchmark file running a case, or the working tree's build running one case of a `compare()`. + */ +interface PageSlot { + /** How the slot is named in the report: `current`, `baseline`, or the compared case's name. */ + variant: string; + url: string; + caseName: string; +} + interface CaseMeasurement { rounds: Round[]; metrics: Record; } /** - * Measures one case in fresh pages that stay open for the whole case, so iterations stay warm and + * Measures one benchmark in fresh pages that stay open for all of it, so iterations stay warm and * module-scope data is built once. Warmup and measured rounds alike run every slot once, in a * shuffled order. */ -async function measureBenchCase( +async function measureBenchmark( browser: Browser, slots: PageSlot[], - name: string, options: RunInterleavedOptions, ): Promise { - const targets = await openBenchPages(browser, slots); + const targets = await openBenchPages( + browser, + slots.map((slot) => slot.url), + ); + const sampleSlot = (index: number, warmup: boolean) => + sampleBenchCase(targets[index].page, slots[index].caseName, warmup); try { for (let roundIndex = 0; roundIndex < options.warmup; roundIndex += 1) { for (const index of shuffledIndices(targets.length)) { // eslint-disable-next-line no-await-in-loop - await sampleBenchCase(targets[index], name, true); + await sampleSlot(index, true); } } const rounds: Round[] = []; @@ -204,7 +216,7 @@ async function measureBenchCase( const round: Round = []; for (const index of shuffledIndices(targets.length)) { // eslint-disable-next-line no-await-in-loop - round[index] = (await sampleBenchCase(targets[index], name, false)) ?? {}; + round[index] = (await sampleSlot(index, false)) ?? {}; } rounds.push(round); } @@ -217,7 +229,7 @@ async function measureBenchCase( } } -/** The round-aligned samples of a case, per variant and metric. */ +/** The round-aligned samples of a benchmark, per variant and metric. */ function samplesOf(slots: PageSlot[], rounds: Round[]): RunBenchmark['samples'] { const metrics = stableMetrics(rounds); return Object.fromEntries( @@ -236,81 +248,35 @@ interface CaseResult { metrics: Record; } -interface CaseGroup { - kind: RunBenchmark['kind']; - file: string; - nameOf: (caseName: string) => string; -} +type BenchmarkEntry = Pick; -/** - * Runs every case the first listed page defines across all of them, paired by name. The first page - * is the reference; a case another page lacks is reported as an error rather than compared with - * nothing. - */ -async function runListedCases( - browser: Browser, - listed: OpenedPage[], - group: CaseGroup, - options: RunInterleavedOptions, -): Promise { - const slots = listed.map((target) => target.slot); - const variants = slots.map((slot) => slot.variant); - const [reference, ...others] = listed; - const results: CaseResult[] = []; - for (const caseName of reference.caseNames) { - const entry = { name: group.nameOf(caseName), file: group.file, kind: group.kind, variants }; - const missing = others.find((target) => !target.caseNames.includes(caseName)); - if (missing) { - // A case added by the change under test, or one library lacks, has nothing to compare with. - results.push({ - benchmark: { - ...entry, - error: `"${caseName}" does not exist in [${missing.slot.variant}].`, - }, - metrics: {}, - }); - continue; - } - - console.log(chalk.cyan(`\nRunning "${entry.name}" (${group.file})…`)); - try { - // eslint-disable-next-line no-await-in-loop - const measured = await measureBenchCase(browser, slots, caseName, options); - results.push({ - benchmark: { ...entry, samples: samplesOf(slots, measured.rounds) }, - metrics: measured.metrics, - }); - } catch (error) { - console.error(chalk.red(` ${errorMessage(error)}`)); - results.push({ benchmark: { ...entry, error: errorMessage(error) }, metrics: {} }); - } - } - return results; -} - -/** Opens a set of pages just to learn what they define, then runs their cases. */ -async function listAndRun( +/** Measures one benchmark across its slots, or reports the error that stopped it. */ +async function runBenchmark( browser: Browser, + entry: BenchmarkEntry, slots: PageSlot[], - group: CaseGroup, options: RunInterleavedOptions, -): Promise<{ results: CaseResult[]; comparisons: OpenedPage['comparisons'] }> { - // Every case then measures in fresh pages of its own. - const listed = await openBenchPages(browser, slots); - await closeBenchPages(listed); - return { - results: await runListedCases(browser, listed, group, options), - comparisons: listed[0].comparisons, - }; +): Promise { + console.log(chalk.cyan(`\nRunning "${entry.name}" (${entry.file})…`)); + try { + const measured = await measureBenchmark(browser, slots, options); + return { + benchmark: { ...entry, samples: samplesOf(slots, measured.rounds) }, + metrics: measured.metrics, + }; + } catch (error) { + console.error(chalk.red(` ${errorMessage(error)}`)); + return { benchmark: { ...entry, error: errorMessage(error) }, metrics: {} }; + } } -function benchPageUrl(origin: string, ref: ResolvedRef, benchFile: BenchFile): URL { - return new URL(`${ref.id}/${benchFile.page}`, `${origin}/`); +function benchPageUrl(origin: string, ref: ResolvedRef, benchFile: BenchFile): string { + return new URL(`${ref.id}/${benchFile.page}`, `${origin}/`).href; } /** - * Runs a benchmark file: its own `benchmark()` cases across the builds, and each `compare()` in it - * across its variants, every variant built from the working tree. + * Runs a benchmark file: each case on its own across the builds, paired by name with the working + * tree as the reference, and each `compare()` across its cases on the working tree's build. */ async function runBenchFile( browser: Browser, @@ -318,51 +284,40 @@ async function runBenchFile( benchFile: BenchFile, options: RunInterleavedOptions, ): Promise { - const buildSlots = options.benchRefs.map((ref): PageSlot => ({ - variant: variantOf(ref), - url: benchPageUrl(origin, ref, benchFile).href, - })); - const { results, comparisons } = await listAndRun( - browser, - buildSlots, - { kind: 'baseline', file: benchFile.file, nameOf: (caseName) => caseName }, - options, - ); + const urls = options.benchRefs.map((ref) => benchPageUrl(origin, ref, benchFile)); + const variants = options.benchRefs.map(variantOf); + const [reference, ...others] = await listBenchPages(browser, urls); - const [worktreeRef] = options.benchRefs; - for (const { name: comparison, variants } of comparisons) { - const variantSlots = variants.map((variant): PageSlot => { - const url = benchPageUrl(origin, worktreeRef, benchFile); - url.searchParams.set('compare', comparison); - url.searchParams.set('variant', variant); - return { variant, url: url.href }; - }); - try { - // eslint-disable-next-line no-await-in-loop - const compared = await listAndRun( - browser, - variantSlots, - { - kind: 'compare', - file: benchFile.file, - nameOf: (caseName) => `${comparison} / ${caseName}`, - }, - options, - ); - results.push(...compared.results); - } catch (error) { - console.error(chalk.red(` ${errorMessage(error)}`)); + const results: CaseResult[] = []; + for (const caseName of reference.caseNames) { + const entry: BenchmarkEntry = { + name: caseName, + file: benchFile.file, + kind: 'baseline', + variants, + }; + const missing = others.findIndex((listing) => !listing.caseNames.includes(caseName)); + if (missing !== -1) { + // A case added by the change under test has nothing to compare with. results.push({ benchmark: { - name: comparison, - file: benchFile.file, - kind: 'compare', - variants, - error: errorMessage(error), + ...entry, + error: `"${caseName}" does not exist in [${variants[missing + 1]}].`, }, metrics: {}, }); + continue; } + const slots = urls.map((url, index) => ({ variant: variants[index], url, caseName })); + // eslint-disable-next-line no-await-in-loop + results.push(await runBenchmark(browser, entry, slots, options)); + } + + for (const { name, cases } of reference.comparisons) { + const entry: BenchmarkEntry = { name, file: benchFile.file, kind: 'compare', variants: cases }; + const slots = cases.map((caseName) => ({ variant: caseName, url: urls[0], caseName })); + // eslint-disable-next-line no-await-in-loop + results.push(await runBenchmark(browser, entry, slots, options)); } return results; } diff --git a/test/benchmark/README.md b/test/benchmark/README.md index fdb473486c..ed6dbd3cfc 100644 --- a/test/benchmark/README.md +++ b/test/benchmark/README.md @@ -17,15 +17,15 @@ For the real thing, see the consumers this tooling was extracted from — `base- | :---------------------------- | :------------------------------------------------------------------------ | | `workload/workload.bench.tsx` | The ordinary regression shape: current vs baseline, two sizes of workload | | `list/list.bench.tsx` | A mount, and a trusted scroll gesture with a custom metric around it | -| `lists/lists.bench.tsx` | `compare()` of two implementations, each a module of `benchmark()` cases | -| `libs/sort/sort.bench.tsx` | `compare()` of three variants, standing in for competitor libraries | +| `lists/lists.bench.tsx` | Two `compare()`s of two list implementations, one per scenario | +| `libs/sort/sort.bench.tsx` | `compare()` of three sorts, standing in for competitor libraries | `workload` and `list` import `@mui/internal-test-utils` and put its value on screen. That is the load-bearing part: every ref — the working tree included — resolves it from a packed tarball in that ref's own tree, so both sides of a comparison consume the library the way a consumer does. If that path breaks the page fails to build, rather than quietly measuring nothing. -The `compare()` files cover the other axis, where nothing is compared across commits. Their variants +The `compare()` files cover the other axis, where nothing is compared across commits. Their cases are built from the same commit and differ only in how they render or sort. A difference there is genuinely `worse` yet must never be reported as a regression, because both sides came from the same build. diff --git a/test/benchmark/src/libs/sort/sort.alpha.tsx b/test/benchmark/src/libs/sort/sort.alpha.tsx index 9a5cdf9642..74f8df3277 100644 --- a/test/benchmark/src/libs/sort/sort.alpha.tsx +++ b/test/benchmark/src/libs/sort/sort.alpha.tsx @@ -1,12 +1,9 @@ import * as React from 'react'; -import { reactBenchmark } from '@mui/internal-benchmark/page'; import { makeValues, report } from './shared'; /** Sorts the boxed array in place, paying a comparator call per comparison. */ -function Sorted() { +export function AlphaSorted() { const values = makeValues(); values.sort((left, right) => left - right); return

    {report('alpha', values)}

    ; } - -reactBenchmark('sort', () => ); diff --git a/test/benchmark/src/libs/sort/sort.bench.tsx b/test/benchmark/src/libs/sort/sort.bench.tsx index c75f71da1c..2c547ea032 100644 --- a/test/benchmark/src/libs/sort/sort.bench.tsx +++ b/test/benchmark/src/libs/sort/sort.bench.tsx @@ -1,12 +1,16 @@ -import { compare } from '@mui/internal-benchmark/page'; +import * as React from 'react'; +import { compare, reactBenchmark } from '@mui/internal-benchmark/page'; +import { AlphaSorted } from './sort.alpha'; +import { BetaSorted } from './sort.beta'; +import { OursSorted } from './sort.ours'; /** * Three sorts compared with each other rather than across builds — the shape a comparison between * libraries takes. This repository has no competitor libraries to install, so the "libraries" here * are three sorts with genuinely different constant factors. `ours` is the reference. */ -compare('libs-sort', { - ours: () => import('./sort.ours'), - alpha: () => import('./sort.alpha'), - beta: () => import('./sort.beta'), -}); +compare('libs-sort', [ + reactBenchmark('ours', () => ), + reactBenchmark('alpha', () => ), + reactBenchmark('beta', () => ), +]); diff --git a/test/benchmark/src/libs/sort/sort.beta.tsx b/test/benchmark/src/libs/sort/sort.beta.tsx index c4a3d9dfad..a3ee1990d7 100644 --- a/test/benchmark/src/libs/sort/sort.beta.tsx +++ b/test/benchmark/src/libs/sort/sort.beta.tsx @@ -1,5 +1,4 @@ import * as React from 'react'; -import { reactBenchmark } from '@mui/internal-benchmark/page'; import { makeValues, report } from './shared'; /** A merge sort written in JavaScript, allocating as it goes — the slow variant of the three. */ @@ -29,8 +28,6 @@ function mergeSort(values: number[]): number[] { return merged; } -function Sorted() { +export function BetaSorted() { return

    {report('beta', mergeSort(makeValues()))}

    ; } - -reactBenchmark('sort', () => ); diff --git a/test/benchmark/src/libs/sort/sort.ours.tsx b/test/benchmark/src/libs/sort/sort.ours.tsx index dd7cf0a21c..a7722f2ae2 100644 --- a/test/benchmark/src/libs/sort/sort.ours.tsx +++ b/test/benchmark/src/libs/sort/sort.ours.tsx @@ -1,12 +1,9 @@ import * as React from 'react'; -import { reactBenchmark } from '@mui/internal-benchmark/page'; import { makeValues, report } from './shared'; /** Copies into a typed array first, so the engine sorts unboxed doubles with no comparator calls. */ -function Sorted() { +export function OursSorted() { const sorted = Float64Array.from(makeValues()); sorted.sort(); return

    {report('ours', sorted)}

    ; } - -reactBenchmark('sort', () => ); diff --git a/test/benchmark/src/lists/lists.bench.tsx b/test/benchmark/src/lists/lists.bench.tsx index 09fe0947de..c6d0b78842 100644 --- a/test/benchmark/src/lists/lists.bench.tsx +++ b/test/benchmark/src/lists/lists.bench.tsx @@ -1,14 +1,51 @@ -import { compare } from '@mui/internal-benchmark/page'; +import * as React from 'react'; +import { compare, reactBenchmark } from '@mui/internal-benchmark/page'; +import type { BenchmarkInteraction } from '@mui/internal-benchmark/page'; +import { ScrollingList } from '../_shared/ScrollingList'; +import { TableList } from './lists.table'; +import { UlList } from './lists.ul'; /** * Two implementations of the same list, compared with each other rather than across builds — the - * shape a comparison between libraries takes. Each variant is a module of plain `reactBenchmark()` cases; - * cases with the same name are paired, and the first variant is the reference. - * - * Every variant gets a page of its own that loads only its own module, so neither implementation's - * code or styles are present while the other is measured. + * shape a comparison between libraries takes. Each `compare()` is one scenario; the first case is + * the reference. */ -compare('lists', { - ul: () => import('./lists.ul'), - table: () => import('./lists.table'), -}); + +const scroll: BenchmarkInteraction = async ({ + input, + resumeReactRecording, + waitForElementTiming, +}) => { + // The scroller has to be painted before a gesture can hit it. + await waitForElementTiming('default'); + resumeReactRecording(); + await input.scroll({ x: 200, y: 150, deltaY: 1200, speed: 2400 }); +}; + +compare('lists mount', [ + reactBenchmark('ul mount', () => ), + reactBenchmark('table mount', () => ), +]); + +compare('lists scroll', [ + reactBenchmark( + 'ul scroll', + () => ( + + + + ), + scroll, + { reactRecordingPaused: true }, + ), + reactBenchmark( + 'table scroll', + () => ( + + + + ), + scroll, + { reactRecordingPaused: true }, + ), +]); diff --git a/test/benchmark/src/lists/lists.table.tsx b/test/benchmark/src/lists/lists.table.tsx index aa54c9230b..e01bb828b0 100644 --- a/test/benchmark/src/lists/lists.table.tsx +++ b/test/benchmark/src/lists/lists.table.tsx @@ -1,10 +1,8 @@ import * as React from 'react'; -import { reactBenchmark } from '@mui/internal-benchmark/page'; -import { ScrollingList } from '../_shared/ScrollingList'; const ROWS = 1000; -function List() { +export function TableList() { return ( @@ -18,21 +16,3 @@ function List() {
    ); } - -reactBenchmark('mount', () => ); - -reactBenchmark( - 'scroll', - () => ( - - - - ), - async ({ input, resumeReactRecording, waitForElementTiming }) => { - // The scroller has to be painted before a gesture can hit it. - await waitForElementTiming('default'); - resumeReactRecording(); - await input.scroll({ x: 200, y: 150, deltaY: 1200, speed: 2400 }); - }, - { reactRecordingPaused: true }, -); diff --git a/test/benchmark/src/lists/lists.ul.tsx b/test/benchmark/src/lists/lists.ul.tsx index 62ad2a5521..84db62b44b 100644 --- a/test/benchmark/src/lists/lists.ul.tsx +++ b/test/benchmark/src/lists/lists.ul.tsx @@ -1,10 +1,8 @@ import * as React from 'react'; -import { reactBenchmark } from '@mui/internal-benchmark/page'; -import { ScrollingList } from '../_shared/ScrollingList'; const ROWS = 1000; -function List() { +export function UlList() { return (
      {Array.from({ length: ROWS }, (_, index) => ( @@ -15,21 +13,3 @@ function List() {
    ); } - -reactBenchmark('mount', () => ); - -reactBenchmark( - 'scroll', - () => ( - - - - ), - async ({ input, resumeReactRecording, waitForElementTiming }) => { - // The scroller has to be painted before a gesture can hit it. - await waitForElementTiming('default'); - resumeReactRecording(); - await input.scroll({ x: 200, y: 150, deltaY: 1200, speed: 2400 }); - }, - { reactRecordingPaused: true }, -); From 31a73b9f80e962c337a606a7c3e316b7909b7ba7 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:04:47 +0200 Subject: [PATCH 086/173] [benchmark] Simplify the page runtime and its runner - sample() returns the values the case recorded, summed per series, so the runner no longer scrapes every performance.measure on the page; the measures stay for DevTools traces. - compare() keeps one case -> comparison map, and the page exposes its case names and comparisons as plain data. - A file of compare()s alone never loads the baseline's page. - BenchmarkOptions and VariantLoader are exported by the Vitest entry only, since the page runtime ignores them. - The lists demo shares its scroll case, and the docs no longer describe the removed isolated mode. --- packages/benchmark/README.md | 12 ++-- packages/benchmark/src/caseRuntime.tsx | 3 - packages/benchmark/src/index.tsx | 12 ++-- packages/benchmark/src/page/page.tsx | 69 ++++++++----------- packages/benchmark/src/publicApi.ts | 2 +- .../benchmark/src/runner/runInterleaved.ts | 55 ++++----------- test/benchmark/README.md | 3 +- test/benchmark/src/lists/lists.bench.tsx | 24 +++---- 8 files changed, 64 insertions(+), 116 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index b0efa0e6ad..e4be291557 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -437,10 +437,10 @@ iteration is a warmup, so the `await import()` is never measured. ### How it measures The plugin generates a page per benchmark file under `src/__bench__/`. A benchmark file that imports -`vitest`, or the Vitest entry `@mui/internal-benchmark`, fails the build. The runner drives Chromium through Playwright and gives every variant — the -current and the baseline build, or each case of a `compare()` — a page of its own, in a browser -context of its own, that stays open for the whole benchmark: every sample is one warm iteration, and -module-scope data is built once. +`vitest`, or the Vitest entry `@mui/internal-benchmark`, fails the build. The runner drives Chromium +through Playwright and gives every variant — the current and the baseline build, or each case of a +`compare()` — a page of its own, in a browser context of its own, that stays open for the whole +benchmark: every sample is one warm iteration, and module-scope data is built once. Every variant is sampled once per round, in a shuffled order, and a difference is judged on the per-round differences rather than on two independent sets of samples. Whatever the machine was doing @@ -486,8 +486,8 @@ Packed builds are cached by commit SHA under `.benchmark/packed/`, so repeating the same commit skips the rebuild; the working tree is never cached. In CI, cache that directory keyed on the baseline SHA, and check out with full history so a fork point can be resolved. -A run resolves **in place**: it pins the packed build in the repository's own -`pnpm-workspace.yaml`, installs it there, and resolution is then ordinary. The repository is put +A run pins the packed build in the repository's own `pnpm-workspace.yaml`, installs it there, and +resolution is then ordinary. The repository is put back, and reinstalled, when the run ends — including when it fails, and at the start of the next run if one was killed outright. Two things follow from it. A tracked file names tarballs under `.benchmark/` until the run ends, so do not commit while one is in flight. And the pins are global to diff --git a/packages/benchmark/src/caseRuntime.tsx b/packages/benchmark/src/caseRuntime.tsx index c5fe6ea021..bb3636542c 100644 --- a/packages/benchmark/src/caseRuntime.tsx +++ b/packages/benchmark/src/caseRuntime.tsx @@ -58,9 +58,6 @@ export interface BenchmarkOptions extends CaseOptions { warmupRuns?: number; } -/** Loads a `compare()` variant's module, whose `benchmark()` calls define that variant's cases. */ -export type VariantLoader = () => Promise; - /** Splits `benchmark()`-style arguments, where the interaction may be left out. */ export function splitCaseArgs( interactionOrOptions: BenchmarkInteraction | Options | undefined, diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index 36c0d2bf6e..e3efa643a7 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -17,19 +17,17 @@ import { splitCaseArgs, warnIfNoGc, } from './caseRuntime'; -import type { - BenchmarkInteraction, - BenchmarkOptions, - CaseOptions, - VariantLoader, -} from './caseRuntime'; +import type { BenchmarkInteraction, BenchmarkOptions, CaseOptions } from './caseRuntime'; // Installs the Vitest metric recorder. import './Metric'; // Import for TaskMeta augmentation side effect import './taskMetaAugmentation'; export * from './publicApi'; -export type { VariantLoader }; +export type { BenchmarkOptions }; + +/** Loads a `compare()` variant's module, whose `benchmark()` calls define that variant's cases. */ +export type VariantLoader = () => Promise; // When true, `benchmark()` opens an interactive profiling session in a headed // browser instead of running the automated measurement loop. Enabled by diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 984262f6c1..de41a41fc4 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -41,8 +41,10 @@ export function benchmark(name: string, run: BenchmarkRun): BenchmarkCase { return { name }; } -/** Each `compare()`'s cases, by name, the reference first. */ -const comparisons = new Map(); +/** The `compare()` each compared case belongs to, by case name. */ +const comparedIn = new Map(); +/** Each `compare()`, with its cases' names, the reference first. */ +const comparisons: Array<{ name: string; cases: string[] }> = []; /** * Compares cases with each other — one library's implementation against another's, say — on the @@ -53,32 +55,26 @@ const comparisons = new Map(); * pages. */ export function compare(name: string, compared: BenchmarkCase[]): void { - if (comparisons.has(name)) { + if (comparisons.some((comparison) => comparison.name === name)) { throw new Error(`Two comparisons share the name "${name}". Comparison names must be unique.`); } if (compared.length < 2) { throw new Error(`Comparison "${name}" needs at least two cases.`); } for (const benchCase of compared) { - const owner = [...comparisons].find(([, names]) => names.includes(benchCase.name)); - if (owner) { - throw new Error(`"${benchCase.name}" is already compared in "${owner[0]}".`); + const owner = comparedIn.get(benchCase.name); + if (owner !== undefined) { + throw new Error(`"${benchCase.name}" is already compared in "${owner}".`); } + comparedIn.set(benchCase.name, name); } - comparisons.set( - name, - compared.map((benchCase) => benchCase.name), - ); + comparisons.push({ name, cases: compared.map((benchCase) => benchCase.name) }); } -interface RecordedValue { - name: string; - value: number; -} - -// Values recorded during a measured sample. `null` outside one (module scope, warmup), where +// Values recorded during a sample, summed per series: a metric recorded more than once in an +// iteration counts its total for that iteration. `null` outside a sample (module scope), where // recorded values are dropped. -let sampleValues: RecordedValue[] | null = null; +let sampleValues: Map | null = null; // Every metric recorded so far, by name, as the report describes it. const metricDefinitions = new Map(); @@ -91,21 +87,21 @@ setMetricRecorder((metric, value, options) => { alarm: metric.config.alarm, }); } - sampleValues?.push({ name: seriesName(metric.name, options?.id), value }); + if (sampleValues) { + const series = seriesName(metric.name, options?.id); + sampleValues.set(series, (sampleValues.get(series) ?? 0) + value); + } }); export interface BenchPage { /** The cases measured across the builds: every case no `compare()` took. */ - caseNames: () => string[]; + caseNames: string[]; /** The file's `compare()` calls, with their cases' names in order. */ - comparisons: () => Array<{ name: string; cases: string[] }>; + comparisons: Array<{ name: string; cases: string[] }>; /** Every metric recorded so far, by name. */ metricDefinitions: () => Record; - /** - * Runs one iteration of a case. A measured sample leaves its results as `performance.measure` - * entries for the runner to collect; a warmup sample leaves none. - */ - sample: (name: string, options: { warmup: boolean }) => Promise; + /** Runs one iteration of a case, and returns what it recorded, by series. */ + sample: (name: string) => Promise>; } declare global { @@ -124,7 +120,7 @@ const input = createInput((method, params) => { return window.benchmarkCdp(method, params); }); -async function sample(name: string, { warmup }: { warmup: boolean }): Promise { +async function sample(name: string): Promise> { const run = cases.get(name); if (!run) { throw new Error(`No benchmark named "${name}". Known: ${[...cases.keys()].join(', ')}`); @@ -132,9 +128,7 @@ async function sample(name: string, { warmup }: { warmup: boolean }): Promise(); sampleValues = values; try { await run({ input }); @@ -142,20 +136,17 @@ async function sample(name: string, { warmup }: { warmup: boolean }): Promise [...cases.keys()].filter((name) => !compared.has(name)), - comparisons: () => - [...comparisons].map(([name, comparedNames]) => ({ name, cases: comparedNames })), + caseNames: [...cases.keys()].filter((name) => !comparedIn.has(name)), + comparisons, metricDefinitions: () => Object.fromEntries(metricDefinitions), sample, }; diff --git a/packages/benchmark/src/publicApi.ts b/packages/benchmark/src/publicApi.ts index 8b40bb8e50..9ac9721d99 100644 --- a/packages/benchmark/src/publicApi.ts +++ b/packages/benchmark/src/publicApi.ts @@ -10,7 +10,7 @@ export type { MetricDefinition, } from './types'; export type { BenchmarkInput } from './input'; -export type { BenchmarkInteraction, BenchmarkOptions } from './caseRuntime'; +export type { BenchmarkInteraction } from './caseRuntime'; export { ElementTiming } from './ElementTiming'; export { Metric, type MetricRecordOptions } from './metricCore'; export { ScalarMetric } from './ScalarMetric'; diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index fbe721990d..0fbdff0487 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -8,8 +8,7 @@ import type { BenchFile } from './benchFiles'; import { variantOf } from './refs'; import type { ResolvedRef } from './refs'; import { serveDirectory } from './serveDirectory'; -// Types for `window.benchmarkPage`, which the evaluated functions below call into. -import type {} from '../page/page'; +import type { BenchPage } from '../page/page'; /** * Runs the cases of `*.bench.tsx` files in Playwright. Every variant gets a page of its own, in a @@ -74,12 +73,7 @@ function stableMetrics(rounds: Round[]): string[] { } /** What a benchmark file's page defines. */ -interface PageListing { - /** The cases measured across the builds. */ - caseNames: string[]; - /** The cases measured against each other, on the working tree's build. */ - comparisons: Array<{ name: string; cases: string[] }>; -} +type PageListing = Pick; interface OpenedPage { context: BrowserContext; @@ -107,8 +101,8 @@ async function openBenchPage(browser: Browser, url: string): Promise } return { visibility: document.visibilityState, - caseNames: window.benchmarkPage.caseNames(), - comparisons: window.benchmarkPage.comparisons(), + caseNames: window.benchmarkPage.caseNames, + comparisons: window.benchmarkPage.comparisons, }; }); const state = await ready.jsonValue(); @@ -144,32 +138,9 @@ async function listBenchPages(browser: Browser, urls: string[]): Promise target.listing); } -/** - * Runs one iteration of a case. A measured one comes back as its `performance.measure` values by - * name, read in the same round trip. - */ -function sampleBenchCase( - page: Page, - name: string, - warmup: boolean, -): Promise | null> { - return page.evaluate( - async (args) => { - await window.benchmarkPage!.sample(args.name, { warmup: args.warmup }); - if (args.warmup) { - return null; - } - const values: Record = {}; - for (const measure of performance.getEntriesByType('measure') as PerformanceMeasure[]) { - const detailValue: unknown = measure.detail?.value; - const value = typeof detailValue === 'number' ? detailValue : measure.duration; - // A metric recorded more than once in an iteration counts its total for that iteration. - values[measure.name] = (values[measure.name] ?? 0) + value; - } - return values; - }, - { name, warmup }, - ); +/** Runs one iteration of a case, and returns what it recorded, by series. */ +function sampleBenchCase(page: Page, name: string): Promise> { + return page.evaluate((caseName) => window.benchmarkPage!.sample(caseName), name); } /** @@ -202,13 +173,12 @@ async function measureBenchmark( browser, slots.map((slot) => slot.url), ); - const sampleSlot = (index: number, warmup: boolean) => - sampleBenchCase(targets[index].page, slots[index].caseName, warmup); + const sampleSlot = (index: number) => sampleBenchCase(targets[index].page, slots[index].caseName); try { for (let roundIndex = 0; roundIndex < options.warmup; roundIndex += 1) { for (const index of shuffledIndices(targets.length)) { // eslint-disable-next-line no-await-in-loop - await sampleSlot(index, true); + await sampleSlot(index); } } const rounds: Round[] = []; @@ -216,7 +186,7 @@ async function measureBenchmark( const round: Round = []; for (const index of shuffledIndices(targets.length)) { // eslint-disable-next-line no-await-in-loop - round[index] = (await sampleSlot(index, false)) ?? {}; + round[index] = await sampleSlot(index); } rounds.push(round); } @@ -286,7 +256,10 @@ async function runBenchFile( ): Promise { const urls = options.benchRefs.map((ref) => benchPageUrl(origin, ref, benchFile)); const variants = options.benchRefs.map(variantOf); - const [reference, ...others] = await listBenchPages(browser, urls); + // The other builds only matter for cases measured across them; a file of `compare()`s alone + // never loads them. + const [reference] = await listBenchPages(browser, urls.slice(0, 1)); + const others = reference.caseNames.length > 0 ? await listBenchPages(browser, urls.slice(1)) : []; const results: CaseResult[] = []; for (const caseName of reference.caseNames) { diff --git a/test/benchmark/README.md b/test/benchmark/README.md index ed6dbd3cfc..f460092e7d 100644 --- a/test/benchmark/README.md +++ b/test/benchmark/README.md @@ -21,8 +21,7 @@ For the real thing, see the consumers this tooling was extracted from — `base- | `libs/sort/sort.bench.tsx` | `compare()` of three sorts, standing in for competitor libraries | `workload` and `list` import `@mui/internal-test-utils` and put its value on screen. That is the -load-bearing part: every ref — the working tree included — resolves it from a packed tarball in that -ref's own tree, so both sides of a comparison consume the library the way a consumer does. If that +load-bearing part: every ref — the working tree included — resolves it from a packed tarball, so both sides of a comparison consume the library the way a consumer does. If that path breaks the page fails to build, rather than quietly measuring nothing. The `compare()` files cover the other axis, where nothing is compared across commits. Their cases diff --git a/test/benchmark/src/lists/lists.bench.tsx b/test/benchmark/src/lists/lists.bench.tsx index c6d0b78842..1113643d39 100644 --- a/test/benchmark/src/lists/lists.bench.tsx +++ b/test/benchmark/src/lists/lists.bench.tsx @@ -27,25 +27,17 @@ compare('lists mount', [ reactBenchmark('table mount', () => ), ]); -compare('lists scroll', [ - reactBenchmark( - 'ul scroll', +function scrollCase(name: string, List: React.ComponentType) { + return reactBenchmark( + name, () => ( - + ), scroll, { reactRecordingPaused: true }, - ), - reactBenchmark( - 'table scroll', - () => ( - - - - ), - scroll, - { reactRecordingPaused: true }, - ), -]); + ); +} + +compare('lists scroll', [scrollCase('ul scroll', UlList), scrollCase('table scroll', TableList)]); From e99fdb98fdabe97ef9293b99c67c21aea6114f97 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:33:31 +0200 Subject: [PATCH 087/173] [benchmark] Upload the tachometer report to this pull request's preview --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 31c5ea6812..a25f49fd2a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -147,7 +147,7 @@ jobs: # TEMPORARY: production does not know the tachometer report type until this pull # request lands, so it would reject the upload. Points at this pull request's own # dashboard preview until then — remove the line before merging. - CI_REPORT_API_URL: https://code-infra-dashboard-pr-1829.onrender.com + CI_REPORT_API_URL: https://code-infra-dashboard-pr-1898.onrender.com command: pnpm test:tacho --baseline "$(cat .tachometer-baseline-sha)" - save_cache: name: Save the packed-baseline cache From 300b204120ef628796ff92bbdcd86df0f50aaaa6 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 15:50:47 +0200 Subject: [PATCH 088/173] [benchmark] Sample benchmark run adaptively, like tachometer After sampleSize rounds, a benchmark keeps adding rounds while any difference is unresolved against an autoSampleConditions horizon, for up to timeout minutes. The options take tachometer's names and defaults and are set per benchmark: benchmark() and compare() take them last, and reactBenchmark() among its options. Resolution is judged on the report's own comparisons for the metrics that alarm, or every metric when none does. The report's sampling now records only the warmup, since rounds vary per benchmark. --- .../benchmarkRun/buildMarkdownReport.test.ts | 2 +- .../src/views/BenchmarkRunDetails.tsx | 5 +- packages/benchmark/README.md | 29 +++- packages/benchmark/src/page/index.ts | 1 + packages/benchmark/src/page/page.tsx | 40 +++-- .../benchmark/src/page/reactBenchmark.tsx | 16 +- .../src/runReport/analyzeRun.test.ts | 2 +- .../benchmark/src/runReport/analyzeRun.ts | 21 ++- packages/benchmark/src/runReport/schema.ts | 3 +- .../benchmark/src/runner/runBenchmarks.ts | 5 +- .../benchmark/src/runner/runInterleaved.ts | 100 +++++++----- packages/benchmark/src/sampling.test.ts | 149 ++++++++++++++++++ packages/benchmark/src/sampling.ts | 86 ++++++++++ test/benchmark/src/_shared/sampling.ts | 8 + test/benchmark/src/json/json.bench.tsx | 15 +- test/benchmark/src/list/list.bench.tsx | 5 +- test/benchmark/src/lists/lists.bench.tsx | 19 ++- .../benchmark/src/workload/workload.bench.tsx | 5 +- 18 files changed, 419 insertions(+), 92 deletions(-) create mode 100644 packages/benchmark/src/sampling.test.ts create mode 100644 packages/benchmark/src/sampling.ts create mode 100644 test/benchmark/src/_shared/sampling.ts diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index 8518795d40..e8ec8eb50a 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -10,7 +10,7 @@ function reportOf(benchmarks: RunBenchmark[]): BenchmarkRunReport { generatedAt: '2026-01-01T00:00:00.000Z', head: { sha: 'abc' }, environment: { browser: 'Chromium 151', platform: 'linux', arch: 'x64', launchArgs: [] }, - sampling: { samples: 4, warmup: 2 }, + sampling: { warmup: 2 }, builds: { current: { label: 'working tree' }, baseline: { sha: 'def', label: 'HEAD~1' } }, metrics: { render: { kind: 'scalar', format: MS, alarm: {} }, diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index bf33b91e60..b86b944419 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -185,9 +185,8 @@ export default function BenchmarkRunDetails() { {RUN_REPORT_FOOTNOTE}     - {report.sampling.samples} rounds after {report.sampling.warmup} warmup ·{' '} - {report.environment.browser} on {report.environment.platform}/ - {report.environment.arch} + {report.sampling.warmup} warmup rounds per benchmark · {report.environment.browser} on{' '} + {report.environment.platform}/{report.environment.arch} {report.environment.launchArgs.length > 0 && ` · ${report.environment.launchArgs.join(' ')}`} diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index e4be291557..d78e5acb0d 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -445,7 +445,34 @@ benchmark: every sample is one warm iteration, and module-scope data is built on Every variant is sampled once per round, in a shuffled order, and a difference is judged on the per-round differences rather than on two independent sets of samples. Whatever the machine was doing during a round — thermal throttling, a background process — then affects both sides of it and cancels -out. 10 warmup rounds are discarded once per benchmark, then 30 rounds are measured. +out. 10 warmup rounds are discarded once per benchmark before any is measured. + +### Sampling + +How many rounds a benchmark is measured for adapts to its results, the way tachometer's +auto-sampling does and under tachometer's names. After `sampleSize` rounds, rounds keep being added +while any difference is unresolved against an `autoSampleConditions` horizon, for up to `timeout` +minutes. A difference is resolved against a horizon once its confidence interval lies entirely on +one side of it; `'10%'` stands for both `'-10%'` and `'+10%'`. The metrics that alarm decide, or every +metric when none does. + +| Option | Default | Meaning | +| :--------------------- | :------- | :-------------------------------------------------------------------- | +| `sampleSize` | `30` | Rounds measured before deciding whether to continue | +| `timeout` | `3` | Minutes to keep sampling while a difference is unresolved | +| `autoSampleConditions` | `['0%']` | Horizons to resolve: by default, until each change is better or worse | + +They are set per benchmark: the last argument of `benchmark()` and `compare()`, and among +`reactBenchmark()`'s options. + +```tsx +reactBenchmark('mount', () => , { timeout: 1, autoSampleConditions: ['5%'] }); + +compare('scatter', [ours, other], { sampleSize: 50 }); +``` + +Two builds that perform the same never resolve against `0%`, so an unchanged benchmark samples until +its timeout; a horizon such as `'5%'` settles once a difference is known to be smaller than that. ### The report diff --git a/packages/benchmark/src/page/index.ts b/packages/benchmark/src/page/index.ts index 5a7ce5c0a5..2d5138cd95 100644 --- a/packages/benchmark/src/page/index.ts +++ b/packages/benchmark/src/page/index.ts @@ -5,3 +5,4 @@ export { benchmark, compare, markPageReady } from './page'; export type { BenchmarkCase, BenchmarkContext, BenchmarkRun, BenchPage } from './page'; export { reactBenchmark } from './reactBenchmark'; export type { ReactBenchmarkOptions } from './reactBenchmark'; +export type { SamplingOptions } from '../sampling'; diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index de41a41fc4..c6e4e5838e 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -1,6 +1,8 @@ import { createInput } from '../input'; import type { BenchmarkInput } from '../input'; import { seriesName, setMetricRecorder } from '../metricCore'; +import { validateSampling } from '../sampling'; +import type { SamplingOptions } from '../sampling'; import type { MetricDefinition } from '../types'; // The page runtime `benchmark run` measures. `benchmark()` registers a case, and the runner calls @@ -26,25 +28,30 @@ export interface BenchmarkCase { } /** Every case the file registered, by name. */ -const cases = new Map(); +const cases = new Map(); /** - * Registers a benchmark case. The runner calls `run` once per sample, warmup included, and decides - * how many samples to take. A case on its own is measured across the builds; one passed to - * `compare()` is measured against the other cases there instead. + * Registers a benchmark case. The runner calls `run` once per sample, warmup included, for as many + * rounds as `sampling` asks. A case on its own is measured across the builds; one passed to + * `compare()` is measured against the other cases there instead, sampled as the `compare()` asks. */ -export function benchmark(name: string, run: BenchmarkRun): BenchmarkCase { +export function benchmark( + name: string, + run: BenchmarkRun, + sampling: SamplingOptions = {}, +): BenchmarkCase { if (cases.has(name)) { throw new Error(`Two benchmarks share the name "${name}". Benchmark names must be unique.`); } - cases.set(name, run); + validateSampling(sampling); + cases.set(name, { run, sampling }); return { name }; } /** The `compare()` each compared case belongs to, by case name. */ const comparedIn = new Map(); /** Each `compare()`, with its cases' names, the reference first. */ -const comparisons: Array<{ name: string; cases: string[] }> = []; +const comparisons: Array<{ name: string; cases: string[]; sampling: SamplingOptions }> = []; /** * Compares cases with each other — one library's implementation against another's, say — on the @@ -54,13 +61,18 @@ const comparisons: Array<{ name: string; cases: string[] }> = []; * case needs inside that case (`await import()` resolves during warmup) to keep it out of the others' * pages. */ -export function compare(name: string, compared: BenchmarkCase[]): void { +export function compare( + name: string, + compared: BenchmarkCase[], + sampling: SamplingOptions = {}, +): void { if (comparisons.some((comparison) => comparison.name === name)) { throw new Error(`Two comparisons share the name "${name}". Comparison names must be unique.`); } if (compared.length < 2) { throw new Error(`Comparison "${name}" needs at least two cases.`); } + validateSampling(sampling); for (const benchCase of compared) { const owner = comparedIn.get(benchCase.name); if (owner !== undefined) { @@ -68,7 +80,7 @@ export function compare(name: string, compared: BenchmarkCase[]): void { } comparedIn.set(benchCase.name, name); } - comparisons.push({ name, cases: compared.map((benchCase) => benchCase.name) }); + comparisons.push({ name, cases: compared.map((benchCase) => benchCase.name), sampling }); } // Values recorded during a sample, summed per series: a metric recorded more than once in an @@ -95,9 +107,9 @@ setMetricRecorder((metric, value, options) => { export interface BenchPage { /** The cases measured across the builds: every case no `compare()` took. */ - caseNames: string[]; + cases: Array<{ name: string; sampling: SamplingOptions }>; /** The file's `compare()` calls, with their cases' names in order. */ - comparisons: Array<{ name: string; cases: string[] }>; + comparisons: Array<{ name: string; cases: string[]; sampling: SamplingOptions }>; /** Every metric recorded so far, by name. */ metricDefinitions: () => Record; /** Runs one iteration of a case, and returns what it recorded, by series. */ @@ -121,7 +133,7 @@ const input = createInput((method, params) => { }); async function sample(name: string): Promise> { - const run = cases.get(name); + const run = cases.get(name)?.run; if (!run) { throw new Error(`No benchmark named "${name}". Known: ${[...cases.keys()].join(', ')}`); } @@ -155,7 +167,9 @@ async function sample(name: string): Promise> { */ export function markPageReady(): void { window.benchmarkPage = { - caseNames: [...cases.keys()].filter((name) => !comparedIn.has(name)), + cases: [...cases] + .filter(([name]) => !comparedIn.has(name)) + .map(([name, { sampling }]) => ({ name, sampling })), comparisons, metricDefinitions: () => Object.fromEntries(metricDefinitions), sample, diff --git a/packages/benchmark/src/page/reactBenchmark.tsx b/packages/benchmark/src/page/reactBenchmark.tsx index d4f1bc9b63..febe9871df 100644 --- a/packages/benchmark/src/page/reactBenchmark.tsx +++ b/packages/benchmark/src/page/reactBenchmark.tsx @@ -10,10 +10,11 @@ import { import type { BenchmarkInteraction, CaseOptions } from '../caseRuntime'; import { DiscreteMetric } from '../DiscreteMetric'; import { ScalarMetric } from '../ScalarMetric'; +import type { SamplingOptions } from '../sampling'; import { benchmark } from './page'; -import type { BenchmarkCase } from './page'; +import type { BenchmarkCase, BenchmarkContext } from './page'; -export type ReactBenchmarkOptions = CaseOptions; +export interface ReactBenchmarkOptions extends CaseOptions, SamplingOptions {} // Render time and render count alarm on any resolved change for the worse; paint dominates each // case's duration and duplicates that signal, so it is informational, as is the per-phase split. @@ -49,10 +50,12 @@ export function reactBenchmark( checkedGc = true; warnIfNoGc(); } - const { interaction, options } = splitCaseArgs(interactionOrOptions, maybeOptions); + const { interaction, options = {} } = splitCaseArgs(interactionOrOptions, maybeOptions); + const { sampleSize, timeout, autoSampleConditions, ...caseOptions } = options; + const sampling = { sampleSize, timeout, autoSampleConditions }; - return benchmark(name, async ({ input }) => { - const result = await measureIteration(renderFn, interaction, options, input); + const run = async ({ input }: BenchmarkContext) => { + const result = await measureIteration(renderFn, interaction, caseOptions, input); if (result.renderError) { throw result.renderError; } @@ -81,5 +84,6 @@ export function reactBenchmark( for (const { id, start, end } of result.paints) { paintMetric.record(end - start, id === undefined ? undefined : { id }); } - }); + }; + return benchmark(name, run, sampling); } diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index 8fbb092d16..81dde64c7a 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -26,7 +26,7 @@ function reportOf(benchmarks: RunBenchmark[], metrics: BenchmarkRunReport['metri generatedAt: '2026-01-01T00:00:00.000Z', head: { sha: 'abc' }, environment: { browser: 'Chromium', platform: 'linux', arch: 'x64', launchArgs: [] }, - sampling: { samples: 10, warmup: 2 }, + sampling: { warmup: 2 }, builds: { current: { label: 'working tree' }, baseline: { sha: 'def', label: 'HEAD~1' } }, metrics, benchmarks, diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index 213fd1586f..195cc9ad29 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -99,9 +99,12 @@ function summarizeSamples(values: number[]): SampleSummary { const DEFAULT_DEFINITION: RunMetricDefinition = { kind: 'scalar' }; /** A metric's definition; a `name#id` sub-series is defined by its base metric. */ -function definitionOf(report: BenchmarkRunReport, metric: string): RunMetricDefinition { +function definitionOf( + metrics: Record, + metric: string, +): RunMetricDefinition { const [base] = metric.split('#'); - return report.metrics[metric] ?? report.metrics[base] ?? DEFAULT_DEFINITION; + return metrics[metric] ?? metrics[base] ?? DEFAULT_DEFINITION; } /** @@ -176,7 +179,11 @@ function pairsOf(benchmark: RunBenchmark): Array<{ subject: string; against: str : others.map((other) => ({ subject: other, against: reference })); } -function analyzeBenchmark(report: BenchmarkRunReport, benchmark: RunBenchmark): BenchmarkAnalysis { +/** Analyses one benchmark, its metrics described by `metrics` (a report's, or a run's so far). */ +export function analyzeBenchmark( + metrics: Record, + benchmark: RunBenchmark, +): BenchmarkAnalysis { const { samples } = benchmark; if (!samples) { return { benchmark, metrics: [] }; @@ -184,8 +191,8 @@ function analyzeBenchmark(report: BenchmarkRunReport, benchmark: RunBenchmark): const [reference] = benchmark.variants; const metricNames = Object.keys(samples[reference] ?? {}); - const metrics = metricNames.map((metric): MetricAnalysis => { - const definition = definitionOf(report, metric); + const analyses = metricNames.map((metric): MetricAnalysis => { + const definition = definitionOf(metrics, metric); const valuesOf = (variant: string) => samples[variant]?.[metric] ?? []; return { metric, @@ -202,11 +209,11 @@ function analyzeBenchmark(report: BenchmarkRunReport, benchmark: RunBenchmark): ), }; }); - return { benchmark, metrics }; + return { benchmark, metrics: analyses }; } export function analyzeRun(report: BenchmarkRunReport): BenchmarkAnalysis[] { - return report.benchmarks.map((benchmark) => analyzeBenchmark(report, benchmark)); + return report.benchmarks.map((benchmark) => analyzeBenchmark(report.metrics, benchmark)); } export interface Regression { diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts index 8a5905e727..e708fcfdcc 100644 --- a/packages/benchmark/src/runReport/schema.ts +++ b/packages/benchmark/src/runReport/schema.ts @@ -70,7 +70,8 @@ export const benchmarkRunReportSchema = z.object({ arch: z.string(), launchArgs: z.array(z.string()), }), - sampling: z.object({ samples: z.number(), warmup: z.number() }), + /** Rounds per benchmark vary with its sampling options; warmup is the same for every one. */ + sampling: z.object({ warmup: z.number() }), /** What the `baseline`-kind variant names refer to. */ builds: z.record(z.string(), buildSchema), /** Every metric the benchmarks report, keyed by name; `name#id` sub-series share `name`'s entry. */ diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index 6d9d8430e4..a64d609178 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -40,8 +40,6 @@ export interface RunBenchmarksOptions { upload?: boolean; } -/** Measured rounds per benchmark; each round samples every variant once. */ -const SAMPLES = 30; /** Discarded rounds before measuring, once per benchmark. */ const WARMUP = 10; @@ -139,7 +137,6 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise [variantOf(ref), { sha: ref.sha, label: refLabel(ref) }]), ), diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index 0fbdff0487..a9fe6ad843 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -3,7 +3,10 @@ import chalk from 'chalk'; import type { Browser, BrowserContext, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS } from '../launchArgs'; +import { analyzeBenchmark } from '../runReport'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; +import { differencesResolved, parseHorizons, resolveSampling } from '../sampling'; +import type { SamplingOptions } from '../sampling'; import type { BenchFile } from './benchFiles'; import { variantOf } from './refs'; import type { ResolvedRef } from './refs'; @@ -26,8 +29,7 @@ export interface RunInterleavedOptions { benchRefs: ResolvedRef[]; browserBinary: string; launchArgs: string[]; - samples: number; - /** Discarded rounds before measuring, once per case. */ + /** Discarded rounds before measuring, once per benchmark. */ warmup: number; } @@ -73,7 +75,7 @@ function stableMetrics(rounds: Round[]): string[] { } /** What a benchmark file's page defines. */ -type PageListing = Pick; +type PageListing = Pick; interface OpenedPage { context: BrowserContext; @@ -101,7 +103,7 @@ async function openBenchPage(browser: Browser, url: string): Promise } return { visibility: document.visibilityState, - caseNames: window.benchmarkPage.caseNames, + cases: window.benchmarkPage.cases, comparisons: window.benchmarkPage.comparisons, }; }); @@ -154,46 +156,71 @@ interface PageSlot { caseName: string; } -interface CaseMeasurement { - rounds: Round[]; +/** A benchmark as it goes into the report, with the definitions of the metrics it reported. */ +interface CaseResult { + benchmark: RunBenchmark; metrics: Record; } +type BenchmarkEntry = Pick; + /** * Measures one benchmark in fresh pages that stay open for all of it, so iterations stay warm and * module-scope data is built once. Warmup and measured rounds alike run every slot once, in a - * shuffled order. + * shuffled order. After `sampleSize` rounds it keeps adding rounds while a difference is unresolved + * against the horizons, until the timeout. */ async function measureBenchmark( browser: Browser, + entry: BenchmarkEntry, slots: PageSlot[], + sampling: SamplingOptions, options: RunInterleavedOptions, -): Promise { +): Promise { + const { sampleSize, timeout, autoSampleConditions } = resolveSampling(sampling); + const horizons = parseHorizons(autoSampleConditions); const targets = await openBenchPages( browser, slots.map((slot) => slot.url), ); - const sampleSlot = (index: number) => sampleBenchCase(targets[index].page, slots[index].caseName); + const runRound = async (): Promise => { + const round: Round = []; + for (const index of shuffledIndices(targets.length)) { + // eslint-disable-next-line no-await-in-loop + round[index] = await sampleBenchCase(targets[index].page, slots[index].caseName); + } + return round; + }; try { for (let roundIndex = 0; roundIndex < options.warmup; roundIndex += 1) { - for (const index of shuffledIndices(targets.length)) { - // eslint-disable-next-line no-await-in-loop - await sampleSlot(index); - } + // eslint-disable-next-line no-await-in-loop + await runRound(); } const rounds: Round[] = []; - for (let roundIndex = 0; roundIndex < options.samples; roundIndex += 1) { - const round: Round = []; - for (const index of shuffledIndices(targets.length)) { - // eslint-disable-next-line no-await-in-loop - round[index] = await sampleSlot(index); - } - rounds.push(round); + for (let roundIndex = 0; roundIndex < sampleSize; roundIndex += 1) { + // eslint-disable-next-line no-await-in-loop + rounds.push(await runRound()); } // Read from the reference page: a metric whose config the change under test altered is - // described the way the change describes it. + // described the way the change describes it. Every metric has been recorded by now. const metrics = await targets[0].page.evaluate(() => window.benchmarkPage!.metricDefinitions()); - return { rounds, metrics }; + const benchmarkOf = (): RunBenchmark => ({ ...entry, samples: samplesOf(slots, rounds) }); + + const deadline = Date.now() + timeout * 60_000; + let resolved = differencesResolved(analyzeBenchmark(metrics, benchmarkOf()), horizons); + while (!resolved && Date.now() < deadline) { + // eslint-disable-next-line no-await-in-loop + rounds.push(await runRound()); + resolved = differencesResolved(analyzeBenchmark(metrics, benchmarkOf()), horizons); + } + if (rounds.length > sampleSize || !resolved) { + console.log( + chalk.dim( + ` ${rounds.length} rounds, ${resolved ? 'resolved' : `unresolved after ${timeout} min`}`, + ), + ); + } + return { benchmark: benchmarkOf(), metrics }; } finally { await closeBenchPages(targets); } @@ -212,28 +239,17 @@ function samplesOf(slots: PageSlot[], rounds: Round[]): RunBenchmark['samples'] ); } -/** A benchmark as it goes into the report, with the definitions of the metrics it reported. */ -interface CaseResult { - benchmark: RunBenchmark; - metrics: Record; -} - -type BenchmarkEntry = Pick; - /** Measures one benchmark across its slots, or reports the error that stopped it. */ async function runBenchmark( browser: Browser, entry: BenchmarkEntry, slots: PageSlot[], + sampling: SamplingOptions, options: RunInterleavedOptions, ): Promise { console.log(chalk.cyan(`\nRunning "${entry.name}" (${entry.file})…`)); try { - const measured = await measureBenchmark(browser, slots, options); - return { - benchmark: { ...entry, samples: samplesOf(slots, measured.rounds) }, - metrics: measured.metrics, - }; + return await measureBenchmark(browser, entry, slots, sampling, options); } catch (error) { console.error(chalk.red(` ${errorMessage(error)}`)); return { benchmark: { ...entry, error: errorMessage(error) }, metrics: {} }; @@ -259,17 +275,19 @@ async function runBenchFile( // The other builds only matter for cases measured across them; a file of `compare()`s alone // never loads them. const [reference] = await listBenchPages(browser, urls.slice(0, 1)); - const others = reference.caseNames.length > 0 ? await listBenchPages(browser, urls.slice(1)) : []; + const others = reference.cases.length > 0 ? await listBenchPages(browser, urls.slice(1)) : []; const results: CaseResult[] = []; - for (const caseName of reference.caseNames) { + for (const { name: caseName, sampling } of reference.cases) { const entry: BenchmarkEntry = { name: caseName, file: benchFile.file, kind: 'baseline', variants, }; - const missing = others.findIndex((listing) => !listing.caseNames.includes(caseName)); + const missing = others.findIndex( + (listing) => !listing.cases.some((benchCase) => benchCase.name === caseName), + ); if (missing !== -1) { // A case added by the change under test has nothing to compare with. results.push({ @@ -283,14 +301,14 @@ async function runBenchFile( } const slots = urls.map((url, index) => ({ variant: variants[index], url, caseName })); // eslint-disable-next-line no-await-in-loop - results.push(await runBenchmark(browser, entry, slots, options)); + results.push(await runBenchmark(browser, entry, slots, sampling, options)); } - for (const { name, cases } of reference.comparisons) { + for (const { name, cases, sampling } of reference.comparisons) { const entry: BenchmarkEntry = { name, file: benchFile.file, kind: 'compare', variants: cases }; const slots = cases.map((caseName) => ({ variant: caseName, url: urls[0], caseName })); // eslint-disable-next-line no-await-in-loop - results.push(await runBenchmark(browser, entry, slots, options)); + results.push(await runBenchmark(browser, entry, slots, sampling, options)); } return results; } diff --git a/packages/benchmark/src/sampling.test.ts b/packages/benchmark/src/sampling.test.ts new file mode 100644 index 0000000000..70c6b127de --- /dev/null +++ b/packages/benchmark/src/sampling.test.ts @@ -0,0 +1,149 @@ +import { describe, expect, it } from 'vitest'; +import { analyzeBenchmark } from './runReport/analyzeRun'; +import type { RunBenchmark, RunMetricDefinition } from './runReport'; +import { + DEFAULT_SAMPLING, + differencesResolved, + isResolved, + parseHorizons, + resolveSampling, + validateSampling, +} from './sampling'; + +describe('resolveSampling', () => { + it('defaults an option left undefined, not only one left out', () => { + expect( + resolveSampling({ sampleSize: undefined, timeout: 1, autoSampleConditions: undefined }), + ).toEqual({ ...DEFAULT_SAMPLING, timeout: 1 }); + }); +}); + +describe('parseHorizons', () => { + it('expands an unsigned percentage to both signs', () => { + expect(parseHorizons(['10%'])).toEqual([-10, 10]); + }); + + it('keeps a signed percentage to its own side', () => { + expect(parseHorizons(['+5%', '-2.5%'])).toEqual([5, -2.5]); + }); + + it('reads zero as a single horizon', () => { + expect(parseHorizons(['0%'])).toEqual([0]); + }); + + it('rejects anything but a percentage', () => { + expect(() => parseHorizons(['5ms'])).toThrow(/Invalid auto-sample condition "5ms"/); + expect(() => parseHorizons(['%'])).toThrow(/Invalid auto-sample condition "%"/); + expect(() => parseHorizons(['ten%'])).toThrow(/Invalid auto-sample condition "ten%"/); + }); +}); + +describe('isResolved', () => { + it('is resolved when the interval lies on one side of every horizon', () => { + expect(isResolved({ low: 2, high: 8 }, [0])).toBe(true); + expect(isResolved({ low: -8, high: -2 }, [0])).toBe(true); + expect(isResolved({ low: -4, high: 4 }, [-10, 10])).toBe(true); + }); + + it('is unresolved while the interval straddles a horizon', () => { + expect(isResolved({ low: -1, high: 3 }, [0])).toBe(false); + expect(isResolved({ low: 5, high: 15 }, [-10, 10])).toBe(false); + }); + + it('treats an exact, zero-width difference as resolved', () => { + expect(isResolved({ low: 0, high: 0 }, [0])).toBe(true); + }); +}); + +describe('validateSampling', () => { + it('accepts the documented ranges', () => { + expect(() => + validateSampling({ sampleSize: 2, timeout: 0, autoSampleConditions: ['1%'] }), + ).not.toThrow(); + }); + + it('rejects a sample size that cannot form an interval', () => { + expect(() => validateSampling({ sampleSize: 1 })).toThrow(/Invalid sampleSize 1/); + expect(() => validateSampling({ sampleSize: 2.5 })).toThrow(/Invalid sampleSize 2.5/); + }); + + it('rejects a negative timeout', () => { + expect(() => validateSampling({ timeout: -1 })).toThrow(/Invalid timeout -1/); + }); +}); + +describe('differencesResolved', () => { + const noisy = [10, 12, 9, 11, 10, 13, 8, 11]; + const clearlySlower = noisy.map((value) => value * 2); + const sameish = [11, 9, 10, 12, 11, 10, 9, 12]; + + function benchmarkOf(samples: Record): RunBenchmark { + return { + name: 'mount', + file: 'button.bench.tsx', + kind: 'baseline', + variants: ['current', 'baseline'], + samples: { + current: { render: samples.current, paint: samples.current }, + baseline: { render: samples.baseline, paint: samples.baseline }, + }, + }; + } + + const alarmedRender: Record = { + render: { kind: 'scalar', alarm: {} }, + paint: { kind: 'scalar' }, + }; + + it('stops once every alarmed difference is clear of the horizon', () => { + const analysis = analyzeBenchmark( + alarmedRender, + benchmarkOf({ current: clearlySlower, baseline: noisy }), + ); + expect(differencesResolved(analysis, [0])).toBe(true); + }); + + it('keeps sampling while an alarmed difference straddles the horizon', () => { + const analysis = analyzeBenchmark( + alarmedRender, + benchmarkOf({ current: sameish, baseline: noisy }), + ); + expect(differencesResolved(analysis, [0])).toBe(false); + }); + + it('ignores metrics that do not alarm when some do', () => { + const analysis = analyzeBenchmark( + { render: { kind: 'scalar', alarm: {} }, paint: { kind: 'scalar' } }, + { + ...benchmarkOf({ current: clearlySlower, baseline: noisy }), + samples: { + current: { render: clearlySlower, paint: sameish }, + baseline: { render: noisy, paint: noisy }, + }, + }, + ); + expect(differencesResolved(analysis, [0])).toBe(true); + }); + + it('judges every metric when none alarms', () => { + const analysis = analyzeBenchmark( + { render: { kind: 'scalar' }, paint: { kind: 'scalar' } }, + { + ...benchmarkOf({ current: clearlySlower, baseline: noisy }), + samples: { + current: { render: clearlySlower, paint: sameish }, + baseline: { render: noisy, paint: noisy }, + }, + }, + ); + expect(differencesResolved(analysis, [0])).toBe(false); + }); + + it('resolves a small difference against a wider horizon', () => { + const analysis = analyzeBenchmark( + alarmedRender, + benchmarkOf({ current: sameish, baseline: noisy }), + ); + expect(differencesResolved(analysis, [-50, 50])).toBe(true); + }); +}); diff --git a/packages/benchmark/src/sampling.ts b/packages/benchmark/src/sampling.ts new file mode 100644 index 0000000000..736886d016 --- /dev/null +++ b/packages/benchmark/src/sampling.ts @@ -0,0 +1,86 @@ +import type { BenchmarkAnalysis } from './runReport/analyzeRun'; + +/** + * How many rounds a benchmark is sampled for: tachometer's knobs, under tachometer's names. After + * `sampleSize` rounds, sampling continues while any difference is still unresolved against an + * `autoSampleConditions` horizon, for up to `timeout` minutes. + */ +export interface SamplingOptions { + /** Rounds measured before deciding whether to continue. Defaults to 30. */ + sampleSize?: number; + /** + * Minutes to keep sampling after `sampleSize` rounds while a difference is unresolved. `0` + * measures exactly `sampleSize` rounds. Defaults to 3. + */ + timeout?: number; + /** + * Relative differences to resolve before `timeout`. A difference is resolved against a horizon + * once its confidence interval lies entirely on one side of it. `'10%'` stands for both `'-10%'` + * and `'+10%'`. Defaults to `['0%']`: sample until each change is known to be better or worse. + * + * Judged on the metrics that alarm — the ones a change can be flagged on — or on every metric when + * none does. + */ + autoSampleConditions?: string[]; +} + +export const DEFAULT_SAMPLING: Required = { + sampleSize: 30, + timeout: 3, + autoSampleConditions: ['0%'], +}; + +/** The options with a default for every one left out, or left `undefined`. */ +export function resolveSampling(options: SamplingOptions): Required { + return { + sampleSize: options.sampleSize ?? DEFAULT_SAMPLING.sampleSize, + timeout: options.timeout ?? DEFAULT_SAMPLING.timeout, + autoSampleConditions: options.autoSampleConditions ?? DEFAULT_SAMPLING.autoSampleConditions, + }; +} + +/** Parses `autoSampleConditions` into signed horizons, in percent. */ +export function parseHorizons(conditions: string[]): number[] { + return conditions.flatMap((condition) => { + const trimmed = condition.trim(); + const value = Number(trimmed.slice(0, -1)); + if (!trimmed.endsWith('%') || trimmed === '%' || Number.isNaN(value)) { + throw new Error( + `Invalid auto-sample condition "${condition}": expected a percentage, like "5%", "+5%" or "-5%".`, + ); + } + const signed = trimmed.startsWith('+') || trimmed.startsWith('-'); + return signed || value === 0 ? [value] : [-value, value]; + }); +} + +/** Checks sampling options when a case registers, so a mistake fails the page, not a round. */ +export function validateSampling(options: SamplingOptions): void { + const { sampleSize, timeout, autoSampleConditions } = options; + if (sampleSize !== undefined && !(Number.isInteger(sampleSize) && sampleSize >= 2)) { + throw new Error(`Invalid sampleSize ${sampleSize}: expected an integer of at least 2.`); + } + if (timeout !== undefined && !(timeout >= 0)) { + throw new Error(`Invalid timeout ${timeout}: expected a number of minutes, 0 or more.`); + } + if (autoSampleConditions !== undefined) { + parseHorizons(autoSampleConditions); + } +} + +/** Whether an interval lies entirely on one side of every horizon. */ +export function isResolved(interval: { low: number; high: number }, horizons: number[]): boolean { + return horizons.every((horizon) => !(interval.low < horizon && horizon < interval.high)); +} + +/** + * Whether sampling a benchmark can stop: every comparison it reports has resolved against the + * horizons, on the metrics that alarm, or on every metric when none does. + */ +export function differencesResolved(analysis: BenchmarkAnalysis, horizons: number[]): boolean { + const alarmed = analysis.metrics.filter((metric) => metric.definition.alarm !== undefined); + const judged = alarmed.length > 0 ? alarmed : analysis.metrics; + return judged.every((metric) => + metric.comparisons.every((comparison) => isResolved(comparison.relative, horizons)), + ); +} diff --git a/test/benchmark/src/_shared/sampling.ts b/test/benchmark/src/_shared/sampling.ts new file mode 100644 index 0000000000..67774258c0 --- /dev/null +++ b/test/benchmark/src/_shared/sampling.ts @@ -0,0 +1,8 @@ +import type { SamplingOptions } from '@mui/internal-benchmark/page'; + +/** + * Sampling for this smoke-test harness. Both sides of its baseline comparisons run identical code, so + * no difference ever resolves against the default `0%` horizon and every case would sample until + * its timeout. A `10%` horizon settles as soon as a difference is known to be smaller than that. + */ +export const SMOKE_SAMPLING: SamplingOptions = { timeout: 1, autoSampleConditions: ['10%'] }; diff --git a/test/benchmark/src/json/json.bench.tsx b/test/benchmark/src/json/json.bench.tsx index 670ca28321..d9b755e03c 100644 --- a/test/benchmark/src/json/json.bench.tsx +++ b/test/benchmark/src/json/json.bench.tsx @@ -1,4 +1,5 @@ import { benchmark, ScalarMetric } from '@mui/internal-benchmark/page'; +import { SMOKE_SAMPLING } from '../_shared/sampling'; /** * A `benchmark()` case without React: every sample runs the function once, and what it measures is @@ -19,8 +20,12 @@ const payload = JSON.stringify( })), ); -benchmark('parse', () => { - parseTime.time(); - JSON.parse(payload); - parseTime.timeEnd(); -}); +benchmark( + 'parse', + () => { + parseTime.time(); + JSON.parse(payload); + parseTime.timeEnd(); + }, + SMOKE_SAMPLING, +); diff --git a/test/benchmark/src/list/list.bench.tsx b/test/benchmark/src/list/list.bench.tsx index 87931b5bd0..5bd36b503a 100644 --- a/test/benchmark/src/list/list.bench.tsx +++ b/test/benchmark/src/list/list.bench.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { reactBenchmark, ScalarMetric } from '@mui/internal-benchmark/page'; import { reactMajor } from '@mui/internal-test-utils/env'; import { ScrollingList } from '../_shared/ScrollingList'; +import { SMOKE_SAMPLING } from '../_shared/sampling'; /** * `reactBenchmark()` cases. Under `benchmark run` each case runs one iteration at a time in a page @@ -21,7 +22,7 @@ function List({ count }: { count: number }) { ); } -reactBenchmark('list mount', () => ); +reactBenchmark('list mount', () => , SMOKE_SAMPLING); const scrollDuration = new ScalarMetric({ name: 'scroll_gesture', @@ -45,5 +46,5 @@ reactBenchmark( await input.scroll({ x: 200, y: 150, deltaY: 1200, speed: 2400 }); scrollDuration.timeEnd(); }, - { reactRecordingPaused: true }, + { reactRecordingPaused: true, ...SMOKE_SAMPLING }, ); diff --git a/test/benchmark/src/lists/lists.bench.tsx b/test/benchmark/src/lists/lists.bench.tsx index 1113643d39..62c87ecea5 100644 --- a/test/benchmark/src/lists/lists.bench.tsx +++ b/test/benchmark/src/lists/lists.bench.tsx @@ -2,6 +2,7 @@ import * as React from 'react'; import { compare, reactBenchmark } from '@mui/internal-benchmark/page'; import type { BenchmarkInteraction } from '@mui/internal-benchmark/page'; import { ScrollingList } from '../_shared/ScrollingList'; +import { SMOKE_SAMPLING } from '../_shared/sampling'; import { TableList } from './lists.table'; import { UlList } from './lists.ul'; @@ -22,10 +23,14 @@ const scroll: BenchmarkInteraction = async ({ await input.scroll({ x: 200, y: 150, deltaY: 1200, speed: 2400 }); }; -compare('lists mount', [ - reactBenchmark('ul mount', () => ), - reactBenchmark('table mount', () => ), -]); +compare( + 'lists mount', + [ + reactBenchmark('ul mount', () => ), + reactBenchmark('table mount', () => ), + ], + SMOKE_SAMPLING, +); function scrollCase(name: string, List: React.ComponentType) { return reactBenchmark( @@ -40,4 +45,8 @@ function scrollCase(name: string, List: React.ComponentType) { ); } -compare('lists scroll', [scrollCase('ul scroll', UlList), scrollCase('table scroll', TableList)]); +compare( + 'lists scroll', + [scrollCase('ul scroll', UlList), scrollCase('table scroll', TableList)], + SMOKE_SAMPLING, +); diff --git a/test/benchmark/src/workload/workload.bench.tsx b/test/benchmark/src/workload/workload.bench.tsx index 4c01263150..b7879df731 100644 --- a/test/benchmark/src/workload/workload.bench.tsx +++ b/test/benchmark/src/workload/workload.bench.tsx @@ -1,6 +1,7 @@ import * as React from 'react'; import { reactBenchmark } from '@mui/internal-benchmark/page'; import { reactMajor } from '@mui/internal-test-utils/env'; +import { SMOKE_SAMPLING } from '../_shared/sampling'; /** * A deterministic CPU workload, sized per case. @@ -43,5 +44,5 @@ function Workload({ size }: { size: number }) { ); } -reactBenchmark('workload', () => ); -reactBenchmark('workload-large', () => ); +reactBenchmark('workload', () => , SMOKE_SAMPLING); +reactBenchmark('workload-large', () => , SMOKE_SAMPLING); From 381087cb8db8579b0dc1792c040ef9a056c8026c Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:03:13 +0200 Subject: [PATCH 089/173] [benchmark] Resolve sampling where a case registers; build dashboard deps on Render - benchmark() and compare() validate their sampling options and fill in the defaults, so the runner gets complete options and reactBenchmark() passes its own straight through. - compare() rejects a case that sets its own sampling, which it would otherwise ignore. - The runner checks resolution in one place and reports a failure from the function that measures. - The Render build builds the dashboard's workspace dependencies first: it imports @mui/internal-benchmark at runtime, which resolves through its build output, and the preview failed to deploy without it. --- packages/benchmark/README.md | 3 +- packages/benchmark/src/page/page.tsx | 38 ++++++++--- .../benchmark/src/page/reactBenchmark.tsx | 8 +-- .../benchmark/src/runner/runInterleaved.ts | 46 ++++++------- packages/benchmark/src/sampling.test.ts | 64 ++++++++++--------- packages/benchmark/src/sampling.ts | 40 +++++++----- render.yaml | 2 + test/benchmark/src/_shared/sampling.ts | 7 +- 8 files changed, 114 insertions(+), 94 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index d78e5acb0d..6c35991029 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -463,7 +463,8 @@ metric when none does. | `autoSampleConditions` | `['0%']` | Horizons to resolve: by default, until each change is better or worse | They are set per benchmark: the last argument of `benchmark()` and `compare()`, and among -`reactBenchmark()`'s options. +`reactBenchmark()`'s options. A compared case is sampled as its `compare()` asks, so setting its own +is an error. ```tsx reactBenchmark('mount', () => , { timeout: 1, autoSampleConditions: ['5%'] }); diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index c6e4e5838e..3d42f0c0b9 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -1,7 +1,7 @@ import { createInput } from '../input'; import type { BenchmarkInput } from '../input'; import { seriesName, setMetricRecorder } from '../metricCore'; -import { validateSampling } from '../sampling'; +import { hasSampling, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { MetricDefinition } from '../types'; @@ -28,7 +28,10 @@ export interface BenchmarkCase { } /** Every case the file registered, by name. */ -const cases = new Map(); +const cases = new Map< + string, + { run: BenchmarkRun; sampling: Required; ownSampling: boolean } +>(); /** * Registers a benchmark case. The runner calls `run` once per sample, warmup included, for as many @@ -43,15 +46,22 @@ export function benchmark( if (cases.has(name)) { throw new Error(`Two benchmarks share the name "${name}". Benchmark names must be unique.`); } - validateSampling(sampling); - cases.set(name, { run, sampling }); + cases.set(name, { + run, + sampling: resolveSampling(sampling), + ownSampling: hasSampling(sampling), + }); return { name }; } /** The `compare()` each compared case belongs to, by case name. */ const comparedIn = new Map(); /** Each `compare()`, with its cases' names, the reference first. */ -const comparisons: Array<{ name: string; cases: string[]; sampling: SamplingOptions }> = []; +const comparisons: Array<{ + name: string; + cases: string[]; + sampling: Required; +}> = []; /** * Compares cases with each other — one library's implementation against another's, say — on the @@ -72,15 +82,25 @@ export function compare( if (compared.length < 2) { throw new Error(`Comparison "${name}" needs at least two cases.`); } - validateSampling(sampling); + const resolved = resolveSampling(sampling); for (const benchCase of compared) { + if (cases.get(benchCase.name)?.ownSampling) { + throw new Error( + `"${benchCase.name}" sets its own sampling, but a compared case is sampled as its ` + + `compare() asks: pass the options to compare("${name}", …) instead.`, + ); + } const owner = comparedIn.get(benchCase.name); if (owner !== undefined) { throw new Error(`"${benchCase.name}" is already compared in "${owner}".`); } comparedIn.set(benchCase.name, name); } - comparisons.push({ name, cases: compared.map((benchCase) => benchCase.name), sampling }); + comparisons.push({ + name, + cases: compared.map((benchCase) => benchCase.name), + sampling: resolved, + }); } // Values recorded during a sample, summed per series: a metric recorded more than once in an @@ -107,9 +127,9 @@ setMetricRecorder((metric, value, options) => { export interface BenchPage { /** The cases measured across the builds: every case no `compare()` took. */ - cases: Array<{ name: string; sampling: SamplingOptions }>; + cases: Array<{ name: string; sampling: Required }>; /** The file's `compare()` calls, with their cases' names in order. */ - comparisons: Array<{ name: string; cases: string[]; sampling: SamplingOptions }>; + comparisons: Array<{ name: string; cases: string[]; sampling: Required }>; /** Every metric recorded so far, by name. */ metricDefinitions: () => Record; /** Runs one iteration of a case, and returns what it recorded, by series. */ diff --git a/packages/benchmark/src/page/reactBenchmark.tsx b/packages/benchmark/src/page/reactBenchmark.tsx index febe9871df..7c448d9649 100644 --- a/packages/benchmark/src/page/reactBenchmark.tsx +++ b/packages/benchmark/src/page/reactBenchmark.tsx @@ -50,12 +50,10 @@ export function reactBenchmark( checkedGc = true; warnIfNoGc(); } - const { interaction, options = {} } = splitCaseArgs(interactionOrOptions, maybeOptions); - const { sampleSize, timeout, autoSampleConditions, ...caseOptions } = options; - const sampling = { sampleSize, timeout, autoSampleConditions }; + const { interaction, options } = splitCaseArgs(interactionOrOptions, maybeOptions); const run = async ({ input }: BenchmarkContext) => { - const result = await measureIteration(renderFn, interaction, caseOptions, input); + const result = await measureIteration(renderFn, interaction, options, input); if (result.renderError) { throw result.renderError; } @@ -85,5 +83,5 @@ export function reactBenchmark( paintMetric.record(end - start, id === undefined ? undefined : { id }); } }; - return benchmark(name, run, sampling); + return benchmark(name, run, options); } diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index a9fe6ad843..400377879c 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -5,7 +5,7 @@ import type { Browser, BrowserContext, CDPSession, Page } from '@playwright/test import { BENCHMARK_LAUNCH_ARGS } from '../launchArgs'; import { analyzeBenchmark } from '../runReport'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; -import { differencesResolved, parseHorizons, resolveSampling } from '../sampling'; +import { differencesResolved, parseHorizons } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { BenchFile } from './benchFiles'; import { variantOf } from './refs'; @@ -168,21 +168,19 @@ type BenchmarkEntry = Pick; * Measures one benchmark in fresh pages that stay open for all of it, so iterations stay warm and * module-scope data is built once. Warmup and measured rounds alike run every slot once, in a * shuffled order. After `sampleSize` rounds it keeps adding rounds while a difference is unresolved - * against the horizons, until the timeout. + * against the horizons, until the timeout. A failure is reported as the benchmark's error. */ -async function measureBenchmark( +async function runBenchmark( browser: Browser, entry: BenchmarkEntry, slots: PageSlot[], - sampling: SamplingOptions, + sampling: Required, options: RunInterleavedOptions, ): Promise { - const { sampleSize, timeout, autoSampleConditions } = resolveSampling(sampling); + console.log(chalk.cyan(`\nRunning "${entry.name}" (${entry.file})…`)); + const { sampleSize, timeout, autoSampleConditions } = sampling; const horizons = parseHorizons(autoSampleConditions); - const targets = await openBenchPages( - browser, - slots.map((slot) => slot.url), - ); + let targets: OpenedPage[] = []; const runRound = async (): Promise => { const round: Round = []; for (const index of shuffledIndices(targets.length)) { @@ -192,6 +190,10 @@ async function measureBenchmark( return round; }; try { + targets = await openBenchPages( + browser, + slots.map((slot) => slot.url), + ); for (let roundIndex = 0; roundIndex < options.warmup; roundIndex += 1) { // eslint-disable-next-line no-await-in-loop await runRound(); @@ -206,12 +208,14 @@ async function measureBenchmark( const metrics = await targets[0].page.evaluate(() => window.benchmarkPage!.metricDefinitions()); const benchmarkOf = (): RunBenchmark => ({ ...entry, samples: samplesOf(slots, rounds) }); + const isSettled = () => differencesResolved(analyzeBenchmark(metrics, benchmarkOf()), horizons); + const deadline = Date.now() + timeout * 60_000; - let resolved = differencesResolved(analyzeBenchmark(metrics, benchmarkOf()), horizons); + let resolved = isSettled(); while (!resolved && Date.now() < deadline) { // eslint-disable-next-line no-await-in-loop rounds.push(await runRound()); - resolved = differencesResolved(analyzeBenchmark(metrics, benchmarkOf()), horizons); + resolved = isSettled(); } if (rounds.length > sampleSize || !resolved) { console.log( @@ -221,6 +225,9 @@ async function measureBenchmark( ); } return { benchmark: benchmarkOf(), metrics }; + } catch (error) { + console.error(chalk.red(` ${errorMessage(error)}`)); + return { benchmark: { ...entry, error: errorMessage(error) }, metrics: {} }; } finally { await closeBenchPages(targets); } @@ -239,23 +246,6 @@ function samplesOf(slots: PageSlot[], rounds: Round[]): RunBenchmark['samples'] ); } -/** Measures one benchmark across its slots, or reports the error that stopped it. */ -async function runBenchmark( - browser: Browser, - entry: BenchmarkEntry, - slots: PageSlot[], - sampling: SamplingOptions, - options: RunInterleavedOptions, -): Promise { - console.log(chalk.cyan(`\nRunning "${entry.name}" (${entry.file})…`)); - try { - return await measureBenchmark(browser, entry, slots, sampling, options); - } catch (error) { - console.error(chalk.red(` ${errorMessage(error)}`)); - return { benchmark: { ...entry, error: errorMessage(error) }, metrics: {} }; - } -} - function benchPageUrl(origin: string, ref: ResolvedRef, benchFile: BenchFile): string { return new URL(`${ref.id}/${benchFile.page}`, `${origin}/`).href; } diff --git a/packages/benchmark/src/sampling.test.ts b/packages/benchmark/src/sampling.test.ts index 70c6b127de..e5d8932f27 100644 --- a/packages/benchmark/src/sampling.test.ts +++ b/packages/benchmark/src/sampling.test.ts @@ -7,17 +7,8 @@ import { isResolved, parseHorizons, resolveSampling, - validateSampling, } from './sampling'; -describe('resolveSampling', () => { - it('defaults an option left undefined, not only one left out', () => { - expect( - resolveSampling({ sampleSize: undefined, timeout: 1, autoSampleConditions: undefined }), - ).toEqual({ ...DEFAULT_SAMPLING, timeout: 1 }); - }); -}); - describe('parseHorizons', () => { it('expands an unsigned percentage to both signs', () => { expect(parseHorizons(['10%'])).toEqual([-10, 10]); @@ -55,20 +46,33 @@ describe('isResolved', () => { }); }); -describe('validateSampling', () => { +describe('resolveSampling', () => { + it('fills in a default for every option left out or undefined', () => { + expect(resolveSampling({ timeout: 1, autoSampleConditions: undefined })).toEqual({ + ...DEFAULT_SAMPLING, + timeout: 1, + }); + }); + it('accepts the documented ranges', () => { expect(() => - validateSampling({ sampleSize: 2, timeout: 0, autoSampleConditions: ['1%'] }), + resolveSampling({ sampleSize: 2, timeout: 0, autoSampleConditions: ['1%'] }), ).not.toThrow(); }); it('rejects a sample size that cannot form an interval', () => { - expect(() => validateSampling({ sampleSize: 1 })).toThrow(/Invalid sampleSize 1/); - expect(() => validateSampling({ sampleSize: 2.5 })).toThrow(/Invalid sampleSize 2.5/); + expect(() => resolveSampling({ sampleSize: 1 })).toThrow(/Invalid sampleSize 1/); + expect(() => resolveSampling({ sampleSize: 2.5 })).toThrow(/Invalid sampleSize 2.5/); }); it('rejects a negative timeout', () => { - expect(() => validateSampling({ timeout: -1 })).toThrow(/Invalid timeout -1/); + expect(() => resolveSampling({ timeout: -1 })).toThrow(/Invalid timeout -1/); + }); + + it('rejects an invalid condition', () => { + expect(() => resolveSampling({ autoSampleConditions: ['5ms'] })).toThrow( + /Invalid auto-sample condition "5ms"/, + ); }); }); @@ -77,15 +81,19 @@ describe('differencesResolved', () => { const clearlySlower = noisy.map((value) => value * 2); const sameish = [11, 9, 10, 12, 11, 10, 9, 12]; - function benchmarkOf(samples: Record): RunBenchmark { + /** A current-vs-baseline benchmark; `paint` follows `render` unless given. */ + function benchmarkOf( + render: { current: number[]; baseline: number[] }, + paint = render, + ): RunBenchmark { return { name: 'mount', file: 'button.bench.tsx', kind: 'baseline', variants: ['current', 'baseline'], samples: { - current: { render: samples.current, paint: samples.current }, - baseline: { render: samples.baseline, paint: samples.baseline }, + current: { render: render.current, paint: paint.current }, + baseline: { render: render.baseline, paint: paint.baseline }, }, }; } @@ -114,13 +122,10 @@ describe('differencesResolved', () => { it('ignores metrics that do not alarm when some do', () => { const analysis = analyzeBenchmark( { render: { kind: 'scalar', alarm: {} }, paint: { kind: 'scalar' } }, - { - ...benchmarkOf({ current: clearlySlower, baseline: noisy }), - samples: { - current: { render: clearlySlower, paint: sameish }, - baseline: { render: noisy, paint: noisy }, - }, - }, + benchmarkOf( + { current: clearlySlower, baseline: noisy }, + { current: sameish, baseline: noisy }, + ), ); expect(differencesResolved(analysis, [0])).toBe(true); }); @@ -128,13 +133,10 @@ describe('differencesResolved', () => { it('judges every metric when none alarms', () => { const analysis = analyzeBenchmark( { render: { kind: 'scalar' }, paint: { kind: 'scalar' } }, - { - ...benchmarkOf({ current: clearlySlower, baseline: noisy }), - samples: { - current: { render: clearlySlower, paint: sameish }, - baseline: { render: noisy, paint: noisy }, - }, - }, + benchmarkOf( + { current: clearlySlower, baseline: noisy }, + { current: sameish, baseline: noisy }, + ), ); expect(differencesResolved(analysis, [0])).toBe(false); }); diff --git a/packages/benchmark/src/sampling.ts b/packages/benchmark/src/sampling.ts index 736886d016..02bb547b7e 100644 --- a/packages/benchmark/src/sampling.ts +++ b/packages/benchmark/src/sampling.ts @@ -30,15 +30,6 @@ export const DEFAULT_SAMPLING: Required = { autoSampleConditions: ['0%'], }; -/** The options with a default for every one left out, or left `undefined`. */ -export function resolveSampling(options: SamplingOptions): Required { - return { - sampleSize: options.sampleSize ?? DEFAULT_SAMPLING.sampleSize, - timeout: options.timeout ?? DEFAULT_SAMPLING.timeout, - autoSampleConditions: options.autoSampleConditions ?? DEFAULT_SAMPLING.autoSampleConditions, - }; -} - /** Parses `autoSampleConditions` into signed horizons, in percent. */ export function parseHorizons(conditions: string[]): number[] { return conditions.flatMap((condition) => { @@ -54,18 +45,33 @@ export function parseHorizons(conditions: string[]): number[] { }); } -/** Checks sampling options when a case registers, so a mistake fails the page, not a round. */ -export function validateSampling(options: SamplingOptions): void { - const { sampleSize, timeout, autoSampleConditions } = options; - if (sampleSize !== undefined && !(Number.isInteger(sampleSize) && sampleSize >= 2)) { +/** + * Checks sampling options and fills in the defaults. Called when a case registers, so a mistake fails + * the page rather than a round, and the runner gets complete options. + */ +export function resolveSampling(options: SamplingOptions): Required { + const { + sampleSize = DEFAULT_SAMPLING.sampleSize, + timeout = DEFAULT_SAMPLING.timeout, + autoSampleConditions = DEFAULT_SAMPLING.autoSampleConditions, + } = options; + if (!(Number.isInteger(sampleSize) && sampleSize >= 2)) { throw new Error(`Invalid sampleSize ${sampleSize}: expected an integer of at least 2.`); } - if (timeout !== undefined && !(timeout >= 0)) { + if (!(timeout >= 0)) { throw new Error(`Invalid timeout ${timeout}: expected a number of minutes, 0 or more.`); } - if (autoSampleConditions !== undefined) { - parseHorizons(autoSampleConditions); - } + parseHorizons(autoSampleConditions); + return { sampleSize, timeout, autoSampleConditions }; +} + +/** Whether any sampling option is set. */ +export function hasSampling(options: SamplingOptions): boolean { + return ( + options.sampleSize !== undefined || + options.timeout !== undefined || + options.autoSampleConditions !== undefined + ); } /** Whether an interval lies entirely on one side of every horizon. */ diff --git a/render.yaml b/render.yaml index 30d28192bf..d76b8341c9 100644 --- a/render.yaml +++ b/render.yaml @@ -14,6 +14,8 @@ services: corepack enable pnpm --version pnpm install --frozen-lockfile + # The dashboard imports workspace packages at runtime, and they resolve through their build output. + pnpm --filter "code-infra-dashboard^..." build pnpm build startCommand: pnpm start plan: starter diff --git a/test/benchmark/src/_shared/sampling.ts b/test/benchmark/src/_shared/sampling.ts index 67774258c0..1dfd88ce8f 100644 --- a/test/benchmark/src/_shared/sampling.ts +++ b/test/benchmark/src/_shared/sampling.ts @@ -1,8 +1,9 @@ import type { SamplingOptions } from '@mui/internal-benchmark/page'; /** - * Sampling for this smoke-test harness. Both sides of its baseline comparisons run identical code, so - * no difference ever resolves against the default `0%` horizon and every case would sample until - * its timeout. A `10%` horizon settles as soon as a difference is known to be smaller than that. + * Sampling for this smoke-test harness, which should stay quick. Its baseline comparisons run + * identical code on both sides, so they never resolve against the default `0%` horizon and would + * sample until the timeout; a close `compare()` can take as long. A `10%` horizon settles as soon as + * a difference is known to be larger or smaller than that. */ export const SMOKE_SAMPLING: SamplingOptions = { timeout: 1, autoSampleConditions: ['10%'] }; From 5beba9277fdf6355796f45b72177a8d26313a286 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:08:35 +0200 Subject: [PATCH 090/173] [benchmark] Default sampleSize to 50, like tachometer --- packages/benchmark/README.md | 4 ++-- packages/benchmark/src/sampling.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 6c35991029..cb70c6ad5d 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -458,7 +458,7 @@ metric when none does. | Option | Default | Meaning | | :--------------------- | :------- | :-------------------------------------------------------------------- | -| `sampleSize` | `30` | Rounds measured before deciding whether to continue | +| `sampleSize` | `50` | Rounds measured before deciding whether to continue | | `timeout` | `3` | Minutes to keep sampling while a difference is unresolved | | `autoSampleConditions` | `['0%']` | Horizons to resolve: by default, until each change is better or worse | @@ -469,7 +469,7 @@ is an error. ```tsx reactBenchmark('mount', () => , { timeout: 1, autoSampleConditions: ['5%'] }); -compare('scatter', [ours, other], { sampleSize: 50 }); +compare('scatter', [ours, other], { sampleSize: 100 }); ``` Two builds that perform the same never resolve against `0%`, so an unchanged benchmark samples until diff --git a/packages/benchmark/src/sampling.ts b/packages/benchmark/src/sampling.ts index 02bb547b7e..f852b2927d 100644 --- a/packages/benchmark/src/sampling.ts +++ b/packages/benchmark/src/sampling.ts @@ -6,7 +6,7 @@ import type { BenchmarkAnalysis } from './runReport/analyzeRun'; * `autoSampleConditions` horizon, for up to `timeout` minutes. */ export interface SamplingOptions { - /** Rounds measured before deciding whether to continue. Defaults to 30. */ + /** Rounds measured before deciding whether to continue. Defaults to 50. */ sampleSize?: number; /** * Minutes to keep sampling after `sampleSize` rounds while a difference is unresolved. `0` @@ -25,7 +25,7 @@ export interface SamplingOptions { } export const DEFAULT_SAMPLING: Required = { - sampleSize: 30, + sampleSize: 50, timeout: 3, autoSampleConditions: ['0%'], }; From 73596748a9ae489ba8c6768e16301a11afb74737 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:15:08 +0200 Subject: [PATCH 091/173] [benchmark] Remove the tachometer runner from this pull request #1829 carries it; this one is the interleaved runner alone. The shared core both build on (code-infra baseline, the pack/git/exec utils and the CI upload path) stays in both until #1903 lands. Removes src/tachometer, the benchmark tacho commands and exports, test/tachometer and its CI job, chromedriver, and the dashboard's tachometer report type, section and details page. --- .circleci/config.yml | 42 - .gitignore | 5 - .../[owner]/[repo]/page.tsx | 16 - .../api/ci-reports/sync-pr-comment/route.ts | 72 +- .../app/api/ci-reports/upload/route.ts | 9 +- apps/code-infra-dashboard/src/constants.ts | 3 - .../src/lib/ciReports/tachometerReport.ts | 34 - .../tachometer/buildMarkdownReport.test.ts | 251 --- .../src/lib/tachometer/buildMarkdownReport.ts | 179 -- .../src/utils/fetchCiReport.ts | 2 - .../src/views/TachometerDetails.tsx | 349 ---- package.json | 2 - packages/benchmark/README.md | 98 +- packages/benchmark/package.json | 3 - packages/benchmark/src/cli/index.ts | 10 - packages/benchmark/src/cli/tachoReport.ts | 51 - packages/benchmark/src/cli/tachoRun.ts | 70 - .../benchmark/src/tachometer/browser.test.ts | 172 -- packages/benchmark/src/tachometer/browser.ts | 160 -- .../benchmark/src/tachometer/buildPages.ts | 322 ---- .../benchmark/src/tachometer/ciReport.test.ts | 141 -- packages/benchmark/src/tachometer/ciReport.ts | 124 -- .../src/tachometer/discoverCases.test.ts | 577 ------ .../benchmark/src/tachometer/discoverCases.ts | 321 ---- .../benchmark/src/tachometer/format.test.ts | 100 -- packages/benchmark/src/tachometer/format.ts | 93 - .../benchmark/src/tachometer/outputDir.ts | 36 - .../benchmark/src/tachometer/plugin.test.ts | 186 -- packages/benchmark/src/tachometer/plugin.ts | 242 --- .../benchmark/src/tachometer/refs.test.ts | 124 -- packages/benchmark/src/tachometer/refs.ts | 70 - .../benchmark/src/tachometer/renderReport.ts | 241 --- .../benchmark/src/tachometer/runTachometer.ts | 322 ---- .../src/tachometer/summarizeCase.test.ts | 203 --- .../benchmark/src/tachometer/summarizeCase.ts | 184 -- pnpm-lock.yaml | 1541 +---------------- pnpm-workspace.yaml | 1 - test/tachometer/README.md | 115 -- test/tachometer/package.json | 22 - test/tachometer/scripts/runBenchmark.mjs | 12 - test/tachometer/src/_shared/measure.ts | 51 - test/tachometer/src/libs/sort/alpha.html | 11 - test/tachometer/src/libs/sort/alpha.ts | 10 - test/tachometer/src/libs/sort/beta.html | 11 - test/tachometer/src/libs/sort/beta.ts | 33 - test/tachometer/src/libs/sort/ours.html | 11 - test/tachometer/src/libs/sort/ours.ts | 13 - test/tachometer/src/libs/sort/shared.ts | 51 - test/tachometer/src/libs/sort/tachometer.json | 22 - .../src/workload-large/tachometer.json | 18 - test/tachometer/src/workload/index.html | 11 - test/tachometer/src/workload/index.ts | 43 - test/tachometer/src/workload/tachometer.json | 18 - test/tachometer/tsconfig.json | 15 - test/tachometer/vite.config.mjs | 6 - tsconfig.json | 3 +- 56 files changed, 38 insertions(+), 6794 deletions(-) delete mode 100644 apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx delete mode 100644 apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts delete mode 100644 apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts delete mode 100644 apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts delete mode 100644 apps/code-infra-dashboard/src/views/TachometerDetails.tsx delete mode 100644 packages/benchmark/src/cli/tachoReport.ts delete mode 100644 packages/benchmark/src/cli/tachoRun.ts delete mode 100644 packages/benchmark/src/tachometer/browser.test.ts delete mode 100644 packages/benchmark/src/tachometer/browser.ts delete mode 100644 packages/benchmark/src/tachometer/buildPages.ts delete mode 100644 packages/benchmark/src/tachometer/ciReport.test.ts delete mode 100644 packages/benchmark/src/tachometer/ciReport.ts delete mode 100644 packages/benchmark/src/tachometer/discoverCases.test.ts delete mode 100644 packages/benchmark/src/tachometer/discoverCases.ts delete mode 100644 packages/benchmark/src/tachometer/format.test.ts delete mode 100644 packages/benchmark/src/tachometer/format.ts delete mode 100644 packages/benchmark/src/tachometer/outputDir.ts delete mode 100644 packages/benchmark/src/tachometer/plugin.test.ts delete mode 100644 packages/benchmark/src/tachometer/plugin.ts delete mode 100644 packages/benchmark/src/tachometer/refs.test.ts delete mode 100644 packages/benchmark/src/tachometer/refs.ts delete mode 100644 packages/benchmark/src/tachometer/renderReport.ts delete mode 100644 packages/benchmark/src/tachometer/runTachometer.ts delete mode 100644 packages/benchmark/src/tachometer/summarizeCase.test.ts delete mode 100644 packages/benchmark/src/tachometer/summarizeCase.ts delete mode 100644 test/tachometer/README.md delete mode 100644 test/tachometer/package.json delete mode 100644 test/tachometer/scripts/runBenchmark.mjs delete mode 100644 test/tachometer/src/_shared/measure.ts delete mode 100644 test/tachometer/src/libs/sort/alpha.html delete mode 100644 test/tachometer/src/libs/sort/alpha.ts delete mode 100644 test/tachometer/src/libs/sort/beta.html delete mode 100644 test/tachometer/src/libs/sort/beta.ts delete mode 100644 test/tachometer/src/libs/sort/ours.html delete mode 100644 test/tachometer/src/libs/sort/ours.ts delete mode 100644 test/tachometer/src/libs/sort/shared.ts delete mode 100644 test/tachometer/src/libs/sort/tachometer.json delete mode 100644 test/tachometer/src/workload-large/tachometer.json delete mode 100644 test/tachometer/src/workload/index.html delete mode 100644 test/tachometer/src/workload/index.ts delete mode 100644 test/tachometer/src/workload/tachometer.json delete mode 100644 test/tachometer/tsconfig.json delete mode 100644 test/tachometer/vite.config.mjs diff --git a/.circleci/config.yml b/.circleci/config.yml index a25f49fd2a..823cbd4b74 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -115,46 +115,6 @@ jobs: # `BENCHMARK_UPLOAD: 'true'` here before merging. command: pnpm -F ./test/performance benchmark - test_tachometer: - <<: *default-job - executor: - name: code-infra/mui-node-browser - playwright-img-version: 'v1.62.1-noble' - steps: - - checkout - - code-infra/install-deps - - run: - name: Build packages - command: pnpm release:build - - run: - name: Resolve the baseline commit - command: | - set -e - # Named, because a CI checkout records no `origin/HEAD` to detect it from. Fetching it - # is the command's own job, so there is nothing to keep in step with it here. - BASE_SHA=$(pnpm code-infra baseline --base-branch master) - echo "Baseline commit: $BASE_SHA" - echo "$BASE_SHA" > .tachometer-baseline-sha - - restore_cache: - name: Restore the packed-baseline cache - keys: - - tachometer-cache-v1-{{ checksum ".tachometer-baseline-sha" }} - - run: - name: Run the tachometer smoke test - no_output_timeout: 30m - environment: - TACHO_UPLOAD: 'true' - # TEMPORARY: production does not know the tachometer report type until this pull - # request lands, so it would reject the upload. Points at this pull request's own - # dashboard preview until then — remove the line before merging. - CI_REPORT_API_URL: https://code-infra-dashboard-pr-1898.onrender.com - command: pnpm test:tacho --baseline "$(cat .tachometer-baseline-sha)" - - save_cache: - name: Save the packed-baseline cache - key: tachometer-cache-v1-{{ checksum ".tachometer-baseline-sha" }} - paths: - - test/tachometer/.tachometer/packed - test_benchmark_run: <<: *default-job executor: @@ -234,8 +194,6 @@ workflows: <<: *default-context - test_benchmark: <<: *default-context - - test_tachometer: - <<: *default-context - test_benchmark_run: <<: *default-context - test_browser: diff --git a/.gitignore b/.gitignore index e6c210e268..a5d602a451 100644 --- a/.gitignore +++ b/.gitignore @@ -67,8 +67,3 @@ next-env.d.ts # .gitignore and never reads nested ones. /test/benchmark/.benchmark/ /test/benchmark/src/__bench__/ - -# Everything the tachometer harness writes: built pages per ref, packed workspace tarballs, the -# isolated installs and the run reports. The directory ignores itself from the inside too, but the -# entry is listed here because ESLint is fed from the root .gitignore and never reads nested ones. -/test/tachometer/.tachometer/ diff --git a/apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx b/apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx deleted file mode 100644 index 6d1a42140e..0000000000 --- a/apps/code-infra-dashboard/app/(dashboard)/tachometer-details/[owner]/[repo]/page.tsx +++ /dev/null @@ -1,16 +0,0 @@ -import * as React from 'react'; -import type { Metadata } from 'next'; -import TachometerDetails from '@/views/TachometerDetails'; - -export async function generateMetadata({ - params, -}: { - params: Promise<{ owner: string; repo: string }>; -}): Promise { - const { owner, repo } = await params; - return { title: `Tachometer Details - ${owner}/${repo}` }; -} - -export default function TachometerDetailsPage() { - return ; -} diff --git a/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts b/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts index 0cedc5ddec..4542ce2e75 100644 --- a/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts +++ b/apps/code-infra-dashboard/app/api/ci-reports/sync-pr-comment/route.ts @@ -10,10 +10,6 @@ import { } from '@/lib/ciReports/bundleSizeReport'; import { generateBenchmarkReport, BENCHMARK_SECTION_TITLE } from '@/lib/ciReports/benchmarkReport'; import { generateDeployPreviewReport } from '@/lib/ciReports/deployPreviewReport'; -import { - generateTachometerReport, - TACHOMETER_SECTION_TITLE, -} from '@/lib/ciReports/tachometerReport'; import type { ReportResult } from '@/lib/ciReports/types'; import { fetchParentCommits } from '@/utils/fetchCiReportWithFallback'; import { getOctokit } from '@/lib/github'; @@ -122,50 +118,36 @@ export async function POST(request: NextRequest) { // Generate all configured report sections in parallel. // Each generator is wrapped in .catch() so a failure in one doesn't block the others. - const [bundleSizeReport, benchmarkReportResult, tachometerReportResult, deployPreviewReport] = - await Promise.all([ - prCommentConfig?.bundleSize - ? generateBundleSizeReport(reportOptions).catch((error): ReportResult => { - console.error('Failed to generate bundle size report:', error); - return { - content: `## ${BUNDLE_SIZE_SECTION_TITLE}\n\n:warning: Failed to generate bundle size report.`, - }; - }) - : null, - prCommentConfig?.benchmark - ? generateBenchmarkReport(reportOptions).catch((error): ReportResult => { - console.error('Failed to generate benchmark report:', error); - return { - content: `## ${BENCHMARK_SECTION_TITLE}\n\n:warning: Failed to generate benchmark report.`, - }; - }) - : null, - prCommentConfig?.tachometer - ? generateTachometerReport(reportOptions).catch((error): ReportResult => { - console.error('Failed to generate tachometer report:', error); - return { - content: `## ${TACHOMETER_SECTION_TITLE}\n\n:warning: Failed to generate tachometer report.`, - }; - }) - : null, - prCommentConfig?.netlifyDocs - ? generateDeployPreviewReport(reportOptions).catch((error): ReportResult => { - console.error('Failed to generate deploy preview report:', error); - return { - content: `## Deploy preview\n\n:warning: Failed to generate deploy preview.`, - }; - }) - : null, - ]); + const [bundleSizeReport, benchmarkReportResult, deployPreviewReport] = await Promise.all([ + prCommentConfig?.bundleSize + ? generateBundleSizeReport(reportOptions).catch((error): ReportResult => { + console.error('Failed to generate bundle size report:', error); + return { + content: `## ${BUNDLE_SIZE_SECTION_TITLE}\n\n:warning: Failed to generate bundle size report.`, + }; + }) + : null, + prCommentConfig?.benchmark + ? generateBenchmarkReport(reportOptions).catch((error): ReportResult => { + console.error('Failed to generate benchmark report:', error); + return { + content: `## ${BENCHMARK_SECTION_TITLE}\n\n:warning: Failed to generate benchmark report.`, + }; + }) + : null, + prCommentConfig?.netlifyDocs + ? generateDeployPreviewReport(reportOptions).catch((error): ReportResult => { + console.error('Failed to generate deploy preview report:', error); + return { + content: `## Deploy preview\n\n:warning: Failed to generate deploy preview.`, + }; + }) + : null, + ]); const trailer = `
    \n\nCheck out the [code infra dashboard](${DASHBOARD_ORIGIN}/repository/${prRepo}/prs/${pr.number}) for more information about this PR.`; - const commentBody = [ - deployPreviewReport, - bundleSizeReport, - benchmarkReportResult, - tachometerReportResult, - ] + const commentBody = [deployPreviewReport, bundleSizeReport, benchmarkReportResult] .filter((report): report is ReportResult => report !== null) .map((report) => report.content) .concat(trailer) diff --git a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts index e672f85a19..fb0e6eb338 100644 --- a/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts +++ b/apps/code-infra-dashboard/app/api/ci-reports/upload/route.ts @@ -5,7 +5,7 @@ import { uploadReport } from '@/lib/ciReports/s3'; import { verifyOidcToken } from '@/lib/ciReports/oidcAuth'; import { findAssociatedPr } from '@/lib/ciReports/findAssociatedPr'; -const VALID_REPORT_TYPES = new Set(['size-snapshot', 'benchmark', 'tachometer']); +const VALID_REPORT_TYPES = new Set(['size-snapshot', 'benchmark']); const uploadSchema = z.object({ version: z.number(), @@ -55,10 +55,9 @@ export async function POST(request: NextRequest) { const { commitSha, repo, reportType, branch, report } = parsed.data; - // A benchmark report is stored as the whole envelope, every other type as just the inner report. - // Historic, and kept because artifacts written that way are still read back. A tachometer report - // needs none of it: it carries its own `version`, `reportType` and `head`, the repo is in the S3 - // key, and the generator reads the rest from its own route params. + // For benchmark uploads, store the full wrapper (version, timestamp, commitSha, + // repo, branch, prNumber, reportType, report, base). Other report types keep + // their historic "just the inner report" storage. const storedBody = reportType === 'benchmark' ? JSON.stringify(parsed.data) : JSON.stringify(report); diff --git a/apps/code-infra-dashboard/src/constants.ts b/apps/code-infra-dashboard/src/constants.ts index 08e8bf2f43..1bd59b2047 100644 --- a/apps/code-infra-dashboard/src/constants.ts +++ b/apps/code-infra-dashboard/src/constants.ts @@ -25,7 +25,6 @@ export interface Repository { prComment?: { bundleSize?: boolean; benchmark?: boolean; - tachometer?: boolean; netlifyDocs?: | boolean | { @@ -167,7 +166,6 @@ export const repositories = new Map( prComment: { bundleSize: true, benchmark: true, - tachometer: true, }, }, { @@ -195,7 +193,6 @@ export const repositories = new Map( prComment: { bundleSize: true, benchmark: true, - tachometer: true, netlifyDocs: { siteId: 'mui-internal', formatDocPath: (filePath) => { diff --git a/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts b/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts deleted file mode 100644 index 7224b5c0ee..0000000000 --- a/apps/code-infra-dashboard/src/lib/ciReports/tachometerReport.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { fetchCiReport } from '@/utils/fetchCiReport'; -import { - buildTachometerMarkdownReport, - TACHOMETER_SECTION_TITLE, -} from '@/lib/tachometer/buildMarkdownReport'; -import { DASHBOARD_ORIGIN } from '@/constants'; - -import type { TachometerReport } from '@mui/internal-benchmark/tachometerReport'; -import type { ReportOptions, ReportResult } from './types'; - -export { TACHOMETER_SECTION_TITLE }; - -/** - * Generates the tachometer section of the pull request comment. - */ -export async function generateTachometerReport( - options: ReportOptions, -): Promise { - const { repo, prNumber, commitSha } = options; - - const report: TachometerReport | null = await fetchCiReport(repo, commitSha, 'tachometer.json'); - if (!report?.cases) { - return null; - } - - const detailsUrl = new URL(`${DASHBOARD_ORIGIN}/tachometer-details/${repo}`); - detailsUrl.searchParams.set('sha', commitSha); - // The details view reads this to title the page and link back to the PR overview. - detailsUrl.searchParams.set('prNumber', String(prNumber)); - - return { - content: buildTachometerMarkdownReport(report, { detailsUrl: detailsUrl.href }), - }; -} diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts deleted file mode 100644 index a086e22459..0000000000 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.test.ts +++ /dev/null @@ -1,251 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import type { - CaseResult, - MeasurementResult, - TachometerReport, - Verdict, -} from '@mui/internal-benchmark/tachometerReport'; -import { buildTachometerMarkdownReport, findRegressions } from './buildMarkdownReport'; - -function variant(name: string) { - return { - variant: name, - meanMs: { low: 20, high: 22 }, - samples: 120, - }; -} - -/** The verdict the other way round; an unresolved difference is unresolved from both sides. */ -function mirrorOf(verdict: Verdict): Verdict { - if (verdict === 'slower') { - return 'faster'; - } - return verdict === 'faster' ? 'slower' : verdict; -} - -/** - * A comparison of the reference against `name`, carrying both directions the way a real report - * does. The two are not negations of each other — each has its own denominator — so the numbers - * here differ, which is what lets a test tell which direction was rendered. - */ -function comparison(name: string, verdict: Verdict) { - const sign = verdict === 'slower' ? 1 : -1; - return { - variant: name, - verdict, - absoluteMs: { low: sign * 6.1, high: sign * 15.2 }, - percentChange: { low: sign * 2.7, high: sign * 6.8 }, - versusReference: { - verdict: mirrorOf(verdict), - absoluteMs: { low: sign * -6.1, high: sign * -15.2 }, - percentChange: { low: sign * -2.6, high: sign * -6.4 }, - }, - }; -} - -/** A case measuring the working tree against the baseline, where a difference is a regression. */ -function regressionCase(name: string, verdicts: Record): CaseResult { - const measurements: MeasurementResult[] = Object.entries(verdicts).map( - ([measurement, verdict]) => ({ - name: measurement, - variants: [variant(`${name} [current]`), variant(`${name} [baseline]`)], - comparisons: [comparison(`${name} [baseline]`, verdict)], - }), - ); - return { name, comparison: 'baseline', reference: `${name} [current]`, measurements }; -} - -/** A case comparing libraries with each other, every variant built from the working tree. */ -function libraryCase(name: string, verdict: Verdict): CaseResult { - return { - name, - comparison: 'variants', - reference: `${name} [mosaic]`, - measurements: [ - { - name: 'mount', - variants: [variant(`${name} [mosaic]`), variant(`${name} [ag-grid]`)], - comparisons: [comparison(`${name} [ag-grid]`, verdict)], - }, - ], - }; -} - -function report(cases: CaseResult[]): TachometerReport { - return { - version: 1, - reportType: 'tachometer', - generatedAt: '2026-09-02T00:00:00.000Z', - head: { ref: 'HEAD', sha: 'a'.repeat(40), branch: 'feature' }, - refs: [ - { id: 'current', kind: 'worktree' }, - { id: 'git-abc123456', kind: 'git', requested: 'HEAD~1', sha: 'b'.repeat(40) }, - ], - cases, - raw: {}, - }; -} - -/** Everything a reader sees without expanding anything. */ -function visiblePart(markdown: string): string { - return markdown.split('
    ')[0]; -} - -/** The collapsed table's row for `variant`, rather than any prose that names it too. */ -function tableRowFor(markdown: string, variantName: string): string | undefined { - return markdown - .split('
    ')[1] - ?.split('\n') - .find((line) => line.startsWith('|') && line.includes(variantName)); -} - -describe('findRegressions', () => { - it('finds a slower verdict across two builds', () => { - const found = findRegressions(report([regressionCase('workload', { mount: 'slower' })])); - - expect(found).toMatchObject([{ caseName: 'workload', measurement: 'mount' }]); - }); - - it.each(['unsure', 'faster'])('does not count a %s verdict', (verdict) => { - expect(findRegressions(report([regressionCase('workload', { mount: verdict })]))).toEqual([]); - }); - - it('ignores a slower verdict between variants on the same build', () => { - // A cross-library case runs every variant from one build, so "slower than ag-grid" says nothing - // about what this pull request changed. Flagging it would warn on every comment forever. - expect(findRegressions(report([libraryCase('libs-mount', 'slower')]))).toEqual([]); - }); - - it('reports only the measurement that regressed', () => { - const found = findRegressions( - report([regressionCase('workload', { mount: 'slower', scroll: 'unsure' })]), - ); - - expect(found).toHaveLength(1); - expect(found[0].measurement).toBe('mount'); - }); -}); - -describe('buildTachometerMarkdownReport', () => { - it('states a regression before the collapsed block, where it can be seen', () => { - // GitHub renders
    closed, so a regression inside it is invisible. - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'slower' })]), - ); - - expect(visiblePart(markdown)).toContain('workload'); - expect(visiblePart(markdown)).toContain('mount'); - expect(visiblePart(markdown)).toContain('+2.7% – +6.8%'); - }); - - it('marks the heading when something regressed', () => { - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'slower' })]), - ); - - expect(markdown.split('\n')[0]).toBe('## Tachometer ⚠️'); - }); - - it('leaves the heading unmarked when nothing regressed', () => { - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'unsure' })]), - ); - - expect(markdown.split('\n')[0]).toBe('## Tachometer'); - expect(visiblePart(markdown)).not.toContain('⚠️'); - }); - - it('summarizes a clean run in one line', () => { - const markdown = buildTachometerMarkdownReport( - report([ - regressionCase('workload', { mount: 'unsure' }), - regressionCase('scroll', { mount: 'unsure' }), - ]), - ); - - expect(visiblePart(markdown)).toContain('2 cases measured · 2 unchanged'); - }); - - it('counts a case that regressed in two measurements once', () => { - // The total beside it counts cases, so counting measurements here stopped the line adding up. - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'slower', 'cold-start': 'slower' })]), - ); - - expect(visiblePart(markdown)).toContain('1 case measured · 1 slower'); - }); - - it('does not count beating another library as having got faster', () => { - const markdown = buildTachometerMarkdownReport( - report([ - libraryCase('libs-mount', 'faster'), - regressionCase('workload', { mount: 'unsure' }), - ]), - ); - - expect(visiblePart(markdown)).toContain('2 cases measured · 2 unchanged'); - expect(visiblePart(markdown)).not.toContain('faster'); - }); - - it('counts a faster case separately', () => { - const markdown = buildTachometerMarkdownReport( - report([ - regressionCase('workload', { mount: 'faster' }), - regressionCase('scroll', { mount: 'unsure' }), - ]), - ); - - expect(visiblePart(markdown)).toContain('1 faster'); - }); - - it('reports a case that produced no result', () => { - const markdown = buildTachometerMarkdownReport( - report([{ name: 'broken', comparison: 'baseline', error: 'produced no benchmarks' }]), - ); - - expect(visiblePart(markdown)).toContain('broken'); - expect(visiblePart(markdown)).toContain('produced no benchmarks'); - }); - - it('puts every variant and measurement in the collapsed table', () => { - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'unsure', scroll: 'unsure' })]), - ); - - const details = markdown.split('
    ')[1]; - expect(details).toContain('mount'); - expect(details).toContain('scroll'); - expect(details).toContain('[current]'); - expect(details).toContain('[baseline]'); - }); - - it('states each row as the variant against the reference, not the reference against it', () => { - // The row sits under a "vs reference" heading next to the variant's own mean, so it has to - // read as a statement about that variant. Rendering the direction a regression is stated in - // makes a variant that is slower than the reference look faster than it. - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'slower' })]), - ); - - expect(tableRowFor(markdown, '[baseline]')).toContain('faster `-2.6% – -6.4%`'); - }); - - it('leaves the comparison out when the report has only one direction', () => { - // Older reports, written before both directions were captured. - const oneDirection = report([regressionCase('workload', { mount: 'slower' })]); - delete oneDirection.cases[0].measurements![0].comparisons[0].versusReference; - - const markdown = buildTachometerMarkdownReport(oneDirection); - - expect(tableRowFor(markdown, '[baseline]')).toContain('—'); - }); - - it('links to the full run when given a url', () => { - const markdown = buildTachometerMarkdownReport( - report([regressionCase('workload', { mount: 'unsure' })]), - { detailsUrl: 'https://example.test/run' }, - ); - - expect(markdown).toContain('[See the full run](https://example.test/run)'); - }); -}); diff --git a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts deleted file mode 100644 index cd3ff28bc5..0000000000 --- a/apps/code-infra-dashboard/src/lib/tachometer/buildMarkdownReport.ts +++ /dev/null @@ -1,179 +0,0 @@ -import { formatMarkdownTable } from '@/utils/formatters'; -import type { - ConfidenceInterval, - TachometerReport, -} from '@mui/internal-benchmark/tachometerReport'; -import { - formatMean, - formatPercent, - formatSignedMs, - isSummarized, - shortNameOf, -} from '@mui/internal-benchmark/tachometerFormat'; -import type { SummarizedCase } from '@mui/internal-benchmark/tachometerFormat'; - -export const TACHOMETER_SECTION_TITLE = 'Tachometer'; - -interface Regression { - caseName: string; - measurement: string; - variant: string; - percentChange: ConfidenceInterval; - absoluteMs: ConfidenceInterval; -} - -interface BuildOptions { - /** Where the full results can be inspected. Omitted in tests. */ - detailsUrl?: string; -} - -/** - * The comparisons that say something about this pull request. - * - * Only a case measuring the working tree against the baseline does. One comparing pages built from - * the same tree — this library against another — is a standing measurement, and a difference there - * is the point rather than a regression. - */ -function baselineComparisons(entry: SummarizedCase) { - if (entry.comparison !== 'baseline') { - return []; - } - return entry.measurements.flatMap((measurement) => - measurement.comparisons.map((comparison) => ({ measurement, comparison })), - ); -} - -export function findRegressions(report: TachometerReport): Regression[] { - const regressions: Regression[] = []; - - for (const entry of report.cases) { - if (!isSummarized(entry)) { - continue; - } - for (const { measurement, comparison } of baselineComparisons(entry)) { - if (comparison.verdict === 'slower') { - regressions.push({ - caseName: entry.name, - measurement: measurement.name, - variant: shortNameOf(entry.name, comparison.variant), - percentChange: comparison.percentChange, - absoluteMs: comparison.absoluteMs, - }); - } - } - } - - return regressions; -} - -/** One flat table per case: every measurement, every variant, every number the report holds. */ -function renderCaseTable(entry: SummarizedCase): string { - const rows = entry.measurements.flatMap((measurement) => { - const byVariant = new Map( - measurement.comparisons.map((comparison) => [comparison.variant, comparison]), - ); - return measurement.variants.map((variant) => { - const againstReference = byVariant.get(variant.variant)?.versusReference; - let versus = '—'; - if (variant.variant === entry.reference) { - versus = '_reference_'; - } else if (againstReference) { - versus = `${againstReference.verdict} \`${formatPercent(againstReference.percentChange)}\``; - } - return { - measurement: measurement.name, - variant: shortNameOf(entry.name, variant.variant), - mean: formatMean(variant.meanMs), - versus, - samples: variant.samples, - }; - }); - }); - - return `**${entry.name}**\n\n${formatMarkdownTable( - [ - { field: 'measurement', header: 'Measurement', align: 'left' }, - { field: 'variant', header: 'Variant', align: 'left' }, - { field: 'mean', header: 'Mean (95% CI)', align: 'right' }, - { field: 'versus', header: 'vs reference', align: 'left' }, - { field: 'samples', header: 'Samples', align: 'right' }, - ], - rows, - )}`; -} - -/** - * Renders the tachometer section of the pull request comment. - * - * Regressions are stated above the collapsed block, and mark the heading, because a reader has to - * see them without expanding anything — GitHub renders `
    ` closed, so anything inside it is - * invisible until someone chooses to look. - */ -export function buildTachometerMarkdownReport( - report: TachometerReport, - options: BuildOptions = {}, -): string { - const regressions = findRegressions(report); - const summarized = report.cases.filter(isSummarized); - const failed = report.cases.filter((entry) => !isSummarized(entry)); - - const heading = `## ${TACHOMETER_SECTION_TITLE}${regressions.length > 0 ? ' ⚠️' : ''}`; - const lines = [heading, '']; - - if (regressions.length > 0) { - for (const regression of regressions) { - lines.push( - `⚠️ **${regression.caseName}** · ${regression.measurement} · slower than ` + - `${regression.variant} \`${formatPercent(regression.percentChange)}\` ` + - `(\`${formatSignedMs(regression.absoluteMs)}\`)`, - ); - } - lines.push(''); - } - - // Everything that did not regress collapses to a single line, so the regressions are what the eye - // lands on. `unsure` is the expected result for two equivalent builds, not a warning. Counted in - // cases, like the total beside them, and over the same baseline comparisons a regression is read - // from — a case that beats a competing library has not got faster. - const regressedCases = new Set(regressions.map((regression) => regression.caseName)); - const faster = summarized.filter( - (entry) => - !regressedCases.has(entry.name) && - baselineComparisons(entry).some(({ comparison }) => comparison.verdict === 'faster'), - ).length; - const unchanged = summarized.length - regressedCases.size - faster; - const summary = [ - `${summarized.length} case${summarized.length === 1 ? '' : 's'} measured`, - unchanged > 0 ? `${unchanged} unchanged` : null, - faster > 0 ? `${faster} faster` : null, - regressedCases.size > 0 ? `${regressedCases.size} slower` : null, - ] - .filter(Boolean) - .join(' · '); - lines.push(summary, ''); - - for (const entry of failed) { - lines.push(`❌ **${entry.name}**: ${entry.error ?? 'produced no result'}`); - } - if (failed.length > 0) { - lines.push(''); - } - - if (summarized.length > 0) { - lines.push('
    ', 'Full results', ''); - lines.push(summarized.map(renderCaseTable).join('\n\n')); - lines.push( - '', - '_Each cell is a 95% confidence interval for the mean; one sample is one page load._', - '_"unsure" means the interval still straddles zero — the expected result for two equivalent builds._', - '', - '
    ', - ); - } - - if (options.detailsUrl) { - lines.push('', `[See the full run](${options.detailsUrl})`); - } - - return lines.join('\n'); -} diff --git a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts index ea37f3799e..90d3f5d542 100644 --- a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts +++ b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts @@ -1,7 +1,6 @@ import type { SizeSnapshotWithMetadata } from '@/lib/bundleSize/types'; import type { BenchmarkReport, BenchmarkUpload } from '@/lib/benchmark/types'; import type { BenchmarkRunUpload } from '@mui/internal-benchmark/runReport'; -import type { TachometerReport } from '@mui/internal-benchmark/tachometerReport'; import { migrateBenchmarkReport } from '@/lib/benchmark/migrateBenchmarkReport'; export interface CiReportTypes { @@ -11,7 +10,6 @@ export interface CiReportTypes { */ 'benchmark.json': BenchmarkUpload | BenchmarkRunUpload; 'size-snapshot.json': SizeSnapshotWithMetadata; - 'tachometer.json': TachometerReport; } export type CiReportName = keyof CiReportTypes; diff --git a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx b/apps/code-infra-dashboard/src/views/TachometerDetails.tsx deleted file mode 100644 index d2dbdc7e06..0000000000 --- a/apps/code-infra-dashboard/src/views/TachometerDetails.tsx +++ /dev/null @@ -1,349 +0,0 @@ -'use client'; - -import * as React from 'react'; -import { useParams, useSearchParams } from 'next/navigation'; -import { useQuery } from '@tanstack/react-query'; -import Alert from '@mui/material/Alert'; -import Box from '@mui/material/Box'; -import CircularProgress from '@mui/material/CircularProgress'; -import Paper from '@mui/material/Paper'; -import Table from '@mui/material/Table'; -import TableBody from '@mui/material/TableBody'; -import TableCell from '@mui/material/TableCell'; -import TableContainer from '@mui/material/TableContainer'; -import TableHead from '@mui/material/TableHead'; -import TableRow from '@mui/material/TableRow'; -import Typography from '@mui/material/Typography'; -import { fetchCiReport } from '@/utils/fetchCiReport'; -import type { - ConfidenceInterval, - TachometerReport, - Verdict, -} from '@mui/internal-benchmark/tachometerReport'; -import { - formatMean, - formatPercent, - refLabel, - groupCasesByVariantSet, - isSummarized, - shortNameOf, -} from '@mui/internal-benchmark/tachometerFormat'; -import type { SummarizedCase } from '@mui/internal-benchmark/tachometerFormat'; -import Heading from '../components/Heading'; -import ReportHeader from '../components/ReportHeader'; -import ErrorDisplay from '../components/ErrorDisplay'; - -/** - * The tables the run prints in CI, as HTML: cases grouped by their variant set, a group of two - * variants read down its measurements, a group of more read across its variants. - */ - -/** Only a resolved difference is coloured; `unsure` is the expected result and stays neutral. */ -function verdictColor(verdict: Verdict): 'error' | 'success' | 'text.secondary' { - if (verdict === 'slower') { - return 'error'; - } - if (verdict === 'faster') { - return 'success'; - } - return 'text.secondary'; -} - -/** Auto-sampling stops per case, so the counts are normally equal; when they are not, say both. */ -function formatSamples(samples: number[]): string { - return [...new Set(samples)].join('/'); -} - -function VerdictText({ - verdict, - percentChange, -}: { - verdict: Verdict; - percentChange: ConfidenceInterval; -}) { - return ( - - {verdict} {formatPercent(percentChange)} - - ); -} - -function Muted({ children }: { children: React.ReactNode }) { - return ( - - {children} - - ); -} - -/** - * One case as a table of variants, for a comparison with more than two of them. - */ -function VariantTable({ entry, variants }: { entry: SummarizedCase; variants: string[] }) { - const [reference] = variants; - const samplesOf = (variantName: string) => - entry.measurements.flatMap((measurement) => - measurement.variants - .filter((variant) => shortNameOf(entry.name, variant.variant) === variantName) - .map((variant) => variant.samples), - ); - - return ( - - - - - {entry.name} - {entry.measurements.map((measurement) => ( - - {measurement.name} - vs {reference} - - ))} - Samples - - - - {variants.map((variantName) => { - return ( - - {variantName} - {entry.measurements.map((measurement) => { - const found = measurement.variants.find( - (candidate) => shortNameOf(entry.name, candidate.variant) === variantName, - ); - // The comparison holds the reference relative to this variant; a row about the - // variant needs the direction whose subject is the variant. - const againstReference = measurement.comparisons.find( - (candidate) => shortNameOf(entry.name, candidate.variant) === variantName, - )?.versusReference; - - let versus = —; - if (variantName === reference) { - versus = reference; - } else if (againstReference) { - versus = ( - - ); - } - - return ( - - - {found ? formatMean(found.meanMs) : —} - - {versus} - - ); - })} - {formatSamples(samplesOf(variantName))} - - ); - })} - -
    -     - ); -} - -/** - * A group of cases sharing a variant set, one row per case and measurement. - */ -function CaseGroupTable({ cases, variants }: { cases: SummarizedCase[]; variants: string[] }) { - const [reference, ...others] = variants; - - return ( - - - - - Case - Measurement - {variants.map((variantName) => ( - - {variantName} - - ))} - {others.map((variantName) => ( - Δ vs {variantName} - ))} - Samples - - - - {cases.flatMap((entry, caseIndex) => - entry.measurements.map((measurement, measurementIndex) => { - const byVariant = new Map( - measurement.variants.map((variant) => [ - shortNameOf(entry.name, variant.variant), - variant, - ]), - ); - const byComparison = new Map( - measurement.comparisons.map((comparison) => [ - shortNameOf(entry.name, comparison.variant), - comparison, - ]), - ); - // A rule where the next case starts, since its name only appears on its first row. - const startsCase = measurementIndex === 0 && caseIndex > 0; - - return ( - - {measurementIndex === 0 ? entry.name : ''} - {measurement.name} - {variants.map((variantName) => { - const found = byVariant.get(variantName); - return ( - - {found ? formatMean(found.meanMs) : —} - - ); - })} - {others.map((variantName) => { - const comparison = byComparison.get(variantName); - return ( - - {comparison ? ( - - ) : ( - — - )} - - ); - })} - - {formatSamples(measurement.variants.map((variant) => variant.samples))} - - - ); - }), - )} - -
    - - Δ is tachometer's confidence interval on the difference, {reference} relative to the - other variant{others.length > 1 ? 's' : ''} — negative is faster. - -     - ); -} - -export default function TachometerDetails() { - const params = useParams<{ owner: string; repo: string }>(); - const searchParams = useSearchParams(); - - if (!params.owner || !params.repo) { - throw new Error('Missing required path parameters'); - } - - const repo = `${params.owner}/${params.repo}`; - const sha = searchParams.get('sha'); - const prNumber = searchParams.get('prNumber'); - - const { - data: report, - isLoading, - error, - } = useQuery({ - queryKey: ['tachometer-report', repo, sha], - queryFn: () => fetchCiReport(repo, sha!, 'tachometer.json'), - retry: 1, - enabled: Boolean(sha), - }); - - if (!sha) { - return ( - - Tachometer Details - - Missing required "sha" query parameter. - - - ); - } - - const summarized = (report?.cases ?? []).filter(isSummarized); - const failed = (report?.cases ?? []).filter((entry) => !isSummarized(entry)); - const groups = groupCasesByVariantSet(summarized); - - // A tachometer report carries its own comparison, so the baseline is one of its own refs rather - // than a separately fetched report. - const baselineRef = report?.refs.find((ref) => ref.kind !== 'worktree'); - - return ( - - Tachometer Details - - - - - {isLoading && ( - - - Loading tachometer report… - - )} - - {error && } - - {!isLoading && !error && !report && ( - No tachometer report found for this commit. - )} - - {failed.map((entry) => ( - - {entry.name}: {entry.error ?? 'produced no result'} - - ))} - - {groups.map((group) => - group.variants.length > 2 ? ( - group.cases.map((entry) => ( - - )) - ) : ( - - ), - )} - - {report && ( - - - Each cell is a 95% confidence interval for the mean, in milliseconds; one sample is - one page load. "unsure" means the interval still straddles zero: the - difference did not resolve within the case's sampling budget. - - - Builds: {report.refs.map((ref) => `${ref.id} = ${refLabel(ref)}`).join(' · ')} - - - head: {report.head.sha.slice(0, 9)} ({report.head.branch || '?'}) · measured on the - production bundle, installed from a packed tarball. - - - )} - - - ); -} diff --git a/package.json b/package.json index 86e12c47f7..1b5469edc1 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,6 @@ "docs:lib": "pnpm -F \"./packages/docs-infra\" run build", "test:benchmark": "pnpm -F ./test/performance benchmark", "test:benchmark-run": "pnpm -F ./test/benchmark benchmark:run", - "test:tacho": "pnpm -F ./test/tachometer test:tacho", "org-versions": "tsx scripts/orgVersions.mts" }, "dependencies": { @@ -64,7 +63,6 @@ "@vitest/browser-playwright": "5.0.1", "@vitest/coverage-v8": "5.0.1", "baseline-browser-mapping": "2.11.23", - "chromedriver": "^151.0.5", "eslint-plugin-n": "18.3.0", "pkg-pr-new": "0.0.88", "playwright": "1.63.0", diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index cb70c6ad5d..83633c9f69 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -1,18 +1,15 @@ # Benchmark -Three ways to measure, while repositories move from the first and last to the second. +Two ways to measure, while repositories move from the first to the second. **Under Vitest** — a React component benchmarking tool built on Vitest and Playwright, using React's profiling build to capture render durations against your source. Everything up to [`benchmark run`](#benchmark-run) covers it. It uploads version 1 of the `benchmark` report. -**`benchmark run`** — the same `benchmark()` files, the working tree against a baseline commit, both -**built** and installed the way a consumer gets them, sampled alternately in one browser run and -compared on paired differences. It uploads version 2 of the `benchmark` report. A repository uploads -one version or the other; the dashboard reads each by its version. - -**[Tachometer](#tachometer)** — Google's [tachometer](https://github.com/google/tachometer) driving -`tachometer.json` pages against your built package, through `benchmark tacho run`. +**`benchmark run`** — `benchmark()` and `reactBenchmark()` files, the working tree against a baseline +commit, both **built** and installed the way a consumer gets them, sampled alternately in one browser +run and compared on paired differences. It uploads version 2 of the `benchmark` report. A repository +uploads one version or the other; the dashboard reads each by its version. ## Features @@ -524,91 +521,6 @@ none — so nothing else should build against the same checkout meanwhile. `benchmark run --help` lists the remaining options. -## Tachometer - -`benchmark tacho run` measures a harness package in a real browser, building the workspace once per -commit under comparison and serving the pages from those builds. Run it from the harness. - -A harness is a package holding a `src/` tree of **case folders**, each with a page and a plain, -unmodified [tachometer config](https://github.com/google/tachometer#config-file). The configs are the -source of truth for which pages exist: the runner reads the selected ones, works out which pages they -reference, builds one variant per distinct commit, and rewrites each url to the built page. Filtering -to one case therefore narrows the build too. - -Sampling — `sampleSize`, `autoSampleConditions`, `timeout` — is set per case in its own config, -because tachometer rejects those as flags once a config file is in play. A case samples until its -difference resolves or the timeout is hit; two builds of equal speed cannot be separated, so such a -case reports `unsure`, which is the expected "nothing changed" outcome rather than a failure. - -### What a case compares - -A case is one of two things, and its `expand` decides which. - -**No `expand`** is the ordinary regression shape. It is expanded for you into `[current]` — the -working tree — against `[baseline]`, both loading the same page: - -```jsonc -{ "benchmarks": [{ "name": "init", "url": "./index.html" }] } -``` - -**Its own `expand`** compares the pages it names with each other, every one of them built from the -working tree. That is how one implementation is measured against another: - -```jsonc -"expand": [ - { "name": "[mosaic]", "url": "./index.html" }, - { "name": "[tanstack]", "url": "./tanstack.html" } -] -``` - -Such a case reports no regression, by construction: there is no baseline in it to regress against. -Every run says which shape each case has before it starts, so this is visible rather than inferred -from an empty result. To sweep a parameter and still measure it against the baseline, give each -value its own case folder pointing at the shared page — `expand` would turn the case into the other -shape. - -### Baselines - -`--baseline` names the build every `[baseline]` variant loads: a revision — a SHA, a tag, a branch, -`HEAD~1` — on its own or behind `git:`. It is resolved to an immutable SHA before anything is -cached, and defaults to `HEAD~1`, which needs no policy to decide. - -Which commit a branch should actually be compared against is `code-infra baseline`'s question — it -answers the same way for every job that compares a branch to its base — so pass it in: - -```bash -benchmark tacho run --baseline "$(code-infra baseline)" -``` - -`github:/#` and `preview:` are reserved for a baseline that is not a commit in -this repository. Neither is implemented, and both are rejected by name rather than handed to git. - -### Builds and caching - -Both sides are built the same way, from packed tarballs installed into an isolated tree, so neither -resolves the library through a workspace link. The benchmark pages themselves stay on the current -branch and only the built library changes between refs — so a commit whose public API differs from -today's pages will fail that ref's build, with the error surfaced. - -Packed builds are cached by commit SHA under `.tachometer/packed/`, so repeating a comparison against -the same commit skips the rebuild; the working tree is never cached. In CI, cache that directory -keyed on the baseline SHA, and check out with full history so a fork point can be resolved. A run -never modifies the checkout it was started from. - -By default a run resolves **in place**: it pins the packed build in the repository's own -`pnpm-workspace.yaml`, installs it there, and resolution is then ordinary. The repository is put -back, and reinstalled, when the run ends — including when it fails, and at the start of the next run -if one was killed outright. Two things follow from it. A tracked file names tarballs under -`.tachometer/` until the run ends, so do not commit while one is in flight. And the pins are global -to the workspace for that time, because pnpm scopes an override by parent package name and a harness -has none — so nothing else should build against the same checkout meanwhile. - -`--resolve-mode isolated` avoids both: it installs each ref into a directory of its own and points -resolution there, leaving the repository untouched, at the cost of a second install tree per ref. - -`--no-install` skips a ref's install. `benchmark tacho run --help` lists the rest, and -`benchmark tacho report` prints the table again from a saved JSON report. - ## API - `benchmark` — define a benchmark test case diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index 7ca5a1f731..26c0efa0b4 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -17,9 +17,6 @@ "./page": "./src/page/index.ts", "./vitePlugin": "./src/runner/plugin.ts", "./runReport": "./src/runReport/index.ts", - "./tachometerPlugin": "./src/tachometer/plugin.ts", - "./tachometerReport": "./src/tachometer/renderReport.ts", - "./tachometerFormat": "./src/tachometer/format.ts", "./cli": "./src/cli/index.ts" }, "scripts": { diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index 1ee03a37b4..e77235372b 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -9,8 +9,6 @@ import { hideBin } from 'yargs/helpers'; // cannot reach the manifest from both — and the repository's workspace symlink hides the break. import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; import run from './run'; -import tachoReport from './tachoReport'; -import tachoRun from './tachoRun'; let globalArgv: { verbose?: boolean } = {}; @@ -28,14 +26,6 @@ await yargs(hideBin(process.argv)) globalArgv = argv; }, true) .command(run) - .command({ - command: 'tacho ', - describe: 'Benchmark a tachometer harness, and read the reports it writes.', - builder: (group) => - group.command(tachoRun).command(tachoReport).demandCommand(1, 'Specify a tacho subcommand.'), - // The group itself does nothing; `demandCommand` above means a subcommand always runs. - handler: () => {}, - }) .fail((msg, err, yargsInstance) => { if (msg) { yargsInstance.showHelp(); diff --git a/packages/benchmark/src/cli/tachoReport.ts b/packages/benchmark/src/cli/tachoReport.ts deleted file mode 100644 index 49b59569ec..0000000000 --- a/packages/benchmark/src/cli/tachoReport.ts +++ /dev/null @@ -1,51 +0,0 @@ -import * as path from 'node:path'; -import { readFile } from 'node:fs/promises'; -import { z } from 'zod/v4'; -import type { CommandModule } from 'yargs'; -import { OUTPUT_DIR } from '../tachometer/outputDir'; - -interface Args { - /** Path to a JSON report written by `tacho run`. */ - file: string; -} - -const DEFAULT_REPORT = path.join(OUTPUT_DIR, 'results', 'report.json'); - -const command: CommandModule<{}, Args> = { - command: 'report [file]', - describe: - 'Print the table for a JSON report written by `tacho run`. Reads the last run of the harness in the current directory when no path is given.', - builder: (yargs) => { - return yargs - .positional('file', { - type: 'string', - default: DEFAULT_REPORT, - describe: 'Path to the JSON report', - }) - .epilogue( - 'The same table `tacho run` prints when it finishes, so a report kept from an earlier run — or downloaded from CI — can be read without sampling again.', - ); - }, - handler: async (argv) => { - const reportPath = path.resolve(argv.file); - const raw = await readFile(reportPath, 'utf8').catch(() => { - throw new Error( - `No report at ${reportPath}. Pass a path, or run \`tacho run\` from a harness first.`, - ); - }); - - const { tachometerReportSchema } = await import('../tachometer/ciReport'); - const parsed = tachometerReportSchema.safeParse(JSON.parse(raw)); - if (!parsed.success) { - throw new Error( - `${reportPath} is not a tachometer report this version can read.\n` + - `${z.prettifyError(parsed.error)}`, - ); - } - - const { renderTachometerReport } = await import('../tachometer/renderReport'); - renderTachometerReport(parsed.data); - }, -}; - -export default command; diff --git a/packages/benchmark/src/cli/tachoRun.ts b/packages/benchmark/src/cli/tachoRun.ts deleted file mode 100644 index a6c6343e0d..0000000000 --- a/packages/benchmark/src/cli/tachoRun.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { CommandModule } from 'yargs'; -import type { RunTachometerOptions } from '../tachometer/runTachometer'; - -/** Everything `runTachometer` takes except the directory, which is where the command was run. */ -type Args = Omit; - -const command: CommandModule<{}, Args> = { - command: 'run [filters...]', - describe: - 'Benchmark a tachometer harness across builds of this workspace and write a JSON report. Run from the harness package.', - builder: (yargs) => { - return yargs - .positional('filters', { - type: 'string', - array: true, - describe: 'Only run cases whose path under src contains this substring', - }) - .option('baseline', { - type: 'string', - describe: - 'The build to compare against: a revision, on its own or as git:. Defaults to HEAD~1; `code-infra baseline` resolves a fork point to pass here', - }) - .option('resolve-mode', { - type: 'string', - choices: ['isolated', 'in-place'] as const, - default: 'in-place' as const, - describe: - "How pages find the library: 'isolated' installs each ref beside the repository, 'in-place' pins it in the repository's own pnpm-workspace.yaml for the length of the run", - }) - .option('build-cmd', { - type: 'string', - default: 'pnpm release:build', - describe: - "Command that builds the publishable workspace packages, in each ref's checkout and in the working tree", - }) - .option('install', { - type: 'boolean', - default: true, - describe: "Install dependencies inside a ref's checkout. Use --no-install to skip", - }) - .option('upload', { - type: 'boolean', - default: process.env.TACHO_UPLOAD === 'true', - describe: - 'Upload the report and refresh the PR comment. Defaults to TACHO_UPLOAD=true, so CI can switch it on by environment. Requires CIRCLE_OIDC_TOKEN_V2', - }) - .option('out', { - type: 'string', - describe: 'Write the combined JSON report here. Default: .tachometer/results/report.json', - }) - .epilogue( - 'Sampling (sampleSize, autoSampleConditions, timeout) is configured per case in its own tachometer.json — tachometer rejects those as CLI flags when a config file is used.', - ); - }, - handler: async (argv) => { - const { runTachometer } = await import('../tachometer/runTachometer'); - await runTachometer({ - harnessDir: process.cwd(), - filters: argv.filters ?? [], - baseline: argv.baseline, - resolveMode: argv.resolveMode, - buildCmd: argv.buildCmd, - install: argv.install, - out: argv.out, - upload: argv.upload, - }); - }, -}; - -export default command; diff --git a/packages/benchmark/src/tachometer/browser.test.ts b/packages/benchmark/src/tachometer/browser.test.ts deleted file mode 100644 index 29f970ea1d..0000000000 --- a/packages/benchmark/src/tachometer/browser.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -import * as path from 'node:path'; -import { chmod, mkdir, writeFile } from 'node:fs/promises'; -import { describe, expect, it } from 'vitest'; -import { assertDriverMatchesBrowser, majorVersionOf, withBrowserDefaults } from './browser'; -import { makeTempDir } from '../utils/testUtils'; - -describe('majorVersionOf', () => { - it('reads the major from Chrome for Testing output', () => { - expect(majorVersionOf('Google Chrome for Testing 151.0.7922.34')).toBe(151); - }); - - it('reads the major from plain Chrome output', () => { - expect(majorVersionOf('Google Chrome 152.0.6367.60')).toBe(152); - }); - - it('reads the major from a bare package version', () => { - // chromedriver's package.json version, which is what the driver side is read from. - expect(majorVersionOf('151.0.5')).toBe(151); - }); - - it('returns undefined when there is no version to read', () => { - expect(majorVersionOf('')).toBeUndefined(); - expect(majorVersionOf('no version here')).toBeUndefined(); - }); - - it('ignores a leading number that is not part of a version', () => { - // Requires three dot-separated components, so a stray count cannot be mistaken for a version. - expect(majorVersionOf('2 browsers found: Chrome 151.0.7922.34')).toBe(151); - }); -}); - -describe('withBrowserDefaults', () => { - const BINARY = '/path/to/Chrome for Testing'; - - it('fills in the binary for a case that does not pin one', () => { - expect(withBrowserDefaults({ name: 'chrome', headless: true }, BINARY, false)).toEqual({ - name: 'chrome', - headless: true, - binary: BINARY, - }); - }); - - it('leaves a case that pins its own binary alone', () => { - const pinned = withBrowserDefaults({ binary: '/opt/chrome' }, BINARY, false); - - expect(pinned.binary).toBe('/opt/chrome'); - }); - - it('keeps the sandbox when the run is not root', () => { - // The sandbox is a real protection for the pages a benchmark loads; only root cannot have it. - expect(withBrowserDefaults({ name: 'chrome' }, BINARY, false)).not.toHaveProperty( - 'addArguments', - ); - }); - - it('disables the sandbox when the run is root', () => { - expect(withBrowserDefaults({ name: 'chrome' }, BINARY, true).addArguments).toEqual([ - '--no-sandbox', - ]); - }); - - it("keeps a case's own arguments when it disables the sandbox", () => { - // Cases pass flags that hold their measurements still; dropping them would move the numbers. - const merged = withBrowserDefaults( - { addArguments: ['--disable-renderer-backgrounding'] }, - BINARY, - true, - ); - - expect(merged.addArguments).toEqual(['--disable-renderer-backgrounding', '--no-sandbox']); - }); - - it('does not repeat an argument a case already passes', () => { - const merged = withBrowserDefaults({ addArguments: ['--no-sandbox'] }, BINARY, true); - - expect(merged.addArguments).toEqual(['--no-sandbox']); - }); - - it('names the browser for a case that declares none', () => { - // Tachometer defaults an absent `browser` to chrome, but validates the config against a schema - // where every browser variant requires `name` — so injecting the binary alone is rejected - // before a single sample is taken. - expect(withBrowserDefaults(undefined, BINARY, true)).toEqual({ - name: 'chrome', - binary: BINARY, - addArguments: ['--no-sandbox'], - }); - }); - - it('expands the string shorthand instead of spreading it into characters', () => { - expect(withBrowserDefaults('chrome-headless', BINARY, false)).toEqual({ - name: 'chrome', - headless: true, - binary: BINARY, - }); - }); - - it('keeps the remote url out of the browser name', () => { - expect(withBrowserDefaults('firefox@http://localhost:4444', BINARY, false)).toEqual({ - name: 'firefox', - headless: false, - remoteUrl: 'http://localhost:4444', - binary: BINARY, - }); - }); - - it('names the browser for a case whose object leaves it out', () => { - expect(withBrowserDefaults({ headless: true }, BINARY, false).name).toBe('chrome'); - }); -}); - -describe('assertDriverMatchesBrowser', () => { - /** - * A harness with tachometer installed, and optionally a chromedriver where tachometer can reach - * it — one directory up from its own, the way a workspace root is reachable in a real install. - * - * Real directories and real module resolution rather than a stand-in: what is being checked is - * which chromedriver node finds from tachometer, and faking that would assume the answer. - */ - async function makeHarness(driverVersion?: string): Promise { - const harnessDir = await makeTempDir(); - const modules = path.join(harnessDir, 'node_modules'); - - await writeFile(path.join(harnessDir, 'package.json'), JSON.stringify({ name: 'harness' })); - await mkdir(path.join(modules, 'tachometer'), { recursive: true }); - await writeFile( - path.join(modules, 'tachometer', 'package.json'), - JSON.stringify({ name: 'tachometer', version: '0.7.2' }), - ); - - if (driverVersion) { - await mkdir(path.join(modules, 'chromedriver'), { recursive: true }); - await writeFile( - path.join(modules, 'chromedriver', 'package.json'), - JSON.stringify({ name: 'chromedriver', version: driverVersion }), - ); - } - - return harnessDir; - } - - /** An executable standing in for the browser, reporting `version` to `--version`. */ - async function makeBrowser(version: string): Promise { - const dir = await makeTempDir(); - const binary = path.join(dir, 'chrome'); - await writeFile(binary, `#!/bin/sh\necho "Google Chrome for Testing ${version}"\n`); - await chmod(binary, 0o755); - return binary; - } - - it('passes when the driver and the browser share a major', async () => { - const harnessDir = await makeHarness('151.0.5'); - const binary = await makeBrowser('151.0.7922.34'); - - expect(() => assertDriverMatchesBrowser(harnessDir, binary)).not.toThrow(); - }); - - it('fails when the driver is on a different major', async () => { - const harnessDir = await makeHarness('150.0.3'); - const binary = await makeBrowser('151.0.7922.34'); - - expect(() => assertDriverMatchesBrowser(harnessDir, binary)).toThrow(/cannot drive Chrome/); - }); - - it('fails when no driver is reachable, naming the major to pin', async () => { - // Tachometer would install the newest chromedriver here, which does not drive a pinned Chrome. - const harnessDir = await makeHarness(); - const binary = await makeBrowser('151.0.7922.34'); - - expect(() => assertDriverMatchesBrowser(harnessDir, binary)).toThrow(/"chromedriver": "\^151"/); - }); -}); diff --git a/packages/benchmark/src/tachometer/browser.ts b/packages/benchmark/src/tachometer/browser.ts deleted file mode 100644 index 95e539eebc..0000000000 --- a/packages/benchmark/src/tachometer/browser.ts +++ /dev/null @@ -1,160 +0,0 @@ -import * as path from 'node:path'; -import { createRequire } from 'node:module'; -import { execFileSync } from 'node:child_process'; -import chalk from 'chalk'; -import type * as PlaywrightTest from '@playwright/test'; -import { pathExists } from '../utils/path'; - -/** - * Picking the browser to benchmark in, and checking it can actually be driven. - * - * Two separate programs are involved. The browser itself comes from Playwright, which downloads a - * fixed Chrome build. Tachometer needs `chromedriver` to drive that browser, and supplies it itself: - * it uses whichever one it can find next to itself, and downloads the newest if it finds none. - * - * A chromedriver only drives the Chrome version it was built for. Since Playwright's Chrome is - * deliberately a little behind the latest, letting tachometer fetch the newest driver usually gets a - * pair that cannot work together. - * - * So the consumer supplies it instead: a `chromedriver` devDependency at the workspace root, which - * is the only place tachometer resolves one from. Declared beside `@playwright/test`, so one place - * governs both versions and Renovate moves them together. All this module does is compare the - * version tachometer would resolve against the browser's own — which version that is stays the - * consumer's to choose. - */ - -/** - * Resolves the browser every case runs on: Playwright's pinned Chrome for Testing, rather than - * whatever Chrome the machine has auto-updated to. - * - * The path is machine-specific, so callers inject it into each config instead of committing it. - */ -export async function resolveBrowserBinary(harnessDir: string): Promise { - let playwright: typeof PlaywrightTest; - try { - playwright = await import('@playwright/test'); - } catch { - // An optional peer, so it installs cleanly when absent and only fails here. - throw new Error( - `"@playwright/test" is not installed. Add it as a devDependency of ${harnessDir}.`, - ); - } - - const binary = playwright.chromium.executablePath(); - if (!binary || !(await pathExists(binary))) { - throw new Error( - `Playwright's Chromium is not installed at "${binary}". Run \`pnpm exec playwright install chromium\`.`, - ); - } - return binary; -} - -/** - * The major version in output like `Google Chrome for Testing 151.0.7922.34`, or undefined when - * there is no version in it. - * - * At least three components are required, so a stray count in the same line is not read as one. - */ -export function majorVersionOf(versionOutput: string): number | undefined { - const match = /(\d+)\.\d+\.\d+/.exec(versionOutput); - return match ? Number(match[1]) : undefined; -} - -/** - * Throws when the chromedriver tachometer will use cannot drive `binary`. - */ -export function assertDriverMatchesBrowser(harnessDir: string, binary: string): void { - const browserMajor = majorVersionOf( - execFileSync(binary, ['--version'], { encoding: 'utf8' }).trim(), - ); - - // The same lookup tachometer does before launching: it uses whichever chromedriver resolves from - // its own package root, and installs the newest one when none does. - let driverVersion: string | undefined; - try { - const fromHarness = createRequire(path.join(harnessDir, 'package.json')); - const fromTachometer = createRequire(fromHarness.resolve('tachometer/package.json')); - driverVersion = fromTachometer('chromedriver/package.json').version; - } catch { - throw new Error( - `No chromedriver is reachable from tachometer, so it will install the newest one — which ` + - `will not drive Chrome for Testing ${browserMajor}, since Playwright pins a Chrome behind ` + - `the current release. Add "chromedriver": "^${browserMajor}" to the workspace root, where ` + - `tachometer can resolve it.`, - ); - } - - const driverMajor = majorVersionOf(driverVersion ?? ''); - if (browserMajor === undefined || driverMajor === undefined) { - console.warn( - chalk.yellow('Could not read the browser or chromedriver version; skipping the check.'), - ); - return; - } - if (browserMajor !== driverMajor) { - throw new Error( - `chromedriver ${driverVersion} cannot drive Chrome for Testing ${browserMajor}. ` + - `Update the "chromedriver" devDependency to ${browserMajor}.x, or align ` + - `"@playwright/test" with it — bump the two together.`, - ); - } -} - -/** - * A benchmark's `browser` config. Only the two fields a run touches are named; everything else - * tachometer accepts there (`name`, `headless`, `windowSize`, …) passes through untouched. - */ -export type BrowserConfig = { - binary?: string; - addArguments?: string[]; -} & Record; - -/** Chrome cannot enter its sandbox as root, and refusing to start is all it does about it. */ -const NO_SANDBOX = '--no-sandbox'; - -/** Tachometer's default when a benchmark names no browser. */ -const DEFAULT_BROWSER = 'chrome'; - -/** The suffix tachometer's browser shorthand uses to ask for a headless run. */ -const HEADLESS_SUFFIX = '-headless'; - -/** - * Whatever a case put in `browser` as the object form tachometer's schema requires: `name` is - * mandatory there and unknown keys are rejected, so the string shorthand - * `"[-headless][@]"` and an absent `browser` are both expanded here. - */ -function asBrowserObject(browser: string | BrowserConfig | undefined): BrowserConfig { - if (browser === undefined) { - return { name: DEFAULT_BROWSER }; - } - if (typeof browser !== 'string') { - // An object of its own may still leave out `name`, which the schema requires. - return { name: DEFAULT_BROWSER, ...browser }; - } - const at = browser.indexOf('@'); - const named = at === -1 ? browser : browser.slice(0, at); - const headless = named.endsWith(HEADLESS_SUFFIX); - const base: BrowserConfig = { - name: headless ? named.slice(0, -HEADLESS_SUFFIX.length) : named, - headless, - }; - return at === -1 ? base : { ...base, remoteUrl: browser.slice(at + 1) }; -} - -/** - * Merges what the run decides into a case's own `browser` config: the binary, which is - * machine-specific, and the sandbox flag, which depends on the user the run happens under. - */ -export function withBrowserDefaults( - browser: string | BrowserConfig | undefined, - binary: string, - asRoot: boolean, -): BrowserConfig { - const base = asBrowserObject(browser); - const merged: BrowserConfig = { ...base, binary: base.binary ?? binary }; - const addArguments = merged.addArguments ?? []; - if (asRoot && !addArguments.includes(NO_SANDBOX)) { - merged.addArguments = [...addArguments, NO_SANDBOX]; - } - return merged; -} diff --git a/packages/benchmark/src/tachometer/buildPages.ts b/packages/benchmark/src/tachometer/buildPages.ts deleted file mode 100644 index 4b910cae47..0000000000 --- a/packages/benchmark/src/tachometer/buildPages.ts +++ /dev/null @@ -1,322 +0,0 @@ -/* eslint-disable no-console */ - -import * as path from 'node:path'; -import { createRequire } from 'node:module'; -import { readFile, mkdir, rm, writeFile } from 'node:fs/promises'; -import { pathToFileURL } from 'node:url'; -import chalk from 'chalk'; -import type * as Vite from 'vite'; -import { parse, stringify } from 'yaml'; -// Self-referencing rather than `../../package.json`, which resolves only in the repository — see -// the note in ../cli/index.ts. -import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; -import { run } from '../utils/exec'; -import { tarballFor } from '../utils/packWorkspace'; -import type { PackedPackage } from '../utils/packWorkspace'; -import { installedVersions, readPackageJson, writePackageJson } from '../utils/pnpm'; -import { refLabel } from './format'; -import type { ResolvedRef } from './refs'; -import { pathExists } from '../utils/path'; - -/** - * How a ref's pages find the library under test. - * - * `isolated` installs that ref's packed build into a directory of its own and points resolution - * there, leaving the repository untouched. `in-place` pins the packed build in the repository's own - * `pnpm-workspace.yaml` and installs it, so resolution is ordinary — at the cost of mutating a - * tracked file for as long as the run lasts. - */ -export type ResolveMode = 'isolated' | 'in-place'; - -/** - * Dependencies the run needs but a page never imports, so a ref's tree does without them — - * whichever section of the harness's manifest they sit in, since neither reason below turns on that. - * - * Every one is resolved from the harness rather than from here: the browser and its driver by the - * runner, tachometer by the command that samples, and this package by the harness's own vite config - * — which is loaded from the harness, since the build happens in place. Every one is also expensive - * to install: tachometer pulls a chromedriver, `@playwright/test` a browser, and this package a - * dependency tree many times the size of a harness. - */ -const RUNNER_ONLY_DEPS = ['tachometer', 'chromedriver', '@playwright/test', pkgJson.name]; - -/** - * A dependency map for a ref's tree: every `workspace:` entry points at that ref's packed tarball, - * and everything else is pinned to the version the harness resolved rather than the range it - * declares. - * - * The tree installs without the repository's lockfile, so a range would float to whatever is newest - * at run time — a different library than the repository builds against, measured as if it were - * ours. - */ -function rewriteWorkspaceDeps( - installed: Map, - deps: Partial> | undefined, - packages: PackedPackage[], - omit: string[] = [], -): Record { - const entries = Object.entries(deps ?? {}).filter( - ([name, version]) => version !== undefined && !omit.includes(name), - ) as Array<[string, string]>; - - return Object.fromEntries( - entries.map(([name, version]) => - version.startsWith('workspace:') - ? [name, `file:${tarballFor(packages, name)}`] - : [name, installed.get(name) ?? version], - ), - ); -} - -/** - * Reads the repository's `overrides`, so the ref's tree resolves the same dependency graph the - * repository does. - * - * The tree is its own workspace root and inherits nothing. Without this, a repository that pins a - * transitive dependency through an override would get a *different* graph here than in its real - * install — quietly changing what is being benchmarked — and any policy the override exists to - * satisfy (a blocked resolution, a security pin) would fail here too. - */ -async function readOverrides(repoRoot: string): Promise> { - let raw: string; - try { - raw = await readFile(path.join(repoRoot, 'pnpm-workspace.yaml'), 'utf8'); - } catch (error) { - // A repository without the file simply has no overrides. Anything else — a malformed file, a - // permission error — would otherwise resolve the un-overridden graph and silently benchmark a - // dependency set the repository never ships, which is the failure this function exists to avoid. - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return {}; - } - throw error; - } - return parse(raw)?.overrides ?? {}; -} - -/** - * A vite plugin that resolves every bare import from `treeDir` instead of from the harness. - * - * Bare specifiers only: a relative or absolute id belongs to the harness's own sources and stays - * where it is. Resolution is handed back to vite with the importer rewritten, so vite's own - * conditions (`browser`, `import`) and the target package's `exports` map still decide the answer — - * which is what an alias to a directory would skip. - */ -function resolveFromTree(treeDir: string): Vite.Plugin { - // Resolution needs a base directory, and vite derives one from the importer's path — falling back - // to the project root unless that path exists on disk, which would silently resolve the harness's - // copy of a dependency instead of this ref's. The tree's own manifest is the file that names this - // directory, the same way `createRequire` is pointed at one. - const importer = path.join(treeDir, 'package.json'); - - return { - name: 'mui-benchmark:tachometer-resolve', - enforce: 'pre', - async resolveId(id, _importer, options) { - if (id.startsWith('.') || path.isAbsolute(id) || id.startsWith('\0') || id.includes(':')) { - return null; - } - return this.resolve(id, importer, { ...options, skipSelf: true }); - }, - }; -} - -/** - * The packed pins an in-place run adds to the repository's overrides. - * - * Everything the run itself resolves is left alone, this package above all: pinning it would hand - * the harness's vite config the ref's copy of the very tool doing the building, and a ref old enough - * not to export the plugin then fails to configure at all. - */ -function packedPins(packages: PackedPackage[]): Record { - return Object.fromEntries( - packages - .filter((pkg) => !RUNNER_ONLY_DEPS.includes(pkg.name)) - .map((pkg) => [pkg.name, `file:${pkg.tarball}`]), - ); -} - -/** Where the repository's own manifest is kept while an in-place run has it pinned. */ -function backupPathOf(outputDir: string): string { - return path.join(outputDir, 'pnpm-workspace.yaml.orig'); -} - -/** - * Points the repository's own install at this ref's packed build, through the same overrides the - * isolated mode applies to a workspace of its own. - * - * The manifest is copied aside first and rewritten from that copy each time, so a second ref pins - * over the original rather than over the first ref's pins. A run that dies leaves both the pins and - * the copy: the manifest then names tarballs under the output directory, which fails the next - * install rather than resolving something else and saying nothing. - */ -async function pinPackedPackages( - repoRoot: string, - outputDir: string, - packages: PackedPackage[], -): Promise { - const manifestPath = path.join(repoRoot, 'pnpm-workspace.yaml'); - const backupPath = backupPathOf(outputDir); - if (!(await pathExists(backupPath))) { - await writeFile(backupPath, await readFile(manifestPath, 'utf8')); - } - - const config = parse(await readFile(backupPath, 'utf8')) ?? {}; - await writeFile( - manifestPath, - stringify({ ...config, overrides: { ...config.overrides, ...packedPins(packages) } }), - ); -} - -/** - * Puts the repository's manifest back, and installs from it. - * - * The install is the point: without it the tree on disk still resolves the library to a tarball - * under the output directory, and that surfaces at the next unrelated command instead of here. - * - * `--no-frozen-lockfile` because pnpm turns a frozen install on by itself in CI, and the lockfile - * here still carries the pins this is undoing. - */ -export async function restoreWorkspace(repoRoot: string, outputDir: string): Promise { - const backupPath = backupPathOf(outputDir); - if (!(await pathExists(backupPath))) { - return; - } - console.log(chalk.cyan('\nRestoring the repository install…')); - await writeFile(path.join(repoRoot, 'pnpm-workspace.yaml'), await readFile(backupPath, 'utf8')); - await rm(backupPath, { force: true }); - await run('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], repoRoot); -} - -/** Installs one ref's packed build, with the harness's own dependencies around it. */ -async function installRefTree({ - harnessDir, - repoRoot, - packages, - treeDir, -}: { - /** The harness package directory, whose dependencies are copied. */ - harnessDir: string; - /** The workspace root, whose overrides the tree inherits. */ - repoRoot: string; - /** That ref's packed workspace packages. */ - packages: PackedPackage[]; - /** Where to install. */ - treeDir: string; -}): Promise { - const [harnessPkg, installed] = await Promise.all([ - readPackageJson(harnessDir), - installedVersions(harnessDir), - mkdir(treeDir, { recursive: true }), - ]); - - await writePackageJson(treeDir, { - name: 'tacho-resolve-tree', - private: true, - version: '0.0.0', - dependencies: rewriteWorkspaceDeps( - installed, - harnessPkg.dependencies, - packages, - RUNNER_ONLY_DEPS, - ), - devDependencies: rewriteWorkspaceDeps( - installed, - harnessPkg.devDependencies, - packages, - RUNNER_ONLY_DEPS, - ), - }); - - // Marks this folder as its own workspace root, so pnpm does not walk up into the monorepo. The - // overrides carry the repository's own, then pin every packed workspace package on top — so a - // transitive dependency between them (whose packed `workspace:*` became a concrete, unpublished - // version) resolves to a local build rather than 404ing on the registry. The packed pins win, - // since they are the whole point of this install. - await writeFile( - path.join(treeDir, 'pnpm-workspace.yaml'), - stringify({ - overrides: { - ...(await readOverrides(repoRoot)), - ...Object.fromEntries(packages.map((pkg) => [pkg.name, `file:${pkg.tarball}`])), - }, - }), - ); - - // `--ignore-scripts`: this workspace has no build-script approvals, and pnpm fails an install - // over unapproved ones rather than warning. Nothing installed here needs its scripts either — the - // pages are built from the packages exactly as packed. - await run( - 'pnpm', - ['install', '--prefer-offline', '--ignore-scripts', '--config.engine-strict=false'], - treeDir, - ); -} - -/** - * Builds the benchmark pages against one ref's packed build. - * - * The pages are built **where they live**, from the harness's own directory and its own vite config, - * so a `tacho run` build sees exactly what a plain `vite build` sees — the same tsconfig, the same - * postcss config, the same everything. Only *resolution* differs: a plugin redirects every bare - * import into a small tree installed for this ref, where the library under test is that ref's packed - * build. - * - * That tree holds the harness's own dependencies too, so the pages and the library resolve React - * (and everything else) to one copy. Building one side through a workspace link instead would - * compare two different resolution paths, and that difference would land in the measurement rather - * than in the library. - */ -export async function buildRefPages(options: { - /** The harness package directory, built in place. */ - harnessDir: string; - /** The workspace root, whose overrides the tree inherits. */ - repoRoot: string; - /** The ref being built. */ - ref: ResolvedRef; - /** That ref's packed workspace packages. */ - packages: PackedPackage[]; - /** Persistent directory holding this ref's installed tree. Unused when resolving in place. */ - treeDir: string; - /** The run's output directory, where an in-place run keeps the manifest it replaced. */ - outputDir: string; - /** Absolute output directory for the built pages. */ - outDir: string; - /** How the pages find the library. Defaults to `in-place`. */ - resolveMode?: ResolveMode; -}): Promise { - const { - harnessDir, - repoRoot, - ref, - packages, - treeDir, - outputDir, - outDir, - resolveMode = 'in-place', - } = options; - console.log(chalk.cyan(`\nBuilding benchmark pages for "${refLabel(ref)}"…`)); - - if (resolveMode === 'in-place') { - await pinPackedPackages(repoRoot, outputDir, packages); - // `--no-frozen-lockfile` because changing the overrides is the point, and pnpm turns a frozen - // install on by itself in CI — where it would refuse the very change being made. - await run('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], repoRoot); - } else { - await installRefTree({ harnessDir, repoRoot, packages, treeDir }); - } - - // vite is the harness's own, not this package's: the harness declares the version its pages are - // built with, and the config being run is the harness's. - const requireFromHarness = createRequire(path.join(harnessDir, 'package.json')); - const vite: typeof Vite = await import(pathToFileURL(requireFromHarness.resolve('vite')).href); - - // `root` only points vite's config discovery at the harness; the harness's own plugin then moves - // it to `src/`, the same as for a plain `vite build`. - await vite.build({ - root: harnessDir, - logLevel: 'info', - // Resolving in place needs no plugin: the repository's own install is the one that changed. - plugins: resolveMode === 'in-place' ? [] : [resolveFromTree(treeDir)], - build: { outDir }, - }); -} diff --git a/packages/benchmark/src/tachometer/ciReport.test.ts b/packages/benchmark/src/tachometer/ciReport.test.ts deleted file mode 100644 index 0e10d03b22..0000000000 --- a/packages/benchmark/src/tachometer/ciReport.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { tachometerUploadSchema } from './ciReport'; - -/** - * A report of the shape `runTachometer` writes, trimmed to one case. - * - * @param {Partial} [overrides] - Fields to replace - * @returns {any} - */ -function report(overrides = {}) { - return { - version: 1, - reportType: 'tachometer', - generatedAt: '2026-09-02T00:00:00.000Z', - head: { ref: 'HEAD', sha: 'a'.repeat(40), branch: 'feature' }, - browser: '/path/to/Chrome for Testing', - refs: [ - { id: 'current', kind: 'worktree' }, - { id: 'git-abc123456', kind: 'git', requested: 'HEAD~1', sha: 'b'.repeat(40) }, - ], - cases: [ - { - name: 'workload', - comparison: 'baseline', - reference: 'workload [current]', - measurements: [ - { - name: 'mount', - variants: [ - { - variant: 'workload [current]', - meanMs: { low: 20.5, high: 22.7 }, - samples: 120, - }, - ], - comparisons: [ - { - variant: 'workload [baseline]', - verdict: 'unsure', - absoluteMs: { low: -1.2, high: 2.1 }, - percentChange: { low: -5.2, high: 9.3 }, - versusReference: { - verdict: 'unsure', - absoluteMs: { low: -2.1, high: 1.2 }, - percentChange: { low: -9.3, high: 5.2 }, - }, - }, - ], - }, - ], - }, - ], - raw: { workload: { benchmarks: [] } }, - ...overrides, - }; -} - -/** - * @param {Partial} [overrides] - Envelope fields to replace - * @returns {any} - */ -function envelope(overrides = {}) { - return { - version: 1, - timestamp: 1_770_000_000_000, - commitSha: 'a'.repeat(40), - repo: 'mui/mui-public', - reportType: 'tachometer', - prNumber: 1829, - branch: 'feature', - report: report(), - ...overrides, - }; -} - -describe('tachometerUploadSchema', () => { - it('accepts a report of the shape the runner writes', () => { - expect(tachometerUploadSchema.safeParse(envelope()).success).toBe(true); - }); - - it('carries the envelope metadata through', () => { - const parsed = tachometerUploadSchema.parse(envelope()); - - expect(parsed).toMatchObject({ - repo: 'mui/mui-public', - prNumber: 1829, - branch: 'feature', - reportType: 'tachometer', - }); - }); - - it('accepts a run with no pull request', () => { - // A push to the base branch uploads too; it just has no PR to comment on. - const { prNumber, ...withoutPr } = envelope(); - - expect(tachometerUploadSchema.safeParse(withoutPr).success).toBe(true); - }); - - it('accepts a case that failed to summarize', () => { - // Kept in the report rather than dropped, so the comment can say the case produced nothing. - const withFailure = envelope({ - report: report({ - cases: [{ name: 'broken', comparison: 'baseline', error: 'produced no benchmarks' }], - }), - }); - - expect(tachometerUploadSchema.safeParse(withFailure).success).toBe(true); - }); - - describe('rejects', () => { - it('a report from another benchmark axis', () => { - // The dashboard routes on reportType, so the wrong one would render as the wrong thing. - const wrongType = envelope({ reportType: 'benchmark' }); - - expect(tachometerUploadSchema.safeParse(wrongType).success).toBe(false); - }); - - it('a commit that is not a full sha', () => { - // The artifact key is built from it, and a short sha would not match what the dashboard - // looks up for the head commit. - expect(tachometerUploadSchema.safeParse(envelope({ commitSha: 'abc1234' })).success).toBe( - false, - ); - }); - - it('a repo that is not owner/name', () => { - expect(tachometerUploadSchema.safeParse(envelope({ repo: 'mui-public' })).success).toBe( - false, - ); - }); - - it('a verdict outside the known set', () => { - const badVerdict = report(); - badVerdict.cases[0].measurements[0].comparisons[0].verdict = 'regressed'; - - expect(tachometerUploadSchema.safeParse(envelope({ report: badVerdict })).success).toBe( - false, - ); - }); - }); -}); diff --git a/packages/benchmark/src/tachometer/ciReport.ts b/packages/benchmark/src/tachometer/ciReport.ts deleted file mode 100644 index 8fe6cb09d4..0000000000 --- a/packages/benchmark/src/tachometer/ciReport.ts +++ /dev/null @@ -1,124 +0,0 @@ -/* eslint-disable no-console -- progress belongs in the CI log. */ - -import { z } from 'zod/v4'; -import { postToDashboard } from '../ciApi'; -import { ciReportUploadSchema } from '../ciReport'; - -/** - * The tachometer axis's upload envelope. It differs from the Vitest one only in the shape of - * `report`; the transport, the CI metadata and the pull request sync are shared. - */ - -const confidenceIntervalSchema = z.object({ - low: z.number(), - high: z.number(), -}); - -const differenceSchema = z.object({ - verdict: z.enum(['faster', 'slower', 'unsure']), - absoluteMs: confidenceIntervalSchema, - percentChange: confidenceIntervalSchema, -}); - -const variantResultSchema = z.object({ - variant: z.string(), - meanMs: confidenceIntervalSchema, - samples: z.number(), -}); - -const comparisonSchema = differenceSchema.extend({ - variant: z.string(), - versusReference: differenceSchema.optional(), -}); - -const measurementResultSchema = z.object({ - name: z.string(), - variants: z.array(variantResultSchema), - comparisons: z.array(comparisonSchema), -}); - -// A case that failed to summarize carries an `error` and no measurements. It is kept in the report -// rather than dropped, so the comment can say a case produced nothing instead of quietly omitting it. -const caseResultSchema = z.object({ - name: z.string(), - // `baseline` compares the working tree against the baseline build, so a difference there is a - // regression; `variants` compares pages built from the same tree, where a difference is the - // point. Carried per case because that is where the two are told apart. - comparison: z.enum(['baseline', 'variants']), - reference: z.string().optional(), - measurements: z.array(measurementResultSchema).optional(), - error: z.string().optional(), -}); - -/** - * A build the pages were loaded from. The working tree has no commit behind it and the baseline - * always does, so the two carry different fields. - * - * `requested` is the revision as the run was given it — `HEAD~1`, a tag, a SHA — kept instead of a - * ready-made display string so a renderer decides how it reads. - */ -const refSchema = z.discriminatedUnion('kind', [ - z.object({ kind: z.literal('worktree'), id: z.string() }), - z.object({ - kind: z.literal('git'), - id: z.string(), - sha: z.string(), - requested: z.string(), - }), -]); - -const tachometerReportV1Schema = z.object({ - version: z.literal(1), - reportType: z.literal('tachometer'), - generatedAt: z.string(), - head: z.object({ ref: z.string(), sha: z.string(), branch: z.string().optional() }), - browser: z.string().optional(), - refs: z.array(refSchema), - cases: z.array(caseResultSchema), - // Every sample tachometer produced. Not read by the comment, but kept so a richer view can be - // built later without changing what CI uploads. - raw: z.record(z.string(), z.unknown()), -}); - -/** - * Every report shape the store may hold, keyed on `version`. - * - * Reports written months ago by an older version of this tooling are still in S3 and still get - * rendered. A new shape joins this union with its own literal version rather than replacing the old - * one, so a reader narrows on `version` and the compiler makes it handle each. - */ -export const tachometerReportSchema = z.discriminatedUnion('version', [tachometerReportV1Schema]); - -/** - * The upload envelope. Built from the same factory the Vitest axis uses, so both report types reach - * the dashboard's upload route in the shape it expects. - */ -export const tachometerUploadSchema = ciReportUploadSchema('tachometer', 1, tachometerReportSchema); - -export type TachometerUpload = z.infer; -export type TachometerReport = z.infer; -export type ReportRef = z.infer; -export type CaseResult = z.infer; -export type ConfidenceInterval = z.infer; -export type Verdict = z.infer['verdict']; -export type MeasurementResult = z.infer; -export type VariantResult = z.infer; -export type Comparison = z.infer; - -/** - * Uploads a report to the CI report store. - * - * Validates before sending because the server stores `report` without inspecting it — a malformed - * report would be accepted here and only fail much later, when the dashboard tried to render it. - */ -export async function uploadCiReport(upload: TachometerUpload): Promise { - tachometerUploadSchema.parse(upload); - - const responseText = await postToDashboard( - '/api/ci-reports/upload', - upload, - 'The tachometer upload', - ); - const result = JSON.parse(responseText); - console.log(`Tachometer report uploaded. S3 key: ${result.key}`); -} diff --git a/packages/benchmark/src/tachometer/discoverCases.test.ts b/packages/benchmark/src/tachometer/discoverCases.test.ts deleted file mode 100644 index 08d3285563..0000000000 --- a/packages/benchmark/src/tachometer/discoverCases.test.ts +++ /dev/null @@ -1,577 +0,0 @@ -import * as path from 'node:path'; -import { mkdir, writeFile } from 'node:fs/promises'; -import { describe, expect, it } from 'vitest'; -import { makeTempDir } from '../utils/testUtils'; -import { discoverCases, measurementNameOf, pagesOf } from './discoverCases'; - -/** A case folder to create in a throwaway harness. */ -interface CaseFixture { - config: any; - pages?: string[]; -} - -/** Builds a throwaway harness: `src//tachometer.json` plus whatever pages each case owns. */ -async function makeHarness(cases: Record): Promise { - const harnessDir = await makeTempDir(); - await Promise.all( - Object.entries(cases).map(async ([name, { config, pages = [] }]) => { - const caseDir = path.join(harnessDir, 'src', name); - await mkdir(caseDir, { recursive: true }); - await writeFile(path.join(caseDir, 'tachometer.json'), JSON.stringify(config, null, 2)); - await Promise.all( - pages.map((page) => writeFile(path.join(caseDir, page), '\n')), - ); - }), - ); - return harnessDir; -} - -/** A minimal single-benchmark config. */ -function config(name: string, url: string): any { - return { - $schema: '../../node_modules/tachometer/config.schema.json', - benchmarks: [{ name, url }], - }; -} - -describe('discoverCases', () => { - it('finds every case folder holding a tachometer.json', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - beta: { config: config('beta', './index.html'), pages: ['index.html'] }, - }); - - const cases = await discoverCases({ harnessDir }); - - expect(cases.map((entry) => entry.name)).toEqual(['alpha', 'beta']); - }); - - it('narrows to cases matching a filter', async () => { - const harnessDir = await makeHarness({ - 'grid-init': { config: config('grid-init', './index.html'), pages: ['index.html'] }, - 'grid-scroll': { config: config('grid-scroll', './index.html'), pages: ['index.html'] }, - }); - - const cases = await discoverCases({ harnessDir, filters: ['scroll'] }); - - expect(cases.map((entry) => entry.name)).toEqual(['grid-scroll']); - }); - - it('finds a case nested in a folder', async () => { - const harnessDir = await makeHarness({ - 'libs/mount': { config: config('libs-mount', './index.html'), pages: ['index.html'] }, - }); - - const cases = await discoverCases({ harnessDir }); - - expect(cases.map((entry) => entry.name)).toEqual(['libs-mount']); - }); - - it('takes a case name from its benchmark rather than its folder', async () => { - // The folder is where a case lives; the config is what it calls itself. Keeping them separate - // is what lets a case move between folders without being renamed in every report. - const harnessDir = await makeHarness({ - 'libs/mount': { config: config('libs-mount', './index.html'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.name).toBe('libs-mount'); - }); - - it('selects a whole folder by filtering on the path', async () => { - const harnessDir = await makeHarness({ - 'libs/mount': { config: config('libs-mount', './index.html'), pages: ['index.html'] }, - 'libs/scroll': { config: config('libs-scroll', './index.html'), pages: ['index.html'] }, - regression: { config: config('regression', './index.html'), pages: ['index.html'] }, - }); - - const cases = await discoverCases({ harnessDir, filters: ['libs'] }); - - expect(cases.map((entry) => entry.name)).toEqual(['libs-mount', 'libs-scroll']); - }); - - it('matches a filter case-insensitively', async () => { - const harnessDir = await makeHarness({ - GridScroll: { config: config('grid-scroll', './index.html'), pages: ['index.html'] }, - }); - - const cases = await discoverCases({ harnessDir, filters: ['gridscroll'] }); - - expect(cases.map((entry) => entry.name)).toEqual(['grid-scroll']); - }); - - it('refuses two cases that call themselves the same thing', async () => { - // The name identifies a case in the report and in the file its results are written to, so a - // duplicate would silently overwrite rather than fail. - const harnessDir = await makeHarness({ - 'libs/scroll': { config: config('scroll', './index.html'), pages: ['index.html'] }, - scroll: { config: config('scroll', './index.html'), pages: ['index.html'] }, - }); - - await expect(discoverCases({ harnessDir })).rejects.toThrow( - /Two benchmark cases are both named "scroll"/, - ); - }); - - it('strips $schema, which tachometer rejects in a config it is handed', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.config.$schema).toBeUndefined(); - }); - - it('throws when no case matches', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - await expect(discoverCases({ harnessDir, filters: ['nope'] })).rejects.toThrow( - /No benchmark case .* matches "nope"/, - ); - }); - - describe('auto-expansion', () => { - it('expands a benchmark with no variants into current versus baseline', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.config.benchmarks).toEqual([ - { name: 'alpha [current]', url: './index.html' }, - { name: 'alpha [baseline]', url: './index.html' }, - ]); - }); - - it('points the two variants at the two builds', async () => { - // The temp harness is no repository at all, so anything reaching for git would fail here. - // Which build a variant loads follows from the case's shape, which is what lets a plain - // `vite build` run in a checkout with no history. - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.comparison).toBe('baseline'); - expect(entry.leaves.map((leaf) => leaf.ref)).toEqual(['current', 'baseline']); - }); - - it('hands tachometer benchmarks it cannot expand again', async () => { - // The config is parsed afresh, so a benchmark that still carried `expand` would be expanded a - // second time — and an un-rewritten parent url would point at a page that was never built. - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - for (const benchmark of entry.config.benchmarks) { - expect(benchmark).not.toHaveProperty('expand'); - } - }); - - it('gives both variants the same page, query and fragment', async () => { - // Everything after the page path belongs to the page. A parameterised case that lost it on one - // side would compare the parameters against the defaults and call the difference a regression. - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html?rows=100#state'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - const target = 'alpha/index.html?rows=100#state'; - - expect(entry.leaves.map((leaf) => `${leaf.page}${leaf.suffix}`)).toEqual([target, target]); - }); - - it('merges an expansion over its parent, nearest value winning', async () => { - // Tachometer merges an expansion over its parent before it reads anything, so a field set - // deeper wins and one set only on the parent is inherited. Flattening here has to match, or - // the config handed over would describe different benchmarks than the ones discovered. - const harnessDir = await makeHarness({ - libs: { - config: { - benchmarks: [ - { - name: 'libs', - measurement: { mode: 'performance', entryName: 'mount' }, - expand: [{ name: 'libs [ours]', url: './ours.html' }, { url: './theirs.html' }], - }, - ], - }, - pages: ['ours.html', 'theirs.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.config.benchmarks).toEqual([ - { - name: 'libs [ours]', - url: './ours.html', - measurement: { mode: 'performance', entryName: 'mount' }, - }, - { - name: 'libs', - url: './theirs.html', - measurement: { mode: 'performance', entryName: 'mount' }, - }, - ]); - }); - - it('leaves a benchmark that declares its own variants alone', async () => { - const harnessDir = await makeHarness({ - libs: { - config: { - benchmarks: [ - { - name: 'libs', - expand: [ - { name: 'libs [ours]', url: './ours.html' }, - { name: 'libs [theirs]', url: './theirs.html' }, - ], - }, - ], - }, - pages: ['ours.html', 'theirs.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.leaves.map((leaf) => leaf.page)).toEqual(['libs/ours.html', 'libs/theirs.html']); - // Its own variants, so they are compared with each other and all load the working tree. - expect(entry.comparison).toBe('variants'); - expect(entry.leaves.map((leaf) => leaf.ref)).toEqual(['current', 'current']); - }); - }); - - describe('leaf urls', () => { - it('inherits a parent url through a nested expand', async () => { - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [ - { - name: 'alpha', - url: './index.html', - expand: [ - { name: 'alpha [one]', expand: [{ name: 'alpha [one/a]' }] }, - { name: 'alpha [two]', url: './other.html' }, - ], - }, - ], - }, - pages: ['index.html', 'other.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.leaves.map((leaf) => leaf.page)).toEqual([ - 'alpha/index.html', - 'alpha/other.html', - ]); - }); - - it('carries the query and the fragment through as written', async () => { - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [ - { name: 'alpha', expand: [{ name: 'a', url: './index.html?rows=100&cols=8#x' }] }, - ], - }, - pages: ['index.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.leaves[0].suffix).toBe('?rows=100&cols=8#x'); - }); - - it('resolves a case that owns no page but parameterises a sibling', async () => { - const harnessDir = await makeHarness({ - updates: { config: config('updates', './index.html'), pages: ['index.html'] }, - 'updates-throttled': { - config: { - benchmarks: [ - { - name: 'updates-throttled', - expand: [{ name: 'throttled', url: '../updates/index.html?throttle=16' }], - }, - ], - }, - }, - }); - - const cases = await discoverCases({ harnessDir }); - const throttled = cases.find((entry) => entry.name === 'updates-throttled'); - - expect(throttled?.leaves[0]).toMatchObject({ - page: 'updates/index.html', - suffix: '?throttle=16', - }); - }); - - it('rejects a url resolving outside src', async () => { - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [{ name: 'alpha', expand: [{ name: 'a', url: '../../outside.html' }] }], - }, - }, - }); - - await expect(discoverCases({ harnessDir })).rejects.toThrow(/resolves outside/); - }); - - it('rejects a benchmark that names no url', async () => { - // Named for the benchmark the author wrote rather than for a variant they did not, which is - // what the auto-expanded pair would otherwise be reported as. - const harnessDir = await makeHarness({ - alpha: { config: { benchmarks: [{ name: 'alpha' }] }, pages: ['index.html'] }, - }); - - await expect(discoverCases({ harnessDir })).rejects.toThrow( - /Benchmark "alpha" in .* has no "url"/, - ); - }); - - it('rejects a url pointing at a missing page', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './missing.html') }, - }); - - await expect(discoverCases({ harnessDir })).rejects.toThrow(/missing page/); - }); - }); -}); - -describe('pagesOf', () => { - it('deduplicates the pages a case references from several variants', async () => { - // The auto-expanded [current] and [baseline] variants point at the same page. - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - expect(pagesOf(await discoverCases({ harnessDir }))).toEqual(['alpha/index.html']); - }); - - it('unions the pages of every case, sorted', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - libs: { - config: { - benchmarks: [ - { - name: 'libs', - expand: [ - { name: 'libs [ours]', url: './ours.html' }, - { name: 'libs [theirs]', url: './theirs.html' }, - ], - }, - ], - }, - pages: ['ours.html', 'theirs.html'], - }, - }); - - expect(pagesOf(await discoverCases({ harnessDir }))).toEqual([ - 'alpha/index.html', - 'libs/ours.html', - 'libs/theirs.html', - ]); - }); -}); - -describe('measurementNameOf', () => { - it('uses an explicit name', () => { - expect(measurementNameOf({ mode: 'performance', entryName: 'mount', name: 'paint' })).toBe( - 'paint', - ); - }); - - it('uses the expression for an expression measurement', () => { - expect(measurementNameOf({ mode: 'expression', expression: 'window.total' })).toBe( - 'window.total', - ); - }); - - it('names a callback measurement', () => { - expect(measurementNameOf({ mode: 'callback' })).toBe('callback'); - expect(measurementNameOf('callback')).toBe('callback'); - }); - - it('names the "global" shorthand by the expression tachometer expands it to', () => { - // `global` becomes an expression measurement, and a result is named by the expression — not by - // the shorthand — so taking the shorthand at face value would never match its own result. - expect(measurementNameOf('global')).toBe('window.tachometerResult'); - expect(measurementNameOf('global', 'window.myResult')).toBe('window.myResult'); - }); - - it('falls back to the entry name', () => { - expect(measurementNameOf({ mode: 'performance', entryName: 'mount' })).toBe('mount'); - }); - - it('abbreviates first-contentful-paint the way tachometer does', () => { - expect(measurementNameOf({ mode: 'performance', entryName: 'first-contentful-paint' })).toBe( - 'fcp', - ); - }); -}); - -describe('case variants and measurements', () => { - it('lists the auto-expanded variants in order, the reference first', async () => { - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [ - { - name: 'alpha', - url: './index.html', - measurement: { mode: 'performance', entryName: 'mount' }, - }, - ], - }, - pages: ['index.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.variants).toEqual(['alpha [current]', 'alpha [baseline]']); - expect(entry.measurements).toEqual(['mount']); - }); - - it('falls back to the case name when a benchmark names neither itself nor its variants', async () => { - // The case name is already the fallback for a nameless benchmark, so the variants read - // "alpha [current]" rather than "undefined [current]". - const harnessDir = await makeHarness({ - alpha: { - config: { benchmarks: [{ url: './index.html' }] }, - pages: ['index.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.variants).toEqual(['alpha [current]', 'alpha [baseline]']); - }); - - it('takes a variant name from the nearest node that sets one', async () => { - // Tachometer merges an expansion over its parent, so a name set partway down the tree is what - // it reports — reading only the leaf and the benchmark would look up a name it never used. - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [ - { - name: 'alpha', - expand: [ - { name: 'alpha [ours]', expand: [{ url: './index.html' }] }, - { name: 'alpha [theirs]', expand: [{ url: './other.html' }] }, - ], - }, - ], - }, - pages: ['index.html', 'other.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.variants).toEqual(['alpha [ours]', 'alpha [theirs]']); - }); - - it('collects every measurement a benchmark declares', async () => { - // A page with several measurements is exactly the case whose results must be paired by name. - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [ - { - name: 'alpha', - url: './index.html', - measurement: [ - { mode: 'performance', entryName: 'mount' }, - { mode: 'performance', entryName: 'scroll' }, - ], - }, - ], - }, - pages: ['index.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.measurements).toEqual(['mount', 'scroll']); - }); - - it('collects the measurements an expansion declares for itself', async () => { - // Read from the parent, a case whose children each measure something different reports the - // parent's default while tachometer reports theirs, and pairing the two then finds nothing. - const harnessDir = await makeHarness({ - alpha: { - config: { - benchmarks: [ - { - name: 'alpha', - expand: [ - { - name: 'alpha [one]', - url: './index.html', - measurement: { mode: 'performance', entryName: 'mount' }, - }, - { - name: 'alpha [two]', - url: './other.html', - measurement: { mode: 'performance', entryName: 'paint' }, - }, - ], - }, - ], - }, - pages: ['index.html', 'other.html'], - }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.measurements).toEqual(['mount', 'paint']); - }); - - it('refuses two variants that end up with the same name', async () => { - // An expansion that names itself nothing inherits its parent's name, so this is reachable from - // a config that looks fine. Results are paired back by name, so one would shadow the other. - const harnessDir = await makeHarness({ - libs: { - config: { - benchmarks: [ - { name: 'libs', expand: [{ url: './ours.html' }, { url: './theirs.html' }] }, - ], - }, - pages: ['ours.html', 'theirs.html'], - }, - }); - - await expect(discoverCases({ harnessDir })).rejects.toThrow(/has two variants named "libs"/); - }); - - it('defaults to the callback measurement when none is declared', async () => { - const harnessDir = await makeHarness({ - alpha: { config: config('alpha', './index.html'), pages: ['index.html'] }, - }); - - const [entry] = await discoverCases({ harnessDir }); - - expect(entry.measurements).toEqual(['callback']); - }); -}); diff --git a/packages/benchmark/src/tachometer/discoverCases.ts b/packages/benchmark/src/tachometer/discoverCases.ts deleted file mode 100644 index dbf1b1ce47..0000000000 --- a/packages/benchmark/src/tachometer/discoverCases.ts +++ /dev/null @@ -1,321 +0,0 @@ -import * as path from 'node:path'; -import { readFile } from 'node:fs/promises'; -import { globby } from 'globby'; -import { pathExists } from '../utils/path'; - -/** - * Case discovery: the `tachometer.json` files under a harness's `src/` are the source of truth for - * which benchmark cases exist and which pages they reference. - * - * Both the runner and the vite plugin read them through here, so there is exactly one notion of - * "what pages exist". Discovery reads only the filesystem — which builds a case loads follows from - * its shape rather than from git, so a plain `vite build` needs no checkout with history. - */ - -/** Which of a run's two builds a variant loads. */ -export type BuildRef = 'current' | 'baseline'; - -/** - * What a case compares. - * - * `baseline` is the working tree against the baseline build: the regression case, and what a case - * gets when it declares no variants of its own. `variants` is the pages the case names compared - * against each other, every one of them built from the working tree. - */ -export type CaseComparison = 'baseline' | 'variants'; - -export interface Leaf { - /** The config node whose `url` this is; rewritten in place once builds exist. */ - node: { url?: string }; - /** Page path relative to `src`, posix, e.g. `data-grid-init/index.html`. */ - page: string; - /** The build this variant loads. */ - ref: BuildRef; - /** Everything after the page path: query and fragment, as written. */ - suffix: string; -} - -export interface BenchmarkCase { - /** The case name, as its benchmark declares it. */ - name: string; - /** Absolute path to its `tachometer.json`. */ - configPath: string; - /** The parsed config, mutated in place as urls are rewritten. */ - config: any; - /** What the case compares. */ - comparison: CaseComparison; - /** Every node that selects a page. */ - leaves: Leaf[]; - /** Variant names in order, as tachometer will report them; the first is the reference. */ - variants: string[]; - /** Measurement names, as tachometer will name them in its output. */ - measurements: string[]; -} - -export interface DiscoverCasesOptions { - /** The harness package directory; cases live under its `src/`. */ - harnessDir: string; - /** - * Only include cases whose path under `src` contains one of these substrings, - * case-insensitively. - */ - filters?: string[]; -} - -/** The file that marks a directory under `src/` as a benchmark case. */ -const CONFIG_NAME = 'tachometer.json'; - -/** Tachometer's expression when a `global` measurement does not name one. */ -const DEFAULT_MEASUREMENT_EXPRESSION = 'window.tachometerResult'; - -/** - * The name tachometer gives a measurement: an explicit `name`, else the expression, else the entry - * name. - * - * With more than one measurement on a page, tachometer appends ` []` to each benchmark's - * name, and results have to be paired back by that name — pairing by position would compare - * unrelated measurements. - */ -export function measurementNameOf(measurement: any, measurementExpression?: string): string { - if (typeof measurement === 'string') { - // Tachometer's string shorthands. `callback` and `fcp` name their results the same way the - // shorthand reads, but `global` expands to an *expression* measurement that is named by the - // expression itself — so taking the shorthand at face value would look for a result that is - // never emitted under that name. - return measurement === 'global' - ? (measurementExpression ?? DEFAULT_MEASUREMENT_EXPRESSION) - : measurement; - } - if (measurement.name) { - return measurement.name; - } - if (measurement.mode === 'expression') { - return measurement.expression; - } - if (measurement.mode === 'callback') { - return 'callback'; - } - return measurement.entryName === 'first-contentful-paint' ? 'fcp' : measurement.entryName; -} - -/** - * Flattens a benchmark's `expand` tree into the list of benchmarks it stands for, the way tachometer - * does before it reads anything else. - * - * `expand` is recursive and an expansion is merged over its parent, so the nearest value of every - * field wins — `url`, `name`, `measurement`, `browser` alike. Doing that merge once, here, is what - * keeps the rest of discovery dealing in whole benchmarks: per-field inheritance is a rule that only - * has value while it agrees with tachometer's exactly, and re-deriving it once per field is how the - * two drift apart. - * - * `expand` itself is dropped. Tachometer leaves the parent's on the merged object, where it is inert - * because expansion has already happened — but the config written from this is parsed afresh, and a - * benchmark still carrying `expand` would be expanded a second time. - */ -function flattenExpansions(benchmark: any): any[] { - const { expand, ...rest } = benchmark; - if (!Array.isArray(expand) || expand.length === 0) { - return [rest]; - } - return expand.flatMap((child) => flattenExpansions(child).map((leaf) => ({ ...rest, ...leaf }))); -} - -/** - * Resolves one variant's url into the source page it references and whatever follows it. - * - * The path is carried through as written rather than through `URL`: these urls are relative to their - * own config's directory and traverse out of it (`../row-updates/index.html?throttle=16`), and `URL` - * would clamp that traversal against whatever base it was given. Everything after the path belongs - * to the page and survives the rewrite verbatim, or a parameterised benchmark would silently run its - * defaults. - */ -async function parseLeafUrl( - url: string, - configDir: string, - srcDir: string, -): Promise<{ page: string; suffix: string }> { - const [pathPart] = url.split(/[?#]/); - const suffix = url.slice(pathPart.length); - const absolute = path.resolve(configDir, decodeURIComponent(pathPart)); - - const relative = path.relative(srcDir, absolute); - if (relative.startsWith('..') || path.isAbsolute(relative)) { - throw new Error(`Benchmark url "${url}" resolves outside ${srcDir}.`); - } - if (!(await pathExists(absolute))) { - throw new Error(`Benchmark url "${url}" points at a missing page (${absolute}).`); - } - // Posix, because a page path is a url as often as it is a path — the case index links to it and - // rollup names a chunk after it — and node takes forward slashes on Windows for the rest. - return { page: relative.split(path.sep).join('/'), suffix }; -} - -/** - * Every directory under `srcDir` that holds a `tachometer.json`, as a posix path relative to it. - * - * Nested rather than flat, so a harness can group its cases in folders — which is the whole of the - * partitioning story: the filter matches these paths, and nothing else has to understand what a - * folder means. - */ -async function findCaseLocations(srcDir: string): Promise { - const configs = await globby(`**/${CONFIG_NAME}`, { cwd: srcDir }); - return ( - configs - // `dirname` gives `.` for a config sitting directly in `src`; that case's location is the root. - .map((config) => (path.posix.dirname(config) === '.' ? '' : path.posix.dirname(config))) - // Sorted so a run's case order — and therefore its report — does not depend on the order the - // filesystem happened to return. - .sort() - ); -} - -/** - * Reads every `tachometer.json` under `src/` (optionally filtered by location) and resolves the page - * each of its variants loads. - * - * `expand` decides what a case compares. A benchmark that declares one owns its variant axis, and - * every variant loads the working tree — a comparison between pages, such as one library against - * another. A benchmark without one is expanded into `[current]` and `[baseline]` over the same page, - * which is the regression case. - */ -export async function discoverCases(options: DiscoverCasesOptions): Promise { - const { harnessDir, filters = [] } = options; - const srcDir = path.join(harnessDir, 'src'); - - const located = await findCaseLocations(srcDir); - // Matched case-insensitively against the location, the way vitest matches its file filters: a - // folder name selects everything under it without the filter having to know what a folder means. - const selected = located.filter( - (location) => - filters.length === 0 || - filters.some((filter) => location.toLowerCase().includes(filter.toLowerCase())), - ); - - if (selected.length === 0) { - throw new Error( - filters.length > 0 - ? `No benchmark case under ${srcDir} matches ${filters.map((filter) => `"${filter}"`).join(', ')}.` - : `No benchmark case with a ${CONFIG_NAME} found under ${srcDir}.`, - ); - } - - const configs = await Promise.all( - selected.map(async (location) => { - const configPath = path.join(srcDir, location, CONFIG_NAME); - const config = JSON.parse(await readFile(configPath, 'utf8')); - // `$schema` is for editors; tachometer rejects it in a config it is handed. - delete config.$schema; - // The benchmark names itself, so a case keeps its identity wherever its folder is moved to, - // and the ` []` prefix that tachometer builds from that name is strippable for - // display by construction rather than by the folder happening to agree with it. - const name: string = config.benchmarks?.[0]?.name ?? path.basename(location); - return { name, configPath, config }; - }), - ); - - const byName = new Map(); - for (const entry of configs) { - const clash = byName.get(entry.name); - if (clash) { - throw new Error( - `Two benchmark cases are both named "${entry.name}": ${clash} and ${entry.configPath}. ` + - `Names identify a case in the report and in its results file, so they have to be unique.`, - ); - } - byName.set(entry.name, entry.configPath); - } - - const cases: BenchmarkCase[] = []; - for (const { name, configPath, config } of configs) { - const configDir = path.dirname(configPath); - - const measurements = new Set(); - const nodes: Array<{ node: any; ref: BuildRef }> = []; - for (const benchmark of config.benchmarks ?? []) { - // A benchmark need not name itself, and the case name is the same fallback its own name gets - // — without it the auto-expanded variants would read "undefined [current]". Set before - // flattening so every benchmark it expands to inherits it. - benchmark.name ??= name; - - const declaresVariants = Array.isArray(benchmark.expand) && benchmark.expand.length > 0; - const expanded = flattenExpansions(benchmark); - - // Read from the flattened nodes, not the parent: an expansion may set its own `measurement`, - // and a parent that names none would otherwise contribute `callback` while tachometer reports - // what the child asked for. `measurement` may also be a single entry or a list, and a page - // with several is exactly the case whose results are paired by name rather than by position. - for (const node of expanded) { - for (const measurement of [node.measurement ?? 'callback'].flat()) { - measurements.add(measurementNameOf(measurement, node.measurementExpression)); - } - } - if (declaresVariants) { - for (const node of expanded) { - nodes.push({ node, ref: 'current' }); - } - } else { - // Without `expand` exactly one benchmark comes back, and the same page is measured twice. - const [base] = expanded; - if (base.url === undefined) { - throw new Error(`Benchmark "${benchmark.name}" in ${configPath} has no "url".`); - } - nodes.push( - { node: { ...base, name: `${benchmark.name} [current]` }, ref: 'current' }, - { node: { ...base, name: `${benchmark.name} [baseline]` }, ref: 'baseline' }, - ); - } - } - - // A declared `expand` tree may bottom out in a variant that names no url and inherits none. - for (const { node } of nodes) { - if (node.url === undefined) { - throw new Error( - `Benchmark "${node.name}" in ${configPath} expands to a variant with no "url".`, - ); - } - } - - // Results are paired back to what was declared by variant name, and an expansion that sets no - // name of its own inherits its parent's — so two variants can end up sharing one, which would - // quietly report the last one's numbers for both. - const named = new Set(); - for (const { node } of nodes) { - if (named.has(node.name)) { - throw new Error( - `Benchmark case "${name}" has two variants named "${node.name}" (${configPath}). ` + - `Names identify a variant in the report and in its results, so they have to be unique.`, - ); - } - named.add(node.name); - } - - // The flattened list is what tachometer is handed: it expands to exactly this, and every later - // step — the url rewrite, the browser defaults — then addresses one whole benchmark at a time. - config.benchmarks = nodes.map((entry) => entry.node); - - // eslint-disable-next-line no-await-in-loop - const leaves: Leaf[] = await Promise.all( - nodes.map(async ({ node, ref }) => ({ - node, - ref, - ...(await parseLeafUrl(node.url, configDir, srcDir)), - })), - ); - - cases.push({ - name, - configPath, - config, - comparison: leaves.some((leaf) => leaf.ref === 'baseline') ? 'baseline' : 'variants', - leaves, - variants: nodes.map((entry) => entry.node.name as string), - measurements: [...measurements], - }); - } - return cases; -} - -/** The distinct pages referenced by a set of cases, as paths relative to `src`, sorted. */ -export function pagesOf(cases: BenchmarkCase[]): string[] { - return [...new Set(cases.flatMap((entry) => entry.leaves.map((leaf) => leaf.page)))].sort(); -} diff --git a/packages/benchmark/src/tachometer/format.test.ts b/packages/benchmark/src/tachometer/format.test.ts deleted file mode 100644 index 19cca7d4a0..0000000000 --- a/packages/benchmark/src/tachometer/format.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { groupCasesByVariantSet, shortNameOf, variantsOf } from './format'; -import type { SummarizedCase } from './format'; - -function variant(name: string) { - return { - variant: name, - meanMs: { low: 20, high: 22 }, - samples: 120, - }; -} - -function caseWith(name: string, variantNames: string[], measurements = ['mount']): SummarizedCase { - return { - name, - comparison: 'baseline', - reference: `${name} ${variantNames[0]}`, - measurements: measurements.map((measurement) => ({ - name: measurement, - variants: variantNames.map((short) => variant(`${name} ${short}`)), - comparisons: [], - })), - }; -} - -describe('shortNameOf', () => { - it('drops the case prefix tachometer puts on a variant', () => { - expect(shortNameOf('libs-mount', 'libs-mount [ag-grid]')).toBe('[ag-grid]'); - }); - - it('leaves a name that does not carry the prefix alone', () => { - expect(shortNameOf('scroll', 'row-updates [current]')).toBe('row-updates [current]'); - }); -}); - -describe('variantsOf', () => { - it('lists the variants in the order they first appear', () => { - expect(variantsOf(caseWith('libs-mount', ['[mosaic]', '[ag-grid]', '[mui-x]']))).toEqual([ - '[mosaic]', - '[ag-grid]', - '[mui-x]', - ]); - }); - - it('counts a variant once across measurements', () => { - const entry = caseWith('workload', ['[current]', '[baseline]'], ['mount', 'cold-start']); - - expect(variantsOf(entry)).toEqual(['[current]', '[baseline]']); - }); -}); - -describe('groupCasesByVariantSet', () => { - it('puts cases sharing a variant set in one group', () => { - // Auto-expanded cases all carry [current]/[baseline], so they belong in one table. - const groups = groupCasesByVariantSet([ - caseWith('workload', ['[current]', '[baseline]']), - caseWith('scroll', ['[current]', '[baseline]']), - ]); - - expect(groups).toHaveLength(1); - expect(groups[0].cases.map((entry) => entry.name)).toEqual(['workload', 'scroll']); - }); - - it('gives a case with a different variant set its own group', () => { - // A table over the union would leave a blank cell in every row, and could name only one - // reference for both. - const groups = groupCasesByVariantSet([ - caseWith('workload', ['[current]', '[baseline]']), - caseWith('libs-mount', ['[mosaic]', '[ag-grid]']), - ]); - - expect(groups.map((group) => group.variants)).toEqual([ - ['[current]', '[baseline]'], - ['[mosaic]', '[ag-grid]'], - ]); - }); - - it('separates variant sets that differ only in order', () => { - // The first variant is the reference, so the same names in another order is another table. - const groups = groupCasesByVariantSet([ - caseWith('a', ['[current]', '[baseline]']), - caseWith('b', ['[baseline]', '[current]']), - ]); - - expect(groups).toHaveLength(2); - }); - - it('keeps the groups in order of first appearance', () => { - const groups = groupCasesByVariantSet([ - caseWith('libs-mount', ['[mosaic]', '[ag-grid]']), - caseWith('workload', ['[current]', '[baseline]']), - caseWith('libs-scroll', ['[mosaic]', '[ag-grid]']), - ]); - - expect(groups.map((group) => group.cases.map((entry) => entry.name))).toEqual([ - ['libs-mount', 'libs-scroll'], - ['workload'], - ]); - }); -}); diff --git a/packages/benchmark/src/tachometer/format.ts b/packages/benchmark/src/tachometer/format.ts deleted file mode 100644 index 6bdd4f56b8..0000000000 --- a/packages/benchmark/src/tachometer/format.ts +++ /dev/null @@ -1,93 +0,0 @@ -import type { CaseResult, MeasurementResult, ReportRef, ConfidenceInterval } from './ciReport'; - -/** - * How a report's numbers and names are written, for every renderer of one. - * - * The terminal table and the pull request comment show the same run, so they format it here rather - * than each their own way. Pure, and free of anything node-only, so a browser can import it. - */ - -/** A case that produced results, as opposed to one carrying only the error that stopped it. */ -export type SummarizedCase = CaseResult & { measurements: MeasurementResult[] }; - -export interface VariantGroup { - /** The variant set the group's cases share, reference first. */ - variants: string[]; - cases: SummarizedCase[]; -} - -/** `20.50 – 22.67 ms`, with the unit written once. */ -export function formatMean(interval: ConfidenceInterval): string { - return `${interval.low.toFixed(2)} – ${interval.high.toFixed(2)} ms`; -} - -/** Both bounds signed, so the direction of a difference reads without the verdict beside it. */ -function signedInterval(interval: ConfidenceInterval, unit: string, digits: number): string { - const signed = (value: number) => `${value >= 0 ? '+' : ''}${value.toFixed(digits)}${unit}`; - return `${signed(interval.low)} – ${signed(interval.high)}`; -} - -/** `+2.7% – +6.8%`. */ -export function formatPercent(interval: ConfidenceInterval): string { - return signedInterval(interval, '%', 1); -} - -/** `+6.10 ms – +15.20 ms`. */ -export function formatSignedMs(interval: ConfidenceInterval): string { - return signedInterval(interval, ' ms', 2); -} - -/** What a ref is called in a report: the working tree, or the revision the run was given. */ -export function refLabel(ref: ReportRef): string { - return ref.kind === 'worktree' ? 'working tree' : ref.requested; -} - -/** - * Tachometer names each variant ` []`, and the case is named by the table itself, so - * the prefix is dropped for display. Dropping it is also what lets cases share a table: every - * auto-expanded case then contributes the same `[current]`/`[baseline]` pair rather than a pair of - * its own. - */ -export function shortNameOf(caseName: string, variant: string): string { - return variant.startsWith(`${caseName} `) ? variant.slice(caseName.length + 1) : variant; -} - -/** Whether a case produced results, as opposed to carrying only the error that stopped it. */ -export function isSummarized(entry: CaseResult): entry is SummarizedCase { - return entry.measurements !== undefined && entry.measurements.length > 0; -} - -/** The distinct variants a case reports, in order of first appearance. */ -export function variantsOf(entry: SummarizedCase): string[] { - const variants: string[] = []; - for (const measurement of entry.measurements) { - for (const variant of measurement.variants) { - const short = shortNameOf(entry.name, variant.variant); - if (!variants.includes(short)) { - variants.push(short); - } - } - } - return variants; -} - -/** - * Splits cases into the groups that each get their own table. - * - * A regression case carries `[current]`/`[baseline]`, a cross-library case `[ours]`/`[theirs]`, and - * each set has its own reference, the first variant. One table over the union of the columns would - * leave a blank cell in every row, and could name only one reference for all of them. - */ -export function groupCasesByVariantSet(cases: SummarizedCase[]): VariantGroup[] { - const groups = new Map(); - - for (const entry of cases) { - const variants = variantsOf(entry); - const key = JSON.stringify(variants); - const group = groups.get(key) ?? { variants, cases: [] }; - group.cases.push(entry); - groups.set(key, group); - } - - return [...groups.values()]; -} diff --git a/packages/benchmark/src/tachometer/outputDir.ts b/packages/benchmark/src/tachometer/outputDir.ts deleted file mode 100644 index fee71d0e3e..0000000000 --- a/packages/benchmark/src/tachometer/outputDir.ts +++ /dev/null @@ -1,36 +0,0 @@ -import * as path from 'node:path'; -import { mkdir, writeFile } from 'node:fs/promises'; - -/** - * Everything a run writes — packed tarballs, per-ref installs, built pages, the report — lives - * under this one directory inside the harness, so a repository has a single thing to ignore and - * deleting it is the whole reset story. - */ -export const OUTPUT_DIR = '.tachometer'; - -/** - * Creates the output directory, marking it ignored from the inside. - * - * A `.gitignore` holding `*` ignores the directory's contents and itself, the way pytest marks its - * cache, so a harness that forgets to list it does not end up offering built pages and tarballs for - * commit. It does not replace an entry in the repository's own ignore file: tools that read only - * the root one — this repository's ESLint config among them — never see a nested `.gitignore`. - */ -export async function prepareOutputDir(harnessDir: string): Promise { - const outputDir = path.join(harnessDir, OUTPUT_DIR); - await mkdir(outputDir, { recursive: true }); - await writeFile(path.join(outputDir, '.gitignore'), '*\n'); - return outputDir; -} - -/** - * Where the pages built for `refId` go, or the root of every ref's pages when it is omitted. - * - * Named here rather than composed at each call site because the runner and the vite plugin have to - * agree on it: the runner points tachometer's static server at the output directory and rewrites - * each url under this path, and a plain `vite build` has to land somewhere that does not collide. - */ -export function buildsDirOf(harnessDir: string, refId?: string): string { - const builds = path.join(harnessDir, OUTPUT_DIR, 'builds'); - return refId ? path.join(builds, refId) : builds; -} diff --git a/packages/benchmark/src/tachometer/plugin.test.ts b/packages/benchmark/src/tachometer/plugin.test.ts deleted file mode 100644 index 49175b5582..0000000000 --- a/packages/benchmark/src/tachometer/plugin.test.ts +++ /dev/null @@ -1,186 +0,0 @@ -import * as path from 'node:path'; -import { mkdir, symlink, writeFile } from 'node:fs/promises'; -import { describe, expect, it } from 'vitest'; -import { makeTempDir } from '../utils/testUtils'; -import { tachometer } from './plugin'; - -/** A case folder to create in a throwaway harness. */ -interface CaseFixture { - config: any; - pages?: string[]; -} - -/** Builds a throwaway harness with a package.json, so the plugin can read its dependencies. */ -async function makeHarness({ - cases, - dependencies = {}, -}: { - cases: Record; - dependencies?: Record; -}): Promise { - const harnessDir = await makeTempDir(); - await writeFile( - path.join(harnessDir, 'package.json'), - JSON.stringify({ name: 'harness', private: true, dependencies }, null, 2), - ); - await Promise.all( - Object.entries(cases).map(async ([name, { config, pages = [] }]) => { - const caseDir = path.join(harnessDir, 'src', name); - await mkdir(caseDir, { recursive: true }); - await writeFile(path.join(caseDir, 'tachometer.json'), JSON.stringify(config, null, 2)); - await Promise.all( - pages.map((page) => writeFile(path.join(caseDir, page), '\n')), - ); - }), - ); - return harnessDir; -} - -/** Invokes the plugin's `config` hook. */ -async function callConfig( - harnessDir: string, - command: 'build' | 'serve', - userConfig: any = {}, -): Promise { - const plugin = tachometer({ harnessDir }); - const hook = plugin.config as any; - return hook(userConfig, { command, mode: command === 'build' ? 'production' : 'development' }); -} - -const oneCase = { - alpha: { - config: { benchmarks: [{ name: 'alpha', url: './index.html' }] }, - pages: ['index.html'], - }, -}; - -describe('tachometer plugin', () => { - describe('config hook', () => { - it('roots the app at src and declares it multi-page', async () => { - const harnessDir = await makeHarness({ cases: oneCase }); - - const config = await callConfig(harnessDir, 'serve'); - - expect(config.root).toBe(path.join(harnessDir, 'src')); - // Vite defaults to 'spa', whose fallback rewrites unmatched urls toward a root index.html - // that does not exist in this harness. - expect(config.appType).toBe('mpa'); - }); - - it('derives no build input while serving', async () => { - const harnessDir = await makeHarness({ cases: oneCase }); - - const config = await callConfig(harnessDir, 'serve'); - - expect(config.build?.rollupOptions).toBeUndefined(); - }); - - it('points serving at the output directory too, so preview finds the built pages', async () => { - // `vite preview` runs as `serve` but serves `build.outDir`. Leaving it unset there sends - // preview to `/dist` — inside `src/` — and it exits instead of serving the build. - const harnessDir = await makeHarness({ cases: oneCase }); - - const config = await callConfig(harnessDir, 'serve'); - - expect(config.build?.outDir).toBe(path.join(harnessDir, '.tachometer', 'builds', 'manual')); - }); - - it('uses relative asset urls for a build', async () => { - // Tachometer serves the build directory and pages live under //, so absolute - // "/assets/…" paths would 404. - const harnessDir = await makeHarness({ cases: oneCase }); - - const config = await callConfig(harnessDir, 'build'); - - expect(config.base).toBe('./'); - }); - - it('defaults the output directory to the run output folder, outside the source root', async () => { - // Vite would otherwise emit into /dist, i.e. inside src/ next to the case sources. - const harnessDir = await makeHarness({ cases: oneCase }); - - const config = await callConfig(harnessDir, 'build'); - - expect(config.build.outDir).toBe(path.join(harnessDir, '.tachometer', 'builds', 'manual')); - }); - - it('leaves an output directory the caller chose alone', async () => { - // A plugin's returned config is merged *over* the inline config, so defaulting this - // unconditionally would silently override `vite build --outDir` — which is how `tacho run` - // directs each ref's build into its own directory. - const harnessDir = await makeHarness({ cases: oneCase }); - - const config = await callConfig(harnessDir, 'build', { - build: { outDir: '/somewhere/builds/current' }, - }); - - expect(config.build.outDir).toBe('/somewhere/builds/current'); - }); - - it('derives the build input from the discovered cases', async () => { - const harnessDir = await makeHarness({ cases: oneCase }); - - const config = await callConfig(harnessDir, 'build'); - - expect(config.build.rollupOptions.input).toEqual({ - 'alpha/index': path.join(harnessDir, 'src', 'alpha', 'index.html'), - }); - }); - - it('keys several pages in one case folder separately', async () => { - // A distinct key per page is what stops a cross-library case's two pages colliding into one - // entry. - const harnessDir = await makeHarness({ - cases: { - libs: { - config: { - benchmarks: [ - { - name: 'libs', - expand: [ - { name: 'libs [ours]', url: './ours.html' }, - { name: 'libs [theirs]', url: './theirs.html' }, - ], - }, - ], - }, - pages: ['ours.html', 'theirs.html'], - }, - }, - }); - - const config = await callConfig(harnessDir, 'build'); - - expect(Object.keys(config.build.rollupOptions.input).sort()).toEqual([ - 'libs/ours', - 'libs/theirs', - ]); - }); - }); - - describe('workspace link check', () => { - it('passes when the harness links to nothing', async () => { - const harnessDir = await makeHarness({ cases: oneCase }); - - await expect(callConfig(harnessDir, 'serve')).resolves.toBeTruthy(); - }); - - it('explains that the library has to be built when its link dangles', async () => { - // A harness resolves the library through its build output, so before the first build the - // link points at a directory that does not exist yet. - const harnessDir = await makeHarness({ - cases: oneCase, - dependencies: { '@scope/library': 'workspace:*' }, - }); - await mkdir(path.join(harnessDir, 'node_modules', '@scope'), { recursive: true }); - await symlink( - path.join(harnessDir, 'packages', 'library', 'build'), - path.join(harnessDir, 'node_modules', '@scope', 'library'), - ); - - await expect(callConfig(harnessDir, 'serve')).rejects.toThrow( - /"@scope\/library" is linked into .* but its target does not exist/, - ); - }); - }); -}); diff --git a/packages/benchmark/src/tachometer/plugin.ts b/packages/benchmark/src/tachometer/plugin.ts deleted file mode 100644 index 0d2a7b72e6..0000000000 --- a/packages/benchmark/src/tachometer/plugin.ts +++ /dev/null @@ -1,242 +0,0 @@ -import * as path from 'node:path'; -import { readFile, realpath } from 'node:fs/promises'; -import type { Plugin } from 'vite'; -import { discoverCases, pagesOf } from './discoverCases'; -import type { BenchmarkCase } from './discoverCases'; -import { buildsDirOf, prepareOutputDir, OUTPUT_DIR } from './outputDir'; - -/** - * Vite plugin for a tachometer benchmark harness. - * - * The harness keeps its own vite config and runs plain `vite` / `vite build` / `vite preview`; this - * plugin only contributes what is derived from the benchmark cases: - * - * - `root` is the harness's `src/`, and the app is an MPA (vite defaults to `spa`, whose fallback - * would rewrite unmatched urls toward a root `index.html` that does not exist here). - * - The build's entry points come from the `tachometer.json` files, so the configs are the single - * source of truth for which pages exist. - * - The dev server serves a generated index listing every case, including cases that own no page of - * their own and only parameterise a sibling's. - * - * It deliberately never resolves refs, so a plain `vite build` needs no git history. - */ - -export interface TachometerPluginOptions { - /** - * The harness package directory. Defaults to the current working directory, which is where vite - * is run from. - */ - harnessDir?: string; -} - -/** - * Checks that the workspace packages the harness links to have actually been built. - * - * A harness depends on the library under test via `workspace:*`, and `publishConfig.directory` - * makes that link point at the package's build output — which does not exist until the library is - * built. The symlink is then dangling, and vite's failure ("failed to resolve import") does not - * say why. - */ -async function assertWorkspaceDepsBuilt( - harnessDir: string, - deps: Record, -): Promise { - const linked = Object.entries(deps).filter(([, version]) => version.startsWith('workspace:')); - await Promise.all( - linked.map(async ([name]) => { - try { - await realpath(path.join(harnessDir, 'node_modules', ...name.split('/'))); - } catch { - throw new Error( - `"${name}" is linked into ${harnessDir} but its target does not exist. ` + - `Build the workspace packages first (e.g. \`pnpm release:build\`) — the harness ` + - `resolves the library through its build output, not its source.`, - ); - } - }), - ); -} - -/** A case name and a query both come from a config file, so neither is trusted as markup. */ -function escapeHtml(text: string): string { - return text - .replace(/&/g, '&') - .replace(//g, '>') - .replace(/"/g, '"'); -} - -/** - * Renders the case index: one link per case, carrying whatever query the case parameterises its - * page with. - */ -function renderIndex(cases: BenchmarkCase[], note: string): string { - const items = cases - .map((entry) => { - // A case's variants can point at several pages (a cross-library comparison) or at one page - // with different queries (a parameterised case), so dedupe on the pair. - const targets = [...new Set(entry.leaves.map((leaf) => `${leaf.page}${leaf.suffix}`))].sort(); - const links = targets - .map( - (target) => - `
  • ${escapeHtml(target)}
    • `, - ) - .join('\n '); - return `
  • \n ${escapeHtml(entry.name)}\n
      \n ${links}\n
    \n
    • `; - }) - .join('\n'); - - return ` - - - - Tachometer benchmark cases - - - -

    Tachometer benchmark cases

    -

    ${note}

    -
      -${items} -
    - - -`; -} - -/** - * Whether a case's own page occupies the `index.html` the case list would be served at. - * - * A `tachometer.json` directly in `src/` is a supported layout, and such a case may own the root - * page. Asked in one place because the build and the dev server both have to give the same answer: - * otherwise `/` shows the case under one command and the list under the other. - */ -function aCaseOwnsIndex(cases: BenchmarkCase[]): boolean { - return pagesOf(cases).includes('index.html'); -} - -/** Creates the tachometer harness plugin. */ -export function tachometer(options: TachometerPluginOptions = {}): Plugin { - const harnessDir = options.harnessDir ?? process.cwd(); - const srcDir = path.join(harnessDir, 'src'); - let buildCases: BenchmarkCase[] = []; - - return { - name: 'mui-benchmark:tachometer', - - async config(userConfig, env) { - const pkg = JSON.parse(await readFile(path.join(harnessDir, 'package.json'), 'utf8')); - await assertWorkspaceDepsBuilt(harnessDir, { - ...pkg.dependencies, - ...pkg.devDependencies, - }); - - // The run output directory, next to the per-ref directories the runner writes — vite would - // otherwise default to `/dist`, i.e. inside `src/` next to the case sources. - // - // Resolved for every command, not just `build`: `vite preview` runs as `serve` but serves - // `build.outDir`, so leaving it unset there sends preview looking for `/dist` and it - // exits rather than serving the pages that were just built. - // - // Only fill it in when the caller said nothing: a plugin's returned config is merged *over* - // the inline config, so unconditionally setting it here would silently override - // `vite build --outDir`, which is exactly how `tacho run` directs each ref's build. - const outDir = userConfig.build?.outDir ?? buildsDirOf(harnessDir, 'manual'); - - if (env.command !== 'build') { - return { root: srcDir, appType: 'mpa', build: { outDir } }; - } - - // Mark the output directory ignored from the inside before the build writes into it, so a - // harness that does not list `.tachometer` is not offered its built pages for commit. Keyed on - // where the output actually lands rather than on who chose it: `tacho run` passes its own - // `outDir` under the same directory and prepares it itself, and any other caller that writes - // there deserves the marker too. Vite resolves a relative `outDir` against `root`. - if (path.resolve(srcDir, outDir).startsWith(path.join(harnessDir, OUTPUT_DIR))) { - await prepareOutputDir(harnessDir); - } - - const cases = await discoverCases({ harnessDir }); - buildCases = cases; - // Key the input map by the page path without its extension. Vite emits each page at its path - // relative to `root` regardless, so `/index.html` lands at `//index.html`; - // a distinct key per page is what lets one case folder hold several pages — a cross-library - // comparison with `mosaic.html` and `tanstack.html` side by side — without them colliding. - const input = Object.fromEntries( - pagesOf(cases).map((page) => [page.replace(/\.html$/, ''), path.join(srcDir, page)]), - ); - - return { - root: srcDir, - appType: 'mpa', - // Relative asset urls: tachometer serves the build directory and the pages live under - // `//`, so absolute "/assets/…" paths would 404. - base: './', - build: { - outDir, - // The output directory sits outside `root`, which is `src/`; allow vite to clean it anyway. - emptyOutDir: true, - chunkSizeWarningLimit: 9999, - rollupOptions: { input }, - }, - }; - }, - - // The same index the dev server renders, emitted as a real file so the built directory is - // navigable too — under `vite preview`, under any static server, or opened from disk. A page-less - // case is only reachable through this list, so a build without it can only be entered by typing - // urls from memory. - generateBundle() { - // Rollup drops an emit whose name a bundle output already claims, with no error and no - // warning, so the list would simply vanish. Say so instead. - if (aCaseOwnsIndex(buildCases)) { - this.warn( - 'A case owns index.html, so the case list was not emitted. Move the case into a subfolder of src/ to get it back.', - ); - return; - } - - this.emitFile({ - type: 'asset', - fileName: 'index.html', - source: renderIndex( - buildCases, - 'These are production bundles — the same build tachometer measures.', - ), - }); - }, - - configureServer(server) { - server.middlewares.use(async (req, res, next) => { - const url = (req.url ?? '/').split('?')[0]; - if (url !== '/' && url !== '/index.html') { - next(); - return; - } - try { - const cases = await discoverCases({ harnessDir }); - if (aCaseOwnsIndex(cases)) { - // Let vite serve the case's own page, the way the build does. - next(); - return; - } - res.setHeader('Content-Type', 'text/html; charset=utf-8'); - res.end( - renderIndex( - cases, - "Pages are served from the working tree's build of the library. Rebuild it and reload to pick up a source change.", - ), - ); - } catch (error) { - next(error); - } - }); - }, - }; -} diff --git a/packages/benchmark/src/tachometer/refs.test.ts b/packages/benchmark/src/tachometer/refs.test.ts deleted file mode 100644 index 4d55ec205c..0000000000 --- a/packages/benchmark/src/tachometer/refs.test.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { execa } from 'execa'; -import { makeTempDir } from '../utils/testUtils'; -import { resolveBaselineRef } from './refs'; - -/** - * A repository with three commits on `main`, the first of them carrying an annotated tag. - * - * A real repository rather than a stand-in for git: every behaviour here is a question about what - * git reports — what a tag resolves to, which refnames exist — and a fake would only restate the - * answer the code already assumes. - */ -async function makeRepo(): Promise<{ repoRoot: string; shas: string[] }> { - const repoRoot = await makeTempDir(); - // Identity through the environment rather than `git config`, so the repo needs no extra processes - // and nothing depends on the machine's own git configuration. - const git = (...args: string[]) => - execa('git', args, { - cwd: repoRoot, - env: { - GIT_AUTHOR_NAME: 'Fixture', - GIT_AUTHOR_EMAIL: 'fixture@example.com', - GIT_COMMITTER_NAME: 'Fixture', - GIT_COMMITTER_EMAIL: 'fixture@example.com', - }, - }); - - await git('init', '--initial-branch=main'); - // Empty commits: nothing here reads a file, only the commits' identities matter. - for (const message of ['first', 'second', 'third']) { - // eslint-disable-next-line no-await-in-loop - await git('commit', '--allow-empty', '-m', message); - } - const { stdout } = await git('log', '--format=%H', '--reverse'); - const shas = stdout.trim().split('\n'); - - // Annotated (`-m`), so the tag is its own object with its own SHA. - await git('tag', '-a', 'v1.0.0', '-m', 'release', shas[0]); - - return { repoRoot, shas }; -} - -describe('resolveBaselineRef', () => { - it('takes a bare value as a revision', async () => { - const { repoRoot, shas } = await makeRepo(); - - const ref = await resolveBaselineRef(shas[0], repoRoot); - - expect(ref.sha).toBe(shas[0]); - }); - - it('takes the same revision behind the git scheme', async () => { - const { repoRoot, shas } = await makeRepo(); - - const ref = await resolveBaselineRef(`git:${shas[0]}`, repoRoot); - - expect(ref.sha).toBe(shas[0]); - }); - - it('resolves an annotated tag to the commit it points at', async () => { - const { repoRoot, shas } = await makeRepo(); - - const ref = await resolveBaselineRef('v1.0.0', repoRoot); - - // Without `^{commit}` this is the tag object's own SHA, which is not a commit at all. - expect(ref.sha).toBe(shas[0]); - }); - - it('falls back to the previous commit when nothing names one', async () => { - // Which commit a branch should be compared against is `code-infra baseline`'s question, and the - // harness scripts pass the answer in. Unbound, the previous commit is the only choice here that - // needs no policy of its own. - const { repoRoot, shas } = await makeRepo(); - - const ref = await resolveBaselineRef(undefined, repoRoot); - - expect(ref.sha).toBe(shas[1]); - }); - - it('keeps the revision it was given, without the scheme', async () => { - const { repoRoot } = await makeRepo(); - - // The report shows this, so a tag stays a tag rather than becoming the SHA it points at. - expect((await resolveBaselineRef('v1.0.0', repoRoot)).requested).toBe('v1.0.0'); - expect((await resolveBaselineRef('git:v1.0.0', repoRoot)).requested).toBe('v1.0.0'); - }); - - it('names a revision and a tag the same build, so the commit is built once', async () => { - const { repoRoot, shas } = await makeRepo(); - - const viaTag = await resolveBaselineRef('v1.0.0', repoRoot); - const viaSha = await resolveBaselineRef(shas[0], repoRoot); - - // `id` is the build directory: two ids for one commit means packing and building it twice. - expect(viaTag.id).toBe(viaSha.id); - }); - - it.each(['github:owner/repo#abc1234', 'preview:abc1234'])( - 'reports %s as recognised but not implemented', - async (token) => { - const { repoRoot } = await makeRepo(); - - await expect(() => resolveBaselineRef(token, repoRoot)).rejects.toThrow( - /recognised but not implemented/, - ); - }, - ); - - it('rejects the git scheme with no revision after it', async () => { - const { repoRoot } = await makeRepo(); - - await expect(() => resolveBaselineRef('git:', repoRoot)).rejects.toThrow(/names no revision/); - }); - - it('reports what git said about a revision it cannot resolve', async () => { - const { repoRoot } = await makeRepo(); - - // Git is the authority on what a revision is, so an unknown one surfaces as its own failure - // rather than as a grammar this code would have to keep in step with git's. - await expect(() => resolveBaselineRef('no-such-thing', repoRoot)).rejects.toThrow( - /Could not resolve git ref "no-such-thing"/, - ); - }); -}); diff --git a/packages/benchmark/src/tachometer/refs.ts b/packages/benchmark/src/tachometer/refs.ts deleted file mode 100644 index ac94c1c6c8..0000000000 --- a/packages/benchmark/src/tachometer/refs.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { resolveCommit } from '../utils/git'; - -/** - * The builds a run measures: the working tree, and the baseline it is compared against. - * - * Those are the only two. A case either compares the working tree against the baseline or compares - * the pages it names against each other, all built from the working tree — so the only build anyone - * chooses is the baseline, and `--baseline` is where it is named. - */ - -/** - * A build with its identity. The working tree has no commit behind it and a git revision always - * does, so the two carry different fields rather than sharing optional ones a reader has to pair - * with the right `kind` by hand. - * - * `id` is the build directory name — `current`, or `git-230342ee2`. `requested` is the revision as - * it was given, which is what a report shows. - */ -export type ResolvedRef = - | { kind: 'worktree'; id: string; sha?: undefined; requested?: undefined } - | { kind: 'git'; id: string; sha: string; requested: string }; - -/** The working tree, which every run builds. Never cached, having no immutable identity. */ -export const WORKTREE_REF: ResolvedRef = { kind: 'worktree', id: 'current' }; - -/** Schemes a baseline may eventually name, each rejected until it does something. */ -const RESERVED_SCHEMES = ['github', 'preview']; - -/** - * The revision a baseline token names. - * - * A bare value is a revision and `git:` says so explicitly, which leaves room for a baseline that is - * not a commit in this repository — a published preview, another repository — to arrive as its own - * scheme without the flag changing shape. Anything else goes to git, which is the authority on what - * a revision is. - */ -function committishOf(token: string | undefined): string { - if (!token) { - // Which commit a pull request should be measured against is not decided here: `code-infra - // baseline` answers that for every job that compares against a base — bundle size as well as - // benchmarks — and the harness scripts pass the answer in. Without one, the previous commit is - // the only choice that needs no policy. - return 'HEAD~1'; - } - if (token.startsWith('git:')) { - const revision = token.slice('git:'.length); - if (!revision) { - throw new Error('The baseline "git:" names no revision.'); - } - return revision; - } - const reserved = RESERVED_SCHEMES.find((scheme) => token.startsWith(`${scheme}:`)); - if (reserved) { - throw new Error( - `Baseline scheme "${reserved}:" is recognised but not implemented yet (in "${token}"). ` + - `Pass a git revision, on its own or as "git:".`, - ); - } - return token; -} - -/** Resolves the baseline to the commit behind it, so a run stays interpretable after the fact. */ -export async function resolveBaselineRef( - token: string | undefined, - repoRoot: string, -): Promise { - const committish = committishOf(token); - const sha = await resolveCommit(repoRoot, committish); - return { kind: 'git', id: `git-${sha.slice(0, 9)}`, sha, requested: committish }; -} diff --git a/packages/benchmark/src/tachometer/renderReport.ts b/packages/benchmark/src/tachometer/renderReport.ts deleted file mode 100644 index 1ad46cd643..0000000000 --- a/packages/benchmark/src/tachometer/renderReport.ts +++ /dev/null @@ -1,241 +0,0 @@ -/* eslint-disable no-console -- a reporter: printing the report is its output. */ - -// Column widths have to measure visible characters, so chalk's escapes come off first. -import { stripVTControlCharacters as stripAnsi } from 'node:util'; -import chalk from 'chalk'; -import { - formatMean, - formatPercent, - groupCasesByVariantSet, - isSummarized, - refLabel, - shortNameOf, -} from './format'; -import type { SummarizedCase } from './format'; -import type { TachometerReport, Verdict } from './ciReport'; - -/** The report's types, from the schema that defines it. */ -export type { - TachometerReport, - CaseResult, - MeasurementResult, - VariantResult, - Comparison, - ReportRef, - ConfidenceInterval, - Verdict, -} from './ciReport'; - -/** Prints a table, padding each column to its widest visible cell. */ -function printTable(headers: string[], rows: string[][], leftColumns: number): void { - const widths = headers.map((header, index) => - Math.max(stripAnsi(header).length, ...rows.map((row) => stripAnsi(row[index] ?? '').length)), - ); - const pad = (text: string, width: number, left: boolean) => { - const fill = ' '.repeat(Math.max(0, width - stripAnsi(text).length)); - return left ? text + fill : fill + text; - }; - const line = (cells: string[]) => - cells - .map((cell, index) => pad(cell ?? '', widths[index], index < leftColumns)) - .join(' ') - .trimEnd(); - - console.log(line(headers)); - console.log(widths.map((width) => '─'.repeat(width)).join(' ')); - for (const row of rows) { - console.log(line(row)); - } -} - -function colorVerdict(verdict: Verdict, text: string): string { - if (verdict === 'faster') { - return chalk.green(text); - } - if (verdict === 'slower') { - return chalk.red(text); - } - return chalk.dim(text); -} - -/** - * Renders one case as a table of variants, for a comparison with more than two of them. - * - * Rows are variants, not cases: with several libraries, a column per variant plus a Δ column per - * pair would run off the screen. Each measurement then shows the variant's interval next to its - * difference *relative to the reference* — the direction a row about that library reads in — taken - * from the variant's own row of tachometer's matrix. - */ -function printVariantTable(entry: SummarizedCase, variants: string[]): void { - const [reference] = variants; - const headers = [ - entry.name, - ...entry.measurements.flatMap((measurement) => [measurement.name, `vs ${reference}`]), - 'Samples', - ]; - - const rows = variants.map((variant) => { - // Looked up once per row rather than accumulated while the measurement cells render, so the - // last two columns do not depend on the cell loop having run. - const perMeasurement = entry.measurements.map((measurement) => ({ - found: measurement.variants.find( - (candidate) => shortNameOf(entry.name, candidate.variant) === variant, - ), - comparison: measurement.comparisons.find( - (candidate) => shortNameOf(entry.name, candidate.variant) === variant, - ), - })); - const sampleCounts = new Set( - perMeasurement.flatMap(({ found }) => (found ? [found.samples] : [])), - ); - - const cells = perMeasurement.flatMap(({ found, comparison }) => { - let delta = chalk.dim('—'); - if (variant === reference) { - delta = chalk.dim('reference'); - } else if (comparison?.versusReference) { - const { verdict, percentChange } = comparison.versusReference; - delta = colorVerdict(verdict, `${verdict} ${formatPercent(percentChange)}`); - } - return [found ? formatMean(found.meanMs) : chalk.dim('—'), delta]; - }); - - return [variant, ...cells, [...sampleCounts].join('/')]; - }); - - printTable(headers, rows, 1); - console.log( - chalk.dim( - `vs ${reference} is tachometer's confidence interval on the difference, that variant ` + - `relative to ${reference} — negative is faster than ${reference}.`, - ), - ); -} - -/** Renders a group of cases sharing a variant set, one row per case and measurement. */ -function printCaseTable(cases: SummarizedCase[], variants: string[]): void { - const [reference, ...others] = variants; - const headers = [ - 'Case', - 'Measurement', - ...variants, - ...others.map((variant) => `Δ vs ${variant}`), - 'Samples', - ]; - - const rows: string[][] = []; - for (const [caseIndex, entry] of cases.entries()) { - if (caseIndex > 0) { - rows.push(headers.map(() => '')); - } - let firstRow = true; - for (const measurement of entry.measurements) { - const byVariant = new Map( - measurement.variants.map((variant) => [shortNameOf(entry.name, variant.variant), variant]), - ); - const byComparison = new Map( - measurement.comparisons.map((comparison) => [ - shortNameOf(entry.name, comparison.variant), - comparison, - ]), - ); - // Auto-sampling stops per case, not per variant, so the counts are normally equal; when they - // are not, saying so beats printing one number that describes neither side. - const sampleCounts = [...new Set(measurement.variants.map((variant) => variant.samples))]; - - rows.push([ - firstRow ? entry.name : '', - measurement.name, - ...variants.map((variant) => { - const found = byVariant.get(variant); - return found ? formatMean(found.meanMs) : chalk.dim('—'); - }), - ...others.map((variant) => { - const comparison = byComparison.get(variant); - if (!comparison) { - return chalk.dim('—'); - } - return colorVerdict( - comparison.verdict, - `${comparison.verdict} ${formatPercent(comparison.percentChange)}`, - ); - }), - sampleCounts.join('/'), - ]); - firstRow = false; - } - } - - printTable(headers, rows, 2); - console.log( - chalk.dim( - `Δ is tachometer's confidence interval on the difference, ${reference ?? 'the reference'} ` + - `relative to the other variant${others.length > 1 ? 's' : ''} — negative is faster.`, - ), - ); -} - -/** - * Prints a tachometer report: one table per variant set, with a column per variant holding its 95% - * confidence interval and a Δ column per non-reference variant. - * - * The Δ column is **tachometer's own confidence interval on the difference**, not a delta of the - * two means. That is the whole reason this suite exists next to a profiling one: variants are - * sampled round-robin in a single session and auto-sampled until that interval clears the case's - * `autoSampleConditions`, so the interval — not a point estimate plus a post-hoc test — is the - * result. `unsure` means it never resolved within the sampling budget, which for two builds that - * really are equivalent is the expected outcome rather than a failure. - * - * Signs are the reference relative to the variant, so a faster reference reads negative; the - * verdict word carries the direction so the sign never has to be read twice. - */ -export function renderTachometerReport(report: TachometerReport): void { - // One predicate for both halves: partitioning on two conditions that have to agree left a case - // with an empty `measurements` array rendered nowhere and reported nowhere. - const usable = report.cases.filter(isSummarized); - const failed = report.cases.filter((entry) => !isSummarized(entry)); - - if (usable.length === 0) { - console.log(chalk.dim('No tachometer results to show.')); - for (const entry of failed) { - console.log(chalk.red(` ${entry.name}: ${entry.error ?? 'no result'}`)); - } - return; - } - - for (const [groupIndex, group] of groupCasesByVariantSet(usable).entries()) { - if (groupIndex > 0) { - console.log(''); - } - if (group.variants.length > 2) { - for (const [caseIndex, entry] of group.cases.entries()) { - if (caseIndex > 0) { - console.log(''); - } - printVariantTable(entry, group.variants); - } - continue; - } - printCaseTable(group.cases, group.variants); - } - - const notes = [ - 'Each cell is a 95% confidence interval for the mean, in milliseconds; one sample = one page load.', - '"unsure" means the interval still straddles zero: the difference did not resolve within the case\'s sampling budget.', - ]; - - // No short SHA appended: `id` already ends in one. - const refLabels = report.refs.map((ref) => `${ref.id} = ${refLabel(ref)}`); - if (refLabels.length > 0) { - notes.push(`Builds: ${refLabels.join(' · ')}`); - } - notes.push( - `head: ${report.head.sha.slice(0, 9)} (${report.head.branch || '?'}) · ` + - `measured on the production bundle, installed from a packed tarball.`, - ); - console.log(chalk.dim(`\n${notes.join('\n')}`)); - - for (const entry of failed) { - console.log(chalk.red(`\n${entry.name}: ${entry.error ?? 'no result'}`)); - } -} diff --git a/packages/benchmark/src/tachometer/runTachometer.ts b/packages/benchmark/src/tachometer/runTachometer.ts deleted file mode 100644 index 45ffd3ccc2..0000000000 --- a/packages/benchmark/src/tachometer/runTachometer.ts +++ /dev/null @@ -1,322 +0,0 @@ -/* eslint-disable no-console */ - -import * as path from 'node:path'; -import * as os from 'node:os'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import chalk from 'chalk'; -import { execaSync } from 'execa'; -import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; -import { packRef, packWorkingTree } from '../utils/packWorkspace'; -import type { PackedPackage } from '../utils/packWorkspace'; -import { resolveBaselineRef, WORKTREE_REF } from './refs'; -import type { ResolvedRef } from './refs'; -import { discoverCases, pagesOf } from './discoverCases'; -import type { BenchmarkCase, Leaf } from './discoverCases'; -import { isSummarized, refLabel } from './format'; -import { buildRefPages, restoreWorkspace } from './buildPages'; -import type { ResolveMode } from './buildPages'; -import { assertDriverMatchesBrowser, resolveBrowserBinary, withBrowserDefaults } from './browser'; -import { summarizeCase } from './summarizeCase'; -import { renderTachometerReport } from './renderReport'; -import { getCiMetadata } from '../ciReport'; -import { syncPrComment } from '../syncPrComment'; -import { uploadCiReport } from './ciReport'; -import type { CaseResult, TachometerReport } from './ciReport'; -import { buildsDirOf, prepareOutputDir } from './outputDir'; -import { run } from '../utils/exec'; - -/** - * A case's name as a filename component. Names come from the benchmark's own `name`, so they are - * prose rather than identifiers and may hold anything — a slash included. - */ -function fileSlugOf(name: string): string { - return name.replace(/[^a-zA-Z0-9._-]+/g, '-'); -} - -/** - * The builds a run measures, and which of them a given variant loads. - * - * The baseline is resolved only when a case compares against it: a harness that only puts pages - * side by side needs no second commit, and asking for one would fail in a clone too shallow to hold - * it. The lookup comes back with the list so that "there is no baseline" is settled once, here, - * rather than being a case every later step has to allow for. - */ -async function resolveBuilds( - cases: BenchmarkCase[], - baseline: string | undefined, - repoRoot: string, -): Promise<{ refs: ResolvedRef[]; buildFor: (leaf: Leaf) => ResolvedRef }> { - if (!cases.some((entry) => entry.comparison === 'baseline')) { - return { refs: [WORKTREE_REF], buildFor: () => WORKTREE_REF }; - } - const baselineRef = await resolveBaselineRef(baseline, repoRoot); - return { - refs: [WORKTREE_REF, baselineRef], - buildFor: (leaf) => (leaf.ref === 'baseline' ? baselineRef : WORKTREE_REF), - }; -} - -export interface RunTachometerOptions { - /** The harness package directory (where the command was run). */ - harnessDir: string; - /** - * Only run cases whose path under `src` contains one of these substrings, case-insensitively. - */ - filters?: string[]; - /** - * The build every `[baseline]` variant loads: a revision, on its own or as `git:`. Defaults - * to `HEAD~1`. - */ - baseline?: string; - /** - * Command that builds the publishable workspace packages. Run for every ref — in each ref's own - * checkout, and in the working tree — so both sides of a comparison are built the same way. - * Defaults to `pnpm release:build`. - */ - buildCmd?: string; - /** Whether to install inside a ref's checkout. Defaults to true. */ - install?: boolean; - /** Where to write the combined JSON report. Defaults to `.tachometer/results/report.json`. */ - out?: string; - /** Upload the report and refresh the pull request comment. */ - upload?: boolean; - /** How a ref's pages find the library under test. Defaults to `in-place`. */ - resolveMode?: ResolveMode; -} - -/** - * Benchmarks the harness's pages across one or more builds of the workspace with tachometer, and - * writes a combined JSON report. - * - * Cases are the source of truth: the selected `tachometer.json` files determine which pages to - * build, one variant is built per distinct ref, each url is then rewritten to that ref's built page, - * and each case runs as its own tachometer invocation so its result is a clean table for that - * scenario rather than a grid comparing unrelated scenarios against each other. - */ -export async function runTachometer(options: RunTachometerOptions): Promise { - const { - harnessDir, - filters = [], - baseline, - buildCmd = 'pnpm release:build', - install = true, - out, - upload = false, - resolveMode = 'in-place', - } = options; - - const repoRoot = await findWorkspaceDir(harnessDir); - if (!repoRoot) { - throw new Error(`Could not find a pnpm workspace root above ${harnessDir}.`); - } - - // Everything a run writes goes under one directory, so a harness has one thing to ignore and - // deleting it is the whole reset story. `packed` holds tarballs — a ref's keyed by commit SHA, - // the working tree's by content hash — and is the one worth caching in CI; `trees` holds the - // install each ref's pages resolve through, cheap to recreate and not portable between - // containers, since its links point into a pnpm store; `builds` holds the pages built per ref; - // `results` the report. - const outputDir = await prepareOutputDir(harnessDir); - const buildsDir = buildsDirOf(harnessDir); - const packedDir = path.join(outputDir, 'packed'); - const treesDir = path.join(outputDir, 'trees'); - - // A run killed outright — Ctrl-C, not a thrown error — never reaches the restore below, leaving - // the repository pinned to tarballs. The copy it left behind is how that is noticed, so put it - // back before doing anything else, whatever mode this run is in. - await restoreWorkspace(repoRoot, outputDir); - - const cases = await discoverCases({ harnessDir, filters }); - const { refs, buildFor } = await resolveBuilds(cases, baseline, repoRoot); - - // The commit follows the name, so a revision like `HEAD~1` still says which commit it landed on. - const describeRef = (ref: ResolvedRef) => - ref.sha ? `${refLabel(ref)} (${ref.sha.slice(0, 9)})` : refLabel(ref); - - // Before any build: a driver that cannot open the browser fails the run either way, and finding - // out now costs seconds instead of minutes of packing and installing. - const browserBinary = await resolveBrowserBinary(harnessDir); - assertDriverMatchesBrowser(harnessDir, browserBinary); - // `getuid` is POSIX-only; on Windows nobody is root. - const asRoot = process.getuid?.() === 0; - - // What each case compares, so one that carries no baseline says so up front rather than through - // an empty regression list at the end. - const describeCase = (entry: BenchmarkCase) => - entry.comparison === 'baseline' - ? `${entry.name} (vs baseline)` - : `${entry.name} (${entry.variants.length} variants)`; - - console.log(chalk.cyan(`Cases: ${cases.map(describeCase).join(', ')}`)); - console.log(chalk.cyan(`Pages: ${pagesOf(cases).join(', ')}`)); - console.log(chalk.cyan(`Refs: ${refs.map(describeRef).join(', ')}`)); - console.log( - chalk.cyan(`Browser: ${browserBinary}${asRoot ? ' (as root, so without its sandbox)' : ''}`), - ); - - const tmpBase = await mkdtemp(path.join(os.tmpdir(), 'tacho-')); - - try { - // Build one variant per distinct ref, each resolving through its own install so both sides - // of a comparison resolve the library identically. - for (const ref of refs) { - let packages: PackedPackage[]; - if (ref.kind === 'worktree') { - // eslint-disable-next-line no-await-in-loop - packages = await packWorkingTree({ - repoRoot, - outRoot: path.join(packedDir, 'current'), - buildCmd, - }); - } else { - // packRef caches a ref's tarballs by SHA; a hit skips the checkout, install, and build. - // eslint-disable-next-line no-await-in-loop - const packed = await packRef({ - repoRoot, - ref: ref.sha, - outRoot: packedDir, - // An empty install command means "skip"; otherwise packRef's default install runs. - installCmd: install ? undefined : '', - buildCmd, - }); - packages = packed.packages; - } - - // eslint-disable-next-line no-await-in-loop - await buildRefPages({ - harnessDir, - repoRoot, - ref, - packages, - treeDir: path.join(treesDir, ref.id), - outputDir, - outDir: path.join(buildsDir, ref.id), - resolveMode, - }); - } - - // Rewrite each leaf url to its ref's built page, then run each case's own config. Tachometer - // resolves relative urls against the config file's directory and these configs are written to a - // temp dir, so the rewritten urls are absolute. - const results: Array<{ entry: BenchmarkCase; json: any }> = []; - for (const entry of cases) { - for (const leaf of entry.leaves) { - leaf.node.url = `${path.join(buildsDir, buildFor(leaf).id, leaf.page)}${leaf.suffix}`; - } - // Discovery flattened `expand` away, so every benchmark already carries its own effective - // browser and there is no inheritance left to walk. - for (const benchmark of entry.config.benchmarks ?? []) { - benchmark.browser = withBrowserDefaults(benchmark.browser, browserBinary, asRoot); - } - const slug = fileSlugOf(entry.name); - const configPath = path.join(tmpBase, `tachometer-${slug}.json`); - // eslint-disable-next-line no-await-in-loop - await writeFile( - configPath, - // Served from the output directory rather than the harness: tachometer's static server is - // koa-static with its default `hidden: false`, so any dot-prefixed segment in the *served* - // path is refused. Rooting here keeps `.tachometer` out of the url entirely. - `${JSON.stringify({ ...entry.config, root: outputDir }, null, 2)}\n`, - ); - - console.log(chalk.cyan(`\nRunning "${entry.name}"…`)); - const jsonPath = path.join(tmpBase, `result-${slug}.json`); - // eslint-disable-next-line no-await-in-loop - await run( - 'pnpm', - ['exec', 'tachometer', '--config', configPath, '--json-file', jsonPath], - harnessDir, - { verbose: true }, - ); - // Cases run sequentially on purpose — concurrent browser benchmarks would contend for the - // same machine and skew timings — so reading each result in turn is fine. - // eslint-disable-next-line no-await-in-loop - results.push({ entry, json: JSON.parse(await readFile(jsonPath, 'utf8')) }); - } - - const outPath = out ? path.resolve(out) : path.join(outputDir, 'results', 'report.json'); - const report = { - // Consumers render reports from several benchmark axes; the pair identifies which one this - // is and how to read it. - version: 1 as const, - reportType: 'tachometer' as const, - generatedAt: new Date().toISOString(), - head: { - ref: 'HEAD', - sha: execaSync('git', ['rev-parse', 'HEAD'], { cwd: repoRoot }).stdout.trim(), - branch: execaSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], { - cwd: repoRoot, - }).stdout.trim(), - }, - browser: browserBinary, - refs, - // Summarising is best-effort per case: a case that produced no usable benchmarks must not - // cost the whole run its report, since `raw` below is the only surviving copy of every other - // case's samples once the temp dir is cleaned up. - cases: results.map(({ entry, json }): CaseResult => { - try { - return summarizeCase(entry, json); - } catch (error) { - const message = error instanceof Error ? error.message : String(error); - console.error(chalk.yellow(`Could not summarize "${entry.name}": ${message}`)); - return { name: entry.name, comparison: entry.comparison, error: message }; - } - }), - raw: Object.fromEntries(results.map(({ entry, json }) => [entry.name, json])), - }; - await mkdir(path.dirname(outPath), { recursive: true }); - await writeFile(outPath, `${JSON.stringify(report, null, 2)}\n`); - console.log(''); - renderTachometerReport(report); - console.log(chalk.green(`\nWrote JSON report to ${outPath}`)); - - if (upload) { - await publishReport(report); - } - - // Written and uploaded first, so the errors are readable and the comment says what happened — - // then fail, because a harness that measured nothing at all otherwise reports a passing job. - if (report.cases.every((entry) => !isSummarized(entry))) { - throw new Error('No case produced any measurement.'); - } - } finally { - await rm(tmpBase, { recursive: true, force: true }); - // Always, including after a failure: an in-place run leaves the repository resolving the library - // to a tarball until this puts it back. - if (resolveMode === 'in-place') { - await restoreWorkspace(repoRoot, outputDir); - } - } -} - -/** - * Uploads a report and refreshes the pull request comment. - * - * A failure to refresh the comment does not fail the run: the numbers are already measured, written - * and uploaded, and losing them to a comment API hiccup would waste the whole benchmark. - */ -async function publishReport(report: TachometerReport): Promise { - const metadata = await getCiMetadata(); - if (!metadata.repo) { - console.warn( - chalk.yellow('Skipping upload: no repository detected, which usually means this is not CI.'), - ); - return; - } - - await uploadCiReport({ version: 1, reportType: 'tachometer', ...metadata, report }); - - try { - console.log('Syncing PR comment via the dashboard API…'); - const result = await syncPrComment(metadata.repo); - console.log( - result.skipped ? 'No open PR found for this branch, skipping.' : 'PR comment synced.', - ); - } catch (error) { - console.error( - chalk.yellow( - `Failed to sync the PR comment: ${error instanceof Error ? error.message : error}`, - ), - ); - } -} diff --git a/packages/benchmark/src/tachometer/summarizeCase.test.ts b/packages/benchmark/src/tachometer/summarizeCase.test.ts deleted file mode 100644 index 11678777dc..0000000000 --- a/packages/benchmark/src/tachometer/summarizeCase.test.ts +++ /dev/null @@ -1,203 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { summarizeCase, verdictOf } from './summarizeCase'; - -type FakeDifference = { ms: [number, number]; pct: [number, number] } | null; - -/** A tachometer benchmark entry. */ -function benchmark({ - name, - measurement, - differences = [], -}: { - /** Entry name, as tachometer reports it. */ - name: string; - measurement?: string; - /** Differences against every entry, by flat index. */ - differences?: FakeDifference[]; -}): any { - return { - name, - mean: { low: 1, high: 2 }, - samples: [1, 2, 3], - measurement: measurement === undefined ? undefined : { name: measurement }, - differences: differences.map((difference) => - difference - ? { - absolute: { low: difference.ms[0], high: difference.ms[1] }, - percentChange: { low: difference.pct[0], high: difference.pct[1] }, - } - : null, - ), - }; -} - -/** - * A discovered case, as `discoverCases` would return it. `variants` is in order; the first is the - * reference. - */ -function discovered(variants: string[], measurements: string[]): any { - return { - name: 'example', - configPath: '/tmp/tachometer.json', - config: {}, - comparison: 'baseline', - leaves: [], - variants, - measurements, - }; -} - -describe('verdictOf', () => { - it('is slower when both intervals sit above zero', () => { - expect(verdictOf({ absolute: { low: 1, high: 3 }, percentChange: { low: 2, high: 5 } })).toBe( - 'slower', - ); - }); - - it('is faster when both intervals sit below zero', () => { - expect( - verdictOf({ absolute: { low: -3, high: -1 }, percentChange: { low: -5, high: -2 } }), - ).toBe('faster'); - }); - - it('is unsure when the intervals disagree', () => { - // The two are computed separately from the same samples, so near zero one can exclude it while - // the other does not. A verdict resting on only one of them would flip depending which is read. - expect(verdictOf({ absolute: { low: 1, high: 3 }, percentChange: { low: -1, high: 5 } })).toBe( - 'unsure', - ); - }); - - it('is unsure when both intervals straddle zero', () => { - expect(verdictOf({ absolute: { low: -1, high: 3 }, percentChange: { low: -2, high: 5 } })).toBe( - 'unsure', - ); - }); -}); - -describe('summarizeCase', () => { - it('reports every variant of a single-measurement case', () => { - const entry = discovered(['a', 'b'], ['mount']); - const summary = summarizeCase(entry, { - benchmarks: [ - benchmark({ - name: 'a', - measurement: 'mount', - differences: [null, { ms: [-3, -1], pct: [-5, -2] }], - }), - benchmark({ - name: 'b', - measurement: 'mount', - differences: [{ ms: [1, 3], pct: [2, 5] }, null], - }), - ], - }); - - expect(summary.reference).toBe('a'); - expect(summary.measurements).toHaveLength(1); - expect(summary.measurements[0].variants.map((variant) => variant.variant)).toEqual(['a', 'b']); - expect(summary.measurements[0].comparisons).toMatchObject([ - { variant: 'b', verdict: 'faster' }, - ]); - }); - - it('pairs results by measurement rather than by position', () => { - // Tachometer flattens variants × measurements into one list and names entries - // ` []`. Reading the difference matrix positionally would compare `mount` - // against `scroll`. Here the flat order deliberately interleaves the two measurements. - const entry = discovered(['a', 'b'], ['mount', 'scroll']); - const summary = summarizeCase(entry, { - benchmarks: [ - // index 0 - benchmark({ - name: 'a [mount]', - measurement: 'mount', - differences: [null, null, { ms: [-3, -1], pct: [-5, -2] }, null], - }), - // index 1 - benchmark({ - name: 'a [scroll]', - measurement: 'scroll', - differences: [null, null, null, { ms: [1, 3], pct: [2, 5] }], - }), - // index 2 - benchmark({ name: 'b [mount]', measurement: 'mount', differences: [] }), - // index 3 - benchmark({ name: 'b [scroll]', measurement: 'scroll', differences: [] }), - ], - }); - - const [mount, scroll] = summary.measurements; - expect(mount.name).toBe('mount'); - expect(mount.comparisons).toMatchObject([{ variant: 'b', verdict: 'faster' }]); - expect(scroll.name).toBe('scroll'); - expect(scroll.comparisons).toMatchObject([{ variant: 'b', verdict: 'slower' }]); - }); - - it('strips the measurement suffix when pairing variants back', () => { - const entry = discovered(['a', 'b'], ['mount']); - const summary = summarizeCase(entry, { - benchmarks: [ - benchmark({ - name: 'a [mount]', - measurement: 'mount', - differences: [null, { ms: [-3, -1], pct: [-5, -2] }], - }), - benchmark({ name: 'b [mount]', measurement: 'mount', differences: [] }), - ], - }); - - expect(summary.measurements[0].variants.map((variant) => variant.variant)).toEqual(['a', 'b']); - }); - - it('records the reverse comparison, which is not the negation of the forward one', () => { - // Each direction has its own denominator. - const entry = discovered(['a', 'b'], ['mount']); - const summary = summarizeCase(entry, { - benchmarks: [ - benchmark({ - name: 'a', - measurement: 'mount', - differences: [null, { ms: [-3, -1], pct: [-5, -2] }], - }), - benchmark({ - name: 'b', - measurement: 'mount', - differences: [{ ms: [1, 3], pct: [3, 9] }, null], - }), - ], - }); - - expect(summary.measurements[0].comparisons[0]).toMatchObject({ - verdict: 'faster', - versusReference: { verdict: 'slower', percentChange: { low: 3, high: 9 } }, - }); - }); - - it('skips a variant with no result for a measurement instead of failing the case', () => { - const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); - const entry = discovered(['a', 'b'], ['mount']); - - const summary = summarizeCase(entry, { - benchmarks: [benchmark({ name: 'a', measurement: 'mount', differences: [null] })], - }); - - expect(summary.measurements[0].variants.map((variant) => variant.variant)).toEqual(['a']); - expect(warn).toHaveBeenCalled(); - warn.mockRestore(); - }); - - it('throws when the reference itself has no result for a measurement', () => { - const entry = discovered(['a'], ['mount']); - - expect(() => summarizeCase(entry, { benchmarks: [] })).toThrow( - 'Case "example": no "mount" result for the reference "a".', - ); - }); - - it('throws when the case declares no variants', () => { - expect(() => summarizeCase(discovered([], ['mount']), { benchmarks: [] })).toThrow( - 'Case "example" declares no variants.', - ); - }); -}); diff --git a/packages/benchmark/src/tachometer/summarizeCase.ts b/packages/benchmark/src/tachometer/summarizeCase.ts deleted file mode 100644 index 19ab0c9da6..0000000000 --- a/packages/benchmark/src/tachometer/summarizeCase.ts +++ /dev/null @@ -1,184 +0,0 @@ -import chalk from 'chalk'; -import type { BenchmarkCase, CaseComparison } from './discoverCases'; - -/** Reduces tachometer's raw JSON output to a flat, interpretable summary. */ - -export interface ConfidenceInterval { - low: number; - high: number; -} - -export interface TachometerDifference { - /** Absolute difference, in milliseconds. */ - absolute: ConfidenceInterval; - /** Relative difference, as a percentage. */ - percentChange: ConfidenceInterval; -} - -export interface TachometerBenchmark { - /** Variant name, suffixed with ` []` when a page has several. */ - name: string; - /** Mean duration, as a 95% confidence interval. */ - mean: ConfidenceInterval; - samples: number[]; - /** The measurement this entry reports. */ - measurement?: { name?: string }; - /** This entry against every other, by flat index. */ - differences: Array; -} - -export interface TachometerJson { - /** One entry per variant × measurement. */ - benchmarks: TachometerBenchmark[]; -} - -export type Verdict = 'faster' | 'slower' | 'unsure'; - -export interface VariantSummary { - variant: string; - meanMs: ConfidenceInterval; - samples: number; -} - -export interface ComparisonSummary { - variant: string; - verdict: Verdict; - absoluteMs: ConfidenceInterval; - percentChange: ConfidenceInterval; - versusReference?: { - verdict: Verdict; - absoluteMs: ConfidenceInterval; - percentChange: ConfidenceInterval; - }; -} - -export interface MeasurementSummary { - name: string; - variants: VariantSummary[]; - comparisons: ComparisonSummary[]; -} - -export interface CaseSummary { - name: string; - /** What the case compares, which is what says whether a difference is a regression. */ - comparison: CaseComparison; - reference: string; - measurements: MeasurementSummary[]; -} - -/** - * Judges a difference. - * - * A difference only counts as `faster`/`slower` when the millisecond interval and the percent - * interval both exclude zero on the same side. They come from the same samples but are computed - * separately, so near zero one can exclude it while the other does not — and a verdict that rests - * on only one of them flips depending on which is read. - */ -export function verdictOf(difference: TachometerDifference): Verdict { - if (difference.absolute.low > 0 && difference.percentChange.low > 0) { - return 'slower'; - } - if (difference.absolute.high < 0 && difference.percentChange.high < 0) { - return 'faster'; - } - return 'unsure'; -} - -/** - * Indexes tachometer's flat benchmark list by measurement and then by variant, keeping each entry's - * position so the difference matrix can be read back. - */ -function indexByMeasurement( - json: TachometerJson, -): Map> { - const byMeasurement = new Map< - string, - Map - >(); - json.benchmarks.forEach((benchmark, index) => { - const measurement = benchmark.measurement?.name ?? ''; - const suffix = ` [${measurement}]`; - const variant = benchmark.name.endsWith(suffix) - ? benchmark.name.slice(0, -suffix.length) - : benchmark.name; - const forMeasurement = byMeasurement.get(measurement) ?? new Map(); - forMeasurement.set(variant, { benchmark, index }); - byMeasurement.set(measurement, forMeasurement); - }); - return byMeasurement; -} - -/** - * Reduces one case's tachometer output to a per-measurement summary. - * - * Tachometer flattens variants × measurements into one list, naming entries - * ` []` when a page declares more than one, with a difference matrix over the - * whole flat list. Results are paired back by (variant, measurement) name — pairing by position - * would compare unrelated measurements as soon as a case has more than one. - * - * The first declared variant is the reference, because that is the one under test: for the - * auto-expanded regression case it is `[current]`. - */ -export function summarizeCase(entry: BenchmarkCase, json: TachometerJson): CaseSummary { - const [reference] = entry.variants; - if (!reference) { - throw new Error(`Case "${entry.name}" declares no variants.`); - } - - const byMeasurement = indexByMeasurement(json); - - const measurements = entry.measurements.map((measurement): MeasurementSummary => { - const byVariant = byMeasurement.get(measurement); - const lookup = (variant: string) => byVariant?.get(variant); - - const referenceResult = lookup(reference); - if (!referenceResult) { - throw new Error( - `Case "${entry.name}": no "${measurement}" result for the reference "${reference}".`, - ); - } - - const variants: VariantSummary[] = []; - const comparisons: ComparisonSummary[] = []; - for (const variant of entry.variants) { - const found = lookup(variant); - if (!found) { - console.warn( - chalk.yellow(` ${entry.name}: no "${measurement}" result for "${variant}"; skipping.`), - ); - continue; - } - variants.push({ - variant, - meanMs: found.benchmark.mean, - samples: found.benchmark.samples.length, - }); - if (variant === reference) { - continue; - } - const difference = referenceResult.benchmark.differences[found.index]; - if (!difference) { - continue; - } - // The same pair read from the variant's own matrix row. Each direction has its own - // denominator, so it is not the negation of `difference`. - const reverse = found.benchmark.differences[referenceResult.index]; - comparisons.push({ - variant, - verdict: verdictOf(difference), - absoluteMs: difference.absolute, - percentChange: difference.percentChange, - versusReference: reverse - ? { - verdict: verdictOf(reverse), - absoluteMs: reverse.absolute, - percentChange: reverse.percentChange, - } - : undefined, - }); - } - return { name: measurement, variants, comparisons }; - }); - - return { name: entry.name, comparison: entry.comparison, reference, measurements }; -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7c86a565f7..e2f7cc9ef6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -245,9 +245,6 @@ importers: baseline-browser-mapping: specifier: 2.11.23 version: 2.11.23 - chromedriver: - specifier: ^151.0.5 - version: 151.0.5(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) eslint-plugin-n: specifier: 18.3.0 version: 18.3.0(@typescript/typescript6@6.0.2)(eslint@10.10.0(jiti@2.7.0)(supports-color@10.2.2)) @@ -1299,31 +1296,6 @@ importers: specifier: 5.0.1 version: 5.0.1(@types/node@22.20.2)(@vitest/browser-playwright@5.0.1)(@vitest/coverage-v8@5.0.1)(@vitest/ui@5.0.1)(jsdom@30.0.1)(vite@8.3.0(@types/node@22.20.2)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.1)) - test/tachometer: - dependencies: - '@mui/internal-test-utils': - specifier: workspace:* - version: link:../../packages/test-utils/build - react: - specifier: 19.2.8 - version: 19.2.8 - devDependencies: - '@mui/internal-benchmark': - specifier: workspace:* - version: link:../../packages/benchmark/build - '@playwright/test': - specifier: 1.62.1 - version: 1.62.1 - '@types/react': - specifier: 19.2.18 - version: 19.2.18 - tachometer: - specifier: ^0.7.2 - version: 0.7.2(supports-color@10.2.2) - vite: - specifier: ^8.2.2 - version: 8.3.0(@types/node@22.20.2)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.13)(yaml@2.9.1) - packages: '@-xun/debug@2.0.2': @@ -2149,9 +2121,6 @@ packages: '@types/react': optional: true - '@bazel/runfiles@6.5.0': - resolution: {integrity: sha512-RzahvqTkfpY2jsDxo8YItPX+/iZ6hbiikw1YhE0bA9EKBR5Og8Pa6FHn9PO9M0zaXRVsr0GFQLKbB/0rzy9SzA==} - '@bcoe/v8-coverage@1.0.2': resolution: {integrity: sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==} engines: {node: '>=18'} @@ -2602,9 +2571,6 @@ packages: '@floating-ui/utils@0.2.12': resolution: {integrity: sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==} - '@hapi/bourne@3.0.0': - resolution: {integrity: sha512-Waj1cwPXJDucOib4a3bAISsKJVb15MKi9IvmTI/7ssVEm6sywXGjVJDhl6/umt1pK1ZS7PacXU3A1PmFKHEZ2w==} - '@html-validate/stylish@6.0.0': resolution: {integrity: sha512-d1/lI3qIXhGVJF3+MmKEBn1dRX6U4Z+BDaOdDI3E6SXcO+OQ7hC/3mSo6BIjwQlcuV6NdDOKm6QCrzIQL1EezQ==} engines: {node: ^22.16.0 || >= 24.0.0} @@ -3821,10 +3787,6 @@ packages: '@sinclair/typebox@0.34.52': resolution: {integrity: sha512-XiMQh7qqVlxZzcVD+kkGMNGMzcTrDMLWI7S4x7z1MkCkbDPrekpZXEUK0eZqZFMuHQg2a2DZOcDIh9o5v3Gonw==} - '@sindresorhus/is@5.6.0': - resolution: {integrity: sha512-TV7t8GKYaJWsn00tFDqBw8+Uqmr8A0fRU1tvTQhyZzGv0sJCGRQL3JGMI3ucuKo3XIZdUP+Lx7/gh2t3lewy7g==} - engines: {node: '>=14.16'} - '@sindresorhus/merge-streams@4.0.0': resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} @@ -3859,10 +3821,6 @@ packages: '@swc/helpers@0.5.23': resolution: {integrity: sha512-5lSsMOTXURePglDfvuAQUqkGek9Hg2kksOYay2m0+XR++b2NWYL/4sWyuvVBIs8oKnJaxkdi9whaL/sqN13afw==} - '@szmarczak/http-timer@5.0.1': - resolution: {integrity: sha512-+PmQX0PiAYPMeVYe237LJAYvOMYW1j2rH5YROyS3b4CTVJum34HfRvKvAzozHAQG0TnHNdUfY9nCeUyRAs//cw==} - engines: {node: '>=14.16'} - '@tanstack/query-core@5.102.8': resolution: {integrity: sha512-ZNjkJ33CqvPNec/6lZBnHqLc3EVGPZ9ySLhYahU9TcuRFdmwXewuj0c4hwSWcGHqEUwcSrKeZ+oGcvPBqXcQcg==} @@ -3871,9 +3829,6 @@ packages: peerDependencies: react: ^18 || ^19 - '@testim/chrome-version@1.1.4': - resolution: {integrity: sha512-kIhULpw9TrGYnHp/8VfdcneIcxKnLixmADtukQRtJUmsVlMg0niMkwV0xZmi8hqa57xqilIHjWFA0GKvEjVU5g==} - '@testing-library/dom@10.4.1': resolution: {integrity: sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==} engines: {node: '>=18'} @@ -3995,9 +3950,6 @@ packages: '@types/etag@1.8.4': resolution: {integrity: sha512-f1z/UMth8gQ6636NBqhFmJ3zES7EuDcUnV6K1gl1osHp+85KPKX+VixYWUpqLkw1fftCagyHJjJOZjZkEi2rHw==} - '@types/execa@0.9.0': - resolution: {integrity: sha512-mgfd93RhzjYBUHHV532turHC2j4l/qxsF/PbfDmprHDEUHmNZGlDn1CEsulGK3AfsPdhkWzZQT/S/k0UGhLGsA==} - '@types/find-package-json@1.2.7': resolution: {integrity: sha512-zHPw+sj/9u0cHV8cS99fizuCSEo3K5pk6JxDu6K3DEJAEZgdvjoHnFJz/C1Vzwr75GSPlAcDZR9O30/4CuQo9Q==} @@ -4013,9 +3965,6 @@ packages: '@types/hast@3.0.5': resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==} - '@types/http-cache-semantics@4.2.0': - resolution: {integrity: sha512-L3LgimLHXtGkWikKnsPg0/VFx9OGZaC+eN1u4r+OB1XRqH3meBIAVC2zr1WdMH+RHmnRkqliQAOHNJ/E0j/e0Q==} - '@types/is-empty@1.2.3': resolution: {integrity: sha512-4J1l5d79hoIvsrKh5VUKVRA1aIdsOb10Hu5j3J2VfP/msDnfTdGPmNp2E1Wg+vs97Bktzo+MZePFFXSGoykYJw==} @@ -4043,9 +3992,6 @@ packages: '@types/ms@2.1.0': resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} - '@types/node@11.15.54': - resolution: {integrity: sha512-1RWYiq+5UfozGsU6MwJyFX6BtktcT10XRjvcAQmskCtMcW3tPske88lM/nHv7BQG1w9KBXI1zPGuu5PnNCX14g==} - '@types/node@14.18.63': resolution: {integrity: sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==} @@ -4066,9 +4012,6 @@ packages: resolution: {integrity: sha512-EULJ8LApcVEPbrfND0cRQqutIOdiIgJ1Mgrhpy755r14xMohPTEpkV/k28SJvuOs9bHRFW8x+KeDAEPiGQPB9Q==} deprecated: This is a stub types definition. parse-path provides its own type definitions, so you do not need this installed. - '@types/parse5@5.0.3': - resolution: {integrity: sha512-kUNnecmtkunAoQ3CnjmMkzNU/gtxG8guhi+Fk2U/kOpIKjIMKnXGp4IJCgQJrXSgMsWYimYG4TGjz/UzbGEBTw==} - '@types/prop-types@15.7.15': resolution: {integrity: sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==} @@ -4606,10 +4549,6 @@ packages: resolution: {integrity: sha512-6/mh1E2u2YgEsCHdY0Yx5oW+61gZU+1vXaoiHHrpKeuRNNgFvS+/jrwHiQhB5apAf5oB7UB7E19ol2R2LKH8hQ==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} - accepts@1.3.8: - resolution: {integrity: sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==} - engines: {node: '>= 0.6'} - acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -4624,18 +4563,6 @@ packages: engines: {node: '>=0.4.0'} hasBin: true - adm-zip@0.6.1: - resolution: {integrity: sha512-Xwrja8nx9e5o2N1my4DsKCeKpdrnACyr1wtbPxBDgGzKzKyE9kRtBFA8mWldI+RVlD7CBZNWY/wQ2+ydwOR6kQ==} - engines: {node: '>=14.0'} - - agent-base@6.0.2: - resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} - engines: {node: '>= 6.0.0'} - - agent-base@9.0.0: - resolution: {integrity: sha512-TQf59BsZnytt8GdJKLPfUZ54g/iaUL2OWDSFCCvMOhsHduDQxO8xC4PNeyIkVcA5KwL2phPSv0douC0fgWzmnA==} - engines: {node: '>= 20'} - ajv-formats@3.0.1: resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} peerDependencies: @@ -4661,15 +4588,6 @@ packages: ansi-align@3.0.1: resolution: {integrity: sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w==} - ansi-escape-sequences@6.2.4: - resolution: {integrity: sha512-2KJQAG1Nk4Iyu0dJENKXQJE9smEASrpu/E0F7LSnR72tQXngKGLqfRkHbkinjNct5vvAQY4BwQNt+4Tvg73nDQ==} - engines: {node: '>=12.17'} - peerDependencies: - '@75lb/nature': latest - peerDependenciesMeta: - '@75lb/nature': - optional: true - ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -4678,10 +4596,6 @@ packages: resolution: {integrity: sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ==} engines: {node: '>=12'} - ansi-styles@3.2.1: - resolution: {integrity: sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==} - engines: {node: '>=4'} - ansi-styles@4.3.0: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} @@ -4713,18 +4627,6 @@ packages: resolution: {integrity: sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==} engines: {node: '>= 0.4'} - array-back@3.1.0: - resolution: {integrity: sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==} - engines: {node: '>=6'} - - array-back@4.0.2: - resolution: {integrity: sha512-NbdMezxqf94cnNfWLL7V/im0Ub+Anbb0IoZhvzie8+4HJ4nMQuzHuy49FkGYCJK2yAloZ3meiB6AVMClbrI1vg==} - engines: {node: '>=8'} - - array-back@6.2.3: - resolution: {integrity: sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==} - engines: {node: '>=12.17'} - array-buffer-byte-length@1.0.2: resolution: {integrity: sha512-LHE+8BuR7RYGDKvnrmcuSq3tDcKv9OFEXQt/HpbZhY7V6h0zlUXutnAD82GiFx9rdieCMjkvtcsPqBwgUl1Iiw==} engines: {node: '>= 0.4'} @@ -4770,10 +4672,6 @@ packages: ast-types-flow@0.0.8: resolution: {integrity: sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==} - ast-types@0.13.4: - resolution: {integrity: sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==} - engines: {node: '>=4'} - ast-v8-to-istanbul@1.0.6: resolution: {integrity: sha512-fvpl29helSO2w/z7utIbrkNXILdrLwDwAMH2I/zPKlGf5244+gf+B4cyS1sANcrPY2h+hWCGSgC8N61s/+AF9A==} @@ -4789,9 +4687,6 @@ packages: resolution: {integrity: sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA==} engines: {node: '>= 0.4'} - asynckit@0.4.0: - resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} - autoprefixer@10.6.0: resolution: {integrity: sha512-A26d6qs9kqGgkmImIXMYvXTzqb4Qv7AVgpY1NXzr9Y659J8qHHnLOO/zE8ewIGFMprOolAoRAQYDgNryXIJKBw==} engines: {node: ^10 || ^12 || >=14} @@ -4811,9 +4706,6 @@ packages: resolution: {integrity: sha512-UzGt8zg7Ny8djbYMhxl2zuEevVa7r2gJjYY5Lwr1xM7+XU2nd6CkIWFTVcCIbAP63vSz71NaVyyuSk9lHKcy0A==} engines: {node: '>=4'} - axios@1.20.0: - resolution: {integrity: sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg==} - axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} @@ -4868,10 +4760,6 @@ packages: engines: {node: '>=6.0.0'} hasBin: true - basic-ftp@5.3.1: - resolution: {integrity: sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw==} - engines: {node: '>=10.0.0'} - bcrypt-pbkdf@1.0.2: resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} @@ -4913,9 +4801,6 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true - buffer-equal-constant-time@1.0.1: - resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} - buffer-from@1.1.2: resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} @@ -4938,18 +4823,6 @@ packages: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} - cache-content-type@1.0.1: - resolution: {integrity: sha512-IKufZ1o4Ut42YUrZSo8+qnMTrFuKkvyoLXUywKz9GJ5BrhOFGhLdkx9sG4KAnVvbY6kEcSFjLQul+DVmBm2bgA==} - engines: {node: '>= 6.0.0'} - - cacheable-lookup@7.0.0: - resolution: {integrity: sha512-+qJyx4xiKra8mZrcwhjMRMUhD5NR1R8esPkzIYxX96JiecFoxAXFuz/GpR3+ev4PE1WamHip78wV0vcmPQtp8w==} - engines: {node: '>=14.16'} - - cacheable-request@10.2.14: - resolution: {integrity: sha512-zkDT5WAF4hSSoUgyfg5tFIxz8XQK+25W/TLVojJTMKBaxevLBBtLxgqguAuVQB8PVW79FVjHcU+GJ9tVbDZ9mQ==} - engines: {node: '>=14.16'} - cacheable@2.5.0: resolution: {integrity: sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g==} @@ -4997,10 +4870,6 @@ packages: resolution: {integrity: sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==} engines: {node: '>=12'} - chalk@2.4.2: - resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} - engines: {node: '>=4'} - chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -5037,11 +4906,6 @@ packages: resolution: {integrity: sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ==} engines: {node: '>=6.0'} - chromedriver@151.0.5: - resolution: {integrity: sha512-oPI/VtTeLePbkTOI5o4yvEOlcdpmqEdqQNLZbyu5ha6kpDQYfDRuliisDiph29hBK/UzHSmOyZFKX1mE3efYOQ==} - engines: {node: '>=22'} - hasBin: true - ci-info@4.4.0: resolution: {integrity: sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==} engines: {node: '>=8'} @@ -5084,14 +4948,6 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} - co-body@6.2.0: - resolution: {integrity: sha512-Kbpv2Yd1NdL1V/V4cwLVxraHDV6K8ayohr2rmH0J87Er8+zJjcTa6dAn9QMPC9CRgU8+aNajKbSf1TzDB1yKPA==} - engines: {node: '>=8.0.0'} - - co@4.6.0: - resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} - engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} - codsen-format-diagnostic-value@1.1.1: resolution: {integrity: sha512-h/5wY3WiUCVOYvJG5aWu9kh/+Ik+rv3qEvM0UNner4E4bPPbs6YYzvIt/Ilz9gnkNp7Tw5aC1nMqDkUGQ57k7g==} engines: {node: '>=18.20.8'} @@ -5103,37 +4959,19 @@ packages: collapse-white-space@2.1.0: resolution: {integrity: sha512-loKTxY1zCOuG4j9f6EPnuyyYkf58RnhhWTvRoZEokgB+WbdXehfjFviyOVYkqzEWz1Q5kRiZdBYS5SwxbQYwzw==} - color-convert@1.9.3: - resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==} - color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} - color-name@1.1.3: - resolution: {integrity: sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==} - color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} colord@2.10.0: resolution: {integrity: sha512-AidJptpBJmjTclAp9BkLwJi0T93fo5epJnbaZslpg6QVzpHjAiveF55mE9AcUJiGMqRHgMDY8soMsQtuNYMHfw==} - combined-stream@1.0.8: - resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} - engines: {node: '>= 0.8'} - comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} - command-line-args@5.2.1: - resolution: {integrity: sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==} - engines: {node: '>=4.0.0'} - - command-line-usage@6.1.3: - resolution: {integrity: sha512-sH5ZSPr+7UStsloltmDh7Ce5fb8XPlHyoPzTpyyMuYCtervL65+ubVZ6Q61cFtFl62UyJlc8/JwERRbAFPUqgw==} - engines: {node: '>=8.0.0'} - commander@14.0.3: resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} engines: {node: '>=20'} @@ -5152,9 +4990,6 @@ packages: commondir@1.0.1: resolution: {integrity: sha512-W9pAhw0ja1Edb5GVdIF1mjZw/ASI0AlShXM83UUGe2DVr5TdAPEA1OA8m/g8zWp9x6On7gqufY+FatDbC3MDQg==} - compare-versions@6.1.1: - resolution: {integrity: sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg==} - compressible@2.0.18: resolution: {integrity: sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==} engines: {node: '>= 0.6'} @@ -5174,10 +5009,6 @@ packages: resolution: {integrity: sha512-kRGRZw3bLlFISDBgwTSA1TMBFN6J6GWDeubmDE3AF+3+yXL8hTWv8r5rkLbqYXY4RjPk/EzHnClI3zQf1cFmHA==} engines: {node: '>= 0.6'} - content-type@1.0.5: - resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} - engines: {node: '>= 0.6'} - content-type@3.1.0: resolution: {integrity: sha512-z26wZR7iXahgpSK7XgnB4IbRVmUNSODvsX9qC9TAru7AcWGiqn5YKy+OH3PLRrwyAnewykKhCqnD+VV3a8uT8Q==} engines: {node: '>=22'} @@ -5192,13 +5023,6 @@ packages: resolution: {integrity: sha512-JtXpxqDqJ8P0UwEHwhxLzCIXQy97vlYBZR222Sbzb1q1Erex9ASrztJ29SyhWFQjod1AeFBaPzEEC8YvtZMIYg==} engines: {node: '>=6'} - cookies@0.9.2: - resolution: {integrity: sha512-8BIbcC6tPZMv2/PD4PrB3CtKks7LmAWhA7FRZK/6/b/WJ2RUz00KCga7gq50S4/RhZ22kxefRFGgsuts0v3eoA==} - engines: {node: '>= 0.8'} - - copy-to@2.0.1: - resolution: {integrity: sha512-3DdaFaU/Zf1AnpLiFDeNCD4TOWe3Zl2RZaTzUvWiIk5ERzcCodOE20Vqq4fzCbNoHURFHT4/us/Lfq+S2zyY4w==} - core-js-compat@3.50.0: resolution: {integrity: sha512-XGpFGbMLHwSt74YLTKho7Ib242qi6O8MSX+sRokV4oz7iKXvQWGYZthjIhjRGMxjzVkAubBO512dKGYcefmX3Q==} engines: {node: '>=6.4.0'} @@ -5226,10 +5050,6 @@ packages: resolution: {integrity: sha512-9IkYqtX3YHPCzoVg1Py+o9057a3i0fp7S530UWokCSaFVTc7CwXPRiOjRjBQQ18ZCNafx78YfnG+HALxtVmOGA==} engines: {node: '>=10.0.0'} - cross-spawn@6.0.6: - resolution: {integrity: sha512-VqCUuhcd1iB+dsv8gxPttb5iZh/D0iubSP21g36KXdEuf6I5JiioesUVjpCdHV9MZRUfVFlvwtIUyPfxo5trtw==} - engines: {node: '>=4.8'} - cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -5264,9 +5084,6 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} - csv-stringify@6.9.0: - resolution: {integrity: sha512-6b3e29CILFfSQ2nDRuZMRROzQOpPuJ+s8zPB5H7eK5ncXBbFOyVuRLU5Sx/zAUmVri3Ly49GyC8NEme4SVSAbA==} - d3-array@3.2.4: resolution: {integrity: sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==} engines: {node: '>=12'} @@ -5314,10 +5131,6 @@ packages: damerau-levenshtein@1.0.8: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} - data-uri-to-buffer@8.0.0: - resolution: {integrity: sha512-6UHfyCux51b8PTGDgveqtz1tvphBku5DrMKKJbFAZAJOI2zsjDpDoYE1+QGj7FOMS4BdTFNJsJiR3zEB0xH0yQ==} - engines: {node: '>= 20'} - data-urls@7.0.0: resolution: {integrity: sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==} engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0} @@ -5356,15 +5169,6 @@ packages: supports-color: optional: true - debug@4.3.1: - resolution: {integrity: sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -5384,13 +5188,6 @@ packages: decode-named-character-reference@1.3.0: resolution: {integrity: sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==} - decompress-response@6.0.0: - resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} - engines: {node: '>=10'} - - deep-equal@1.0.1: - resolution: {integrity: sha512-bHtC0iYvWhyaTzvV3CZgPeZQqCOBGyGsVV7v4eevpdkLHfiSrXUdBG+qAuSz4RI70sszvjQ1QSZ98An1yNwpSw==} - deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} @@ -5406,10 +5203,6 @@ packages: resolution: {integrity: sha512-m1pAzaJgZ/gssEqlOhJkPJp8Xly7QyW6xcrkUa2KKcDeDSEMP7X8xipU3snUcfisTQx0w1AGae+9UtJSfVnXGw==} engines: {node: '>=18'} - defer-to-connect@2.0.1: - resolution: {integrity: sha512-4tvttepXG1VaYGrRibk5EwJd1t4udunSOVMdLSAL6mId1ix438oPwPZMALY41FCijukO1L0twNcGsdzS7dHgDg==} - engines: {node: '>=10'} - define-data-property@1.1.4: resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} engines: {node: '>= 0.4'} @@ -5422,35 +5215,10 @@ packages: resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} engines: {node: '>= 0.4'} - degenerator@7.0.1: - resolution: {integrity: sha512-ABErK0IefDSyHjlPH7WUEenIAX2rPPnrDcDM+TS3z3+zu9TfyKKi07BQM+8rmxpdE2y1v5fjjdoAS/x4D2U60w==} - engines: {node: '>= 20'} - peerDependencies: - quickjs-wasi: ^2.2.0 - - delayed-stream@1.0.0: - resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} - engines: {node: '>=0.4.0'} - - delegates@1.0.0: - resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} - - depd@1.1.2: - resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} - engines: {node: '>= 0.6'} - - depd@2.0.0: - resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} - engines: {node: '>= 0.8'} - dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} - destroy@1.2.0: - resolution: {integrity: sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==} - engines: {node: '>= 0.8', npm: 1.2.8000 || >= 1.4.16} - detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} @@ -5501,12 +5269,6 @@ packages: eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} - ecdsa-sig-formatter@1.0.11: - resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} - - ee-first@1.1.1: - resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} - electron-to-chromium@1.5.427: resolution: {integrity: sha512-n14zb3FdsChZ2BNobqNHAJMcP3ifFv4paox2LvCrfVAQcqGiSURgbJl+PfMpHVCNFkStnNc+RRVtPBTVW5PDgw==} @@ -5523,13 +5285,6 @@ packages: resolution: {integrity: sha512-YGRs8knHhKHVShLkFET/rWAU8kmHbOV5LwN938RHI0pljAJ1Gf6SzXsSmRaEzcXTtOOmVqJ5+WtQPL5uigY50Q==} engines: {node: '>=14'} - encodeurl@1.0.2: - resolution: {integrity: sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==} - engines: {node: '>= 0.8'} - - end-of-stream@1.4.5: - resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} - enhanced-resolve@5.25.0: resolution: {integrity: sha512-ghq3mhs649mvbarTCAlZn2wRhbfHmzAFiKxoWA14B3VtqnxtZt+wz8BroKXU0tF3GiJsnUldjVncQGZ8qk4rdA==} engines: {node: '>=10.13.0'} @@ -5617,13 +5372,6 @@ packages: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} engines: {node: '>=6'} - escape-html@1.0.3: - resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} - - escape-string-regexp@1.0.5: - resolution: {integrity: sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==} - engines: {node: '>=0.8.0'} - escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -5632,11 +5380,6 @@ packages: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} - escodegen@2.1.0: - resolution: {integrity: sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==} - engines: {node: '>=6.0'} - hasBin: true - eslint-compat-utils@0.5.1: resolution: {integrity: sha512-3z3vFexKIEnjHE3zCMRo6fn/e44U7T1khUjg+Hp0ZQMCigh28rALD0nPFBcGZuiLC5rLZa2ubQHDRln09JfU2Q==} engines: {node: '>=12'} @@ -5825,11 +5568,6 @@ packages: resolution: {integrity: sha512-7p3DrVEIopW1B1avAGLuCSh1jubc01H2JHc8B4qqGblmg5gI9yumBgACjWo4JlIc04ufug4xJ3SQI8HkS/Rgzw==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} - esprima@4.0.1: - resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} - engines: {node: '>=4'} - hasBin: true - esquery@1.7.0: resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} engines: {node: '>=0.10'} @@ -5875,10 +5613,6 @@ packages: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} - execa@1.0.0: - resolution: {integrity: sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==} - engines: {node: '>=6'} - execa@10.0.1: resolution: {integrity: sha512-ge98qjkRK4IB7tL7Ju/6qmm5LHoH1eEMt5FNZrz3f4UIYhF28lggX20z3FaX1sgc67msLEn0N0BscOs29iuwyw==} engines: {node: '>=22'} @@ -5983,10 +5717,6 @@ packages: find-package-json@1.2.0: resolution: {integrity: sha512-+SOGcLGYDJHtyqHd87ysBhmaeQ95oWspDKnMXBrnQ9Eq4OkLNqejgoaD8xVWu6GPa0B6roa6KinCMEMcVeqONw==} - find-replace@3.0.0: - resolution: {integrity: sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==} - engines: {node: '>=4.0.0'} - find-root@1.1.0: resolution: {integrity: sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==} @@ -5998,10 +5728,6 @@ packages: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} - find-up@6.3.0: - resolution: {integrity: sha512-v2ZsoEuVHYy8ZIlYqwPe/39Cy+cFDzp4dXPaxNvkEuouymu+2Jbz0PxpKarJHYJTmv2HWT3O382qY8l4jMWthw==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - flat-cache@4.0.1: resolution: {integrity: sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw==} engines: {node: '>=16'} @@ -6015,15 +5741,6 @@ packages: flatted@3.4.4: resolution: {integrity: sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==} - follow-redirects@1.16.0: - resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==} - engines: {node: '>=4.0'} - peerDependencies: - debug: '*' - peerDependenciesMeta: - debug: - optional: true - for-each@0.3.5: resolution: {integrity: sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==} engines: {node: '>= 0.4'} @@ -6032,14 +5749,6 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} - form-data-encoder@2.1.4: - resolution: {integrity: sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==} - engines: {node: '>= 14.17'} - - form-data@4.0.6: - resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} - engines: {node: '>= 6'} - format-util@1.0.5: resolution: {integrity: sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==} @@ -6050,14 +5759,6 @@ packages: fraction.js@5.3.4: resolution: {integrity: sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ==} - fresh@0.5.2: - resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} - engines: {node: '>= 0.6'} - - fs-extra@10.1.0: - resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==} - engines: {node: '>=12'} - fsevents@2.3.2: resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -6109,14 +5810,6 @@ packages: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} - get-stream@4.1.0: - resolution: {integrity: sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==} - engines: {node: '>=6'} - - get-stream@5.2.0: - resolution: {integrity: sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==} - engines: {node: '>=8'} - get-stream@6.0.1: resolution: {integrity: sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==} engines: {node: '>=10'} @@ -6136,10 +5829,6 @@ packages: get-tsconfig@4.14.3: resolution: {integrity: sha512-++QEw4DIY7WGoukz+/+A/8dGYPT9l9yIadnmSgZ8Rjr3YVSVDipQSO9CdnJo9ePqFqUUqh+wk9uIaoiAwsiPkA==} - get-uri@8.0.1: - resolution: {integrity: sha512-/5N/P4Lrh0p/mDwlDRi7Y1+P2o/OyzZI3l6Iz1Ov6XXwwm1y3RlZLuo3gVgML99djrEDtV980bBxSuOeHLk8ww==} - engines: {node: '>= 20'} - git-up@8.1.1: resolution: {integrity: sha512-FDenSF3fVqBYSaJoYy1KSc2wosx0gCvKP+c+PRBht7cAaiCeQlBtfBDX9vgnNOHmdePlSFITVcn4pFfcgNvx3g==} @@ -6204,10 +5893,6 @@ packages: resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} engines: {node: '>= 0.4'} - got@12.6.1: - resolution: {integrity: sha512-mThBblvlAF1d4O5oqyvN+ZxLAYwIJK7bpMxgYqPD9okW0C3qm5FFn7k811QrcuEBwaogR3ngOFoCfs6mRv7teQ==} - engines: {node: '>=14.16'} - graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} @@ -6219,10 +5904,6 @@ packages: resolution: {integrity: sha512-R3pbpkcIqv2Pm3dUwgjclDRVmWpTJW2DcMzcIhEXEx1oh/CEMObMm3KLmRJOdvhM7o4uQBnwr8pzRK2sJWIqfg==} engines: {node: '>= 0.4'} - has-flag@3.0.0: - resolution: {integrity: sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==} - engines: {node: '>=4'} - has-flag@4.0.0: resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} engines: {node: '>=8'} @@ -6345,41 +6026,6 @@ packages: html-void-elements@3.0.0: resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==} - http-assert@1.5.0: - resolution: {integrity: sha512-uPpH7OKX4H25hBmU6G1jWNaqJGpTXxey+YOUizJUAgu0AjLUeC8D73hTrhvDS5D+GJN1DN1+hhc/eF/wpxtp0w==} - engines: {node: '>= 0.8'} - - http-cache-semantics@4.2.0: - resolution: {integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==} - - http-errors@1.6.3: - resolution: {integrity: sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==} - engines: {node: '>= 0.6'} - - http-errors@1.8.1: - resolution: {integrity: sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==} - engines: {node: '>= 0.6'} - - http-errors@2.0.1: - resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} - engines: {node: '>= 0.8'} - - http-proxy-agent@9.1.0: - resolution: {integrity: sha512-2NxoveTT58mjYT4n3RPTEfCZGLMbidoO8XEieXfpSYxu+PQJ1qpx4ypwH6N+uF9twBPIvRRgvkvW5HUTYWENig==} - engines: {node: '>= 20'} - - http2-wrapper@2.2.1: - resolution: {integrity: sha512-V5nVw1PAOgfI3Lmeaj2Exmeg7fenjhRUgz1lPSezy1CuhPYbgQtbQj4jZfEAEMlaL+vupsvhjqCyjzob0yxsmQ==} - engines: {node: '>=10.19.0'} - - https-proxy-agent@5.0.1: - resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} - engines: {node: '>= 6'} - - https-proxy-agent@9.1.0: - resolution: {integrity: sha512-ag87y7cJJ9/3+GxFr8Oy4O5faDsGRGnBGsJj/YjOSsSx/5eadKLYTMPlzuR6obgoCDDm0abAAZitXXQkMOPSpA==} - engines: {node: '>= 20'} - human-signals@2.1.0: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} engines: {node: '>=10.17.0'} @@ -6392,10 +6038,6 @@ packages: resolution: {integrity: sha512-eKCa6bwnJhvxj14kZk5NCPc6Hb6BdsU9DZcOnmQKSnO1VKrfV0zCvtttPZUsBvjmNDn8rpcJfpwSYnHBjc95MQ==} engines: {node: '>=18.18.0'} - iconv-lite@0.4.24: - resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} - engines: {node: '>=0.10.0'} - iconv-lite@0.7.3: resolution: {integrity: sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==} engines: {node: '>=0.10.0'} @@ -6432,13 +6074,6 @@ packages: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} engines: {node: '>=0.8.19'} - inflation@2.1.0: - resolution: {integrity: sha512-t54PPJHG1Pp7VQvxyVCJ9mBbjG3Hqryges9bXoOO6GExCPa+//i/d5GSuFtpx3ALLd7lgIAur6zrIlBQyJuMlQ==} - engines: {node: '>= 0.8.0'} - - inherits@2.0.3: - resolution: {integrity: sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==} - inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} @@ -6463,14 +6098,6 @@ packages: resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==} engines: {node: '>=12'} - ip-address@10.7.2: - resolution: {integrity: sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w==} - engines: {node: '>= 12'} - - ip-regex@2.1.0: - resolution: {integrity: sha512-58yWmlHpp7VYfcdTwMTvwMmqx/Elfxjd9RXTDyMsbL7lLWmhMylLEqiYVLKuLzOZqVgiWXD9MfR62Vv89VRxkw==} - engines: {node: '>=4'} - is-alphabetical@2.0.1: resolution: {integrity: sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==} @@ -6629,10 +6256,6 @@ packages: is-ssh@1.4.1: resolution: {integrity: sha512-JNeu1wQsHjyHgn9NcWTaXq6zWSR6hqE0++zhfZlkFBbScNkyvxCdeV8sRkSBaeLKxmbpR21brail63ACNxJ0Tg==} - is-stream@1.1.0: - resolution: {integrity: sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==} - engines: {node: '>=0.10.0'} - is-stream@2.0.1: resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==} engines: {node: '>=8'} @@ -6661,9 +6284,6 @@ packages: resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==} engines: {node: '>=18'} - is-url@1.2.4: - resolution: {integrity: sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==} - is-wayland@0.1.0: resolution: {integrity: sha512-QkbMsWkIfkrzOPxenwye0h56iAXirZYHG9eHVPb22fO9y+wPbaX/CHacOWBa/I++4ohTcByimhM1/nyCsH8KNA==} engines: {node: '>=20'} @@ -6688,10 +6308,6 @@ packages: resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==} engines: {node: '>=16'} - is2@2.0.1: - resolution: {integrity: sha512-+WaJvnaA7aJySz2q/8sLjMb2Mw14KTplHmSwcSpZ/fWJPkUmqw3YTzSWbPJ7OAwRvdYTWF2Wg+yYJ1AdP5Z8CA==} - engines: {node: '>=v0.10.0'} - is64bit@2.0.0: resolution: {integrity: sha512-jv+8jaWCl0g2lSBkNSVXdzfBA0npK1HGC2KtWM9FumFRoGS94g3NbCCLVnCYHLjp4GrW2KZeeSTMo5ddtznmGw==} engines: {node: '>=18'} @@ -6791,19 +6407,6 @@ packages: engines: {node: ^18.0.0 || >=20.0.0} hasBin: true - jsonfile@6.2.1: - resolution: {integrity: sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==} - - jsonschema@1.5.0: - resolution: {integrity: sha512-K+A9hhqbn0f3pJX17Q/7H6yQfD/5OXgdrR5UE12gMXCiN9D5Xq2o5mddV2QEcX/bjla99ASsAAQUyMCCRWAEhw==} - - jsonwebtoken@9.0.3: - resolution: {integrity: sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==} - engines: {node: '>=12', npm: '>=6'} - - jstat@1.9.6: - resolution: {integrity: sha512-rPBkJbK2TnA8pzs93QcDDPlKcrtZWuuCo2dVR0TFLOJSxhqfWOVCSp8aV3/oSbn+4uY4yw1URtLpHQedtmXfug==} - jsx-ast-utils@3.3.5: resolution: {integrity: sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==} engines: {node: '>=4.0'} @@ -6811,19 +6414,9 @@ packages: jszip@3.10.2: resolution: {integrity: sha512-3l+rb15IOWtUhU0H5MFqES/T6Kh7abYwjosBey/vD6hDt8zoEffkSC5Ws5SGtgVw3gBx2NEbhTeSW1+kWkpyTQ==} - jwa@2.0.1: - resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} - - jws@4.0.1: - resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} - kebab-case@2.0.2: resolution: {integrity: sha512-NhIP7vecGtqJfNZ85ct0yRQfx//gCJHapJCBZP6/BBcBw/U218Jw7YX7rO6TI5/cCp5dJvlIjN3vbcRTqq7nWA==} - keygrip@1.1.0: - resolution: {integrity: sha512-iYSchDJ+liQ8iwbSI2QqsQOvqv58eJCEanyJPJi+Khyu8smkcKSFUCbPwzFcL7YVtZ6eONjqRX/38caJ7QjRAQ==} - engines: {node: '>= 0.6'} - keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -6842,36 +6435,6 @@ packages: resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} engines: {node: '>=6'} - koa-bodyparser@4.4.1: - resolution: {integrity: sha512-kBH3IYPMb+iAXnrxIhXnW+gXV8OTzCu8VPDqvcDHW9SQrbkHmqPQtiZwrltNmSq6/lpipHnT7k7PsjlVD7kK0w==} - engines: {node: '>=8.0.0'} - - koa-compose@4.1.0: - resolution: {integrity: sha512-8ODW8TrDuMYvXRwra/Kh7/rJo9BtOfPc6qO8eAfC80CnCvSjSl0bkRM24X6/XBBEyj0v1nRUQ1LyOy3dbqOWXw==} - - koa-convert@2.0.0: - resolution: {integrity: sha512-asOvN6bFlSnxewce2e/DK3p4tltyfC4VM7ZwuTuepI7dEQVcvpyFuBcEARu1+Hxg8DIwytce2n7jrZtRlPrARA==} - engines: {node: '>= 10'} - - koa-mount@4.2.0: - resolution: {integrity: sha512-2iHQc7vbA9qLeVq5gKAYh3m5DOMMlMfIKjW/REPAS18Mf63daCJHHVXY9nbu7ivrnYn5PiPC4CE523Tf5qvjeQ==} - engines: {node: '>= 7.6.0'} - - koa-node-resolve@1.0.0-pre.9: - resolution: {integrity: sha512-WKgqe5TGVD6zuR3NrKnmbb/NNHIbWOCezQVqqnyQLdtLLXWgiothlUQT23S5qQGE0Z623jp6jxpMjvAqyrcZFQ==} - - koa-send@5.0.1: - resolution: {integrity: sha512-tmcyQ/wXXuxpDxyNXv5yNNkdAMdFRqwtegBXUaowiQzUKqJehttS0x2j0eOZDQAyloAth5w6wwBImnFzkUz3pQ==} - engines: {node: '>= 8'} - - koa-static@5.0.0: - resolution: {integrity: sha512-UqyYyH5YEXaJrf9S8E23GoJFQZXkBVJ9zYYMPGz919MSX1KuvAcycIuS0ci150HCoPf4XQVhQ84Qf8xRPWxFaQ==} - engines: {node: '>= 7.6.0'} - - koa@2.16.4: - resolution: {integrity: sha512-3An0GCLDSR34tsCO4H8Tef8Pp2ngtaZDAZnsWJYelqXUK5wyiHvGItgK/xcSkmHLSTn1Jcho1mRQs2ehRzvKKw==} - engines: {node: ^4.8.4 || ^6.10.1 || ^7.10.1 || >= 8.1.4} - language-subtag-registry@0.3.23: resolution: {integrity: sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ==} @@ -6978,13 +6541,6 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} - locate-path@7.2.0: - resolution: {integrity: sha512-gvVijfZvn7R+2qyPX8mAuKcFGDf6Nc61GdvGafQsHL0sBIxfKzA+usWn4GFC/bk+QdwPUD4kWFJLhElipq+0VA==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - - lodash.camelcase@4.3.0: - resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} - lodash.clonedeep@4.5.0: resolution: {integrity: sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==} @@ -6997,24 +6553,6 @@ packages: lodash.groupby@4.6.0: resolution: {integrity: sha512-5dcWxm23+VAoz+awKmBaiBvzox8+RqMgFhi7UvX9DHZr2HdxHXM/Wrf8cfKpsW37RNrvtPn6hSwNqurSILbmJw==} - lodash.includes@4.3.0: - resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} - - lodash.isboolean@3.0.3: - resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} - - lodash.isinteger@4.0.4: - resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==} - - lodash.isnumber@3.0.3: - resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==} - - lodash.isplainobject@4.0.6: - resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} - - lodash.isstring@4.0.1: - resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==} - lodash.memoize@4.1.2: resolution: {integrity: sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==} @@ -7024,9 +6562,6 @@ packages: lodash.mergewith@4.6.2: resolution: {integrity: sha512-GK3g5RPZWTRSeLSpgP8Xhra+pnjBC56q9FZYe1d5RN3TJ35dbkGy3YqBSMbyCrlbi+CM9Z3Jk5yTL7RCsqboyQ==} - lodash.once@4.1.1: - resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==} - lodash.truncate@4.4.2: resolution: {integrity: sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==} @@ -7046,10 +6581,6 @@ packages: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true - lowercase-keys@3.0.0: - resolution: {integrity: sha512-ozCC6gdQ+glXOQsveKD0YsDy8DSQFjDTz4zyzEHNV5+JP5D62LmfDZ6o1cycFx9ouG940M5dE8C8CTewdj2YWQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} @@ -7060,10 +6591,6 @@ packages: lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} - lru-cache@7.18.3: - resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==} - engines: {node: '>=12'} - lru.min@1.1.5: resolution: {integrity: sha512-5J9ysMYUpYIg9RF2vJpy9SinEmSviFSe0GyPpCQ4L5QSkLAgeLXlTAOu2ZwWUU5m+0SBl6gUU1R1ZQB3aKypfA==} engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} @@ -7165,10 +6692,6 @@ packages: mdn-data@2.27.1: resolution: {integrity: sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==} - media-typer@0.3.0: - resolution: {integrity: sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==} - engines: {node: '>= 0.6'} - meow@14.1.0: resolution: {integrity: sha512-EDYo6VlmtnumlcBCbh1gLJ//9jvM/ndXHfVXIFrZVr6fGcwTUyCTFNTLCKuY3ffbK8L/+3Mzqnd58RojiZqHVw==} engines: {node: '>=20'} @@ -7296,10 +6819,6 @@ packages: resolution: {integrity: sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ==} engines: {node: '>= 0.6'} - mime-db@1.52.0: - resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} - engines: {node: '>= 0.6'} - mime-db@1.54.0: resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} engines: {node: '>= 0.6'} @@ -7308,10 +6827,6 @@ packages: resolution: {integrity: sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ==} engines: {node: '>= 0.6'} - mime-types@2.1.35: - resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} - engines: {node: '>= 0.6'} - mime-types@3.0.2: resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} engines: {node: '>=18'} @@ -7324,14 +6839,6 @@ packages: resolution: {integrity: sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==} engines: {node: '>=12'} - mimic-response@3.1.0: - resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} - engines: {node: '>=10'} - - mimic-response@4.0.0: - resolution: {integrity: sha512-e5ISH9xMYU0DzrT+jl8q2ze9D6eWBto+I8CNpe+VI+K2J/F/k3PdkdTdz4wvGVH4NTpo+NRYTVIuMQEMMcsLqg==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - min-indent@1.0.1: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} engines: {node: '>=4'} @@ -7420,9 +6927,6 @@ packages: ms@2.0.0: resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} - ms@2.1.2: - resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} - ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -7459,10 +6963,6 @@ packages: natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} - negotiator@0.6.3: - resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} - engines: {node: '>= 0.6'} - negotiator@0.6.4: resolution: {integrity: sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==} engines: {node: '>= 0.6'} @@ -7470,10 +6970,6 @@ packages: neo-async@2.6.2: resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} - netmask@2.1.1: - resolution: {integrity: sha512-eonl3sLUha+S1GzTPxychyhnUzKyeQkZ7jLjKrBagJgPla13F+uQ71HgpFefyHgqrjEbCPkDArxYsjY8/+gLKA==} - engines: {node: '>= 0.4.0'} - next@16.3.5: resolution: {integrity: sha512-MdtsTgzyfCPRLC6uJ1mN8ao7lyJ4BB0U6Inhnx3gta1UcCIdHK3yxLG0E8OWQteWD8/Q0qb8A5o7wJaL8M9y2w==} engines: {node: '>=20.9.0'} @@ -7495,9 +6991,6 @@ packages: sass: optional: true - nice-try@1.0.5: - resolution: {integrity: sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==} - node-exports-info@1.6.2: resolution: {integrity: sha512-kXs9Go0cah0qHVV2v389IXQLdLCeE1xfFtjOAF+iobu0OIoG1pje8At2vMHyaPMiPMnG/LWP50twML21eMcAag==} engines: {node: '>= 0.4'} @@ -7525,10 +7018,6 @@ packages: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} - normalize-url@8.1.1: - resolution: {integrity: sha512-JYc0DPlpGWB40kH5g07gGTrYuMqV653k3uBKY6uITPWds3M0ov3GaWGp9lbE3Bzngx8+XkfzgvASb9vk9JDFXQ==} - engines: {node: '>=14.16'} - npm-install-checks@6.3.0: resolution: {integrity: sha512-W29RiK/xtpCGqn6f3ixfRYGk+zRyr+Ew9F2E20BfXxT5/euLdA/Nm7fO7OeTGuAmTs30cpgInyJ0cYe708YTZw==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} @@ -7545,10 +7034,6 @@ packages: resolution: {integrity: sha512-nkc+3pIIhqHVQr085X9d2JzPzLyjzQS96zbruppqC9aZRm/x8xx6xhI98gHtsfELP2bE+loHq8ZaHFHhe+NauA==} engines: {node: ^16.14.0 || >=18.0.0} - npm-run-path@2.0.2: - resolution: {integrity: sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==} - engines: {node: '>=4'} - npm-run-path@4.0.1: resolution: {integrity: sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==} engines: {node: '>=8'} @@ -7604,17 +7089,10 @@ packages: resolution: {integrity: sha512-XrsrhT5sybtKI6wakr2SPOlGZWWYbUXZ7a0jT8/QOeAPau+1X/bSegNe5YR75oJmEZQbKningirmGOEJCIk61Q==} engines: {node: '>=12.20.0'} - on-finished@2.4.1: - resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} - engines: {node: '>= 0.8'} - on-headers@1.1.0: resolution: {integrity: sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==} engines: {node: '>= 0.8'} - once@1.4.0: - resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} - onetime@5.1.2: resolution: {integrity: sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==} engines: {node: '>=6'} @@ -7623,9 +7101,6 @@ packages: resolution: {integrity: sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==} engines: {node: '>=12'} - only@0.0.2: - resolution: {integrity: sha512-Fvw+Jemq5fjjyWz6CpKx6w9s7xxqo3+JCyM0WXWeCSOboZ8ABkyvP8ID4CZuChA/wxSx+XSJmdOm8rGVyJ1hdQ==} - open@11.0.3: resolution: {integrity: sha512-b3souUgU0VaAvMmzA4+9cRgrOgMdE0vc9H9VhQ0FYnjjcyZ8R/f9DiwMU9V8rl+vjf05On9UJhbgb06fsTs87Q==} engines: {node: '>=20'} @@ -7648,14 +7123,6 @@ packages: resolution: {integrity: sha512-19YVAg7T+WTrxggPukVq7DjTv6+PJ867TmhCvBsYwmbFCsZd344rq2Ld1p0wo8f8Qrrhgp82c6FJRqdXWtSEhg==} engines: {node: '>= 0.4'} - p-cancelable@3.0.0: - resolution: {integrity: sha512-mlVgR3PGuzlo0MmTdk4cXqXWlwQDLnONTAg6sm62XkMJEiRxN3GL3SffkYvqwonbkJBcrI7Uvv5Zh9yjvn2iUw==} - engines: {node: '>=12.20'} - - p-finally@1.0.0: - resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} - engines: {node: '>=4'} - p-limit@2.3.0: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} engines: {node: '>=6'} @@ -7664,10 +7131,6 @@ packages: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} - p-limit@4.0.0: - resolution: {integrity: sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - p-locate@4.1.0: resolution: {integrity: sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==} engines: {node: '>=8'} @@ -7676,10 +7139,6 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} - p-locate@6.0.0: - resolution: {integrity: sha512-wPrq66Llhl7/4AGC6I+cqxT07LhXvWL08LNXz1fENOw0Ap4sRZZ/gZpTTJ5jpurzzzfS2W/Ge9BY3LgLjCShcw==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - p-retry@8.0.1: resolution: {integrity: sha512-NAigyu8hfvJe7MsS18mnc4is0MZtau3QMdBklaJKK23oBQb6occO3UPM3vHmTckW8KhfyjMZaUOqdTFEYliHgg==} engines: {node: '>=22'} @@ -7688,16 +7147,6 @@ packages: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} - pac-proxy-agent@9.1.0: - resolution: {integrity: sha512-1aU+1mpj3DrQPfo3gh+3Gap3G5x+axnMx1P/y0ZF2ch7kb2meyOCAH8K2k9d27ROsTE7TnAerzxqF9aon2jqnA==} - engines: {node: '>= 20'} - - pac-resolver@9.0.1: - resolution: {integrity: sha512-lJbS008tmkj08VhoM8Hzuv/VE5tK9MS0OIQ/7+s0lIF+BYhiQWFYzkSpML7lXs9iBu2jfmzBTLzhe9n6BX+dYw==} - engines: {node: '>= 20'} - peerDependencies: - quickjs-wasi: ^2.2.0 - package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} @@ -7733,38 +7182,19 @@ packages: resolution: {integrity: sha512-bCgsFI+GeGWPAvAiUv63ZorMeif3/U0zaXABGJbOWt5OH2KCaPHF6S+0ok4aqM9RuIPGyZdx9tR9l13PsW4AYQ==} engines: {node: '>=14.13.0'} - parse5@5.1.1: - resolution: {integrity: sha512-ugq4DFI0Ptb+WWjAdOK16+u/nHfiIrcE+sh8kZMaM0WllQKLI9rOUq6c2b7cwPkXdzfQESqvoqK6ug7U/Yyzug==} - parse5@7.3.0: resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} parse5@8.0.1: resolution: {integrity: sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==} - parseurl@1.3.3: - resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} - engines: {node: '>= 0.8'} - path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} - path-exists@5.0.0: - resolution: {integrity: sha512-RjhtfwJOxzcFmNOi6ltcbcu4Iu+FL3zEj83dk4kAS+fVpTxXLO1b38RvJgT/0QwvV/L3aY9TAnyv0EOqW4GoMQ==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - - path-is-absolute@1.0.1: - resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} - engines: {node: '>=0.10.0'} - path-is-inside@1.0.2: resolution: {integrity: sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==} - path-key@2.0.1: - resolution: {integrity: sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==} - engines: {node: '>=4'} - path-key@3.1.1: resolution: {integrity: sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==} engines: {node: '>=8'} @@ -7820,17 +7250,10 @@ packages: resolution: {integrity: sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==} engines: {node: '>=8'} - pkg-install@1.0.0: - resolution: {integrity: sha512-UGI8bfhrDb1KN01RZ7Bq08GRQc8rmVjxQ2up0g4mUHPCYDTK1FzQ0PMmLOBCHg3yaIijZ2U3Fn9ofLa4N392Ug==} - pkg-pr-new@0.0.88: resolution: {integrity: sha512-Xc6PMJ2gher0WZP+rtjefFk26hIb7V1PTLL30bmy1Z2vsRmSvqiss7Ag1XUdyplTGYzIlFNJp4L3vMNmK44N6g==} hasBin: true - pkg-up@4.0.0: - resolution: {integrity: sha512-N4zdA4sfOe6yCv+ulPCmpnIBQ5I60xfhDr1otdBBhKte9QtEf3bhfrfkW7dTb+IQ0iEx4ZDzas0kc1o5rdWpYg==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - playwright-core@1.62.1: resolution: {integrity: sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==} engines: {node: '>=20'} @@ -7957,10 +7380,6 @@ packages: process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} - progress@2.0.3: - resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} - engines: {node: '>=0.4.0'} - promise-inflight@1.0.1: resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==} peerDependencies: @@ -7989,26 +7408,6 @@ packages: protocols@2.0.2: resolution: {integrity: sha512-hHVTzba3wboROl0/aWRRG9dMytgH6ow//STBZh43l/wQgmMhYhOFi0EHWAPtoCz9IAUymsyP0TSBHkhgMEGNnQ==} - proxy-agent-negotiate@1.1.0: - resolution: {integrity: sha512-N8IBcM3UgCVzz2L2Lqv8DVntDnnC8/hiV4nEDUPkqq72TPUgYWjQc+bdZlBPZK9LzPAvOY//gAt0S0DApoOXWQ==} - engines: {node: '>= 20'} - peerDependencies: - kerberos: ^2.0.0 - peerDependenciesMeta: - kerberos: - optional: true - - proxy-agent@8.0.2: - resolution: {integrity: sha512-idLLRewuemWd7GH/BDJzGiB0dWGfT2SQs3jy6NtZtGWU9uPTTSdeC1/cdbqLwgzhfv027daGFuXX426e2Eg20A==} - engines: {node: '>= 20'} - - proxy-from-env@2.1.0: - resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} - engines: {node: '>=10'} - - pump@3.0.4: - resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} - punycode@1.4.1: resolution: {integrity: sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ==} @@ -8032,13 +7431,6 @@ packages: queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} - quick-lru@5.1.1: - resolution: {integrity: sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==} - engines: {node: '>=10'} - - quickjs-wasi@2.2.0: - resolution: {integrity: sha512-zQxXmQMrEoD3S+jQdYsloq4qAuaxKFHZj6hHqOYGwB2iQZH+q9e/lf5zQPXCKOk0WJuAjzRFbO4KwHIp2D05Iw==} - quotation@2.0.3: resolution: {integrity: sha512-yEc24TEgCFLXx7D4JHJJkK4JFVtatO8fziwUxY4nB/Jbea9o9CVS3gt22mA0W7rPYAGW2fWzYDSOtD94PwOyqA==} @@ -8065,10 +7457,6 @@ packages: rasterizehtml@1.4.1: resolution: {integrity: sha512-mcE6b5cKhQO0Qka9vduSQW/F/crQ7EUjrTpXDPAdcj9uNtYijBiNmUOxOrIuMMq7GO/X3CR6UckHTgh5ZVqqBA==} - raw-body@2.5.3: - resolution: {integrity: sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==} - engines: {node: '>= 0.8'} - rc@1.2.8: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true @@ -8144,10 +7532,6 @@ packages: recma-stringify@1.0.0: resolution: {integrity: sha512-cjwII1MdIIVloKvC9ErQ+OgAtwHBmcZ0Bg4ciz78FtbT8In39aAYbaA7zvxQ61xVMSPE8WxhLwLbhif4Js2C+g==} - reduce-flatten@2.0.0: - resolution: {integrity: sha512-EJ4UNY/U1t2P/2k6oqotuX2Cc3T6nxJwsM0N0asT7dhrtH1ltUxDn4NalSYmPE2rCkVpcf/X6R0wDwcFpzhd4w==} - engines: {node: '>=6'} - reflect.getprototypeof@1.0.10: resolution: {integrity: sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw==} engines: {node: '>= 0.4'} @@ -8279,17 +7663,10 @@ packages: reselect@5.3.0: resolution: {integrity: sha512-XGoLeRAVzUTcJ1qkxPQhDJyIZ5d6zzZD9nT7AEZOaaU9UbWclhycElmhO+VD5bFeLuzhPBaOV2oXC8uG35ZSpg==} - resolve-alpn@1.2.1: - resolution: {integrity: sha512-0a1F4l73/ZFZOakJnQ3FvkJ2+gSTQWz/r2KE5OdDY0TxPm5h4GkqkWWfM47T7HsbnOtcJVEF4epCVy6u7Q3K+g==} - resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} - resolve-path@1.4.0: - resolution: {integrity: sha512-i1xevIst/Qa+nA9olDxLWnLk8YZbi8R/7JPbCMcgyWaFR6bKWaexgJgEB5oc2PKMjYdrHynyz0NY+if+H98t1w==} - engines: {node: '>= 0.8'} - resolve-pkg-maps@1.0.0: resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} @@ -8303,10 +7680,6 @@ packages: engines: {node: '>= 0.4'} hasBin: true - responselike@3.0.0: - resolution: {integrity: sha512-40yHxbNcl2+rzXvZuVkrYohathsSJlMTXKryG5y8uciHv1+xDLHQpgjG64JUO9nrEq2jGLH6IZ8BcZyw3wrweg==} - engines: {node: '>=14.16'} - retry@0.12.0: resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} engines: {node: '>= 4'} @@ -8368,9 +7741,6 @@ packages: sane-domparser-error@0.2.0: resolution: {integrity: sha512-wxjDV5jty95tNv8N/4WA15UNGqqaor/xX7rnNYY961hifN3bheYoKqtXN+V/M6EUgmUAs6pMul3klwUPMEiVXA==} - sanitize-filename@1.6.4: - resolution: {integrity: sha512-9ZyI08PsvdQl2r/bBIGubpVdR3RR9sY6RDiWFPreA21C/EFlQhmgo20UZlNjZMMZNubusLhAQozkA0Od5J21Eg==} - saxes@5.0.1: resolution: {integrity: sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw==} engines: {node: '>=10'} @@ -8389,14 +7759,6 @@ packages: resolution: {integrity: sha512-ZzWFVzFgyRHi/T2Ecxm37lL6J9+hZO0P9L7LCuLxAbC6XWxkvxcaQBCXhQxMGn/Tdt9nD7zIBk0ELwhMQ3UEDg==} engines: {node: '>= 10.13.0'} - selenium-webdriver@4.49.0: - resolution: {integrity: sha512-16XqeOEMj+4+p+pzLLyvNWmF5jCY05EKIl3e1anZBgv4+zq7PUTtiNjKtQhMpwosZ0wIMKDFAQ2xdyogOHD4DA==} - engines: {node: '>= 22.0.0'} - - semver@5.7.2: - resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} - hasBin: true - semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -8435,12 +7797,6 @@ packages: setimmediate@1.0.5: resolution: {integrity: sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==} - setprototypeof@1.1.0: - resolution: {integrity: sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==} - - setprototypeof@1.2.0: - resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} - sharp@0.35.4: resolution: {integrity: sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==} engines: {node: '>=20.9.0'} @@ -8450,18 +7806,10 @@ packages: '@types/node': optional: true - shebang-command@1.2.0: - resolution: {integrity: sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==} - engines: {node: '>=0.10.0'} - shebang-command@2.0.0: resolution: {integrity: sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==} engines: {node: '>=8'} - shebang-regex@1.0.0: - resolution: {integrity: sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==} - engines: {node: '>=0.10.0'} - shebang-regex@3.0.0: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} @@ -8511,18 +7859,6 @@ packages: resolution: {integrity: sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==} engines: {node: '>=10'} - smart-buffer@4.2.0: - resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==} - engines: {node: '>= 6.0.0', npm: '>= 3.0.0'} - - socks-proxy-agent@10.1.0: - resolution: {integrity: sha512-WlMj/67cEJ6MDI1OcsnjuYKDNDoyPCCYZ249kuuXPiMDw9F8PXkVaQ7YWu3siTydfQ/4BEZcvGzu+aYvz7dDCQ==} - engines: {node: '>= 20'} - - socks@2.8.10: - resolution: {integrity: sha512-e0VyvkVTwVYViNovRkZ9aodhxVlyoMn7eJhVUPxZ+eK9P/7CBkxvvsBOHqFPEH416726W8tLXXXjKwqgTErrCQ==} - engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} - source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -8576,14 +7912,6 @@ packages: stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} - statuses@1.5.0: - resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==} - engines: {node: '>= 0.6'} - - statuses@2.0.2: - resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} - engines: {node: '>= 0.8'} - std-env@4.2.0: resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} @@ -8675,10 +8003,6 @@ packages: resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==} engines: {node: '>=4'} - strip-eof@1.0.0: - resolution: {integrity: sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==} - engines: {node: '>=0.10.0'} - strip-final-newline@2.0.0: resolution: {integrity: sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==} engines: {node: '>=6'} @@ -8755,10 +8079,6 @@ packages: resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} engines: {node: '>=18'} - supports-color@5.5.0: - resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} - engines: {node: '>=4'} - supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} @@ -8793,31 +8113,14 @@ packages: resolution: {integrity: sha512-ulAk51I9UVUyJgxlv9M6lFot2WP3e7t8Kz9+IS6D4rVba1tR9kON+Ey69f+1R4Q8cd45Lod6a4IcJIxnzGc/zA==} engines: {node: '>=18'} - systeminformation@5.33.13: - resolution: {integrity: sha512-GtyakkchTQKymM1WeAB652zi3rM1q00RKdc+jZe9KGstuc/mWUwAnQGN/RE5gVYvHIR8xYOUIUD9irVKDtA0wA==} - engines: {node: '>=10.0.0'} - os: [darwin, linux, win32, freebsd, openbsd, netbsd, sunos, android] - hasBin: true - - table-layout@1.0.2: - resolution: {integrity: sha512-qd/R7n5rQTRFi+Zf2sk5XVVd9UQl6ZkduPFC3S7WEGJAmetDTjY3qPN50eSKzwuzEyQKy5TN2TiZdkIjos2L6A==} - engines: {node: '>=8.0.0'} - table@6.9.0: resolution: {integrity: sha512-9kY+CygyYM6j02t5YFHbNz2FN5QmYGv9zAjVp4lCDjlCw7amdckXlEt/bjMhUIfj4ThGRE4gCUH5+yGnNuPo5A==} engines: {node: '>=10.0.0'} - tachometer@0.7.2: - resolution: {integrity: sha512-e9dtDyWuMFyomQxo16vzti6ymoJd3Dp2yGfKrhAiNzCkb2HiP6A0+SWtqE84hcO6tSKZF8eTFqrxM2uKbliJyQ==} - hasBin: true - tapable@2.3.3: resolution: {integrity: sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==} engines: {node: '>=6'} - tcp-port-used@1.0.3: - resolution: {integrity: sha512-4CEQ3qRJYo+mtEbJ+OoQu3dF4TDkwaO3RDVC4UzP5cpAOIUWwuwPjD7sdxDFFqsMUjsXVVYBMlg/boAaloThMA==} - terser@5.51.2: resolution: {integrity: sha512-bWnjSNscmuI+GJze6ZupnHP8G/cTcsJF+bXCeQknk2SHQsgbNJnLrqiH9jZ2W4STPVXH2mDKKRX3iwPhc9Cn/Q==} engines: {node: '>=10'} @@ -8868,10 +8171,6 @@ packages: resolution: {integrity: sha512-m1TdR/rvT7kgGJZhspNtXdsdYk0fddFpJJFlG5s+UkPFo6lkLoZ3YLOaovPYjq1R75NP5JfeTlSHaOsE09peCg==} engines: {node: '>=20'} - toidentifier@1.0.1: - resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} - engines: {node: '>=0.6'} - totalist@3.0.1: resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} engines: {node: '>=6'} @@ -8890,9 +8189,6 @@ packages: trough@2.2.0: resolution: {integrity: sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==} - truncate-utf8-bytes@1.0.2: - resolution: {integrity: sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==} - ts-api-utils@2.5.0: resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} engines: {node: '>=18.12'} @@ -8908,10 +8204,6 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - tsscmp@1.0.6: - resolution: {integrity: sha512-LxhtAkPDTkVCMQjt2h6eBVY28KCjikZqZfMcC15YBeNjkgUpdCfBu5HoiOTDu86v6smE8yOjyEktJ8hlbANHQA==} - engines: {node: '>=0.6.x'} - tsx@4.23.13: resolution: {integrity: sha512-BL5MGkRln6aDYhb0xbQlEAGw743BaZYWdbWtdJOBriYJboKgUUYCadFp2/FpBBZquBC/ezNBn7wMMPx7FDZUDw==} engines: {node: '>=18.0.0'} @@ -8944,10 +8236,6 @@ packages: resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} engines: {node: '>=16'} - type-is@1.6.18: - resolution: {integrity: sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==} - engines: {node: '>= 0.6'} - typed-array-buffer@1.0.3: resolution: {integrity: sha512-nAYYwfY3qnzX30IkA6AQZjVbtK6duGontcQm1WSG1MD94YLqK0515GNApXkoxKOWMusVssAHWLh9SeaoefYFGw==} engines: {node: '>= 0.4'} @@ -8988,18 +8276,6 @@ packages: engines: {node: '>=16.20.0'} hasBin: true - typical@4.0.0: - resolution: {integrity: sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==} - engines: {node: '>=8'} - - typical@5.2.0: - resolution: {integrity: sha512-dvdQgNDNJo+8B2uBQoqdb11eUCE1JQXhvjC/CZtgvZseVd5TYMXnq0+vuUemXbd/Se29cTaUuPX3YIc2xgbvIg==} - engines: {node: '>=8'} - - ua-parser-js@1.0.41: - resolution: {integrity: sha512-LbBDqdIC5s8iROCUjMbW1f5dJQTEFB1+KO9ogbvlb3nm9n4YHa5p4KTvFPWvh2Hs8gZMBuiB1/8+pdfe/tDPug==} - hasBin: true - uint8-to-base64@0.2.1: resolution: {integrity: sha512-uO/84GaoDUfiAxpa8EksjVLE77A9Kc7ZTziN4zRpq4de9yLaLcZn3jx1/sVjyupsywcVX6RKWbqLe7gUNyzH+Q==} @@ -9094,14 +8370,6 @@ packages: universal-user-agent@7.0.3: resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==} - universalify@2.0.1: - resolution: {integrity: sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==} - engines: {node: '>= 10.0.0'} - - unpipe@1.0.0: - resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} - engines: {node: '>= 0.8'} - unrs-resolver@1.12.2: resolution: {integrity: sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ==} @@ -9126,9 +8394,6 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 - utf8-byte-length@1.0.5: - resolution: {integrity: sha512-Xn0w3MtiQ6zoz2vFyUVruaCL53O/DwUvkEeOvj+uulMm0BkUGYWmBYVyElqZaSLhY6ZD0ulfU3aBra2aVT4xfA==} - util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -9369,10 +8634,6 @@ packages: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} engines: {node: '>=0.10.0'} - wordwrapjs@4.0.1: - resolution: {integrity: sha512-kKlNACbvHrkpIw6oPeYDSmdCTu2hdMHoyXLTcUKala++lx5Y+wjJ/e474Jqv5abnVmwxw08DiTuHmw69lJGksA==} - engines: {node: '>=8.0.0'} - wrap-ansi@6.2.0: resolution: {integrity: sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==} engines: {node: '>=8'} @@ -9389,9 +8650,6 @@ packages: resolution: {integrity: sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==} engines: {node: '>=18'} - wrappy@1.0.2: - resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - write-file-atomic@7.0.1: resolution: {integrity: sha512-OTIk8iR8/aCRWBqvxrzxR0hgxWpnYBblY1S5hDWBQfk/VFmJwzmJgQFN3WsoUKHISv2eAwe+PpbUzyL1CKTLXg==} engines: {node: ^20.17.0 || >=22.9.0} @@ -9473,18 +8731,10 @@ packages: resolution: {integrity: sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg==} engines: {node: ^20.19.0 || ^22.12.0 || >=23} - ylru@1.4.0: - resolution: {integrity: sha512-2OQsPNEmBCvXuFlIni/a+Rn+R2pHW9INm0BxXJ4hVDA8TirqMj+J/Rp9ItLatT/5pZqWwefVrTQcHpixsxnVlA==} - engines: {node: '>= 4.0.0'} - yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} - yocto-queue@1.2.2: - resolution: {integrity: sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==} - engines: {node: '>=12.20'} - yoctocolors@2.2.0: resolution: {integrity: sha512-xYqdZFUK/VYazNl/oCDYN+3WloWQwMfZxBoiNt6qNyk+xfOdi598muWE42rNZFp1kNOiqW936q5RhUdnpqElSg==} engines: {node: '>=18'} @@ -10605,8 +9855,6 @@ snapshots: optionalDependencies: '@types/react': 19.3.0 - '@bazel/runfiles@6.5.0': {} - '@bcoe/v8-coverage@1.0.2': {} '@blazediff/core@1.10.0': {} @@ -11012,8 +10260,6 @@ snapshots: '@floating-ui/utils@0.2.12': {} - '@hapi/bourne@3.0.0': {} - '@html-validate/stylish@6.0.0': {} '@humanfs/core@0.19.2': @@ -12122,8 +11368,6 @@ snapshots: '@sinclair/typebox@0.34.52': {} - '@sindresorhus/is@5.6.0': {} - '@sindresorhus/merge-streams@4.0.0': {} '@smithy/core@3.34.1': @@ -12166,10 +11410,6 @@ snapshots: dependencies: tslib: 2.8.1 - '@szmarczak/http-timer@5.0.1': - dependencies: - defer-to-connect: 2.0.1 - '@tanstack/query-core@5.102.8': {} '@tanstack/react-query@5.102.8(react@19.3.0)': @@ -12177,8 +11417,6 @@ snapshots: '@tanstack/query-core': 5.102.8 react: 19.3.0 - '@testim/chrome-version@1.1.4': {} - '@testing-library/dom@10.4.1': dependencies: '@babel/code-frame': 7.29.7 @@ -12317,10 +11555,6 @@ snapshots: dependencies: '@types/node': 22.20.2 - '@types/execa@0.9.0': - dependencies: - '@types/node': 22.20.2 - '@types/find-package-json@1.2.7': dependencies: '@types/node': 22.20.2 @@ -12335,8 +11569,6 @@ snapshots: dependencies: '@types/unist': 3.0.3 - '@types/http-cache-semantics@4.2.0': {} - '@types/is-empty@1.2.3': {} '@types/jsdom@30.0.0': @@ -12364,8 +11596,6 @@ snapshots: '@types/ms@2.1.0': {} - '@types/node@11.15.54': {} - '@types/node@14.18.63': {} '@types/node@18.19.130': @@ -12386,8 +11616,6 @@ snapshots: dependencies: parse-path: 7.1.0 - '@types/parse5@5.0.3': {} - '@types/prop-types@15.7.15': {} '@types/proper-lockfile@4.1.4': @@ -12926,11 +12154,6 @@ snapshots: abbrev@2.0.0: {} - accepts@1.3.8: - dependencies: - mime-types: 2.1.35 - negotiator: 0.6.3 - acorn-jsx@5.3.2(acorn@8.18.0): dependencies: acorn: 8.18.0 @@ -12941,16 +12164,6 @@ snapshots: acorn@8.18.0: {} - adm-zip@0.6.1: {} - - agent-base@6.0.2(supports-color@10.2.2): - dependencies: - debug: 4.4.3(supports-color@10.2.2) - transitivePeerDependencies: - - supports-color - - agent-base@9.0.0: {} - ajv-formats@3.0.1(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -12985,18 +12198,10 @@ snapshots: dependencies: string-width: 4.2.3 - ansi-escape-sequences@6.2.4: - dependencies: - array-back: 6.2.3 - ansi-regex@5.0.1: {} ansi-regex@6.3.0: {} - ansi-styles@3.2.1: - dependencies: - color-convert: 1.9.3 - ansi-styles@4.3.0: dependencies: color-convert: 2.0.1 @@ -13019,12 +12224,6 @@ snapshots: aria-query@5.3.2: {} - array-back@3.1.0: {} - - array-back@4.0.2: {} - - array-back@6.2.3: {} - array-buffer-byte-length@1.0.2: dependencies: call-bound: 1.0.4 @@ -13104,10 +12303,6 @@ snapshots: ast-types-flow@0.0.8: {} - ast-types@0.13.4: - dependencies: - tslib: 2.8.1 - ast-v8-to-istanbul@1.0.6: dependencies: '@jridgewell/trace-mapping': 0.3.31 @@ -13120,8 +12315,6 @@ snapshots: async-function@1.0.0: {} - asynckit@0.4.0: {} - autoprefixer@10.6.0(postcss@8.5.28): dependencies: browserslist: 4.28.9 @@ -13139,16 +12332,6 @@ snapshots: axe-core@4.13.0: {} - axios@1.20.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2): - dependencies: - follow-redirects: 1.16.0(debug@4.4.3(supports-color@10.2.2)) - form-data: 4.0.6 - https-proxy-agent: 5.0.1(supports-color@10.2.2) - proxy-from-env: 2.1.0 - transitivePeerDependencies: - - debug - - supports-color - axobject-query@4.1.0: {} babel-plugin-macros@3.1.0: @@ -13208,8 +12391,6 @@ snapshots: baseline-browser-mapping@2.11.23: {} - basic-ftp@5.3.1: {} - bcrypt-pbkdf@1.0.2: dependencies: tweetnacl: 0.14.5 @@ -13262,8 +12443,6 @@ snapshots: node-releases: 2.0.55 update-browserslist-db: 1.3.3(browserslist@4.28.9) - buffer-equal-constant-time@1.0.1: {} - buffer-from@1.1.2: {} buildcheck@0.0.7: @@ -13281,23 +12460,6 @@ snapshots: bytes@3.1.2: {} - cache-content-type@1.0.1: - dependencies: - mime-types: 2.1.35 - ylru: 1.4.0 - - cacheable-lookup@7.0.0: {} - - cacheable-request@10.2.14: - dependencies: - '@types/http-cache-semantics': 4.2.0 - get-stream: 6.0.1 - http-cache-semantics: 4.2.0 - keyv: 4.5.4 - mimic-response: 4.0.0 - normalize-url: 8.1.1 - responselike: 3.0.0 - cacheable@2.5.0: dependencies: '@cacheable/memory': 2.2.0 @@ -13343,12 +12505,6 @@ snapshots: dependencies: chalk: 4.1.2 - chalk@2.4.2: - dependencies: - ansi-styles: 3.2.1 - escape-string-regexp: 1.0.5 - supports-color: 5.5.0 - chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -13374,20 +12530,6 @@ snapshots: chrome-trace-event@1.0.4: {} - chromedriver@151.0.5(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2): - dependencies: - '@testim/chrome-version': 1.1.4 - adm-zip: 0.6.1 - axios: 1.20.0(debug@4.4.3(supports-color@10.2.2))(supports-color@10.2.2) - compare-versions: 6.1.1 - proxy-agent: 8.0.2(supports-color@10.2.2) - proxy-from-env: 2.1.0 - tcp-port-used: 1.0.3(supports-color@10.2.2) - transitivePeerDependencies: - - debug - - kerberos - - supports-color - ci-info@4.4.0: {} cli-boxes@3.0.0: {} @@ -13431,16 +12573,6 @@ snapshots: clsx@2.1.1: {} - co-body@6.2.0: - dependencies: - '@hapi/bourne': 3.0.0 - inflation: 2.1.0 - qs: 6.16.0 - raw-body: 2.5.3 - type-is: 1.6.18 - - co@4.6.0: {} - codsen-format-diagnostic-value@1.1.1: {} codsen-utils@1.10.2: @@ -13449,40 +12581,16 @@ snapshots: collapse-white-space@2.1.0: {} - color-convert@1.9.3: - dependencies: - color-name: 1.1.3 - color-convert@2.0.1: dependencies: color-name: 1.1.4 - color-name@1.1.3: {} - color-name@1.1.4: {} colord@2.10.0: {} - combined-stream@1.0.8: - dependencies: - delayed-stream: 1.0.0 - comma-separated-tokens@2.0.3: {} - command-line-args@5.2.1: - dependencies: - array-back: 3.1.0 - find-replace: 3.0.0 - lodash.camelcase: 4.3.0 - typical: 4.0.0 - - command-line-usage@6.1.3: - dependencies: - array-back: 4.0.2 - chalk: 2.4.2 - table-layout: 1.0.2 - typical: 5.2.0 - commander@14.0.3: {} commander@2.20.3: {} @@ -13493,8 +12601,6 @@ snapshots: commondir@1.0.1: {} - compare-versions@6.1.1: {} - compressible@2.0.18: dependencies: mime-db: 1.54.0 @@ -13522,8 +12628,6 @@ snapshots: content-disposition@0.5.2: {} - content-type@1.0.5: {} - content-type@3.1.0: {} convert-source-map@1.9.0: {} @@ -13535,13 +12639,6 @@ snapshots: lodash.clonedeep: 4.5.0 yargs-parser: 20.2.9 - cookies@0.9.2: - dependencies: - depd: 2.0.0 - keygrip: 1.1.0 - - copy-to@2.0.1: {} - core-js-compat@3.50.0: dependencies: browserslist: 4.28.9 @@ -13573,14 +12670,6 @@ snapshots: nan: 2.28.0 optional: true - cross-spawn@6.0.6: - dependencies: - nice-try: 1.0.5 - path-key: 2.0.1 - semver: 5.7.2 - shebang-command: 1.2.0 - which: 1.3.1 - cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -13615,8 +12704,6 @@ snapshots: csstype@3.2.3: {} - csv-stringify@6.9.0: {} - d3-array@3.2.4: dependencies: internmap: 2.0.3 @@ -13659,8 +12746,6 @@ snapshots: damerau-levenshtein@1.0.8: {} - data-uri-to-buffer@8.0.0: {} - data-urls@7.0.0: dependencies: whatwg-mimetype: 5.0.0 @@ -13702,12 +12787,6 @@ snapshots: optionalDependencies: supports-color: 10.2.2 - debug@4.3.1(supports-color@10.2.2): - dependencies: - ms: 2.1.2 - optionalDependencies: - supports-color: 10.2.2 - debug@4.4.3(supports-color@10.2.2): dependencies: ms: 2.1.3 @@ -13728,12 +12807,6 @@ snapshots: dependencies: character-entities: 2.0.2 - decompress-response@6.0.0: - dependencies: - mimic-response: 3.1.0 - - deep-equal@1.0.1: {} - deep-extend@0.6.0: {} deep-is@0.1.4: {} @@ -13745,8 +12818,6 @@ snapshots: bundle-name: 4.1.0 default-browser-id: 5.0.1 - defer-to-connect@2.0.1: {} - define-data-property@1.1.4: dependencies: es-define-property: 1.0.1 @@ -13761,25 +12832,8 @@ snapshots: has-property-descriptors: 1.0.2 object-keys: 1.1.1 - degenerator@7.0.1(quickjs-wasi@2.2.0): - dependencies: - ast-types: 0.13.4 - escodegen: 2.1.0 - esprima: 4.0.1 - quickjs-wasi: 2.2.0 - - delayed-stream@1.0.0: {} - - delegates@1.0.0: {} - - depd@1.1.2: {} - - depd@2.0.0: {} - dequal@2.0.3: {} - destroy@1.2.0: {} - detect-libc@2.1.2: {} devlop@1.1.0: @@ -13831,12 +12885,6 @@ snapshots: eastasianwidth@0.2.0: {} - ecdsa-sig-formatter@1.0.11: - dependencies: - safe-buffer: 5.2.1 - - ee-first@1.1.1: {} - electron-to-chromium@1.5.427: {} emoji-regex@10.6.0: {} @@ -13847,12 +12895,6 @@ snapshots: empathic@2.0.1: {} - encodeurl@1.0.2: {} - - end-of-stream@1.4.5: - dependencies: - once: 1.4.0 - enhanced-resolve@5.25.0: dependencies: graceful-fs: 4.2.11 @@ -14037,22 +13079,10 @@ snapshots: escalade@3.2.0: {} - escape-html@1.0.3: {} - - escape-string-regexp@1.0.5: {} - escape-string-regexp@4.0.0: {} escape-string-regexp@5.0.0: {} - escodegen@2.1.0: - dependencies: - esprima: 4.0.1 - estraverse: 5.3.0 - esutils: 2.0.3 - optionalDependencies: - source-map: 0.6.1 - eslint-compat-utils@0.5.1(eslint@10.10.0(jiti@2.7.0)(supports-color@10.2.2)): dependencies: eslint: 10.10.0(jiti@2.7.0)(supports-color@10.2.2) @@ -14382,8 +13412,6 @@ snapshots: acorn-jsx: 5.3.2(acorn@8.18.0) eslint-visitor-keys: 5.0.1 - esprima@4.0.1: {} - esquery@1.7.0: dependencies: estraverse: 5.3.0 @@ -14433,16 +13461,6 @@ snapshots: events@3.3.0: {} - execa@1.0.0: - dependencies: - cross-spawn: 6.0.6 - get-stream: 4.1.0 - is-stream: 1.1.0 - npm-run-path: 2.0.2 - p-finally: 1.0.0 - signal-exit: 3.0.7 - strip-eof: 1.0.0 - execa@10.0.1: dependencies: '@sindresorhus/merge-streams': 4.0.0 @@ -14582,10 +13600,6 @@ snapshots: find-package-json@1.2.0: {} - find-replace@3.0.0: - dependencies: - array-back: 3.1.0 - find-root@1.1.0: {} find-up@4.1.0: @@ -14598,11 +13612,6 @@ snapshots: locate-path: 6.0.0 path-exists: 4.0.0 - find-up@6.3.0: - dependencies: - locate-path: 7.2.0 - path-exists: 5.0.0 - flat-cache@4.0.1: dependencies: flatted: 3.4.4 @@ -14618,10 +13627,6 @@ snapshots: flatted@3.4.4: {} - follow-redirects@1.16.0(debug@4.4.3(supports-color@10.2.2)): - optionalDependencies: - debug: 4.4.3(supports-color@10.2.2) - for-each@0.3.5: dependencies: is-callable: 1.2.7 @@ -14631,30 +13636,12 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 - form-data-encoder@2.1.4: {} - - form-data@4.0.6: - dependencies: - asynckit: 0.4.0 - combined-stream: 1.0.8 - es-set-tostringtag: 2.1.0 - hasown: 2.0.4 - mime-types: 2.1.35 - format-util@1.0.5: {} format@0.2.2: {} fraction.js@5.3.4: {} - fresh@0.5.2: {} - - fs-extra@10.1.0: - dependencies: - graceful-fs: 4.2.11 - jsonfile: 6.2.1 - universalify: 2.0.1 - fsevents@2.3.2: optional: true @@ -14709,14 +13696,6 @@ snapshots: dunder-proto: 1.0.1 es-object-atoms: 1.1.2 - get-stream@4.1.0: - dependencies: - pump: 3.0.4 - - get-stream@5.2.0: - dependencies: - pump: 3.0.4 - get-stream@6.0.1: {} get-stream@8.0.1: {} @@ -14736,14 +13715,6 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 - get-uri@8.0.1(supports-color@10.2.2): - dependencies: - basic-ftp: 5.3.1 - data-uri-to-buffer: 8.0.0 - debug: 4.4.3(supports-color@10.2.2) - transitivePeerDependencies: - - supports-color - git-up@8.1.1: dependencies: is-ssh: 1.4.1 @@ -14815,20 +13786,6 @@ snapshots: gopd@1.2.0: {} - got@12.6.1: - dependencies: - '@sindresorhus/is': 5.6.0 - '@szmarczak/http-timer': 5.0.1 - cacheable-lookup: 7.0.0 - cacheable-request: 10.2.14 - decompress-response: 6.0.0 - form-data-encoder: 2.1.4 - get-stream: 6.0.1 - http2-wrapper: 2.2.1 - lowercase-keys: 3.0.0 - p-cancelable: 3.0.0 - responselike: 3.0.0 - graceful-fs@4.2.11: {} gzip-size@6.0.0: @@ -14837,8 +13794,6 @@ snapshots: has-bigints@1.1.0: {} - has-flag@3.0.0: {} - has-flag@4.0.0: {} has-flag@5.0.1: {} @@ -15017,76 +13972,12 @@ snapshots: html-void-elements@3.0.0: {} - http-assert@1.5.0: - dependencies: - deep-equal: 1.0.1 - http-errors: 1.8.1 - - http-cache-semantics@4.2.0: {} - - http-errors@1.6.3: - dependencies: - depd: 1.1.2 - inherits: 2.0.3 - setprototypeof: 1.1.0 - statuses: 1.5.0 - - http-errors@1.8.1: - dependencies: - depd: 1.1.2 - inherits: 2.0.4 - setprototypeof: 1.2.0 - statuses: 1.5.0 - toidentifier: 1.0.1 - - http-errors@2.0.1: - dependencies: - depd: 2.0.0 - inherits: 2.0.4 - setprototypeof: 1.2.0 - statuses: 2.0.2 - toidentifier: 1.0.1 - - http-proxy-agent@9.1.0(supports-color@10.2.2): - dependencies: - agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) - proxy-agent-negotiate: 1.1.0 - transitivePeerDependencies: - - kerberos - - supports-color - - http2-wrapper@2.2.1: - dependencies: - quick-lru: 5.1.1 - resolve-alpn: 1.2.1 - - https-proxy-agent@5.0.1(supports-color@10.2.2): - dependencies: - agent-base: 6.0.2(supports-color@10.2.2) - debug: 4.4.3(supports-color@10.2.2) - transitivePeerDependencies: - - supports-color - - https-proxy-agent@9.1.0(supports-color@10.2.2): - dependencies: - agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) - proxy-agent-negotiate: 1.1.0 - transitivePeerDependencies: - - kerberos - - supports-color - human-signals@2.1.0: {} human-signals@5.0.0: {} human-signals@8.0.1: {} - iconv-lite@0.4.24: - dependencies: - safer-buffer: 2.1.2 - iconv-lite@0.7.3: dependencies: safer-buffer: 2.1.2 @@ -15112,10 +14003,6 @@ snapshots: imurmurhash@0.1.4: {} - inflation@2.1.0: {} - - inherits@2.0.3: {} - inherits@2.0.4: {} ini@1.3.8: {} @@ -15138,10 +14025,6 @@ snapshots: internmap@2.0.3: {} - ip-address@10.7.2: {} - - ip-regex@2.1.0: {} - is-alphabetical@2.0.1: {} is-alphanumerical@2.0.1: @@ -15279,8 +14162,6 @@ snapshots: dependencies: protocols: 2.0.2 - is-stream@1.1.0: {} - is-stream@2.0.1: {} is-stream@3.0.0: {} @@ -15304,8 +14185,6 @@ snapshots: is-unicode-supported@2.1.0: {} - is-url@1.2.4: {} - is-wayland@0.1.0: {} is-weakmap@2.0.2: {} @@ -15327,12 +14206,6 @@ snapshots: dependencies: is-inside-container: 1.0.0 - is2@2.0.1: - dependencies: - deep-is: 0.1.4 - ip-regex: 2.1.0 - is-url: 1.2.4 - is64bit@2.0.0: dependencies: system-architecture: 0.1.0 @@ -15432,29 +14305,6 @@ snapshots: dependencies: '@dmsnell/diff-match-patch': 1.1.0 - jsonfile@6.2.1: - dependencies: - universalify: 2.0.1 - optionalDependencies: - graceful-fs: 4.2.11 - - jsonschema@1.5.0: {} - - jsonwebtoken@9.0.3: - dependencies: - jws: 4.0.1 - lodash.includes: 4.3.0 - lodash.isboolean: 3.0.3 - lodash.isinteger: 4.0.4 - lodash.isnumber: 3.0.3 - lodash.isplainobject: 4.0.6 - lodash.isstring: 4.0.1 - lodash.once: 4.1.1 - ms: 2.1.3 - semver: 7.8.5 - - jstat@1.9.6: {} - jsx-ast-utils@3.3.5: dependencies: array-includes: 3.2.0 @@ -15469,23 +14319,8 @@ snapshots: readable-stream: 2.3.8 setimmediate: 1.0.5 - jwa@2.0.1: - dependencies: - buffer-equal-constant-time: 1.0.1 - ecdsa-sig-formatter: 1.0.11 - safe-buffer: 5.2.1 - - jws@4.0.1: - dependencies: - jwa: 2.0.1 - safe-buffer: 5.2.1 - kebab-case@2.0.2: {} - keygrip@1.1.0: - dependencies: - tsscmp: 1.0.6 - keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -15500,82 +14335,6 @@ snapshots: kleur@4.1.5: {} - koa-bodyparser@4.4.1: - dependencies: - co-body: 6.2.0 - copy-to: 2.0.1 - type-is: 1.6.18 - - koa-compose@4.1.0: {} - - koa-convert@2.0.0: - dependencies: - co: 4.6.0 - koa-compose: 4.1.0 - - koa-mount@4.2.0(supports-color@10.2.2): - dependencies: - debug: 4.4.3(supports-color@10.2.2) - koa-compose: 4.1.0 - transitivePeerDependencies: - - supports-color - - koa-node-resolve@1.0.0-pre.9(supports-color@10.2.2): - dependencies: - '@babel/generator': 7.29.8 - '@babel/parser': 7.29.8 - '@babel/traverse': 7.29.8(supports-color@10.2.2) - '@types/babel__generator': 7.27.0 - '@types/parse5': 5.0.3 - get-stream: 5.2.0 - parse5: 5.1.1 - resolve: 1.22.12 - transitivePeerDependencies: - - supports-color - - koa-send@5.0.1(supports-color@10.2.2): - dependencies: - debug: 4.4.3(supports-color@10.2.2) - http-errors: 1.8.1 - resolve-path: 1.4.0 - transitivePeerDependencies: - - supports-color - - koa-static@5.0.0(supports-color@10.2.2): - dependencies: - debug: 3.2.7(supports-color@10.2.2) - koa-send: 5.0.1(supports-color@10.2.2) - transitivePeerDependencies: - - supports-color - - koa@2.16.4(supports-color@10.2.2): - dependencies: - accepts: 1.3.8 - cache-content-type: 1.0.1 - content-disposition: 0.5.2 - content-type: 1.0.5 - cookies: 0.9.2 - debug: 4.4.3(supports-color@10.2.2) - delegates: 1.0.0 - depd: 2.0.0 - destroy: 1.2.0 - encodeurl: 1.0.2 - escape-html: 1.0.3 - fresh: 0.5.2 - http-assert: 1.5.0 - http-errors: 1.8.1 - is-generator-function: 1.1.2 - koa-compose: 4.1.0 - koa-convert: 2.0.0 - on-finished: 2.4.1 - only: 0.0.2 - parseurl: 1.3.3 - statuses: 1.5.0 - type-is: 1.6.18 - vary: 1.1.2 - transitivePeerDependencies: - - supports-color - language-subtag-registry@0.3.23: {} language-tags@1.0.9: @@ -15659,12 +14418,6 @@ snapshots: dependencies: p-locate: 5.0.0 - locate-path@7.2.0: - dependencies: - p-locate: 6.0.0 - - lodash.camelcase@4.3.0: {} - lodash.clonedeep@4.5.0: {} lodash.debounce@4.0.8: {} @@ -15673,26 +14426,12 @@ snapshots: lodash.groupby@4.6.0: {} - lodash.includes@4.3.0: {} - - lodash.isboolean@3.0.3: {} - - lodash.isinteger@4.0.4: {} - - lodash.isnumber@3.0.3: {} - - lodash.isplainobject@4.0.6: {} - - lodash.isstring@4.0.1: {} - lodash.memoize@4.1.2: {} lodash.merge@4.6.2: {} lodash.mergewith@4.6.2: {} - lodash.once@4.1.1: {} - lodash.truncate@4.4.2: {} lodash.uniq@4.5.0: {} @@ -15707,8 +14446,6 @@ snapshots: dependencies: js-tokens: 4.0.0 - lowercase-keys@3.0.0: {} - lru-cache@10.4.3: {} lru-cache@11.5.2: {} @@ -15717,8 +14454,6 @@ snapshots: dependencies: yallist: 3.1.1 - lru-cache@7.18.3: {} - lru.min@1.1.5: {} lucide-react@1.45.0(react@19.3.0): @@ -15940,8 +14675,6 @@ snapshots: mdn-data@2.27.1: {} - media-typer@0.3.0: {} - meow@14.1.0: {} merge-stream@2.0.0: {} @@ -16226,18 +14959,12 @@ snapshots: mime-db@1.33.0: {} - mime-db@1.52.0: {} - mime-db@1.54.0: {} mime-types@2.1.18: dependencies: mime-db: 1.33.0 - mime-types@2.1.35: - dependencies: - mime-db: 1.52.0 - mime-types@3.0.2: dependencies: mime-db: 1.54.0 @@ -16246,10 +14973,6 @@ snapshots: mimic-fn@4.0.0: {} - mimic-response@3.1.0: {} - - mimic-response@4.0.0: {} - min-indent@1.0.1: {} minimatch@10.2.6: @@ -16287,8 +15010,6 @@ snapshots: ms@2.0.0: {} - ms@2.1.2: {} - ms@2.1.3: {} mute-stream@3.0.0: {} @@ -16323,14 +15044,10 @@ snapshots: natural-compare@1.4.0: {} - negotiator@0.6.3: {} - negotiator@0.6.4: {} neo-async@2.6.2: {} - netmask@2.1.1: {} - next@16.3.5(@babel/core@7.29.7(supports-color@10.2.2))(@playwright/test@1.63.0)(@types/node@22.20.2)(babel-plugin-macros@3.1.0)(babel-plugin-react-compiler@1.0.0)(react-dom@19.3.0(react@19.3.0))(react@19.3.0): dependencies: '@next/env': 16.3.5 @@ -16358,8 +15075,6 @@ snapshots: - '@types/node' - babel-plugin-macros - nice-try@1.0.5: {} - node-exports-info@1.6.2: dependencies: array.prototype.flatmap: 1.3.3 @@ -16388,8 +15103,6 @@ snapshots: normalize-path@3.0.0: {} - normalize-url@8.1.1: {} - npm-install-checks@6.3.0: dependencies: semver: 7.8.5 @@ -16410,10 +15123,6 @@ snapshots: npm-package-arg: 11.0.3 semver: 7.8.5 - npm-run-path@2.0.2: - dependencies: - path-key: 2.0.1 - npm-run-path@4.0.1: dependencies: path-key: 3.1.1 @@ -16477,16 +15186,8 @@ snapshots: obug@2.2.1: {} - on-finished@2.4.1: - dependencies: - ee-first: 1.1.1 - on-headers@1.1.0: {} - once@1.4.0: - dependencies: - wrappy: 1.0.2 - onetime@5.1.2: dependencies: mimic-fn: 2.1.0 @@ -16495,8 +15196,6 @@ snapshots: dependencies: mimic-fn: 4.0.0 - only@0.0.2: {} - open@11.0.3: dependencies: default-browser: 5.5.1 @@ -16530,10 +15229,6 @@ snapshots: object-keys: 1.1.1 safe-push-apply: 1.0.0 - p-cancelable@3.0.0: {} - - p-finally@1.0.0: {} - p-limit@2.3.0: dependencies: p-try: 2.2.0 @@ -16542,10 +15237,6 @@ snapshots: dependencies: yocto-queue: 0.1.0 - p-limit@4.0.0: - dependencies: - yocto-queue: 1.2.2 - p-locate@4.1.0: dependencies: p-limit: 2.3.0 @@ -16554,36 +15245,12 @@ snapshots: dependencies: p-limit: 3.1.0 - p-locate@6.0.0: - dependencies: - p-limit: 4.0.0 - p-retry@8.0.1: dependencies: is-network-error: 1.3.2 p-try@2.2.0: {} - pac-proxy-agent@9.1.0(supports-color@10.2.2): - dependencies: - agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) - get-uri: 8.0.1(supports-color@10.2.2) - http-proxy-agent: 9.1.0(supports-color@10.2.2) - https-proxy-agent: 9.1.0(supports-color@10.2.2) - pac-resolver: 9.0.1(quickjs-wasi@2.2.0) - quickjs-wasi: 2.2.0 - socks-proxy-agent: 10.1.0(supports-color@10.2.2) - transitivePeerDependencies: - - kerberos - - supports-color - - pac-resolver@9.0.1(quickjs-wasi@2.2.0): - dependencies: - degenerator: 7.0.1(quickjs-wasi@2.2.0) - netmask: 2.1.1 - quickjs-wasi: 2.2.0 - package-json-from-dist@1.0.1: {} pako@1.0.11: {} @@ -16630,8 +15297,6 @@ snapshots: '@types/parse-path': 7.1.0 parse-path: 7.1.0 - parse5@5.1.1: {} - parse5@7.3.0: dependencies: entities: 6.0.1 @@ -16640,18 +15305,10 @@ snapshots: dependencies: entities: 8.1.0 - parseurl@1.3.3: {} - path-exists@4.0.0: {} - path-exists@5.0.0: {} - - path-is-absolute@1.0.1: {} - path-is-inside@1.0.2: {} - path-key@2.0.1: {} - path-key@3.1.1: {} path-key@4.0.0: {} @@ -16692,18 +15349,8 @@ snapshots: dependencies: find-up: 4.1.0 - pkg-install@1.0.0: - dependencies: - '@types/execa': 0.9.0 - '@types/node': 11.15.54 - execa: 1.0.0 - pkg-pr-new@0.0.88: {} - pkg-up@4.0.0: - dependencies: - find-up: 6.3.0 - playwright-core@1.62.1: {} playwright-core@1.63.0: {} @@ -16813,8 +15460,6 @@ snapshots: process-nextick-args@2.0.1: {} - progress@2.0.3: {} - promise-inflight@1.0.1: {} promise-retry@2.0.1: @@ -16843,29 +15488,6 @@ snapshots: protocols@2.0.2: {} - proxy-agent-negotiate@1.1.0: {} - - proxy-agent@8.0.2(supports-color@10.2.2): - dependencies: - agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) - http-proxy-agent: 9.1.0(supports-color@10.2.2) - https-proxy-agent: 9.1.0(supports-color@10.2.2) - lru-cache: 7.18.3 - pac-proxy-agent: 9.1.0(supports-color@10.2.2) - proxy-from-env: 2.1.0 - socks-proxy-agent: 10.1.0(supports-color@10.2.2) - transitivePeerDependencies: - - kerberos - - supports-color - - proxy-from-env@2.1.0: {} - - pump@3.0.4: - dependencies: - end-of-stream: 1.4.5 - once: 1.4.0 - punycode@1.4.1: optional: true @@ -16885,13 +15507,10 @@ snapshots: dependencies: es-define-property: 1.0.1 side-channel: 1.1.1 + optional: true queue-microtask@1.2.3: {} - quick-lru@5.1.1: {} - - quickjs-wasi@2.2.0: {} - quotation@2.0.3: {} range-parser@1.2.0: {} @@ -16924,13 +15543,6 @@ snapshots: xmlserializer: 0.6.1 optional: true - raw-body@2.5.3: - dependencies: - bytes: 3.1.2 - http-errors: 2.0.1 - iconv-lite: 0.4.24 - unpipe: 1.0.0 - rc@1.2.8: dependencies: deep-extend: 0.6.0 @@ -17026,8 +15638,6 @@ snapshots: unified: 11.0.5 vfile: 6.0.3 - reduce-flatten@2.0.0: {} - reflect.getprototypeof@1.0.10: dependencies: call-bind: 1.0.9 @@ -17321,15 +15931,8 @@ snapshots: reselect@5.3.0: {} - resolve-alpn@1.2.1: {} - resolve-from@4.0.0: {} - resolve-path@1.4.0: - dependencies: - http-errors: 1.6.3 - path-is-absolute: 1.0.1 - resolve-pkg-maps@1.0.0: {} resolve@1.22.12: @@ -17348,10 +15951,6 @@ snapshots: path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 - responselike@3.0.0: - dependencies: - lowercase-keys: 3.0.0 - retry@0.12.0: {} reusify@1.1.0: {} @@ -17427,10 +16026,6 @@ snapshots: sane-domparser-error@0.2.0: optional: true - sanitize-filename@1.6.4: - dependencies: - truncate-utf8-bytes: 1.0.2 - saxes@5.0.1: dependencies: xmlchars: 2.2.0 @@ -17450,18 +16045,6 @@ snapshots: ajv-formats: 3.0.1(ajv@8.20.0) ajv-keywords: 5.1.0(ajv@8.20.0) - selenium-webdriver@4.49.0: - dependencies: - '@bazel/runfiles': 6.5.0 - jszip: 3.10.2 - tmp: 0.2.7 - ws: 8.21.3 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - semver@5.7.2: {} - semver@6.3.1: {} semver@7.8.5: {} @@ -17520,10 +16103,6 @@ snapshots: setimmediate@1.0.5: {} - setprototypeof@1.1.0: {} - - setprototypeof@1.2.0: {} - sharp@0.35.4(@types/node@22.20.2): dependencies: '@img/colour': 1.1.0 @@ -17557,16 +16136,10 @@ snapshots: '@img/sharp-win32-x64': 0.35.4 '@types/node': 22.20.2 - shebang-command@1.2.0: - dependencies: - shebang-regex: 1.0.0 - shebang-command@2.0.0: dependencies: shebang-regex: 3.0.0 - shebang-regex@1.0.0: {} - shebang-regex@3.0.0: {} side-channel-list@1.0.1: @@ -17625,21 +16198,6 @@ snapshots: astral-regex: 2.0.0 is-fullwidth-code-point: 3.0.0 - smart-buffer@4.2.0: {} - - socks-proxy-agent@10.1.0(supports-color@10.2.2): - dependencies: - agent-base: 9.0.0 - debug: 4.4.3(supports-color@10.2.2) - socks: 2.8.10 - transitivePeerDependencies: - - supports-color - - socks@2.8.10: - dependencies: - ip-address: 10.7.2 - smart-buffer: 4.2.0 - source-map-js@1.2.1: {} source-map-support@0.5.21: @@ -17685,10 +16243,6 @@ snapshots: stackback@0.0.2: {} - statuses@1.5.0: {} - - statuses@2.0.2: {} - std-env@4.2.0: {} stop-iteration-iterator@1.1.0: @@ -17823,8 +16377,6 @@ snapshots: strip-bom@3.0.0: {} - strip-eof@1.0.0: {} - strip-final-newline@2.0.0: {} strip-final-newline@3.0.0: {} @@ -17924,10 +16476,6 @@ snapshots: supports-color@10.2.2: {} - supports-color@5.5.0: - dependencies: - has-flag: 3.0.0 - supports-color@7.2.0: dependencies: has-flag: 4.0.0 @@ -17955,15 +16503,6 @@ snapshots: system-architecture@0.1.0: {} - systeminformation@5.33.13: {} - - table-layout@1.0.2: - dependencies: - array-back: 4.0.2 - deep-extend: 0.6.0 - typical: 5.2.0 - wordwrapjs: 4.0.1 - table@6.9.0: dependencies: ajv: 8.20.0 @@ -17972,50 +16511,8 @@ snapshots: string-width: 4.2.3 strip-ansi: 6.0.1 - tachometer@0.7.2(supports-color@10.2.2): - dependencies: - ansi-escape-sequences: 6.2.4 - command-line-args: 5.2.1 - command-line-usage: 6.1.3 - csv-stringify: 6.9.0 - fs-extra: 10.1.0 - get-stream: 6.0.1 - got: 12.6.1 - jsonschema: 1.5.0 - jsonwebtoken: 9.0.3 - jstat: 1.9.6 - koa: 2.16.4(supports-color@10.2.2) - koa-bodyparser: 4.4.1 - koa-mount: 4.2.0(supports-color@10.2.2) - koa-node-resolve: 1.0.0-pre.9(supports-color@10.2.2) - koa-send: 5.0.1(supports-color@10.2.2) - koa-static: 5.0.0(supports-color@10.2.2) - pkg-install: 1.0.0 - pkg-up: 4.0.0 - progress: 2.0.3 - sanitize-filename: 1.6.4 - selenium-webdriver: 4.49.0 - semver: 7.8.5 - source-map-support: 0.5.21 - strip-ansi: 7.2.0 - systeminformation: 5.33.13 - table: 6.9.0 - ua-parser-js: 1.0.41 - transitivePeerDependencies: - - '@75lb/nature' - - bufferutil - - supports-color - - utf-8-validate - tapable@2.3.3: {} - tcp-port-used@1.0.3(supports-color@10.2.2): - dependencies: - debug: 4.3.1(supports-color@10.2.2) - is2: 2.0.1 - transitivePeerDependencies: - - supports-color - terser@5.51.2: dependencies: '@jridgewell/source-map': 0.3.11 @@ -18058,8 +16555,6 @@ snapshots: toad-cache@3.7.4: {} - toidentifier@1.0.1: {} - totalist@3.0.1: {} tough-cookie@6.0.2: @@ -18074,10 +16569,6 @@ snapshots: trough@2.2.0: {} - truncate-utf8-bytes@1.0.2: - dependencies: - utf8-byte-length: 1.0.5 - ts-api-utils@2.5.0(@typescript/typescript6@6.0.2): dependencies: typescript: '@typescript/typescript6@6.0.2' @@ -18093,8 +16584,6 @@ snapshots: tslib@2.8.1: {} - tsscmp@1.0.6: {} - tsx@4.23.13: dependencies: esbuild: 0.28.2 @@ -18117,11 +16606,6 @@ snapshots: type-fest@4.41.0: {} - type-is@1.6.18: - dependencies: - media-typer: 0.3.0 - mime-types: 2.1.35 - typed-array-buffer@1.0.3: dependencies: call-bound: 1.0.4 @@ -18198,12 +16682,6 @@ snapshots: '@typescript/typescript-win32-arm64': 7.0.2 '@typescript/typescript-win32-x64': 7.0.2 - typical@4.0.0: {} - - typical@5.2.0: {} - - ua-parser-js@1.0.41: {} - uint8-to-base64@0.2.1: {} unbox-primitive@1.1.0: @@ -18337,10 +16815,6 @@ snapshots: universal-user-agent@7.0.3: {} - universalify@2.0.1: {} - - unpipe@1.0.0: {} - unrs-resolver@1.12.2: dependencies: napi-postinstall: 0.3.4 @@ -18393,8 +16867,6 @@ snapshots: dependencies: react: 19.3.0 - utf8-byte-length@1.0.5: {} - util-deprecate@1.0.2: {} validate-npm-package-license@3.0.4: @@ -18662,11 +17134,6 @@ snapshots: word-wrap@1.2.5: {} - wordwrapjs@4.0.1: - dependencies: - reduce-flatten: 2.0.0 - typical: 5.2.0 - wrap-ansi@6.2.0: dependencies: ansi-styles: 4.3.0 @@ -18691,8 +17158,6 @@ snapshots: string-width: 7.2.0 strip-ansi: 7.2.0 - wrappy@1.0.2: {} - write-file-atomic@7.0.1: dependencies: signal-exit: 4.1.0 @@ -18755,12 +17220,8 @@ snapshots: y18n: 5.0.8 yargs-parser: 22.0.0 - ylru@1.4.0: {} - yocto-queue@0.1.0: {} - yocto-queue@1.2.2: {} - yoctocolors@2.2.0: {} zod-validation-error@3.5.4(zod@3.25.76): diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 8559ab480b..cdf532cc12 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,7 +6,6 @@ packages: engineStrict: true strictDepBuilds: true allowBuilds: - chromedriver: true esbuild: true nx: true sharp: true diff --git a/test/tachometer/README.md b/test/tachometer/README.md deleted file mode 100644 index f17882ea83..0000000000 --- a/test/tachometer/README.md +++ /dev/null @@ -1,115 +0,0 @@ -# Tachometer smoke test - -A working harness for `benchmark tacho run`, used to exercise that tooling end to end. - -This is **not** a meaningful benchmark, and it is not trying to be. This repository ships build and -test tooling, not a browser library, so there is nothing here whose render time is worth tracking. -The cases run a deterministic CPU workload purely so the pipeline has something real to drive: -discovering cases, resolving the baseline, packing a commit's workspace to tarballs, building each -variant, and running Chrome under tachometer. - -For the real thing, see the consumers this tooling was extracted from — `base-ui-mosaic` and -`base-ui-charts` benchmark actual data grids and charts. - -## What it covers - -The cases are chosen to walk the parts of the pipeline most likely to break: - -| Case | Covers | -| :--------------- | :------------------------------------------------------------------------------- | -| `workload` | The ordinary regression shape: one page, auto-expanded into current vs baseline | -| `workload-large` | A case owning **no page of its own**, parameterising its sibling's with `?size=` | -| `libs-sort` | Three variants from **one build**: the cross-library shape, with no baseline | - -`workload-large` exercises that a case's query parameters survive the url rewrite, that a url -traversing out of its own case folder resolves, and that two cases referencing one page build it -only once. - -`libs-sort` covers the other axis, where nothing is compared across commits. Its three variants are -built from the same commit and differ only in how they sort, standing in for the competitor -libraries this repository has no reason to install. It exercises three things the two-variant cases -cannot: a case with more than two variants, which the report draws as a table of variants rather -than a row per case; a second variant set, which gets a table of its own rather than sharing one -full of blank cells; and a verdict that is genuinely `slower` yet must never be reported as a -regression, because both sides came from the same build. - -`workload` and `workload-large` import `@mui/internal-test-utils` and put its value on screen. That -is the load-bearing part: every ref — the working tree included — resolves it from a packed tarball -in that ref's own tree, so both sides of a comparison consume the library the way a consumer does. -If that path breaks the page fails to build, rather than quietly measuring nothing. - -## Running - -```bash -pnpm release:build # the harness resolves workspace packages through their build output -pnpm test:tacho # from the repo root -``` - -Or from here, with the usual filters and flags: - -```bash -pnpm -F ./test/tachometer test:tacho large -pnpm -F ./test/tachometer test:tacho --baseline HEAD -``` - -A filter is a case-insensitive substring of a case's path under `src/`, the way vitest matches test -files. Cases may sit in folders, and since the path is what a filter matches, a folder name selects -everything beneath it — which is how a repository partitions its suite (a quick set for pull -requests, everything for a release) without the runner being taught what any of it means. A case is -named by its benchmark's own `name`, so moving one between folders does not rename it in the report. - -Cases run sequentially, and each auto-samples until its difference resolves or the (deliberately -short) timeout is hit. Because both sides run identical code, the expected verdict is `unsure` — -that is the "no change" outcome, not a failure. - -`benchmark tacho report` prints that table again from the saved JSON, so a run kept from earlier — -or a report downloaded from CI — can be read without sampling again. - -Everything a run writes goes under `.tachometer/`: the report in `results/`, the pages built per ref -in `builds/`, the packed tarballs in `packed/` (the one worth caching in CI, keyed by commit SHA) -and the install each ref resolves through in `trees/`. Deleting that directory resets the harness -completely, and it is the only thing a repository has to ignore. - -### Prerequisites - -Playwright's Chromium, installed with `pnpm exec playwright install chromium`. Runs use Playwright's -pinned Chrome for Testing rather than whatever Chrome the machine has auto-updated to, so a browser -update cannot move the numbers on its own. - -A chromedriver only drives its own Chrome major, so the root `chromedriver` devDependency has to -stay aligned with `@playwright/test` — bump the two together. `tacho run` checks the pair before it -builds anything and fails with both versions named, rather than letting you find out minutes later -through `SessionNotCreatedError`. - -One wrinkle: tachometer installs a chromedriver into its own package directory on demand, and that -copy wins over the root pin. If the check reports a driver you did not install, delete -`node_modules/.pnpm/tachometer@*/node_modules/tachometer/node_modules/chromedriver` and re-run. - -### A note on the baseline - -`tacho run` does not work out which commit to compare against — `code-infra baseline` does, for -every job that compares a branch to its base, so bundle size and the benchmarks agree on the answer -instead of each deriving it. Pass it in: - -```bash -pnpm test:tacho --baseline "$(pnpm code-infra baseline)" -``` - -Without `--baseline` the previous commit is used, which needs no policy to decide. - -A baseline older than the introduction of `benchmark tacho` will fail to build its pages: this -harness's vite config imports a plugin that did not exist at that commit. `--baseline HEAD` compares -the working tree against the current commit while that is still true. - -## Working on a case by hand - -```bash -pnpm -F ./test/tachometer dev -``` - -Every discovered case is listed at `/`, including `workload-large`, which owns no page of its own and -would otherwise be unreachable — its `?size=` query is in the config, not the file tree. The list is -generated from the case configs rather than written by hand, so it cannot go stale. - -`pnpm -F ./test/tachometer build` and `preview` produce and serve the production bundle, which is -what actually gets measured. The index is emitted into that build too, so the same list is there. diff --git a/test/tachometer/package.json b/test/tachometer/package.json deleted file mode 100644 index cca8712442..0000000000 --- a/test/tachometer/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "private": true, - "description": "Tachometer smoke-test harness for the @mui/internal-benchmark tacho tooling", - "scripts": { - "dev": "vite", - "build": "vite build", - "preview": "vite preview", - "typescript": "tsc -noEmit", - "test:tacho": "node ./scripts/runBenchmark.mjs tacho run" - }, - "dependencies": { - "@mui/internal-test-utils": "workspace:*", - "react": "19.2.8" - }, - "devDependencies": { - "@mui/internal-benchmark": "workspace:*", - "@playwright/test": "1.62.1", - "@types/react": "19.2.18", - "tachometer": "^0.7.2", - "vite": "^8.2.2" - } -} diff --git a/test/tachometer/scripts/runBenchmark.mjs b/test/tachometer/scripts/runBenchmark.mjs deleted file mode 100644 index 63da240419..0000000000 --- a/test/tachometer/scripts/runBenchmark.mjs +++ /dev/null @@ -1,12 +0,0 @@ -// Thin wrapper that runs the `@mui/internal-benchmark` CLI from the workspace package's build -// output. Importing the package by name (rather than relying on its `benchmark` bin) avoids -// depending on pnpm linking the bin shim: the package publishes from `build/`, so the workspace -// link points at a directory that does not exist until the first build — and a later build cannot -// create a shim that install already skipped. The same wrapper exists in `docs/scripts` for the -// same reason. -// -// A consumer installing the published package gets a self-contained directory and can use the bin -// directly; only this repository, where the dependency is a workspace link, needs this. -// -// The CLI parses `process.argv` itself on import. -import '@mui/internal-benchmark/cli'; diff --git a/test/tachometer/src/_shared/measure.ts b/test/tachometer/src/_shared/measure.ts deleted file mode 100644 index 00eefa0ffa..0000000000 --- a/test/tachometer/src/_shared/measure.ts +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Shared measurement helper. - * - * Tachometer's `performance` measurement mode polls the page for a `performance.measure()` span of - * a given name, so a page's only obligation is to produce one. - */ - -/** The measure name each case's `tachometer.json` reads. */ -const MEASURE = 'mount'; - -/** - * Runs `work`, writes its result into the page, and closes the `mount` measure once the browser has - * painted it. - * - * The paint is inside the measured window on purpose: a workload whose result is never displayed - * can be optimized away, and the number would then track nothing. - */ -export function measureMount(work: () => string): void { - performance.mark(`${MEASURE}-start`); - const text = work(); - - const root = document.getElementById('root'); - if (!root) { - throw new Error('The page is missing its #root element.'); - } - root.textContent = text; - - // A double rAF closes the measure after the DOM write *and* the paint it triggers; a single one - // would still be inside the frame that performs the paint. - requestAnimationFrame(() => { - requestAnimationFrame(() => { - performance.mark(`${MEASURE}-end`); - performance.measure(MEASURE, `${MEASURE}-start`, `${MEASURE}-end`); - }); - }); -} - -/** - * Reads a positive integer from the page's query string, so one page can back several cases. - */ -export function numberParam(name: string, fallback: number): number { - const raw = new URLSearchParams(window.location.search).get(name); - if (raw === null) { - return fallback; - } - const value = Number(raw); - if (!Number.isInteger(value) || value <= 0) { - throw new Error(`Expected "${name}" to be a positive integer, got "${raw}".`); - } - return value; -} diff --git a/test/tachometer/src/libs/sort/alpha.html b/test/tachometer/src/libs/sort/alpha.html deleted file mode 100644 index 5de8ad125c..0000000000 --- a/test/tachometer/src/libs/sort/alpha.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - libs-sort [alpha] - - -
    - - - diff --git a/test/tachometer/src/libs/sort/alpha.ts b/test/tachometer/src/libs/sort/alpha.ts deleted file mode 100644 index 425c67e246..0000000000 --- a/test/tachometer/src/libs/sort/alpha.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { measureMount } from '../../_shared/measure'; -import { makeValues, report } from './shared'; - -const values = makeValues(); - -/** Sorts the boxed array in place, paying a comparator call per comparison. */ -measureMount(() => { - values.sort((left, right) => left - right); - return report('alpha', values); -}); diff --git a/test/tachometer/src/libs/sort/beta.html b/test/tachometer/src/libs/sort/beta.html deleted file mode 100644 index f0b543d18e..0000000000 --- a/test/tachometer/src/libs/sort/beta.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - libs-sort [beta] - - -
    - - - diff --git a/test/tachometer/src/libs/sort/beta.ts b/test/tachometer/src/libs/sort/beta.ts deleted file mode 100644 index bd68fad1b3..0000000000 --- a/test/tachometer/src/libs/sort/beta.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { measureMount } from '../../_shared/measure'; -import { makeValues, report } from './shared'; - -const values = makeValues(); - -/** A merge sort written in JavaScript, allocating as it goes — the slow variant of the three. */ -function mergeSort(values: number[]): number[] { - if (values.length <= 1) { - return values; - } - const middle = Math.floor(values.length / 2); - const left = mergeSort(values.slice(0, middle)); - const right = mergeSort(values.slice(middle)); - - const merged: number[] = new Array(values.length); - let leftIndex = 0; - let rightIndex = 0; - for (let index = 0; index < merged.length; index += 1) { - if ( - leftIndex < left.length && - (rightIndex >= right.length || left[leftIndex] <= right[rightIndex]) - ) { - merged[index] = left[leftIndex]; - leftIndex += 1; - } else { - merged[index] = right[rightIndex]; - rightIndex += 1; - } - } - return merged; -} - -measureMount(() => report('beta', mergeSort(values))); diff --git a/test/tachometer/src/libs/sort/ours.html b/test/tachometer/src/libs/sort/ours.html deleted file mode 100644 index 68b8517ce4..0000000000 --- a/test/tachometer/src/libs/sort/ours.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - libs-sort [ours] - - -
    - - - diff --git a/test/tachometer/src/libs/sort/ours.ts b/test/tachometer/src/libs/sort/ours.ts deleted file mode 100644 index 15adf702f1..0000000000 --- a/test/tachometer/src/libs/sort/ours.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { measureMount } from '../../_shared/measure'; -import { makeValues, report } from './shared'; - -// Built outside the measured window: every sample is a fresh page load, so the input is still -// generated once per sample — it just no longer counts as part of the sort being measured. -const values = makeValues(); - -/** Copies into a typed array first, so the engine sorts unboxed doubles with no comparator calls. */ -measureMount(() => { - const sorted = Float64Array.from(values); - sorted.sort(); - return report('ours', sorted); -}); diff --git a/test/tachometer/src/libs/sort/shared.ts b/test/tachometer/src/libs/sort/shared.ts deleted file mode 100644 index 56e7dc3ba8..0000000000 --- a/test/tachometer/src/libs/sort/shared.ts +++ /dev/null @@ -1,51 +0,0 @@ -/** - * The workload the three variants share, so they differ only in how they sort. - * - * This case stands in for a cross-library comparison: the shape where every variant is built from - * the *same* commit, and the difference worth knowing is between implementations rather than - * between two builds of ours. This repository has no competitor libraries to install, so the - * "libraries" here are three sorts with genuinely different constant factors. - */ - -/** Values per sample. Large enough that the sort, not the page load, dominates the measurement. */ -export const SIZE = 100_000; - -/** A small linear congruential generator, so every sample sorts identical input. */ -function createRandom(seed: number): () => number { - let state = seed; - return () => { - state = (state * 1664525 + 1013904223) % 0x100000000; - return state / 0x100000000; - }; -} - -/** A fresh unsorted array. Fresh per sample: re-sorting a sorted array measures something else. */ -export function makeValues(): number[] { - const random = createRandom(42); - const values: number[] = new Array(SIZE); - for (let index = 0; index < SIZE; index += 1) { - values[index] = random(); - } - return values; -} - -/** - * A checksum that depends on the order, so a variant that returned an unsorted or truncated array - * could not quietly report the same number as one that sorted correctly. - * - * Takes an `ArrayLike` so a typed array can be checked without being copied back into a boxed one — - * that copy would have landed inside the measured window of the very variant whose point is to - * avoid boxing. - */ -export function checksum(values: ArrayLike): number { - let total = 0; - for (let index = 0; index < values.length; index += 1) { - total += values[index] * index; - } - return total; -} - -/** The line each variant puts on screen, so the three differ only in how they sort. */ -export function report(variant: string, sorted: ArrayLike): string { - return `libs-sort [${variant}] size=${SIZE} checksum=${checksum(sorted).toFixed(3)}`; -} diff --git a/test/tachometer/src/libs/sort/tachometer.json b/test/tachometer/src/libs/sort/tachometer.json deleted file mode 100644 index 1f87085259..0000000000 --- a/test/tachometer/src/libs/sort/tachometer.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "$schema": "../../../node_modules/tachometer/config.schema.json", - "sampleSize": 10, - "timeout": 1, - "autoSampleConditions": ["10%"], - "benchmarks": [ - { - "name": "libs-sort", - "measurement": { "mode": "performance", "entryName": "mount" }, - "browser": { - "name": "chrome", - "headless": true, - "windowSize": { "width": 1024, "height": 768 } - }, - "expand": [ - { "name": "libs-sort [ours]", "url": "./ours.html" }, - { "name": "libs-sort [alpha]", "url": "./alpha.html" }, - { "name": "libs-sort [beta]", "url": "./beta.html" } - ] - } - ] -} diff --git a/test/tachometer/src/workload-large/tachometer.json b/test/tachometer/src/workload-large/tachometer.json deleted file mode 100644 index ae102d0e4d..0000000000 --- a/test/tachometer/src/workload-large/tachometer.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "../../node_modules/tachometer/config.schema.json", - "sampleSize": 10, - "timeout": 1, - "autoSampleConditions": ["10%"], - "benchmarks": [ - { - "name": "workload-large", - "url": "../workload/index.html?size=400000", - "measurement": { "mode": "performance", "entryName": "mount" }, - "browser": { - "name": "chrome", - "headless": true, - "windowSize": { "width": 1024, "height": 768 } - } - } - ] -} diff --git a/test/tachometer/src/workload/index.html b/test/tachometer/src/workload/index.html deleted file mode 100644 index 556d47228b..0000000000 --- a/test/tachometer/src/workload/index.html +++ /dev/null @@ -1,11 +0,0 @@ - - - - - workload - - -
    - - - diff --git a/test/tachometer/src/workload/index.ts b/test/tachometer/src/workload/index.ts deleted file mode 100644 index e20ed2abfc..0000000000 --- a/test/tachometer/src/workload/index.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { reactMajor } from '@mui/internal-test-utils/env'; -import { measureMount, numberParam } from '../_shared/measure'; - -/** - * A deterministic CPU workload, sized by `?size=`. - * - * The point of this page is not the number it produces — this repository ships build tooling, not a - * browser library, so there is nothing here whose render time is worth tracking. It exists to give - * `benchmark tacho run` something real to drive end to end. - * - * It does import a workspace package (`@mui/internal-test-utils`) and put its value on screen. That - * is the part that matters: under `tacho run` every ref — the working tree included — resolves it - * from a packed tarball in an isolated install, so both sides of a comparison resolve identically. - * A plain `vite dev` or `vite build` resolves it through the workspace link instead. If either path - * breaks, this page fails to build rather than silently measuring nothing. - */ - -/** A small linear congruential generator, so every sample does identical work. */ -function createRandom(seed: number): () => number { - let state = seed; - return () => { - state = (state * 1664525 + 1013904223) % 0x100000000; - return state / 0x100000000; - }; -} - -measureMount(() => { - const size = numberParam('size', 50_000); - const random = createRandom(42); - - const values = new Float64Array(size); - for (let index = 0; index < size; index += 1) { - values[index] = random(); - } - values.sort(); - - let checksum = 0; - for (let index = 0; index < size; index += 1) { - checksum += values[index] * index; - } - - return `size=${size} checksum=${checksum.toFixed(3)} react=${reactMajor}`; -}); diff --git a/test/tachometer/src/workload/tachometer.json b/test/tachometer/src/workload/tachometer.json deleted file mode 100644 index 49d6cecbf3..0000000000 --- a/test/tachometer/src/workload/tachometer.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$schema": "../../node_modules/tachometer/config.schema.json", - "sampleSize": 10, - "timeout": 1, - "autoSampleConditions": ["10%"], - "benchmarks": [ - { - "name": "workload", - "url": "./index.html", - "measurement": { "mode": "performance", "entryName": "mount" }, - "browser": { - "name": "chrome", - "headless": true, - "windowSize": { "width": 1024, "height": 768 } - } - } - ] -} diff --git a/test/tachometer/tsconfig.json b/test/tachometer/tsconfig.json deleted file mode 100644 index cc3ed604af..0000000000 --- a/test/tachometer/tsconfig.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "ESNext", - "moduleResolution": "bundler", - "lib": ["ES2022", "DOM", "DOM.Iterable"], - "types": ["vite/client"], - "strict": true, - "noEmit": true, - "isolatedModules": true, - "verbatimModuleSyntax": true, - "skipLibCheck": true - }, - "include": ["src", "vite.config.mjs"] -} diff --git a/test/tachometer/vite.config.mjs b/test/tachometer/vite.config.mjs deleted file mode 100644 index 98416fbdc9..0000000000 --- a/test/tachometer/vite.config.mjs +++ /dev/null @@ -1,6 +0,0 @@ -import { defineConfig } from 'vite'; -import { tachometer } from '@mui/internal-benchmark/tachometerPlugin'; - -// The plugin contributes everything derived from the benchmark cases: the `src/` root, the -// multi-page app type, the build's entry points, and the dev-server case index. -export default defineConfig({ plugins: [tachometer()] }); diff --git a/tsconfig.json b/tsconfig.json index 4ce11c9516..ce12c69e43 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -26,8 +26,7 @@ { "path": "scripts" }, { "path": "docs" }, { "path": "test/performance" }, - { "path": "test/benchmark" }, - { "path": "test/tachometer" } + { "path": "test/benchmark" } ], "files": [] } From 9e0afeca126773adf7b9896727f40395b6b264b5 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 16:24:09 +0200 Subject: [PATCH 092/173] [benchmark] Drop the thin wrappers from the shared core - mapAsync from es-toolkit, as code-infra uses, replaces mapConcurrently. - execa's own verbose: 'short' echoes each command, so the run() wrapper goes; commands whose output is worth watching inherit stdio. - readRawManifest returns null for a missing manifest instead of the caller checking first, and a fresh cache probes its tarballs with access() directly, so pathExists goes too. --- packages/benchmark/package.json | 1 + packages/benchmark/src/runner/browser.ts | 6 +- packages/benchmark/src/runner/buildPages.ts | 37 ++++++++---- packages/benchmark/src/utils/build.ts | 31 ---------- packages/benchmark/src/utils/exec.ts | 34 ----------- packages/benchmark/src/utils/packWorkspace.ts | 59 ++++++++++++------- packages/benchmark/src/utils/path.ts | 11 ---- pnpm-lock.yaml | 3 + 8 files changed, 73 insertions(+), 109 deletions(-) delete mode 100644 packages/benchmark/src/utils/build.ts delete mode 100644 packages/benchmark/src/utils/exec.ts delete mode 100644 packages/benchmark/src/utils/path.ts diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index 26c0efa0b4..c99de1e170 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -29,6 +29,7 @@ "@vitejs/plugin-react": "^6.1.1", "chalk": "^5.6.2", "env-ci": "^11.2.0", + "es-toolkit": "^1.52.0", "execa": "^10.0.1", "globby": "^16.2.4", "yaml": "^2.9.0", diff --git a/packages/benchmark/src/runner/browser.ts b/packages/benchmark/src/runner/browser.ts index d770685040..8de1402dc9 100644 --- a/packages/benchmark/src/runner/browser.ts +++ b/packages/benchmark/src/runner/browser.ts @@ -1,5 +1,5 @@ +import { access } from 'node:fs/promises'; import type * as PlaywrightTest from '@playwright/test'; -import { pathExists } from '../utils/path'; /** * Resolves the browser every benchmark runs on: Playwright's pinned Chrome for Testing, rather than @@ -17,7 +17,9 @@ export async function resolveBrowserBinary(harnessDir: string): Promise } const binary = playwright.chromium.executablePath(); - if (!binary || !(await pathExists(binary))) { + try { + await access(binary); + } catch { throw new Error( `Playwright's Chromium is not installed at "${binary}". Run \`pnpm exec playwright install chromium\`.`, ); diff --git a/packages/benchmark/src/runner/buildPages.ts b/packages/benchmark/src/runner/buildPages.ts index 9096331946..7f76e62940 100644 --- a/packages/benchmark/src/runner/buildPages.ts +++ b/packages/benchmark/src/runner/buildPages.ts @@ -2,19 +2,18 @@ import * as path from 'node:path'; import { createRequire } from 'node:module'; -import { readFile, rm, writeFile } from 'node:fs/promises'; +import { constants, copyFile, readFile, rm, writeFile } from 'node:fs/promises'; import { pathToFileURL } from 'node:url'; import chalk from 'chalk'; +import { execa } from 'execa'; import type * as Vite from 'vite'; import { parse, stringify } from 'yaml'; // Self-referencing rather than `../../package.json`, which resolves only in the repository — see // the note in ../cli/index.ts. import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; -import { run } from '../utils/exec'; import type { PackedPackage } from '../utils/packWorkspace'; import { refLabel } from './refs'; import type { ResolvedRef } from './refs'; -import { pathExists } from '../utils/path'; /** * Packages the run itself resolves from the harness, so they are never pinned to a ref's build: the @@ -57,8 +56,13 @@ async function pinPackedPackages( ): Promise { const manifestPath = path.join(repoRoot, 'pnpm-workspace.yaml'); const backupPath = backupPathOf(outputDir); - if (!(await pathExists(backupPath))) { - await writeFile(backupPath, await readFile(manifestPath, 'utf8')); + // The first ref sets the repository's manifest aside; a later one finds that copy and keeps it. + try { + await copyFile(manifestPath, backupPath, constants.COPYFILE_EXCL); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') { + throw error; + } } const config = parse(await readFile(backupPath, 'utf8')) ?? {}; @@ -79,13 +83,23 @@ async function pinPackedPackages( */ export async function restoreWorkspace(repoRoot: string, outputDir: string): Promise { const backupPath = backupPathOf(outputDir); - if (!(await pathExists(backupPath))) { - return; + let original: string; + try { + original = await readFile(backupPath, 'utf8'); + } catch (error) { + // No copy set aside: nothing was pinned. + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return; + } + throw error; } console.log(chalk.cyan('\nRestoring the repository install…')); - await writeFile(path.join(repoRoot, 'pnpm-workspace.yaml'), await readFile(backupPath, 'utf8')); + await writeFile(path.join(repoRoot, 'pnpm-workspace.yaml'), original); await rm(backupPath, { force: true }); - await run('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], repoRoot); + await execa('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], { + cwd: repoRoot, + verbose: 'short', + }); } /** @@ -120,7 +134,10 @@ export async function buildRefPages(options: { await pinPackedPackages(repoRoot, outputDir, packages); // `--no-frozen-lockfile` because changing the overrides is the point, and pnpm turns a frozen // install on by itself in CI — where it would refuse the very change being made. - await run('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], repoRoot); + await execa('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], { + cwd: repoRoot, + verbose: 'short', + }); // vite is the harness's own, not this package's: the harness declares the version its pages are // built with, and the config being run is the harness's. diff --git a/packages/benchmark/src/utils/build.ts b/packages/benchmark/src/utils/build.ts deleted file mode 100644 index 6900a559d3..0000000000 --- a/packages/benchmark/src/utils/build.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Maps `mapper` over `items` with at most `concurrency` in flight. - * - * Each worker takes the next item as it frees up, so a slow item does not hold back the rest — - * which matters when the items are `pnpm pack` invocations and each is its own node process. - */ -export async function mapConcurrently( - items: T[], - mapper: (item: T) => Promise, - concurrency: number, -): Promise { - if (!items.length) { - return []; - } - const itemIterator = items.entries(); - const count = Math.min(concurrency, items.length); - const results: R[] = new Array(items.length); - const workers: Promise[] = []; - for (let index = 0; index < count; index += 1) { - workers.push( - Promise.resolve().then(async () => { - for (const [itemIndex, item] of itemIterator) { - // eslint-disable-next-line no-await-in-loop -- a worker processes its items in sequence - results[itemIndex] = await mapper(item); - } - }), - ); - } - await Promise.all(workers); - return results; -} diff --git a/packages/benchmark/src/utils/exec.ts b/packages/benchmark/src/utils/exec.ts deleted file mode 100644 index 4ade33d97d..0000000000 --- a/packages/benchmark/src/utils/exec.ts +++ /dev/null @@ -1,34 +0,0 @@ -/* eslint-disable no-console */ - -import chalk from 'chalk'; -import { execa } from 'execa'; - -export interface RunOptions { - /** Extends the current environment rather than replacing it. */ - env?: NodeJS.ProcessEnv; - /** - * Let the command's output through, for one whose progress is worth watching — a build, a sampling - * run. - * - * Otherwise it is captured, so a successful run says nothing. Nothing is lost either way: execa - * puts both streams on the error it throws, so a failure reports everything it printed. - */ - verbose?: boolean; -} - -/** - * Runs a command in `cwd`, throwing on a non-zero exit. - * - * Echoes the command first: a benchmark run shells out a lot, and being able to copy a failing step - * out of the log and re-run it by hand is worth the noise. - */ -export async function run( - file: string, - args: string[], - cwd: string, - options: RunOptions = {}, -): Promise { - const { env, verbose = false } = options; - console.log(chalk.dim(`$ ${file} ${args.join(' ')} (in ${cwd})`)); - await execa(file, args, { cwd, env, stdio: verbose ? 'inherit' : 'pipe' }); -} diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index 05285f9d4c..1bdb8099d5 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -4,13 +4,11 @@ import * as path from 'node:path'; import * as os from 'node:os'; import { createHash } from 'node:crypto'; import { createReadStream } from 'node:fs'; -import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { access, mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; import { pipeline } from 'node:stream/promises'; import chalk from 'chalk'; +import { mapAsync } from 'es-toolkit/array'; import { execa, parseCommandString } from 'execa'; -import { mapConcurrently } from './build'; -import { run } from './exec'; -import { pathExists } from './path'; import { resolveCommit } from './git'; import { listPublishablePackages } from './pnpm'; @@ -124,9 +122,18 @@ async function hashFile(file: string): Promise { return hash.digest('hex').slice(0, 12); } -/** Reads a folder's raw `manifest.json`. */ -async function readRawManifest(dir: string): Promise { - return JSON.parse(await readFile(path.join(dir, MANIFEST), 'utf8')); +/** Reads a folder's raw `manifest.json`, or `null` when it has none. */ +async function readRawManifest(dir: string): Promise { + let text: string; + try { + text = await readFile(path.join(dir, MANIFEST), 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return null; + } + throw error; + } + return JSON.parse(text); } /** Resolves a raw manifest's stored (relative) tarball names to absolute paths under `dir`. */ @@ -152,17 +159,14 @@ function resolveManifest(raw: RawManifest, dir: string): PackedWorkspace { * evicted or deleted. */ export async function readFreshCache(dir: string, buildCmd: string): Promise { - if (!(await pathExists(path.join(dir, MANIFEST)))) { - return null; - } const raw = await readRawManifest(dir); - if (raw.buildCmd !== buildCmd) { + if (raw === null || raw.buildCmd !== buildCmd) { return null; } - const tarballsPresent = await Promise.all( - raw.packages.map((pkg) => pathExists(path.join(dir, pkg.tarball))), + const tarballs = await Promise.allSettled( + raw.packages.map((pkg) => access(path.join(dir, pkg.tarball))), ); - return tarballsPresent.every(Boolean) ? raw : null; + return tarballs.every((result) => result.status === 'fulfilled') ? raw : null; } /** @@ -200,14 +204,14 @@ export async function packBuiltPackages( } // Each `pnpm pack` is its own node process, so a repository with a dozen public packages would // otherwise start a dozen at once. - return mapConcurrently( + return mapAsync( packages, async ({ path: pkgDir, name, version }) => { const tarball = path.join(outDir, tarballName(name)); - await run('pnpm', ['pack', '--out', tarball], pkgDir); + await execa('pnpm', ['pack', '--out', tarball], { cwd: pkgDir, verbose: 'short' }); return { name, version, tarball }; }, - os.availableParallelism(), + { concurrency: os.availableParallelism() }, ); } @@ -252,18 +256,26 @@ export async function packRef(options: PackRefOptions): Promise const checkout = await mkdtemp(path.join(os.tmpdir(), 'pack-workspace-')); try { console.log(chalk.cyan(`\nChecking out "${ref}" (${sha.slice(0, 9)}) at ${checkout}`)); - await run('git', ['worktree', 'add', '--detach', checkout, sha], repoRoot); + await execa('git', ['worktree', 'add', '--detach', checkout, sha], { + cwd: repoRoot, + verbose: 'short', + }); if (installCmd) { console.log(chalk.cyan(`\nInstalling dependencies for ${sha.slice(0, 9)}…`)); const [installFile, ...installArgs] = parseCommandString(installCmd); - await run(installFile, installArgs, checkout, { verbose: true }); + await execa(installFile, installArgs, { cwd: checkout, stdio: 'inherit', verbose: 'short' }); } console.log(chalk.cyan(`\nBuilding packages for ${sha.slice(0, 9)}…`)); // Disable the nx daemon (lerna runs builds through nx): a lingering daemon keeps writing into // the checkout and makes removal fail with "Directory not empty". execa's `env` extends the // current environment, so only NX_DAEMON is overridden. const [buildFile, ...buildArgs] = parseCommandString(buildCmd); - await run(buildFile, buildArgs, checkout, { env: { NX_DAEMON: 'false' }, verbose: true }); + await execa(buildFile, buildArgs, { + cwd: checkout, + env: { NX_DAEMON: 'false' }, + stdio: 'inherit', + verbose: 'short', + }); const packages = await packBuiltPackages(checkout, staging); // Store tarballs by basename so the folder is relocatable; record buildCmd so a later run can @@ -317,7 +329,12 @@ export async function packWorkingTree(options: { console.log(chalk.cyan(`\nBuilding workspace packages for "working tree" (${buildCmd})…`)); // Disable the nx daemon: it keeps writing into the workspace after the build returns. const [file, ...args] = parseCommandString(buildCmd); - await run(file, args, repoRoot, { env: { NX_DAEMON: 'false' }, verbose: true }); + await execa(file, args, { + cwd: repoRoot, + env: { NX_DAEMON: 'false' }, + stdio: 'inherit', + verbose: 'short', + }); console.log(chalk.cyan('\nPacking the working tree…')); // Stage next to the destination so the renames cannot cross filesystems. diff --git a/packages/benchmark/src/utils/path.ts b/packages/benchmark/src/utils/path.ts deleted file mode 100644 index 7a601e7e10..0000000000 --- a/packages/benchmark/src/utils/path.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { access } from 'node:fs/promises'; - -/** True if `target` exists on disk. */ -export async function pathExists(target: string): Promise { - try { - await access(target); - return true; - } catch { - return false; - } -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e2f7cc9ef6..191d51d0ab 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -534,6 +534,9 @@ importers: env-ci: specifier: ^11.2.0 version: 11.2.0 + es-toolkit: + specifier: ^1.52.0 + version: 1.52.0 execa: specifier: ^10.0.1 version: 10.0.1 From 8ee7a2c2b1394bb009bf2b6c20a897af95c40847 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:22:43 +0200 Subject: [PATCH 093/173] [benchmark] Name tarballs scope+name, which no two packages share Keeps the shared core in step with #1903. The CI API's comment no longer talks about more than one benchmark runner. --- packages/benchmark/src/ciApi.ts | 6 +++--- .../benchmark/src/utils/packWorkspace.test.ts | 15 ++++++++----- packages/benchmark/src/utils/packWorkspace.ts | 21 +++++++------------ 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/packages/benchmark/src/ciApi.ts b/packages/benchmark/src/ciApi.ts index 88bccfeec5..46aeb5b84f 100644 --- a/packages/benchmark/src/ciApi.ts +++ b/packages/benchmark/src/ciApi.ts @@ -1,12 +1,12 @@ /* eslint-disable no-console -- progress belongs in the CI log. */ /** - * The dashboard's CI-report API, which both benchmark axes talk to. + * The dashboard's CI-report API. * * Every call is the same shape — a bearer-token POST of one JSON body — and the token is the OIDC * one CircleCI mints for the job. Keeping that in one place is what stops the endpoint host, the - * environment variable and the error handling from drifting between the two axes, where a - * divergence would only ever show up in CI, on whichever one nobody edited. + * environment variable and the error handling from drifting between calls, where a divergence + * would only ever show up in CI. */ const DEFAULT_API_URL = 'https://frontend-public.mui.com'; diff --git a/packages/benchmark/src/utils/packWorkspace.test.ts b/packages/benchmark/src/utils/packWorkspace.test.ts index 659952d57d..a506396ce3 100644 --- a/packages/benchmark/src/utils/packWorkspace.test.ts +++ b/packages/benchmark/src/utils/packWorkspace.test.ts @@ -6,13 +6,18 @@ import { makeTempDir } from './testUtils'; import { packRef, packWorkingTree, readFreshCache, tarballFor, tarballName } from './packWorkspace'; describe('tarballName', () => { - it('flattens a scoped name into a filesystem-safe basename', () => { - expect(tarballName('@base-ui/mosaic')).toBe('base-ui-mosaic.tgz'); + it('joins scope and name with a character no package name contains', () => { + expect(tarballName('@base-ui/mosaic')).toBe('base-ui+mosaic.tgz'); }); - it('leaves an unscoped name alone', () => { + it('keeps an unscoped name as is', () => { expect(tarballName('mosaic')).toBe('mosaic.tgz'); }); + + it('never gives two packages the same tarball', () => { + const names = ['@mui/package', 'mui-package', '@a/b-c', '@a-b/c']; + expect(new Set(names.map(tarballName)).size).toBe(names.length); + }); }); describe('tarballFor', () => { @@ -148,7 +153,7 @@ describe('packRef', () => { // absolute path still resolves. const manifest = JSON.parse(await readFile(path.join(packed.dir, 'manifest.json'), 'utf8')); expect(manifest.packages).toEqual([ - { name: '@fixture/public', version: '1.0.0', tarball: 'fixture-public.tgz' }, + { name: '@fixture/public', version: '1.0.0', tarball: 'fixture+public.tgz' }, ]); expect(manifest.buildCmd).toBe(buildCmd); @@ -217,7 +222,7 @@ describe('packWorkingTree', () => { // An unchanged build keeps its filename, which is what lets a consumer's isolated install // persist: the dependency path does not move, so pnpm has nothing to do. expect(first.map((pkg) => pkg.tarball)).toEqual(unchanged.map((pkg) => pkg.tarball)); - expect(path.basename(first[0].tarball)).toMatch(/^fixture-public-[0-9a-f]{12}\.tgz$/); + expect(path.basename(first[0].tarball)).toMatch(/^fixture\+public-[0-9a-f]{12}\.tgz$/); await expect(stat(first[0].tarball)).resolves.toBeTruthy(); }); diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index 1bdb8099d5..8fda4e925b 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -76,9 +76,14 @@ export interface PackRefOptions { /** Name of the file written into each packed folder describing its contents. */ const MANIFEST = 'manifest.json'; -/** Turns a package name into a filesystem-safe tarball basename (`@scope/pkg` → `scope-pkg.tgz`). */ +/** + * Turns a package name into a filesystem-safe tarball basename: `@scope/pkg` → `scope+pkg.tgz`. + * + * `+` cannot appear in a package name, so no two packages share a tarball — `@mui/pkg` and + * `mui-pkg` stay apart, where a `-` would fold them together. + */ export function tarballName(pkgName: string): string { - return `${pkgName.replace(/^@/, '').replace(/\//g, '-')}.tgz`; + return `${pkgName.replace(/^@/, '').replace('/', '+')}.tgz`; } /** @@ -190,18 +195,6 @@ export async function packBuiltPackages( if (packages.length === 0) { throw new Error(`No public workspace packages found in ${checkoutDir}.`); } - // `@a/b-c` and `@a-b/c` both name `a-b-c.tgz`, and these pack concurrently into one directory — - // so without this the last writer wins and a ref's tree installs one package's build under the - // other's name. - const byTarball = new Map(); - for (const { name } of packages) { - const tarball = tarballName(name); - const clash = byTarball.get(tarball); - if (clash) { - throw new Error(`"${clash}" and "${name}" would both pack to ${tarball}.`); - } - byTarball.set(tarball, name); - } // Each `pnpm pack` is its own node process, so a repository with a dozen public packages would // otherwise start a dozen at once. return mapAsync( From de264550bea8a79bca8c3d76f341871f402a3c7a Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:24:38 +0200 Subject: [PATCH 094/173] [benchmark] Drop tarballFor from the shared core Nothing here looks a tarball up by name; only the tachometer runner in #1829 does, and it keeps its own. --- .../benchmark/src/utils/packWorkspace.test.ts | 22 +------------------ packages/benchmark/src/utils/packWorkspace.ts | 10 --------- 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/packages/benchmark/src/utils/packWorkspace.test.ts b/packages/benchmark/src/utils/packWorkspace.test.ts index a506396ce3..ca5ecbf90d 100644 --- a/packages/benchmark/src/utils/packWorkspace.test.ts +++ b/packages/benchmark/src/utils/packWorkspace.test.ts @@ -3,7 +3,7 @@ import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises'; import { execa } from 'execa'; import { describe, expect, it } from 'vitest'; import { makeTempDir } from './testUtils'; -import { packRef, packWorkingTree, readFreshCache, tarballFor, tarballName } from './packWorkspace'; +import { packRef, packWorkingTree, readFreshCache, tarballName } from './packWorkspace'; describe('tarballName', () => { it('joins scope and name with a character no package name contains', () => { @@ -20,26 +20,6 @@ describe('tarballName', () => { }); }); -describe('tarballFor', () => { - const packages = [{ name: '@scope/one', version: '1.0.0', tarball: '/tmp/scope-one.tgz' }]; - - it('finds a packed package by name', () => { - expect(tarballFor(packages, '@scope/one')).toBe('/tmp/scope-one.tgz'); - }); - - it('lists what was packed when a package is missing', () => { - // A workspace dependency that was never packed is a packaging bug in the ref — usually a - // private package a public one depends on — so the message has to name what is available. - expect(() => tarballFor(packages, '@scope/two')).toThrow( - 'Package "@scope/two" was not packed. Packed: @scope/one.', - ); - }); - - it('says so when nothing was packed at all', () => { - expect(() => tarballFor([], '@scope/two')).toThrow(/Packed: \(none\)\./); - }); -}); - describe('readFreshCache', () => { /** Writes a packed folder with a manifest and its tarballs. */ async function makePackedDir({ diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index 8fda4e925b..52e1b8c78f 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -350,13 +350,3 @@ export async function packWorkingTree(options: { await rm(staging, { recursive: true, force: true }); } } - -/** Looks up one packed package's tarball by name, throwing a clear error if it was not packed. */ -export function tarballFor(packages: PackedPackage[], name: string): string { - const found = packages.find((pkg) => pkg.name === name); - if (!found) { - const available = packages.map((pkg) => pkg.name).join(', ') || '(none)'; - throw new Error(`Package "${name}" was not packed. Packed: ${available}.`); - } - return found.tarball; -} From b4eeb4e8f01a76e1ab264642d04c379df9f2168a Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:37:16 +0200 Subject: [PATCH 095/173] [benchmark] Test the pack API through packRef and packWorkingTree readFreshCache, tarballName and packBuiltPackages are no longer exported just to be tested. The cache cases, a relocated cache and scoped names that used to collide are covered through the two entry points instead. --- .../benchmark/src/utils/packWorkspace.test.ts | 176 ++++++++---------- packages/benchmark/src/utils/packWorkspace.ts | 9 +- 2 files changed, 81 insertions(+), 104 deletions(-) diff --git a/packages/benchmark/src/utils/packWorkspace.test.ts b/packages/benchmark/src/utils/packWorkspace.test.ts index ca5ecbf90d..d8a5367f6e 100644 --- a/packages/benchmark/src/utils/packWorkspace.test.ts +++ b/packages/benchmark/src/utils/packWorkspace.test.ts @@ -1,84 +1,28 @@ import * as path from 'node:path'; -import { mkdir, readFile, readdir, stat, writeFile } from 'node:fs/promises'; +import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises'; import { execa } from 'execa'; import { describe, expect, it } from 'vitest'; import { makeTempDir } from './testUtils'; -import { packRef, packWorkingTree, readFreshCache, tarballName } from './packWorkspace'; +import { packRef, packWorkingTree } from './packWorkspace'; -describe('tarballName', () => { - it('joins scope and name with a character no package name contains', () => { - expect(tarballName('@base-ui/mosaic')).toBe('base-ui+mosaic.tgz'); - }); - - it('keeps an unscoped name as is', () => { - expect(tarballName('mosaic')).toBe('mosaic.tgz'); - }); - - it('never gives two packages the same tarball', () => { - const names = ['@mui/package', 'mui-package', '@a/b-c', '@a-b/c']; - expect(new Set(names.map(tarballName)).size).toBe(names.length); - }); -}); - -describe('readFreshCache', () => { - /** Writes a packed folder with a manifest and its tarballs. */ - async function makePackedDir({ - buildCmd, - withTarball = true, - }: { - /** The build command to record. */ - buildCmd: string; - /** Whether to actually create the tarball file. */ - withTarball?: boolean; - }): Promise { - const dir = await makeTempDir(); - await writeFile( - path.join(dir, 'manifest.json'), - JSON.stringify({ - ref: 'HEAD', - sha: 'abc', - buildCmd, - packages: [{ name: '@scope/one', version: '1.0.0', tarball: 'scope-one.tgz' }], - }), - ); - if (withTarball) { - await writeFile(path.join(dir, 'scope-one.tgz'), 'tarball'); - } - return dir; - } - - it('returns the manifest when the folder matches the requested build', async () => { - const dir = await makePackedDir({ buildCmd: 'pnpm release:build' }); - - expect(await readFreshCache(dir, 'pnpm release:build')).toMatchObject({ sha: 'abc' }); - }); - - it('misses when the folder was built with a different command', async () => { - const dir = await makePackedDir({ buildCmd: 'pnpm build' }); - - expect(await readFreshCache(dir, 'pnpm release:build')).toBeNull(); - }); - - it('misses when a referenced tarball has been evicted', async () => { - const dir = await makePackedDir({ buildCmd: 'pnpm release:build', withTarball: false }); - - expect(await readFreshCache(dir, 'pnpm release:build')).toBeNull(); - }); +interface FixturePackage { + dir: string; + pkg: { name: string; version: string; private?: boolean }; +} - it('misses when there is no manifest', async () => { - expect(await readFreshCache(await makeTempDir(), 'pnpm release:build')).toBeNull(); - }); -}); +const DEFAULT_PACKAGES: FixturePackage[] = [ + { dir: 'public', pkg: { name: '@fixture/public', version: '1.0.0' } }, + { dir: 'private', pkg: { name: '@fixture/private', version: '1.0.0', private: true } }, +]; /** - * Creates a git repository holding a two-package pnpm workspace: one publishable, one private. + * Creates a git repository holding a pnpm workspace of `packages` — by default one publishable and + * one private. * * The build command is a script that appends to `marker`, so a later assertion can tell whether a * build actually ran or the cache was reused. - * - * @returns {Promise<{ repoRoot: string, marker: string, buildCmd: string }>} */ -async function makeFixtureRepo() { +async function makeFixtureRepo(packages: FixturePackage[] = DEFAULT_PACKAGES) { const repoRoot = await makeTempDir(); const marker = path.join(repoRoot, 'builds.log'); @@ -94,14 +38,14 @@ async function makeFixtureRepo() { ); await Promise.all( - [ - { dir: 'public', pkg: { name: '@fixture/public', version: '1.0.0' } }, - { dir: 'private', pkg: { name: '@fixture/private', version: '1.0.0', private: true } }, - ].map(async ({ dir, pkg }) => { + packages.map(async ({ dir, pkg }) => { const pkgDir = path.join(repoRoot, 'packages', dir); await mkdir(pkgDir, { recursive: true }); await writeFile(path.join(pkgDir, 'package.json'), JSON.stringify(pkg, null, 2)); - await writeFile(path.join(pkgDir, 'index.js'), 'export default 1;\n'); + await writeFile( + path.join(pkgDir, 'index.js'), + `export default ${JSON.stringify(pkg.name)};\n`, + ); }), ); @@ -126,17 +70,7 @@ describe('packRef', () => { // The private package is never packed: a release would not publish it. expect(packed.packages.map((pkg) => pkg.name)).toEqual(['@fixture/public']); - expect(packed.dir).toBe(path.join(outRoot, packed.sha)); await expect(stat(packed.packages[0].tarball)).resolves.toBeTruthy(); - - // Tarball names are stored relative to the folder, so a cache restored under a different - // absolute path still resolves. - const manifest = JSON.parse(await readFile(path.join(packed.dir, 'manifest.json'), 'utf8')); - expect(manifest.packages).toEqual([ - { name: '@fixture/public', version: '1.0.0', tarball: 'fixture+public.tgz' }, - ]); - expect(manifest.buildCmd).toBe(buildCmd); - expect(await readFile(marker, 'utf8')).toBe('built\n'); // The temporary checkout is cleaned up, leaving the tarballs as the only artifact. @@ -150,6 +84,30 @@ describe('packRef', () => { }, ); + it( + 'reuses a cache restored under a different path, as a CI cache is', + { timeout: 120_000 }, + async () => { + const { repoRoot, marker, buildCmd } = await makeFixtureRepo(); + const outRoot = path.join(await makeTempDir(), 'cache'); + await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); + + const restoredRoot = path.join(await makeTempDir(), 'restored'); + await rename(outRoot, restoredRoot); + const restored = await packRef({ + repoRoot, + ref: 'HEAD', + outRoot: restoredRoot, + installCmd: '', + buildCmd, + }); + + expect(await readFile(marker, 'utf8')).toBe('built\n'); + expect(restored.packages[0].tarball.startsWith(restoredRoot)).toBe(true); + await expect(stat(restored.packages[0].tarball)).resolves.toBeTruthy(); + }, + ); + it( 'rebuilds when the cached folder used a different build command', { timeout: 120_000 }, @@ -170,6 +128,18 @@ describe('packRef', () => { }, ); + it('rebuilds when a cached tarball has gone missing', { timeout: 120_000 }, async () => { + const { repoRoot, marker, buildCmd } = await makeFixtureRepo(); + const outRoot = path.join(await makeTempDir(), 'cache'); + + const packed = await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); + await rm(packed.packages[0].tarball); + const again = await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); + + expect(await readFile(marker, 'utf8')).toBe('built\nbuilt\n'); + await expect(stat(again.packages[0].tarball)).resolves.toBeTruthy(); + }); + it('leaves no staging directory behind', { timeout: 120_000 }, async () => { const { repoRoot, buildCmd } = await makeFixtureRepo(); const outRoot = path.join(await makeTempDir(), 'cache'); @@ -192,30 +162,40 @@ describe('packRef', () => { }); describe('packWorkingTree', () => { - it('names each tarball by a hash of its content', { timeout: 120_000 }, async () => { + it('keeps a tarball path until its content changes', { timeout: 120_000 }, async () => { const { repoRoot, buildCmd } = await makeFixtureRepo(); const outRoot = path.join(await makeTempDir(), 'packed', 'current'); const first = await packWorkingTree({ repoRoot, outRoot, buildCmd }); const unchanged = await packWorkingTree({ repoRoot, outRoot, buildCmd }); - // An unchanged build keeps its filename, which is what lets a consumer's isolated install - // persist: the dependency path does not move, so pnpm has nothing to do. - expect(first.map((pkg) => pkg.tarball)).toEqual(unchanged.map((pkg) => pkg.tarball)); - expect(path.basename(first[0].tarball)).toMatch(/^fixture\+public-[0-9a-f]{12}\.tgz$/); + // An unchanged build keeps its path, so an install that depends on it has nothing to redo. + expect(unchanged.map((pkg) => pkg.tarball)).toEqual(first.map((pkg) => pkg.tarball)); await expect(stat(first[0].tarball)).resolves.toBeTruthy(); - }); - it('renames when the packed content changes', { timeout: 120_000 }, async () => { - const { repoRoot, buildCmd } = await makeFixtureRepo(); - const outRoot = path.join(await makeTempDir(), 'packed', 'current'); - - const before = await packWorkingTree({ repoRoot, outRoot, buildCmd }); await writeFile(path.join(repoRoot, 'packages', 'public', 'index.js'), 'export default 2;\n'); - const after = await packWorkingTree({ repoRoot, outRoot, buildCmd }); + const changed = await packWorkingTree({ repoRoot, outRoot, buildCmd }); - expect(after[0].tarball).not.toBe(before[0].tarball); + expect(changed[0].tarball).not.toBe(first[0].tarball); // The directory is replaced, so a stale tarball cannot accumulate or be resolved by mistake. - expect(await readdir(outRoot)).toEqual([path.basename(after[0].tarball)]); + expect(await readdir(outRoot)).toEqual([path.basename(changed[0].tarball)]); }); + + it( + 'packs packages whose names only differ in their scope separately', + { timeout: 120_000 }, + async () => { + const { repoRoot, buildCmd } = await makeFixtureRepo([ + { dir: 'scoped', pkg: { name: '@mui/package', version: '1.0.0' } }, + { dir: 'unscoped', pkg: { name: 'mui-package', version: '1.0.0' } }, + ]); + const outRoot = path.join(await makeTempDir(), 'packed', 'current'); + + const packed = await packWorkingTree({ repoRoot, outRoot, buildCmd }); + + expect(packed.map((pkg) => pkg.name).sort()).toEqual(['@mui/package', 'mui-package']); + expect(new Set(packed.map((pkg) => pkg.tarball)).size).toBe(2); + await Promise.all(packed.map((pkg) => expect(stat(pkg.tarball)).resolves.toBeTruthy())); + }, + ); }); diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index 52e1b8c78f..e415376d40 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -82,7 +82,7 @@ const MANIFEST = 'manifest.json'; * `+` cannot appear in a package name, so no two packages share a tarball — `@mui/pkg` and * `mui-pkg` stay apart, where a `-` would fold them together. */ -export function tarballName(pkgName: string): string { +function tarballName(pkgName: string): string { return `${pkgName.replace(/^@/, '').replace('/', '+')}.tgz`; } @@ -163,7 +163,7 @@ function resolveManifest(raw: RawManifest, dir: string): PackedWorkspace { * reusing a folder built with a different build script, or one whose tarballs were partially * evicted or deleted. */ -export async function readFreshCache(dir: string, buildCmd: string): Promise { +async function readFreshCache(dir: string, buildCmd: string): Promise { const raw = await readRawManifest(dir); if (raw === null || raw.buildCmd !== buildCmd) { return null; @@ -186,10 +186,7 @@ export async function readFreshCache(dir: string, buildCmd: string): Promise { +async function packBuiltPackages(checkoutDir: string, outDir: string): Promise { await mkdir(outDir, { recursive: true }); const packages = await listPublishablePackages(checkoutDir); if (packages.length === 0) { From b1a0bcc91fc5b89c47b3577a9e318ef57c0f78cc Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 17:44:03 +0200 Subject: [PATCH 096/173] [code-infra] Test base-branch detection through resolveBaseline detectBaseBranch and closestBaseBranch are internals of resolveBaseline, so they are no longer exported; detection is covered by resolving a baseline without naming the base branch. --- packages/code-infra/src/utils/git.mjs | 4 +-- packages/code-infra/src/utils/git.test.mjs | 34 +++++++++++++--------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index 1f2bff16f1..2aa68b982a 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -110,7 +110,7 @@ function branchNameOf(shortRef) { * @param {string} [cwd=process.cwd()] * @returns {Promise} */ -export async function detectBaseBranch(cwd = process.cwd()) { +async function detectBaseBranch(cwd = process.cwd()) { const listed = (await $({ cwd })`git remote`).stdout.trim().split('\n').filter(Boolean); // `origin` first: it is the one a clone records a HEAD for. const remotes = [...new Set(['origin', ...listed])].filter((remote) => listed.includes(remote)); @@ -145,7 +145,7 @@ export async function detectBaseBranch(cwd = process.cwd()) { * @param {string} [cwd=process.cwd()] * @returns {Promise<{ ref: string, mergeBase: string } | undefined>} */ -export async function closestBaseBranch(baseBranch, cwd = process.cwd()) { +async function closestBaseBranch(baseBranch, cwd = process.cwd()) { // On ties (same merge base), prefer upstream's, then origin's, then a local base branch. const preference = [`upstream/${baseBranch}`, `origin/${baseBranch}`, baseBranch]; /** @param {string} ref */ diff --git a/packages/code-infra/src/utils/git.test.mjs b/packages/code-infra/src/utils/git.test.mjs index 3601563799..1818e9447e 100644 --- a/packages/code-infra/src/utils/git.test.mjs +++ b/packages/code-infra/src/utils/git.test.mjs @@ -3,7 +3,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { describe, expect, it, onTestFinished } from 'vitest'; import { execa } from 'execa'; -import { detectBaseBranch, resolveBaseline } from './git.mjs'; +import { resolveBaseline } from './git.mjs'; /** * A repository with three commits on `main`. @@ -42,7 +42,7 @@ async function makeRepo() { const git = gitWith(undefined); await git('init', '--initial-branch=main'); - // A day apart each: `closestBaseBranch` picks the candidate whose merge base is most recent, and + // A day apart each: the baseline comes from the candidate whose merge base is most recent, and // three commits made in a loop otherwise share a second and leave that comparison to a tie-break. for (const [index, message] of ['first', 'second', 'third'].entries()) { // Empty commits: nothing here reads a file, only the commits' identities matter. @@ -138,24 +138,30 @@ describe('resolveBaseline', () => { }); }); -describe('detectBaseBranch', () => { - it('reads the default branch from origin/HEAD', async () => { - const { repoRoot, git } = await makeRepo(); +describe('resolveBaseline without a base branch named', () => { + // Each default branch points at an older commit than the local `main`, so taking the wrong branch + // as the base would answer a different fork point. + it('forks from the default branch origin/HEAD records', async () => { + const { repoRoot, shas, git } = await makeRepo(); await git('remote', 'add', 'origin', repoRoot); - await git('update-ref', 'refs/remotes/origin/trunk', 'HEAD'); + await git('update-ref', 'refs/remotes/origin/trunk', shas[0]); await git('symbolic-ref', 'refs/remotes/origin/HEAD', 'refs/remotes/origin/trunk'); + await git('checkout', '-b', 'feature'); + await git('commit', '--allow-empty', '-m', 'work'); - expect(await detectBaseBranch(repoRoot)).toBe('trunk'); + expect(await resolveBaseline({ cwd: repoRoot })).toBe(shas[0]); }); - it('reads it from another remote when origin records none', async () => { + it('reads the default branch from another remote when origin records none', async () => { // A remote added by hand has no HEAD unless someone wrote one. If they did, it still counts. - const { repoRoot, git } = await makeRepo(); + const { repoRoot, shas, git } = await makeRepo(); await git('remote', 'add', 'upstream', repoRoot); - await git('update-ref', 'refs/remotes/upstream/release', 'HEAD'); + await git('update-ref', 'refs/remotes/upstream/release', shas[1]); await git('symbolic-ref', 'refs/remotes/upstream/HEAD', 'refs/remotes/upstream/release'); + await git('checkout', '-b', 'feature'); + await git('commit', '--allow-empty', '-m', 'work'); - expect(await detectBaseBranch(repoRoot)).toBe('release'); + expect(await resolveBaseline({ cwd: repoRoot })).toBe(shas[1]); }); it('does not take a conventional name as evidence', async () => { @@ -164,13 +170,15 @@ describe('detectBaseBranch', () => { await git('remote', 'add', 'origin', repoRoot); await git('update-ref', 'refs/remotes/origin/main', 'HEAD'); - await expect(detectBaseBranch(repoRoot)).rejects.toThrow(/no remote records a default branch/); + await expect(resolveBaseline({ cwd: repoRoot })).rejects.toThrow( + /no remote records a default branch/, + ); }); it('fails when there is no remote at all', async () => { const { repoRoot } = await makeRepo(); - await expect(detectBaseBranch(repoRoot)).rejects.toThrow(/Could not tell which branch/); + await expect(resolveBaseline({ cwd: repoRoot })).rejects.toThrow(/Could not tell which branch/); }); }); From c26a1fac4c8a62a36aba712826f2e97baaefc8fd Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 19:00:11 +0200 Subject: [PATCH 097/173] [benchmark] Remove a temporary checkout with rm and prune git worktree remove needed a fallback for when a task runner was still writing into the tree; deleting the directory with retries and pruning does the same in every case. Errors are no longer swallowed. --- packages/benchmark/src/utils/packWorkspace.ts | 32 ++++--------------- 1 file changed, 7 insertions(+), 25 deletions(-) diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index e415376d40..8694e31a98 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -87,32 +87,15 @@ function tarballName(pkgName: string): string { } /** - * Removes a temporary git worktree, best-effort. + * Removes a temporary checkout, and git's record of it. * - * Removal can fail with "Directory not empty" when a background task runner (nx, via lerna) - * repopulates the tree while git is deleting it; fall back to force-removing the directory and - * pruning git's bookkeeping. Never throws — by the time this runs the packed tarballs are already - * safe, so a leftover temp checkout must not fail the run. + * A plain delete and a prune rather than `git worktree remove`, which fails with "Directory not + * empty" when a background task runner (nx, via lerna) is still writing into the tree; `rm` retries + * through that. */ async function removeCheckout(repoRoot: string, checkout: string): Promise { - const removed = await execa('git', ['worktree', 'remove', '--force', checkout], { - cwd: repoRoot, - reject: false, - }); - if (removed.exitCode !== 0) { - try { - await rm(checkout, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); - } catch (error) { - console.warn( - chalk.yellow( - `Could not remove temporary checkout ${checkout}: ${error instanceof Error ? error.message : error}. ` + - `Delete it and prune git's bookkeeping to clean up.`, - ), - ); - } - // Drop git's now-dangling admin entry regardless of whether the directory delete stuck. - await execa('git', ['worktree', 'prune'], { cwd: repoRoot, reject: false }); - } + await rm(checkout, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); + await execa('git', ['worktree', 'prune'], { cwd: repoRoot }); } /** @@ -290,8 +273,7 @@ export async function packRef(options: PackRefOptions): Promise await rm(staging, { recursive: true, force: true }); throw error; } finally { - // Runs even if the checkout failed (leaving an empty temp dir) — removeCheckout falls back to a - // plain delete, so the temp checkout never leaks. + // However the build went, including a checkout that never happened. await removeCheckout(repoRoot, checkout); } } From abfc5bb74d2a19e038bb8185909d72a0680faa00 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 19:02:00 +0200 Subject: [PATCH 098/173] [benchmark] Shorten the removeCheckout comment --- packages/benchmark/src/utils/packWorkspace.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index 8694e31a98..f34da84419 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -86,13 +86,7 @@ function tarballName(pkgName: string): string { return `${pkgName.replace(/^@/, '').replace('/', '+')}.tgz`; } -/** - * Removes a temporary checkout, and git's record of it. - * - * A plain delete and a prune rather than `git worktree remove`, which fails with "Directory not - * empty" when a background task runner (nx, via lerna) is still writing into the tree; `rm` retries - * through that. - */ +/** Removes a temporary checkout, and git's record of it. */ async function removeCheckout(repoRoot: string, checkout: string): Promise { await rm(checkout, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); await execa('git', ['worktree', 'prune'], { cwd: repoRoot }); From 315b88d945a923589ef4f6b9cf550d791d1d6fd2 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 19:04:35 +0200 Subject: [PATCH 099/173] [benchmark] Return the packed packages from packRef Callers only ever read .packages, so packRef returns them the way packWorkingTree does, and PackedWorkspace goes. The manifest keeps only what the cache check reads. --- .../benchmark/src/runner/runBenchmarks.ts | 3 +- .../benchmark/src/utils/packWorkspace.test.ts | 19 ++++----- packages/benchmark/src/utils/packWorkspace.ts | 39 ++++--------------- 3 files changed, 19 insertions(+), 42 deletions(-) diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index a64d609178..ff3930c2aa 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -111,8 +111,7 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise { const packed = await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); // The private package is never packed: a release would not publish it. - expect(packed.packages.map((pkg) => pkg.name)).toEqual(['@fixture/public']); - await expect(stat(packed.packages[0].tarball)).resolves.toBeTruthy(); + expect(packed.map((pkg) => pkg.name)).toEqual(['@fixture/public']); + await expect(stat(packed[0].tarball)).resolves.toBeTruthy(); expect(await readFile(marker, 'utf8')).toBe('built\n'); // The temporary checkout is cleaned up, leaving the tarballs as the only artifact. @@ -79,7 +79,7 @@ describe('packRef', () => { // A second call for the same commit reuses the folder and never builds again. const again = await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); - expect(again.sha).toBe(packed.sha); + expect(again).toEqual(packed); expect(await readFile(marker, 'utf8')).toBe('built\n'); }, ); @@ -103,8 +103,8 @@ describe('packRef', () => { }); expect(await readFile(marker, 'utf8')).toBe('built\n'); - expect(restored.packages[0].tarball.startsWith(restoredRoot)).toBe(true); - await expect(stat(restored.packages[0].tarball)).resolves.toBeTruthy(); + expect(restored[0].tarball.startsWith(restoredRoot)).toBe(true); + await expect(stat(restored[0].tarball)).resolves.toBeTruthy(); }, ); @@ -133,22 +133,23 @@ describe('packRef', () => { const outRoot = path.join(await makeTempDir(), 'cache'); const packed = await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); - await rm(packed.packages[0].tarball); + await rm(packed[0].tarball); const again = await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); expect(await readFile(marker, 'utf8')).toBe('built\nbuilt\n'); - await expect(stat(again.packages[0].tarball)).resolves.toBeTruthy(); + await expect(stat(again[0].tarball)).resolves.toBeTruthy(); }); it('leaves no staging directory behind', { timeout: 120_000 }, async () => { const { repoRoot, buildCmd } = await makeFixtureRepo(); const outRoot = path.join(await makeTempDir(), 'cache'); - const packed = await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); + await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); + const { stdout: sha } = await execa('git', ['rev-parse', 'HEAD'], { cwd: repoRoot }); // The folder is assembled in a staging sibling and atomically renamed, so an interrupted run // can never leave something that looks like a cache hit. - expect(await readdir(outRoot)).toEqual([packed.sha]); + expect(await readdir(outRoot)).toEqual([sha.trim()]); }); it('reports a ref it cannot resolve', async () => { diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index f34da84419..275ad0538d 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -29,16 +29,6 @@ export interface PackedPackage { tarball: string; } -export interface PackedWorkspace { - /** The ref that was packed, verbatim as given. */ - ref: string; - /** The resolved commit SHA. */ - sha: string; - /** Absolute path to the folder holding the tarballs and `manifest.json`. */ - dir: string; - packages: PackedPackage[]; -} - interface RawManifestPackage { name: string; version: string; @@ -47,8 +37,6 @@ interface RawManifestPackage { } interface RawManifest { - ref: string; - sha: string; /** The build command used, part of the cache key. */ buildCmd: string; packages: RawManifestPackage[]; @@ -118,18 +106,9 @@ async function readRawManifest(dir: string): Promise { return JSON.parse(text); } -/** Resolves a raw manifest's stored (relative) tarball names to absolute paths under `dir`. */ -function resolveManifest(raw: RawManifest, dir: string): PackedWorkspace { - return { - ref: raw.ref, - sha: raw.sha, - dir, - packages: raw.packages.map((pkg) => ({ - name: pkg.name, - version: pkg.version, - tarball: path.join(dir, pkg.tarball), - })), - }; +/** A manifest's packages, their tarball names resolved to absolute paths under `dir`. */ +function packagesIn(raw: RawManifest, dir: string): PackedPackage[] { + return raw.packages.map((pkg) => ({ ...pkg, tarball: path.join(dir, pkg.tarball) })); } /** @@ -183,8 +162,8 @@ async function packBuiltPackages(checkoutDir: string, outDir: string): Promise

    /`, returning the folder and a - * manifest of the packed tarballs. + * Packs every public workspace package at `ref` into `/`, returning the packed + * tarballs. * * The build is deterministic for a commit, so the folder is cached by SHA: a later call for the * same commit reuses it and skips the checkout, install, and build entirely — but only when it was @@ -197,7 +176,7 @@ async function packBuiltPackages(checkoutDir: string, outDir: string): Promise

    { +export async function packRef(options: PackRefOptions): Promise { const { repoRoot, ref, @@ -213,7 +192,7 @@ export async function packRef(options: PackRefOptions): Promise const cached = await readFreshCache(dir, buildCmd); if (cached) { console.log(chalk.green(`\nReusing packed workspace for "${ref}" (${sha.slice(0, 9)}).`)); - return resolveManifest(cached, dir); + return packagesIn(cached, dir); } await mkdir(outRoot, { recursive: true }); @@ -248,8 +227,6 @@ export async function packRef(options: PackRefOptions): Promise // Store tarballs by basename so the folder is relocatable; record buildCmd so a later run can // tell whether the cache matches. Write the manifest last — it marks completeness. const manifest: RawManifest = { - ref, - sha, buildCmd, packages: packages.map(({ name, version, tarball }) => ({ name, @@ -262,7 +239,7 @@ export async function packRef(options: PackRefOptions): Promise // rebuild — so the rename can't fail with ENOTEMPTY. await rm(dir, { recursive: true, force: true }); await rename(staging, dir); - return resolveManifest(manifest, dir); + return packagesIn(manifest, dir); } catch (error) { await rm(staging, { recursive: true, force: true }); throw error; From 241b5c8a9eda4e67f9a283f279ecdc7c8e71fb8c Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 19:09:21 +0200 Subject: [PATCH 100/173] [benchmark] Remove packRef's staging folder in finally On success it has already been renamed into place, so removing it there is a no-op, and the catch that only cleaned up and rethrew goes. --- .../benchmark/src/utils/packWorkspace.test.ts | 19 +++++++++++++++++++ packages/benchmark/src/utils/packWorkspace.ts | 10 +++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/benchmark/src/utils/packWorkspace.test.ts b/packages/benchmark/src/utils/packWorkspace.test.ts index c7e913a072..a4f80182b5 100644 --- a/packages/benchmark/src/utils/packWorkspace.test.ts +++ b/packages/benchmark/src/utils/packWorkspace.test.ts @@ -152,6 +152,25 @@ describe('packRef', () => { expect(await readdir(outRoot)).toEqual([sha.trim()]); }); + it('leaves nothing behind when the build fails', { timeout: 120_000 }, async () => { + const { repoRoot } = await makeFixtureRepo(); + const outRoot = path.join(await makeTempDir(), 'cache'); + + await expect( + packRef({ + repoRoot, + ref: 'HEAD', + outRoot, + installCmd: '', + buildCmd: 'node -e process.exit(1)', + }), + ).rejects.toThrow(); + + expect(await readdir(outRoot)).toEqual([]); + const worktrees = await execa('git', ['worktree', 'list'], { cwd: repoRoot }); + expect(worktrees.stdout.trim().split('\n')).toHaveLength(1); + }); + it('reports a ref it cannot resolve', async () => { const { repoRoot, buildCmd } = await makeFixtureRepo(); const outRoot = path.join(await makeTempDir(), 'cache'); diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index 275ad0538d..3ac2b5b5b7 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -240,12 +240,12 @@ export async function packRef(options: PackRefOptions): Promise await rm(dir, { recursive: true, force: true }); await rename(staging, dir); return packagesIn(manifest, dir); - } catch (error) { - await rm(staging, { recursive: true, force: true }); - throw error; } finally { - // However the build went, including a checkout that never happened. - await removeCheckout(repoRoot, checkout); + // However the build went; on success the staging folder has already become `dir`. + await Promise.all([ + rm(staging, { recursive: true, force: true }), + removeCheckout(repoRoot, checkout), + ]); } } From fad0b324e6f5759ff7ff55eb353383d2c713bd81 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 29 Sep 2026 19:19:51 +0200 Subject: [PATCH 101/173] [code-infra] Simplify resolveBaseline to a fetch and a merge-base The baseline is the fork point from the remote's base branch, fetched fresh from upstream when the clone has one, or origin; when HEAD is on that branch, its parent. The default branch comes from the remote itself. That replaces detecting it from local /HEAD refs, ranking every remote's copy of the base branch by merge-base recency, and fetching only when no copy existed. Also trims the shared benchmark utils: one cache read, one build helper, no staging folder in packWorkingTree, the pnpm helpers nothing here uses dropped, and shorter comments. --- packages/benchmark/src/ciApi.ts | 16 +- packages/benchmark/src/utils/git.ts | 12 +- packages/benchmark/src/utils/packWorkspace.ts | 206 +++++--------- packages/benchmark/src/utils/pnpm.ts | 56 +--- packages/code-infra/src/cli/cmdBaseline.mjs | 14 +- packages/code-infra/src/utils/git.mjs | 219 +++------------ packages/code-infra/src/utils/git.test.mjs | 258 ++++++------------ 7 files changed, 194 insertions(+), 587 deletions(-) diff --git a/packages/benchmark/src/ciApi.ts b/packages/benchmark/src/ciApi.ts index 46aeb5b84f..da522e7069 100644 --- a/packages/benchmark/src/ciApi.ts +++ b/packages/benchmark/src/ciApi.ts @@ -1,21 +1,13 @@ /* eslint-disable no-console -- progress belongs in the CI log. */ -/** - * The dashboard's CI-report API. - * - * Every call is the same shape — a bearer-token POST of one JSON body — and the token is the OIDC - * one CircleCI mints for the job. Keeping that in one place is what stops the endpoint host, the - * environment variable and the error handling from drifting between calls, where a divergence - * would only ever show up in CI. - */ +// The dashboard's CI-report API: a bearer-token POST of one JSON body, with the OIDC token +// CircleCI mints for the job. const DEFAULT_API_URL = 'https://frontend-public.mui.com'; /** - * Posts `body` to the dashboard, authenticated as this CI job, and returns the response text. - * - * `what` names the call in both errors it can raise, so a failure says which request failed rather - * than only that one did. + * Posts `body` to the dashboard, authenticated as this CI job, and returns the response text. `what` + * names the request in the errors it raises. */ export async function postToDashboard( pathname: string, diff --git a/packages/benchmark/src/utils/git.ts b/packages/benchmark/src/utils/git.ts index 4072b54810..478c583aa3 100644 --- a/packages/benchmark/src/utils/git.ts +++ b/packages/benchmark/src/utils/git.ts @@ -1,16 +1,6 @@ import { execa } from 'execa'; -/** - * Resolves a committish to the commit it names. - * - * `^{commit}` peels an annotated tag to its commit: `git rev-parse v1.2.3` returns the tag *object*, - * whose SHA is not the commit's, so the same commit reached by a tag and by its SHA would look like - * two different builds. `--verify` turns an unknown ref into a non-zero exit rather than an echoed - * argument. - * - * Shared because two callers resolve a ref — the baseline, and `packRef` to key its cache — and a - * commit they disagreed about would be built twice. - */ +/** Resolves a committish to the commit it names, peeling an annotated tag to its commit. */ export async function resolveCommit(repoRoot: string, ref: string): Promise { const result = await execa('git', ['rev-parse', '--verify', `${ref}^{commit}`], { cwd: repoRoot, diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index 3ac2b5b5b7..45721fb798 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -1,5 +1,8 @@ /* eslint-disable no-console */ +// Packs a workspace's public packages with `pnpm pack`, so each tarball is exactly what +// `pnpm publish` would upload — for installing a build the way a consumer installs a release. + import * as path from 'node:path'; import * as os from 'node:os'; import { createHash } from 'node:crypto'; @@ -12,15 +15,6 @@ import { execa, parseCommandString } from 'execa'; import { resolveCommit } from './git'; import { listPublishablePackages } from './pnpm'; -/** - * Packs the public workspace packages at a given git ref into a folder of tarballs. - * - * The tarballs are produced with `pnpm pack`, so each is exactly what `pnpm publish` would upload — - * honoring the package's `files`, `.npmignore`, and `publishConfig.directory`. That makes them - * suitable for installing a ref's build the way a consumer would install a published release, which - * is what benchmark harnesses and end-to-end install tests need. - */ - export interface PackedPackage { /** Package name from its package.json. */ name: string; @@ -29,17 +23,11 @@ export interface PackedPackage { tarball: string; } -interface RawManifestPackage { - name: string; - version: string; - /** Tarball name, relative to the folder, so the cache is relocatable. */ - tarball: string; -} - -interface RawManifest { +interface Manifest { /** The build command used, part of the cache key. */ buildCmd: string; - packages: RawManifestPackage[]; + /** Tarball paths relative to the folder, so a restored cache still resolves. */ + packages: PackedPackage[]; } export interface PackRefOptions { @@ -61,15 +49,9 @@ export interface PackRefOptions { buildCmd?: string; } -/** Name of the file written into each packed folder describing its contents. */ const MANIFEST = 'manifest.json'; -/** - * Turns a package name into a filesystem-safe tarball basename: `@scope/pkg` → `scope+pkg.tgz`. - * - * `+` cannot appear in a package name, so no two packages share a tarball — `@mui/pkg` and - * `mui-pkg` stay apart, where a `-` would fold them together. - */ +/** `@scope/pkg` → `scope+pkg.tgz`; `+` cannot appear in a package name, so no two collide. */ function tarballName(pkgName: string): string { return `${pkgName.replace(/^@/, '').replace('/', '+')}.tgz`; } @@ -80,76 +62,62 @@ async function removeCheckout(repoRoot: string, checkout: string): Promise await execa('git', ['worktree', 'prune'], { cwd: repoRoot }); } -/** - * Content hash of a file, short enough to live in a filename. - * - * Streamed rather than read whole: these are tarballs of built packages, and every one of them - * would otherwise be resident at once. - */ +/** Runs `buildCmd` in `cwd`, with the nx daemon off so nothing keeps writing after it returns. */ +async function build(cwd: string, buildCmd: string): Promise { + const [file, ...args] = parseCommandString(buildCmd); + await execa(file, args, { + cwd, + env: { NX_DAEMON: 'false' }, + stdio: 'inherit', + verbose: 'short', + }); +} + +/** Content hash of a file, short enough to live in a filename. */ async function hashFile(file: string): Promise { const hash = createHash('sha256'); await pipeline(createReadStream(file), hash); return hash.digest('hex').slice(0, 12); } -/** Reads a folder's raw `manifest.json`, or `null` when it has none. */ -async function readRawManifest(dir: string): Promise { - let text: string; +/** + * `dir`'s packages if it holds a cache of the requested build — built with the same `buildCmd`, + * every tarball still present — or `null` otherwise. + */ +async function readFreshCache(dir: string, buildCmd: string): Promise { + let manifest: Manifest; try { - text = await readFile(path.join(dir, MANIFEST), 'utf8'); + manifest = JSON.parse(await readFile(path.join(dir, MANIFEST), 'utf8')); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') { return null; } throw error; } - return JSON.parse(text); -} - -/** A manifest's packages, their tarball names resolved to absolute paths under `dir`. */ -function packagesIn(raw: RawManifest, dir: string): PackedPackage[] { - return raw.packages.map((pkg) => ({ ...pkg, tarball: path.join(dir, pkg.tarball) })); -} - -/** - * Returns `dir`'s manifest if it is a usable cache for the requested build — produced with the same - * `buildCmd`, with every tarball it references still present — or `null` otherwise. - * - * Returning the parsed manifest lets the caller reuse it without a second read. Guards against - * reusing a folder built with a different build script, or one whose tarballs were partially - * evicted or deleted. - */ -async function readFreshCache(dir: string, buildCmd: string): Promise { - const raw = await readRawManifest(dir); - if (raw === null || raw.buildCmd !== buildCmd) { + if (manifest.buildCmd !== buildCmd) { return null; } - const tarballs = await Promise.allSettled( - raw.packages.map((pkg) => access(path.join(dir, pkg.tarball))), - ); - return tarballs.every((result) => result.status === 'fulfilled') ? raw : null; + const packages = manifest.packages.map((pkg) => ({ + ...pkg, + tarball: path.join(dir, pkg.tarball), + })); + const present = await Promise.allSettled(packages.map((pkg) => access(pkg.tarball))); + return present.every((result) => result.status === 'fulfilled') ? packages : null; } /** - * Packs the already-built public workspace packages of `checkoutDir` into `outDir`, one `.tgz` per - * package. Packages must already be built. - * - * Uses pnpm's own workspace resolution rather than scanning a directory, so the set follows - * `pnpm-workspace.yaml` and mirrors exactly what a release publishes. + * Packs the already-built public workspace packages of `workspaceDir` into `outDir`. * - * A public package that (transitively) depends on a *private* workspace package cannot be resolved - * by a consumer — the private package is never packed, so `pnpm install` looks for it on the - * registry and 404s. That is a real packaging bug in the ref, not something to paper over here: - * make the internal dependency public so it ships alongside the package that needs it. + * A public package that depends on a private workspace package cannot be installed from these — + * the private one is never packed — which is a packaging bug in the ref, not something to hide here. */ -async function packBuiltPackages(checkoutDir: string, outDir: string): Promise { +async function packBuiltPackages(workspaceDir: string, outDir: string): Promise { await mkdir(outDir, { recursive: true }); - const packages = await listPublishablePackages(checkoutDir); + const packages = await listPublishablePackages(workspaceDir); if (packages.length === 0) { - throw new Error(`No public workspace packages found in ${checkoutDir}.`); + throw new Error(`No public workspace packages found in ${workspaceDir}.`); } - // Each `pnpm pack` is its own node process, so a repository with a dozen public packages would - // otherwise start a dozen at once. + // Each `pnpm pack` is a node process of its own. return mapAsync( packages, async ({ path: pkgDir, name, version }) => { @@ -165,16 +133,9 @@ async function packBuiltPackages(checkoutDir: string, outDir: string): Promise

    /`, returning the packed * tarballs. * - * The build is deterministic for a commit, so the folder is cached by SHA: a later call for the - * same commit reuses it and skips the checkout, install, and build entirely — but only when it was - * built with the same `buildCmd` and its tarballs are all present, so the cache can never hand back - * stale or mismatched output. - * - * On a cache miss the ref is checked out in a throwaway location, installed, built, and packed; the - * checkout is then removed, so the small tarballs are the only lasting artifact. The folder is - * assembled in a staging directory and atomically renamed into place, and its `manifest.json` - * stores tarball names relative to the folder — so the cache is safe to move between machines (for - * example restored from a CI cache under a different absolute path). + * The folder is cached by SHA: a later call for the same commit, built with the same `buildCmd`, + * reuses it and skips the checkout, install and build. It is assembled in a staging folder and + * renamed into place, so an interrupted run never leaves something that looks like a cache. */ export async function packRef(options: PackRefOptions): Promise { const { @@ -188,16 +149,13 @@ export async function packRef(options: PackRefOptions): Promise const sha = await resolveCommit(repoRoot, ref); const dir = path.join(outRoot, sha); - // Cache hit: reuse the folder only if it was built the same way and its tarballs are all present. const cached = await readFreshCache(dir, buildCmd); if (cached) { console.log(chalk.green(`\nReusing packed workspace for "${ref}" (${sha.slice(0, 9)}).`)); - return packagesIn(cached, dir); + return cached; } await mkdir(outRoot, { recursive: true }); - // Assemble in a staging sibling on the same filesystem so the final rename into `/` - // is atomic — an interrupted run can never leave a partial folder that looks like a cache hit. const staging = await mkdtemp(path.join(outRoot, '.staging-')); const checkout = await mkdtemp(path.join(os.tmpdir(), 'pack-workspace-')); try { @@ -212,36 +170,20 @@ export async function packRef(options: PackRefOptions): Promise await execa(installFile, installArgs, { cwd: checkout, stdio: 'inherit', verbose: 'short' }); } console.log(chalk.cyan(`\nBuilding packages for ${sha.slice(0, 9)}…`)); - // Disable the nx daemon (lerna runs builds through nx): a lingering daemon keeps writing into - // the checkout and makes removal fail with "Directory not empty". execa's `env` extends the - // current environment, so only NX_DAEMON is overridden. - const [buildFile, ...buildArgs] = parseCommandString(buildCmd); - await execa(buildFile, buildArgs, { - cwd: checkout, - env: { NX_DAEMON: 'false' }, - stdio: 'inherit', - verbose: 'short', - }); + await build(checkout, buildCmd); const packages = await packBuiltPackages(checkout, staging); - // Store tarballs by basename so the folder is relocatable; record buildCmd so a later run can - // tell whether the cache matches. Write the manifest last — it marks completeness. - const manifest: RawManifest = { + const manifest: Manifest = { buildCmd, - packages: packages.map(({ name, version, tarball }) => ({ - name, - version, - tarball: path.basename(tarball), - })), + packages: packages.map((pkg) => ({ ...pkg, tarball: path.basename(pkg.tarball) })), }; + // Written last: it is what marks the folder complete. await writeFile(path.join(staging, MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`); - // Replace any pre-existing folder — a stale/partial one, or a mismatched cache we chose to - // rebuild — so the rename can't fail with ENOTEMPTY. await rm(dir, { recursive: true, force: true }); await rename(staging, dir); - return packagesIn(manifest, dir); + return packages.map((pkg) => ({ ...pkg, tarball: path.join(dir, path.basename(pkg.tarball)) })); } finally { - // However the build went; on success the staging folder has already become `dir`. + // On success the staging folder has already become `dir`. await Promise.all([ rm(staging, { recursive: true, force: true }), removeCheckout(repoRoot, checkout), @@ -250,14 +192,8 @@ export async function packRef(options: PackRefOptions): Promise } /** - * Builds and packs the working tree, naming each tarball by a hash of its bytes. - * - * A tarball path then never points at different content, which is what lets a consumer's installs - * persist across runs: an unchanged build keeps its filename, so `pnpm install` has nothing to do, - * while a changed one swaps a single package. - * - * The tree is mutable, so this always builds; a good `buildCmd` is cached (nx, turbo) and an - * unchanged tree rebuilds in seconds. + * Builds and packs the working tree into `outRoot`, naming each tarball by a hash of its bytes: an + * unchanged build keeps its path, so an install that depends on it has nothing to redo. */ export async function packWorkingTree(options: { /** The workspace to build and pack. */ @@ -270,33 +206,17 @@ export async function packWorkingTree(options: { const { repoRoot, outRoot, buildCmd = 'pnpm release:build' } = options; console.log(chalk.cyan(`\nBuilding workspace packages for "working tree" (${buildCmd})…`)); - // Disable the nx daemon: it keeps writing into the workspace after the build returns. - const [file, ...args] = parseCommandString(buildCmd); - await execa(file, args, { - cwd: repoRoot, - env: { NX_DAEMON: 'false' }, - stdio: 'inherit', - verbose: 'short', - }); + await build(repoRoot, buildCmd); console.log(chalk.cyan('\nPacking the working tree…')); - // Stage next to the destination so the renames cannot cross filesystems. - const packedRoot = path.dirname(outRoot); - await mkdir(packedRoot, { recursive: true }); - const staging = await mkdtemp(path.join(packedRoot, '.pack-')); - try { - const packed = await packBuiltPackages(repoRoot, staging); - await rm(outRoot, { recursive: true, force: true }); - await mkdir(outRoot, { recursive: true }); - return await Promise.all( - packed.map(async (pkg) => { - const hash = await hashFile(pkg.tarball); - const tarball = path.join(outRoot, `${path.basename(pkg.tarball, '.tgz')}-${hash}.tgz`); - await rename(pkg.tarball, tarball); - return { ...pkg, tarball }; - }), - ); - } finally { - await rm(staging, { recursive: true, force: true }); - } + await rm(outRoot, { recursive: true, force: true }); + const packed = await packBuiltPackages(repoRoot, outRoot); + return Promise.all( + packed.map(async (pkg) => { + const hash = await hashFile(pkg.tarball); + const tarball = path.join(outRoot, `${path.basename(pkg.tarball, '.tgz')}-${hash}.tgz`); + await rename(pkg.tarball, tarball); + return { ...pkg, tarball }; + }), + ); } diff --git a/packages/benchmark/src/utils/pnpm.ts b/packages/benchmark/src/utils/pnpm.ts index 8e260e35d9..e8b377a0dc 100644 --- a/packages/benchmark/src/utils/pnpm.ts +++ b/packages/benchmark/src/utils/pnpm.ts @@ -1,5 +1,3 @@ -import * as path from 'node:path'; -import * as fs from 'node:fs/promises'; import { $ } from 'execa'; /** A publishable workspace package, as `pnpm ls` reports it. */ @@ -18,15 +16,8 @@ interface ListedPackage { } /** - * The publishable workspace packages of `cwd`, as `pnpm ls` resolves them. - * - * Asks pnpm rather than scanning directories, so the set follows `pnpm-workspace.yaml` and mirrors - * exactly what a release publishes. A package is skipped when it says it is private, or when it - * lacks the name or version that publishing requires. - * - * Deliberately narrower than the equivalent in `@mui/internal-code-infra`, which also filters by - * pattern and by what changed since a ref: those need git and pnpm-filter semantics nothing here - * wants. Named for what it does so the two cannot be mistaken for each other. + * The publishable workspace packages of `cwd`: those pnpm lists that are not private and have the + * name and version publishing requires. */ export async function listPublishablePackages(cwd: string): Promise { const result = await $({ cwd })`pnpm ls -r --json --depth -1`; @@ -38,46 +29,3 @@ export async function listPublishablePackages(cwd: string): Promise> { - const content = await fs.readFile(path.join(packagePath, 'package.json'), 'utf8'); - return JSON.parse(content); -} - -/** Writes a directory's `package.json`, formatted the way the repository writes manifests. */ -export async function writePackageJson(packagePath: string, packageJson: object): Promise { - const content = `${JSON.stringify(packageJson, null, 2)}\n`; - await fs.writeFile(path.join(packagePath, 'package.json'), content); -} - -/** One project, as `pnpm ls` reports it. */ -interface ListedProject { - dependencies?: Record; - devDependencies?: Record; -} - -/** - * The version the project in `cwd` resolved for each of its direct dependencies. - * - * Asked of pnpm rather than read out of `node_modules`, so the answer follows however pnpm chose to - * lay that directory out. `--filter .` keeps it to the one project; without it pnpm reports every - * workspace member. - */ -export async function installedVersions(cwd: string): Promise> { - const result = await $({ cwd })`pnpm ls --json --depth 0 --filter .`; - const [project]: ListedProject[] = JSON.parse(result.stdout); - const versions = new Map(); - - for (const section of [project?.dependencies, project?.devDependencies]) { - for (const [name, entry] of Object.entries(section ?? {})) { - // A workspace link reports `link:../…` here rather than a version. Callers want a version to - // pin, so leave those out and let them fall back to whatever the manifest declared. - if (entry.version && !entry.version.startsWith('link:')) { - versions.set(name, entry.version); - } - } - } - - return versions; -} diff --git a/packages/code-infra/src/cli/cmdBaseline.mjs b/packages/code-infra/src/cli/cmdBaseline.mjs index 9dcb58f5d5..52d9097465 100644 --- a/packages/code-infra/src/cli/cmdBaseline.mjs +++ b/packages/code-infra/src/cli/cmdBaseline.mjs @@ -13,16 +13,10 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ command: 'baseline', describe: 'Print the commit HEAD should be compared against', builder: (yargs) => { - return yargs - .option('base-branch', { - type: 'string', - description: - "Branch pull requests fork from. Default: read from a remote's HEAD, which a CI checkout does not record", - }) - .example( - 'BASE=$($0 baseline)', - 'Nothing but the commit is printed, so this stays substitutable', - ); + return yargs.option('base-branch', { + type: 'string', + description: "Branch pull requests fork from. Default: the remote's default branch", + }); }, handler: async (argv) => { console.log(await resolveBaseline({ baseBranch: argv.baseBranch })); diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index 2aa68b982a..39ef18ab2f 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -86,220 +86,65 @@ export async function remoteGitTagExists(tagName, cwd = process.cwd()) { return stdout.trim().length > 0; } -const REMOTES_PREFIX = 'refs/remotes/'; - -/** - * `origin/release/7.x` → `release/7.x`. - * - * Only the remote name comes off, so a branch whose own name contains a slash keeps it. Splitting on - * the last slash instead would turn `release/7.x` into `7.x` and match nothing. - * @param {string} shortRef - * @returns {string} - */ -function branchNameOf(shortRef) { - const slash = shortRef.indexOf('/'); - return slash === -1 ? shortRef : shortRef.slice(slash + 1); -} - /** - * The branch pull requests fork from, from `/HEAD` — the only local record of a remote's - * default branch, and often absent: only `git clone` writes one, and only for `origin`. - * - * Throws rather than guessing a conventional name, which resolves to no ref in - * {@link closestBaseBranch} and leaves the baseline as the previous commit. - * @param {string} [cwd=process.cwd()] + * The remote the base branch lives on: `upstream` when there is one, as in a fork's clone, or + * `origin`. + * @param {string} cwd * @returns {Promise} */ -async function detectBaseBranch(cwd = process.cwd()) { - const listed = (await $({ cwd })`git remote`).stdout.trim().split('\n').filter(Boolean); - // `origin` first: it is the one a clone records a HEAD for. - const remotes = [...new Set(['origin', ...listed])].filter((remote) => listed.includes(remote)); - - for (const remote of remotes) { - // Sequential: a handful of remotes, and the first hit wins. - // eslint-disable-next-line no-await-in-loop - const head = await $({ - cwd, - reject: false, - })`git symbolic-ref --short ${`refs/remotes/${remote}/HEAD`}`; - if (head.exitCode === 0) { - return branchNameOf(head.stdout.trim()); - } - } - - throw new Error( - 'Could not tell which branch this one forks from: no remote records a default branch. ' + - 'Run `git remote set-head origin -a` to write one, or name the base branch explicitly.', - ); -} - -/** - * Among the available base branches — every remote's `/` plus a local `` — the - * one whose merge base with HEAD is the most recent commit (the closest fork point), or undefined - * if none exist. - * - * Picking by most-recent merge base prefers an up-to-date upstream over a stale fork's base branch - * without hardcoding which remote is authoritative: locally `origin` may be a months-behind fork - * while `upstream` tracks the real repo, yet in CI `origin` *is* that repo. - * @param {string} baseBranch - * @param {string} [cwd=process.cwd()] - * @returns {Promise<{ ref: string, mergeBase: string } | undefined>} - */ -async function closestBaseBranch(baseBranch, cwd = process.cwd()) { - // On ties (same merge base), prefer upstream's, then origin's, then a local base branch. - const preference = [`upstream/${baseBranch}`, `origin/${baseBranch}`, baseBranch]; - /** @param {string} ref */ - const priority = (ref) => { - const index = preference.indexOf(ref); - return index === -1 ? preference.length : index; - }; - // Match only real base branches from full refnames: a remote's `/` (exactly one - // segment before it) or the local ``. Filtering short names on `/` would also catch a - // local branch literally named e.g. `wip/master`. - // - // Compared as a string rather than through a regex built from `baseBranch`: the branch name is - // arbitrary, and a real one like `v6.x` would make `.` match any character — quietly admitting - // `origin/v6-x` as a baseline candidate. - /** @param {string} refname */ - const isRemoteBase = (refname) => { - if (!refname.startsWith(REMOTES_PREFIX)) { - return false; - } - const withoutPrefix = refname.slice(REMOTES_PREFIX.length); - // A remote-tracking refname always has a remote to strip; `refs/remotes/foo` alone is not one. - return withoutPrefix.includes('/') && branchNameOf(withoutPrefix) === baseBranch; - }; - - const { stdout } = await $({ - cwd, - })`git for-each-ref --format=%(refname) refs/remotes refs/heads`; - const candidates = stdout - .trim() - .split('\n') - .filter((ref) => isRemoteBase(ref) || ref === `refs/heads/${baseBranch}`) - .map((ref) => ref.replace(/^refs\/(remotes|heads)\//, '')) - .sort((a, b) => priority(a) - priority(b)); - - /** @type {{ ref: string, mergeBase: string, when: number } | undefined} */ - let best; - for (const ref of candidates) { - // Sequential on purpose: each iteration is a git process, and the list is a handful of refs. - // eslint-disable-next-line no-await-in-loop - const result = await $({ cwd, reject: false })`git merge-base HEAD ${ref}`; - if (result.exitCode !== 0) { - continue; - } - const mergeBase = result.stdout.trim(); - // `--no-show-signature` because `log.showSignature` would otherwise prepend verification lines - // to the output, and the timestamp would read as NaN — which loses every later comparison and - // leaves whichever candidate came first standing. - // eslint-disable-next-line no-await-in-loop - const shown = await $({ - cwd, - reject: false, - })`git show -s --no-show-signature --format=%ct ${mergeBase}`; - const when = Number(shown.stdout.trim()); - if (shown.exitCode !== 0 || !Number.isFinite(when)) { - continue; - } - // Strictly greater, so on ties the higher-priority (earlier-sorted) ref wins. - if (!best || when > best.when) { - best = { ref, mergeBase, when }; - } - } - return best && { ref: best.ref, mergeBase: best.mergeBase }; +async function baseRemote(cwd) { + const { stdout } = await $({ cwd })`git remote`; + return stdout.split('\n').includes('upstream') ? 'upstream' : 'origin'; } /** - * Makes the base branch reachable, fetching it when nothing local has it yet. - * - * A CI checkout fetches only the branch under test, so the ref a fork point is measured from is - * usually absent there. Fetching it here rather than leaving it to the caller keeps the command's - * one job — say which commit to compare against — from depending on a step somewhere else. - * @param {string} baseBranch + * The default branch of `remote`, as the remote itself reports it. + * @param {string} remote * @param {string} cwd - * @returns {Promise} + * @returns {Promise} */ -async function fetchBaseBranch(baseBranch, cwd) { - const listed = (await $({ cwd })`git remote`).stdout.trim().split('\n').filter(Boolean); - for (const ref of [ - `refs/heads/${baseBranch}`, - ...listed.map((r) => `refs/remotes/${r}/${baseBranch}`), - ]) { - // Sequential on purpose: a handful of refs, and the first hit means there is nothing to fetch. - // eslint-disable-next-line no-await-in-loop - const found = await $({ cwd, reject: false })`git rev-parse --verify --quiet ${ref}`; - if (found.exitCode === 0) { - return; - } - } - if (!listed.includes('origin')) { - return; +async function defaultBranchOf(remote, cwd) { + const { stdout } = await $({ cwd })`git ls-remote --symref ${remote} HEAD`; + const match = /^ref: refs\/heads\/(\S+)\tHEAD$/m.exec(stdout); + if (!match) { + throw new Error(`Remote "${remote}" does not report a default branch.`); } - // Best-effort: a failure here is not the error to report — the caller's own check says plainly - // that no ref for the base branch exists, which is what a reader needs either way. - await $({ - cwd, - reject: false, - })`git fetch --no-tags origin ${`${baseBranch}:refs/remotes/origin/${baseBranch}`}`; + return match[1]; } /** - * The commit HEAD should be compared against: on a feature branch the fork point from the closest - * base branch, so a comparison reflects only what the branch changed; on the base branch itself the - * previous commit, since there is no meaningful fork point there. - * - * One answer for every job that compares a branch to its base — bundle size, benchmarks — which - * each used to derive it in shell, and had drifted apart doing so. + * The commit HEAD should be compared against: where it forked from the base branch, or its parent + * when HEAD is already on the base branch. * @param {Object} [options] * @param {string} [options.cwd] - * @param {string} [options.baseBranch] - Overrides detection from `origin/HEAD`. + * @param {string} [options.baseBranch] - Defaults to the remote's default branch. * @returns {Promise} */ export async function resolveBaseline(options = {}) { const cwd = options.cwd ?? process.cwd(); - const baseBranch = options.baseBranch ?? (await detectBaseBranch(cwd)); - - const previousCommit = async () => { - const result = await $({ cwd, reject: false })`git rev-parse --verify HEAD~1`; - if (result.exitCode !== 0) { - throw new Error( - 'HEAD has no parent commit, so there is nothing here to compare against. A shallow clone ' + - 'is the usual cause — fetch more history, or name the base branch explicitly.', - ); - } - return result.stdout.trim(); - }; - - const branch = (await $({ cwd })`git rev-parse --abbrev-ref HEAD`).stdout.trim(); - if (branch === baseBranch) { - return previousCommit(); + const remote = await baseRemote(cwd); + const baseBranch = options.baseBranch ?? (await defaultBranchOf(remote, cwd)); + + await $({ cwd })`git fetch --no-tags ${remote} ${baseBranch}`; + const [forkPoint, head] = await Promise.all([ + $({ cwd })`git merge-base HEAD FETCH_HEAD`.then((result) => result.stdout.trim()), + getCurrentGitSha(cwd), + ]); + if (forkPoint !== head) { + return forkPoint; } - await fetchBaseBranch(baseBranch, cwd); - const base = await closestBaseBranch(baseBranch, cwd); - if (!base) { - // Not a fallback: being on the base branch and having HEAD already contained in it are both - // handled above, so reaching here means the branch named is one this repository cannot see. - // Answering with the previous commit would be a plausible number for a question left unasked. + const parent = await $({ cwd, reject: false })`git rev-parse --verify HEAD~1`; + if (parent.exitCode !== 0) { throw new Error( - `No ref for base branch "${baseBranch}", so there is no fork point to measure from. ` + - `Name a branch this repository has, or make it fetchable.`, + 'HEAD has no parent commit to compare against. A shallow clone is the usual cause.', ); } - // The fork point is HEAD itself — HEAD is already contained in the base branch, so there is - // nothing this branch changed to compare. - if (base.mergeBase === (await getCurrentGitSha(cwd))) { - return previousCommit(); - } - return base.mergeBase; + return parent.stdout.trim(); } /** * The most recent version tag reachable from HEAD — where a changelog's range starts. - * - * Not a baseline: {@link resolveBaseline} answers where a *branch* diverged, which on a feature - * branch is a different commit entirely, and the two are not interchangeable. * @param {Object} opts * @param {string} opts.cwd * @param {boolean} [opts.fetchAll=true] Whether to fetch all tags from all remotes before finding the latest tag. diff --git a/packages/code-infra/src/utils/git.test.mjs b/packages/code-infra/src/utils/git.test.mjs index 1818e9447e..3cafd0b531 100644 --- a/packages/code-infra/src/utils/git.test.mjs +++ b/packages/code-infra/src/utils/git.test.mjs @@ -5,203 +5,121 @@ import { describe, expect, it, onTestFinished } from 'vitest'; import { execa } from 'execa'; import { resolveBaseline } from './git.mjs'; -/** - * A repository with three commits on `main`. - * - * A real repository rather than a stand-in for git: every question here is about what git reports — - * which refnames exist, where two branches diverge — and faking that would only restate the answer - * the code already assumes. - * @returns {Promise<{ repoRoot: string, shas: string[], git: (...args: string[]) => Promise }>} - */ -async function makeRepo() { - const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'baseline-test-')); - onTestFinished(async () => { - await fs.rm(repoRoot, { recursive: true, force: true }); - }); +// Keeps a contributor's own git config (signing, hooks) out of the fixture commits. +const GIT_ENV = { + GIT_AUTHOR_NAME: 'Fixture', + GIT_AUTHOR_EMAIL: 'fixture@example.com', + GIT_COMMITTER_NAME: 'Fixture', + GIT_COMMITTER_EMAIL: 'fixture@example.com', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_CONFIG_SYSTEM: '/dev/null', +}; + +/** @param {string} prefix */ +async function tempDir(prefix) { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + onTestFinished(() => fs.rm(dir, { recursive: true, force: true })); + return dir; +} - // Everything through the environment rather than `git config`, so the repo needs no extra - // processes. The two `GIT_CONFIG_*` variables are what make the second half true: without them a - // contributor's own `commit.gpgsign` or `core.hooksPath` still applies, and these commits fail - // for a reason that has nothing to do with the code under test. - /** @param {string} [date] Commit date, so ranking by recency is not left to the clock. */ - const gitWith = - (date) => - (/** @type {string[]} */ ...args) => - execa('git', args, { - cwd: repoRoot, - env: { - GIT_AUTHOR_NAME: 'Fixture', - GIT_AUTHOR_EMAIL: 'fixture@example.com', - GIT_COMMITTER_NAME: 'Fixture', - GIT_COMMITTER_EMAIL: 'fixture@example.com', - GIT_CONFIG_GLOBAL: '/dev/null', - GIT_CONFIG_SYSTEM: '/dev/null', - ...(date ? { GIT_AUTHOR_DATE: date, GIT_COMMITTER_DATE: date } : {}), - }, - }); - const git = gitWith(undefined); +/** @param {string} cwd */ +function gitIn(cwd) { + return (/** @type {string[]} */ ...args) => execa('git', args, { cwd, env: GIT_ENV }); +} +/** + * A remote repository with `commits` on `main`. + * @param {string[]} [commits] + */ +async function makeRemote(commits = ['first', 'second', 'third']) { + const dir = await tempDir('baseline-remote-'); + const git = gitIn(dir); await git('init', '--initial-branch=main'); - // A day apart each: the baseline comes from the candidate whose merge base is most recent, and - // three commits made in a loop otherwise share a second and leave that comparison to a tie-break. - for (const [index, message] of ['first', 'second', 'third'].entries()) { - // Empty commits: nothing here reads a file, only the commits' identities matter. + for (const message of commits) { // eslint-disable-next-line no-await-in-loop - await gitWith(`2020-01-0${index + 1}T00:00:00Z`)('commit', '--allow-empty', '-m', message); + await git('commit', '--allow-empty', '-m', message); } const { stdout } = await git('log', '--format=%H', '--reverse'); - return { repoRoot, shas: stdout.trim().split('\n'), git }; + return { dir, git, shas: stdout.trim().split('\n') }; } -describe('resolveBaseline', () => { - it('uses the previous commit when HEAD is on the base branch', async () => { - const { repoRoot, shas } = await makeRepo(); +/** + * Clones `remoteDir`. + * @param {string} remoteDir + * @param {string[]} [cloneArgs] + */ +async function clone(remoteDir, cloneArgs = []) { + const dir = await tempDir('baseline-clone-'); + await execa('git', ['clone', ...cloneArgs, remoteDir, dir], { env: GIT_ENV }); + return { dir, git: gitIn(dir) }; +} - const sha = await resolveBaseline({ cwd: repoRoot, baseBranch: 'main' }); +describe('resolveBaseline', () => { + it('uses the parent commit when HEAD is on the base branch', async () => { + const remote = await makeRemote(); + const work = await clone(remote.dir); - expect(sha).toBe(shas[1]); + expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[1]); }); it('uses the fork point when HEAD is on a branch', async () => { - const { repoRoot, shas, git } = await makeRepo(); - // Branch off the first commit, so the fork point is not simply HEAD~1. - await git('checkout', '-b', 'feature', shas[0]); - await git('commit', '--allow-empty', '-m', 'work'); - - const sha = await resolveBaseline({ cwd: repoRoot, baseBranch: 'main' }); + const remote = await makeRemote(); + const work = await clone(remote.dir); + await work.git('checkout', '-b', 'feature', remote.shas[0]); + await work.git('commit', '--allow-empty', '-m', 'work'); - expect(sha).toBe(shas[0]); + expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[0]); }); - it('prefers the remote whose fork point is most recent', async () => { - const { repoRoot, shas, git } = await makeRepo(); - // `upstream` is the preferred remote on a tie, so it is deliberately the *stale* one here: - // picking by recency has to override that preference, and a "first candidate wins" reading would - // answer `upstream` instead. - await git('update-ref', 'refs/remotes/upstream/main', shas[0]); - await git('update-ref', 'refs/remotes/origin/main', shas[1]); - await git('checkout', '-b', 'feature'); - await git('commit', '--allow-empty', '-m', 'work'); - // Drop the local `main`, which is a candidate in its own right and sits ahead of both remotes — - // it would win on recency and say nothing about which remote was chosen. - await git('branch', '-D', 'main'); - - const sha = await resolveBaseline({ cwd: repoRoot, baseBranch: 'main' }); - - expect(sha).toBe(shas[1]); + it("forks from upstream's base branch over a fork's stale one", async () => { + const remote = await makeRemote(); + const fork = await clone(remote.dir); + await fork.git('reset', '--hard', remote.shas[0]); + const work = await clone(fork.dir); + await work.git('remote', 'add', 'upstream', remote.dir); + await work.git('fetch', 'upstream'); + await work.git('checkout', '-b', 'feature', remote.shas[1]); + await work.git('commit', '--allow-empty', '-m', 'work'); + + expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[1]); }); - it('fetches the base branch when the checkout never pulled it', async () => { - // What CI looks like: the branch under test is there, the branch it forked from is not. Without - // the fetch there is no fork point to find, and the answer silently became the previous commit. - const { repoRoot, shas, git } = await makeRepo(); - const clone = await fs.mkdtemp(path.join(os.tmpdir(), 'baseline-clone-')); - onTestFinished(async () => { - await fs.rm(clone, { recursive: true, force: true }); - }); - - await git('checkout', '-b', 'feature', shas[0]); - await git('commit', '--allow-empty', '-m', 'work'); - const branchTip = (await git('rev-parse', 'HEAD')).stdout.trim(); - - // A single-branch clone, so `origin/main` is absent exactly as it is after a CI checkout. - await execa('git', ['clone', '--single-branch', '--branch', 'feature', repoRoot, clone], { - env: { GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null' }, - }); - expect( - ( - await execa('git', ['rev-parse', '--verify', '--quiet', 'refs/remotes/origin/main'], { - cwd: clone, - reject: false, - }) - ).exitCode, - ).not.toBe(0); - - const sha = await resolveBaseline({ cwd: clone, baseBranch: 'main' }); - - // The fork point, not `branchTip~1` — which is what the previous-commit fallback would give. - expect(sha).toBe(shas[0]); - expect(sha).not.toBe(branchTip); - }); - - it('does not treat a branch matching the base branch as a pattern', async () => { - const { repoRoot, shas, git } = await makeRepo(); - // `v6-x`, not `v6.x` — only a regex reading `.` as a wildcard would accept it. - await git('update-ref', 'refs/remotes/origin/v6-x', shas[0]); - await git('checkout', '-b', 'feature'); + it('fetches the base branch a CI checkout does not have', async () => { + const remote = await makeRemote(); + await remote.git('checkout', '-b', 'feature', remote.shas[0]); + await remote.git('commit', '--allow-empty', '-m', 'work'); + await remote.git('checkout', 'main'); + const work = await clone(remote.dir, ['--single-branch', '--branch', 'feature']); - // No ref matches, which is the answer rather than a fallback. Matching `origin/v6-x` would - // instead have picked its merge base — the first commit — and compared against the wrong one. - await expect(resolveBaseline({ cwd: repoRoot, baseBranch: 'v6.x' })).rejects.toThrow( - /No ref for base branch "v6.x"/, - ); + expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[0]); }); -}); - -describe('resolveBaseline without a base branch named', () => { - // Each default branch points at an older commit than the local `main`, so taking the wrong branch - // as the base would answer a different fork point. - it('forks from the default branch origin/HEAD records', async () => { - const { repoRoot, shas, git } = await makeRepo(); - await git('remote', 'add', 'origin', repoRoot); - await git('update-ref', 'refs/remotes/origin/trunk', shas[0]); - await git('symbolic-ref', 'refs/remotes/origin/HEAD', 'refs/remotes/origin/trunk'); - await git('checkout', '-b', 'feature'); - await git('commit', '--allow-empty', '-m', 'work'); - - expect(await resolveBaseline({ cwd: repoRoot })).toBe(shas[0]); - }); - - it('reads the default branch from another remote when origin records none', async () => { - // A remote added by hand has no HEAD unless someone wrote one. If they did, it still counts. - const { repoRoot, shas, git } = await makeRepo(); - await git('remote', 'add', 'upstream', repoRoot); - await git('update-ref', 'refs/remotes/upstream/release', shas[1]); - await git('symbolic-ref', 'refs/remotes/upstream/HEAD', 'refs/remotes/upstream/release'); - await git('checkout', '-b', 'feature'); - await git('commit', '--allow-empty', '-m', 'work'); - expect(await resolveBaseline({ cwd: repoRoot })).toBe(shas[1]); + it('measures from the base branch it is given', async () => { + const remote = await makeRemote(); + await remote.git('checkout', '-b', 'release', remote.shas[0]); + await remote.git('commit', '--allow-empty', '-m', 'release fix'); + const { stdout: releaseTip } = await remote.git('rev-parse', 'HEAD'); + await remote.git('checkout', 'main'); + const work = await clone(remote.dir); + await work.git('checkout', '-b', 'feature', 'origin/release'); + await work.git('commit', '--allow-empty', '-m', 'work'); + + expect(await resolveBaseline({ cwd: work.dir, baseBranch: 'release' })).toBe(releaseTip.trim()); }); - it('does not take a conventional name as evidence', async () => { - // The branch exists, but nothing says it is the default. - const { repoRoot, git } = await makeRepo(); - await git('remote', 'add', 'origin', repoRoot); - await git('update-ref', 'refs/remotes/origin/main', 'HEAD'); + it('fails for a base branch the remote does not have', async () => { + const remote = await makeRemote(); + const work = await clone(remote.dir); - await expect(resolveBaseline({ cwd: repoRoot })).rejects.toThrow( - /no remote records a default branch/, + await expect(resolveBaseline({ cwd: work.dir, baseBranch: 'no-such-branch' })).rejects.toThrow( + /no-such-branch/, ); }); - it('fails when there is no remote at all', async () => { - const { repoRoot } = await makeRepo(); - - await expect(resolveBaseline({ cwd: repoRoot })).rejects.toThrow(/Could not tell which branch/); - }); -}); - -describe('resolveBaseline without history', () => { it('explains that a root commit has nothing behind it', async () => { - const repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'baseline-root-')); - onTestFinished(async () => { - await fs.rm(repoRoot, { recursive: true, force: true }); - }); - const env = { - GIT_AUTHOR_NAME: 'Fixture', - GIT_AUTHOR_EMAIL: 'fixture@example.com', - GIT_COMMITTER_NAME: 'Fixture', - GIT_COMMITTER_EMAIL: 'fixture@example.com', - GIT_CONFIG_GLOBAL: '/dev/null', - GIT_CONFIG_SYSTEM: '/dev/null', - }; - await execa('git', ['init', '--initial-branch=main'], { cwd: repoRoot, env }); - await execa('git', ['commit', '--allow-empty', '-m', 'first'], { cwd: repoRoot, env }); - - // A shallow clone reaches the same place, which is the common one: the parent was never fetched. - await expect(resolveBaseline({ cwd: repoRoot, baseBranch: 'main' })).rejects.toThrow( - /HEAD has no parent commit/, - ); + const remote = await makeRemote(['first']); + const work = await clone(remote.dir); + + await expect(resolveBaseline({ cwd: work.dir })).rejects.toThrow(/HEAD has no parent commit/); }); }); From 87586530e6ac47992702fd5eded7cba2cec2f874 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 30 Sep 2026 13:06:48 +0200 Subject: [PATCH 102/173] [code-infra] Pick the baseline's remote by URL, via getRepositoryInfo The base branch is fetched from the mui remote whatever it is named, not only upstream or origin. getRepositoryInfo reads each remote's configured URL rather than git remote -v, which prints URLs after insteadOf rewriting. --- packages/code-infra/src/cli/cmdBaseline.mjs | 2 +- packages/code-infra/src/utils/git.mjs | 57 +++++++++------------ packages/code-infra/src/utils/git.test.mjs | 49 ++++++++++++++++-- 3 files changed, 71 insertions(+), 37 deletions(-) diff --git a/packages/code-infra/src/cli/cmdBaseline.mjs b/packages/code-infra/src/cli/cmdBaseline.mjs index 52d9097465..95f63a3a03 100644 --- a/packages/code-infra/src/cli/cmdBaseline.mjs +++ b/packages/code-infra/src/cli/cmdBaseline.mjs @@ -15,7 +15,7 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ builder: (yargs) => { return yargs.option('base-branch', { type: 'string', - description: "Branch pull requests fork from. Default: the remote's default branch", + description: "Branch pull requests fork from. Default: the mui remote's default branch", }); }, handler: async (argv) => { diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index 39ef18ab2f..0e09e2a83f 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -9,7 +9,9 @@ import gitUrlParse from 'git-url-parse'; */ /** - * Get current repository info from git remote + * Get current repository info from git remote. + * + * Reads each remote's configured URL, which `url..insteadOf` does not rewrite. * @param {string} [cwd=process.cwd()] * @returns {Promise} Repository owner and name */ @@ -18,8 +20,11 @@ export async function getRepositoryInfo(cwd = process.cwd()) { * @type {Record} */ const cause = {}; - const { stdout } = await $({ cwd })`git remote -v`; - const lines = stdout.trim().split('\n'); + // Exits 1 when no remote is configured; the error below reports that. + const { stdout } = await $({ + cwd, + reject: false, + })`git config --get-regexp ${'^remote\\..*\\.url$'}`; /** * @type {Set} */ @@ -29,25 +34,22 @@ export async function getRepositoryInfo(cwd = process.cwd()) { */ const validRemotes = new Map(); - for (const line of lines) { - // Match pattern: "remoteName url (fetch|push)" - const [remoteName, url, type] = line.trim().split(/\s+/, 3); + for (const line of stdout.split('\n').filter(Boolean)) { + // "remote..url "; a remote name may itself contain dots. + const [key, url] = line.split(' ', 2); + const remoteName = key.slice('remote.'.length, -'.url'.length); repoRemotes.add(remoteName); - if (type === '(fetch)') { - try { - const parsed = gitUrlParse(url); - if (parsed.source !== 'github.com' || parsed.owner !== 'mui') { - cause[remoteName] = `Remote is not a GitHub repository under 'mui' organization: ${url}`; - continue; - } - if (!validRemotes.has(remoteName)) { - validRemotes.set(remoteName, { owner: parsed.owner, repo: parsed.name }); - } - } catch (error) { - cause[remoteName] = `Failed to parse URL for remote ${remoteName}: ${url}`; + try { + const parsed = gitUrlParse(url); + if (parsed.source !== 'github.com' || parsed.owner !== 'mui') { + cause[remoteName] = `Remote is not a GitHub repository under 'mui' organization: ${url}`; + continue; } - } else if (type !== '(push)') { - throw new Error(`Unexpected line format for "git remote -v": "${line}"`); + if (!validRemotes.has(remoteName)) { + validRemotes.set(remoteName, { owner: parsed.owner, repo: parsed.name }); + } + } catch (error) { + cause[remoteName] = `Failed to parse URL for remote ${remoteName}: ${url}`; } } @@ -86,17 +88,6 @@ export async function remoteGitTagExists(tagName, cwd = process.cwd()) { return stdout.trim().length > 0; } -/** - * The remote the base branch lives on: `upstream` when there is one, as in a fork's clone, or - * `origin`. - * @param {string} cwd - * @returns {Promise} - */ -async function baseRemote(cwd) { - const { stdout } = await $({ cwd })`git remote`; - return stdout.split('\n').includes('upstream') ? 'upstream' : 'origin'; -} - /** * The default branch of `remote`, as the remote itself reports it. * @param {string} remote @@ -117,12 +108,12 @@ async function defaultBranchOf(remote, cwd) { * when HEAD is already on the base branch. * @param {Object} [options] * @param {string} [options.cwd] - * @param {string} [options.baseBranch] - Defaults to the remote's default branch. + * @param {string} [options.baseBranch] - Defaults to the mui remote's default branch. * @returns {Promise} */ export async function resolveBaseline(options = {}) { const cwd = options.cwd ?? process.cwd(); - const remote = await baseRemote(cwd); + const { remoteName: remote } = await getRepositoryInfo(cwd); const baseBranch = options.baseBranch ?? (await defaultBranchOf(remote, cwd)); await $({ cwd })`git fetch --no-tags ${remote} ${baseBranch}`; diff --git a/packages/code-infra/src/utils/git.test.mjs b/packages/code-infra/src/utils/git.test.mjs index 3cafd0b531..14630e8b41 100644 --- a/packages/code-infra/src/utils/git.test.mjs +++ b/packages/code-infra/src/utils/git.test.mjs @@ -44,14 +44,32 @@ async function makeRemote(commits = ['first', 'second', 'third']) { } /** - * Clones `remoteDir`. + * Points `remoteName` of the repository at `dir` to a GitHub URL under `mui`, which `insteadOf` + * sends to `remoteDir` — so it looks like a mui remote and fetches locally. + * @param {(...args: string[]) => Promise} git + * @param {string} remoteName + * @param {string} remoteDir + */ +async function addMuiRemote(git, remoteName, remoteDir) { + const url = `https://github.com/mui/${path.basename(remoteDir)}.git`; + await git('remote', 'add', remoteName, url); + await git('config', `url.${remoteDir}.insteadOf`, url); +} + +/** + * Clones `remoteDir`, as a mui remote named `origin`. * @param {string} remoteDir * @param {string[]} [cloneArgs] */ async function clone(remoteDir, cloneArgs = []) { const dir = await tempDir('baseline-clone-'); await execa('git', ['clone', ...cloneArgs, remoteDir, dir], { env: GIT_ENV }); - return { dir, git: gitIn(dir) }; + const git = gitIn(dir); + await git('remote', 'remove', 'origin'); + await addMuiRemote(git, 'origin', remoteDir); + // `remote remove` drops the tracking config a clone set up; later tests check out `origin/…`. + await git('fetch', 'origin'); + return { dir, git }; } describe('resolveBaseline', () => { @@ -76,7 +94,7 @@ describe('resolveBaseline', () => { const fork = await clone(remote.dir); await fork.git('reset', '--hard', remote.shas[0]); const work = await clone(fork.dir); - await work.git('remote', 'add', 'upstream', remote.dir); + await addMuiRemote(work.git, 'upstream', remote.dir); await work.git('fetch', 'upstream'); await work.git('checkout', '-b', 'feature', remote.shas[1]); await work.git('commit', '--allow-empty', '-m', 'work'); @@ -84,6 +102,31 @@ describe('resolveBaseline', () => { expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[1]); }); + it('finds the mui remote under another name', async () => { + const remote = await makeRemote(); + const fork = await makeRemote(['unrelated']); + const work = await clone(remote.dir); + await work.git('remote', 'set-url', 'origin', 'https://github.com/someone/fork.git'); + await work.git('config', `url.${fork.dir}.insteadOf`, 'https://github.com/someone/fork.git'); + await addMuiRemote(work.git, 'mui', remote.dir); + await work.git('checkout', '-b', 'feature', remote.shas[1]); + await work.git('commit', '--allow-empty', '-m', 'work'); + + expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[1]); + }); + + it('fails without a mui remote', async () => { + const remote = await makeRemote(); + const work = await clone(remote.dir); + // Only a fork: a GitHub repository, but not one under `mui`. + await work.git('remote', 'set-url', 'origin', 'https://github.com/someone/fork.git'); + await work.git('config', `url.${remote.dir}.insteadOf`, 'https://github.com/someone/fork.git'); + + await expect(resolveBaseline({ cwd: work.dir })).rejects.toThrow( + /Failed to find correct remote/, + ); + }); + it('fetches the base branch a CI checkout does not have', async () => { const remote = await makeRemote(); await remote.git('checkout', '-b', 'feature', remote.shas[0]); From 767bc691fcb5e1dfa5981929ed4f821779597f28 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:48:20 +0200 Subject: [PATCH 103/173] [code-infra] Choose the canonical remote by name getCanonicalRemote picks the remote of the canonical repository: the one named by git config code-infra.canonicalRemote, else upstream, else origin, else fail. resolveBaseline, getRepositoryInfo and findLatestTaggedVersion all go through it; getRepositoryInfo then reads only that remote's URL, and no longer requires it to be under mui. --- packages/code-infra/src/cli/cmdBaseline.mjs | 2 +- packages/code-infra/src/utils/git.mjs | 97 +++++++--------- packages/code-infra/src/utils/git.test.mjs | 117 ++++++++++---------- 3 files changed, 102 insertions(+), 114 deletions(-) diff --git a/packages/code-infra/src/cli/cmdBaseline.mjs b/packages/code-infra/src/cli/cmdBaseline.mjs index 95f63a3a03..5a34807cbe 100644 --- a/packages/code-infra/src/cli/cmdBaseline.mjs +++ b/packages/code-infra/src/cli/cmdBaseline.mjs @@ -15,7 +15,7 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ builder: (yargs) => { return yargs.option('base-branch', { type: 'string', - description: "Branch pull requests fork from. Default: the mui remote's default branch", + description: "Branch pull requests fork from. Default: the canonical remote's default branch", }); }, handler: async (argv) => { diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index 0e09e2a83f..a606f5ec63 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -9,62 +9,51 @@ import gitUrlParse from 'git-url-parse'; */ /** - * Get current repository info from git remote. - * - * Reads each remote's configured URL, which `url..insteadOf` does not rewrite. + * The remote of the canonical repository, as opposed to a fork: the one named by + * `git config code-infra.canonicalRemote`, else `upstream`, else `origin`. * @param {string} [cwd=process.cwd()] - * @returns {Promise} Repository owner and name + * @returns {Promise} */ -export async function getRepositoryInfo(cwd = process.cwd()) { - /** - * @type {Record} - */ - const cause = {}; - // Exits 1 when no remote is configured; the error below reports that. - const { stdout } = await $({ - cwd, - reject: false, - })`git config --get-regexp ${'^remote\\..*\\.url$'}`; - /** - * @type {Set} - */ - const repoRemotes = new Set(); - /** - * @type {Map} - */ - const validRemotes = new Map(); - - for (const line of stdout.split('\n').filter(Boolean)) { - // "remote..url "; a remote name may itself contain dots. - const [key, url] = line.split(' ', 2); - const remoteName = key.slice('remote.'.length, -'.url'.length); - repoRemotes.add(remoteName); - try { - const parsed = gitUrlParse(url); - if (parsed.source !== 'github.com' || parsed.owner !== 'mui') { - cause[remoteName] = `Remote is not a GitHub repository under 'mui' organization: ${url}`; - continue; - } - if (!validRemotes.has(remoteName)) { - validRemotes.set(remoteName, { owner: parsed.owner, repo: parsed.name }); - } - } catch (error) { - cause[remoteName] = `Failed to parse URL for remote ${remoteName}: ${url}`; +async function getCanonicalRemote(cwd = process.cwd()) { + const [configured, listed] = await Promise.all([ + // Exits 1 when the key is not set. + $({ cwd, reject: false })`git config --get code-infra.canonicalRemote`, + $({ cwd })`git remote`, + ]); + const remotes = listed.stdout.split('\n').filter(Boolean); + const override = configured.stdout.trim(); + if (override) { + if (!remotes.includes(override)) { + throw new Error( + `code-infra.canonicalRemote names "${override}", which is not a remote of this repository.`, + ); } + return override; } - - const preferredOrder = ['upstream', 'origin', ...validRemotes.keys()]; - for (const name of preferredOrder) { - const match = validRemotes.get(name); - if (match) { - return { ...match, remoteName: name }; - } + const remote = ['upstream', 'origin'].find((name) => remotes.includes(name)); + if (!remote) { + throw new Error( + 'No canonical remote: add an `upstream` or `origin` remote, or name one with ' + + '`git config code-infra.canonicalRemote `.', + ); } + return remote; +} - throw new Error( - `Failed to find correct remote(s) in : ${Array.from(repoRemotes.keys()).join(', ')}`, - { cause }, - ); +/** + * The GitHub repository of the canonical remote (see {@link getCanonicalRemote}). + * @param {string} [cwd=process.cwd()] + * @returns {Promise} + */ +export async function getRepositoryInfo(cwd = process.cwd()) { + const remoteName = await getCanonicalRemote(cwd); + const { stdout } = await $({ cwd })`git remote get-url ${remoteName}`; + const url = stdout.trim(); + const parsed = gitUrlParse(url); + if (parsed.source !== 'github.com') { + throw new Error(`Remote "${remoteName}" is not a GitHub repository: ${url}`); + } + return { owner: parsed.owner, repo: parsed.name, remoteName }; } /** @@ -108,12 +97,12 @@ async function defaultBranchOf(remote, cwd) { * when HEAD is already on the base branch. * @param {Object} [options] * @param {string} [options.cwd] - * @param {string} [options.baseBranch] - Defaults to the mui remote's default branch. + * @param {string} [options.baseBranch] - Defaults to the canonical remote's default branch. * @returns {Promise} */ export async function resolveBaseline(options = {}) { const cwd = options.cwd ?? process.cwd(); - const { remoteName: remote } = await getRepositoryInfo(cwd); + const remote = await getCanonicalRemote(cwd); const baseBranch = options.baseBranch ?? (await defaultBranchOf(remote, cwd)); await $({ cwd })`git fetch --no-tags ${remote} ${baseBranch}`; @@ -145,10 +134,8 @@ export async function findLatestTaggedVersion(opts) { const $$ = $({ cwd: opts.cwd }); const fetchAll = opts.fetchAll ?? true; if (fetchAll) { - const { remoteName } = await getRepositoryInfo(opts.cwd); - // Fetch all tags from the mui remote to ensure we have the latest tags. // --force to update any existing tags that may have changed to avoid the clobering error. - await $$`git fetch --tags --force ${remoteName}`; + await $$`git fetch --tags --force ${await getCanonicalRemote(opts.cwd)}`; } const { stdout } = await $$`git describe --tags --abbrev=0 --match ${'v*'}`; // only include "version-tags" return stdout.trim(); diff --git a/packages/code-infra/src/utils/git.test.mjs b/packages/code-infra/src/utils/git.test.mjs index 14630e8b41..95f21fbe2f 100644 --- a/packages/code-infra/src/utils/git.test.mjs +++ b/packages/code-infra/src/utils/git.test.mjs @@ -44,32 +44,31 @@ async function makeRemote(commits = ['first', 'second', 'third']) { } /** - * Points `remoteName` of the repository at `dir` to a GitHub URL under `mui`, which `insteadOf` - * sends to `remoteDir` — so it looks like a mui remote and fetches locally. - * @param {(...args: string[]) => Promise} git - * @param {string} remoteName - * @param {string} remoteDir - */ -async function addMuiRemote(git, remoteName, remoteDir) { - const url = `https://github.com/mui/${path.basename(remoteDir)}.git`; - await git('remote', 'add', remoteName, url); - await git('config', `url.${remoteDir}.insteadOf`, url); -} - -/** - * Clones `remoteDir`, as a mui remote named `origin`. + * Clones `remoteDir` as `origin`. * @param {string} remoteDir * @param {string[]} [cloneArgs] */ async function clone(remoteDir, cloneArgs = []) { const dir = await tempDir('baseline-clone-'); await execa('git', ['clone', ...cloneArgs, remoteDir, dir], { env: GIT_ENV }); - const git = gitIn(dir); - await git('remote', 'remove', 'origin'); - await addMuiRemote(git, 'origin', remoteDir); - // `remote remove` drops the tracking config a clone set up; later tests check out `origin/…`. - await git('fetch', 'origin'); - return { dir, git }; + return { dir, git: gitIn(dir) }; +} + +/** + * A clone whose `origin` is a fork stuck at the first commit, with `remote` reachable under + * `remoteName`, and a `feature` branch forked from the remote's second commit. + * @param {string} remoteName + */ +async function makeForkClone(remoteName) { + const remote = await makeRemote(); + const fork = await clone(remote.dir); + await fork.git('reset', '--hard', remote.shas[0]); + const work = await clone(fork.dir); + await work.git('remote', 'add', remoteName, remote.dir); + await work.git('fetch', remoteName); + await work.git('checkout', '-b', 'feature', remote.shas[1]); + await work.git('commit', '--allow-empty', '-m', 'work'); + return { remote, work }; } describe('resolveBaseline', () => { @@ -89,44 +88,6 @@ describe('resolveBaseline', () => { expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[0]); }); - it("forks from upstream's base branch over a fork's stale one", async () => { - const remote = await makeRemote(); - const fork = await clone(remote.dir); - await fork.git('reset', '--hard', remote.shas[0]); - const work = await clone(fork.dir); - await addMuiRemote(work.git, 'upstream', remote.dir); - await work.git('fetch', 'upstream'); - await work.git('checkout', '-b', 'feature', remote.shas[1]); - await work.git('commit', '--allow-empty', '-m', 'work'); - - expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[1]); - }); - - it('finds the mui remote under another name', async () => { - const remote = await makeRemote(); - const fork = await makeRemote(['unrelated']); - const work = await clone(remote.dir); - await work.git('remote', 'set-url', 'origin', 'https://github.com/someone/fork.git'); - await work.git('config', `url.${fork.dir}.insteadOf`, 'https://github.com/someone/fork.git'); - await addMuiRemote(work.git, 'mui', remote.dir); - await work.git('checkout', '-b', 'feature', remote.shas[1]); - await work.git('commit', '--allow-empty', '-m', 'work'); - - expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[1]); - }); - - it('fails without a mui remote', async () => { - const remote = await makeRemote(); - const work = await clone(remote.dir); - // Only a fork: a GitHub repository, but not one under `mui`. - await work.git('remote', 'set-url', 'origin', 'https://github.com/someone/fork.git'); - await work.git('config', `url.${remote.dir}.insteadOf`, 'https://github.com/someone/fork.git'); - - await expect(resolveBaseline({ cwd: work.dir })).rejects.toThrow( - /Failed to find correct remote/, - ); - }); - it('fetches the base branch a CI checkout does not have', async () => { const remote = await makeRemote(); await remote.git('checkout', '-b', 'feature', remote.shas[0]); @@ -166,3 +127,43 @@ describe('resolveBaseline', () => { await expect(resolveBaseline({ cwd: work.dir })).rejects.toThrow(/HEAD has no parent commit/); }); }); + +describe('resolveBaseline: the canonical remote', () => { + it('prefers upstream over origin', async () => { + const { remote, work } = await makeForkClone('upstream'); + + expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[1]); + }); + + it('uses the remote code-infra.canonicalRemote names', async () => { + const { remote, work } = await makeForkClone('mui'); + await work.git('config', 'code-infra.canonicalRemote', 'mui'); + + expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[1]); + }); + + it('goes by name, not by what a remote points at', async () => { + // Without the setting, `mui` is just another remote and the fork's `origin` is canonical. + const { remote, work } = await makeForkClone('mui'); + + expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[0]); + }); + + it('fails when code-infra.canonicalRemote names a missing remote', async () => { + const remote = await makeRemote(); + const work = await clone(remote.dir); + await work.git('config', 'code-infra.canonicalRemote', 'mui'); + + await expect(resolveBaseline({ cwd: work.dir })).rejects.toThrow( + /"mui", which is not a remote/, + ); + }); + + it('fails without upstream or origin', async () => { + const remote = await makeRemote(); + const work = await clone(remote.dir); + await work.git('remote', 'rename', 'origin', 'mui'); + + await expect(resolveBaseline({ cwd: work.dir })).rejects.toThrow(/No canonical remote/); + }); +}); From 4e4779a308c02632ee577afb3c946c2f1a96dc03 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 30 Sep 2026 14:56:18 +0200 Subject: [PATCH 104/173] [code-infra] Override the baseline's remote with --remote Replaces the code-infra.canonicalRemote git config key: overrides here are flags, and this one is rarely needed. getCanonicalRemote is upstream, else origin. --- packages/code-infra/src/cli/cmdBaseline.mjs | 16 +++++++---- packages/code-infra/src/utils/git.mjs | 30 +++++---------------- packages/code-infra/src/utils/git.test.mjs | 14 ++++------ 3 files changed, 23 insertions(+), 37 deletions(-) diff --git a/packages/code-infra/src/cli/cmdBaseline.mjs b/packages/code-infra/src/cli/cmdBaseline.mjs index 5a34807cbe..c01fef0fc2 100644 --- a/packages/code-infra/src/cli/cmdBaseline.mjs +++ b/packages/code-infra/src/cli/cmdBaseline.mjs @@ -7,18 +7,24 @@ import { resolveBaseline } from '../utils/git.mjs'; /** * @typedef {Object} Args * @property {string} [baseBranch] - Branch pull requests fork from + * @property {string} [remote] - Remote the base branch is fetched from */ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ command: 'baseline', describe: 'Print the commit HEAD should be compared against', builder: (yargs) => { - return yargs.option('base-branch', { - type: 'string', - description: "Branch pull requests fork from. Default: the canonical remote's default branch", - }); + return yargs + .option('base-branch', { + type: 'string', + description: "Branch pull requests fork from. Default: the remote's default branch", + }) + .option('remote', { + type: 'string', + description: 'Remote the base branch is fetched from. Default: upstream, else origin', + }); }, handler: async (argv) => { - console.log(await resolveBaseline({ baseBranch: argv.baseBranch })); + console.log(await resolveBaseline({ baseBranch: argv.baseBranch, remote: argv.remote })); }, }); diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index a606f5ec63..c11b542d0a 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -9,33 +9,16 @@ import gitUrlParse from 'git-url-parse'; */ /** - * The remote of the canonical repository, as opposed to a fork: the one named by - * `git config code-infra.canonicalRemote`, else `upstream`, else `origin`. + * The remote of the canonical repository, as opposed to a fork: `upstream`, else `origin`. * @param {string} [cwd=process.cwd()] * @returns {Promise} */ async function getCanonicalRemote(cwd = process.cwd()) { - const [configured, listed] = await Promise.all([ - // Exits 1 when the key is not set. - $({ cwd, reject: false })`git config --get code-infra.canonicalRemote`, - $({ cwd })`git remote`, - ]); - const remotes = listed.stdout.split('\n').filter(Boolean); - const override = configured.stdout.trim(); - if (override) { - if (!remotes.includes(override)) { - throw new Error( - `code-infra.canonicalRemote names "${override}", which is not a remote of this repository.`, - ); - } - return override; - } + const { stdout } = await $({ cwd })`git remote`; + const remotes = stdout.split('\n'); const remote = ['upstream', 'origin'].find((name) => remotes.includes(name)); if (!remote) { - throw new Error( - 'No canonical remote: add an `upstream` or `origin` remote, or name one with ' + - '`git config code-infra.canonicalRemote `.', - ); + throw new Error('No canonical remote: this repository has neither `upstream` nor `origin`.'); } return remote; } @@ -97,12 +80,13 @@ async function defaultBranchOf(remote, cwd) { * when HEAD is already on the base branch. * @param {Object} [options] * @param {string} [options.cwd] - * @param {string} [options.baseBranch] - Defaults to the canonical remote's default branch. + * @param {string} [options.remote] - Defaults to the canonical remote. + * @param {string} [options.baseBranch] - Defaults to the remote's default branch. * @returns {Promise} */ export async function resolveBaseline(options = {}) { const cwd = options.cwd ?? process.cwd(); - const remote = await getCanonicalRemote(cwd); + const remote = options.remote ?? (await getCanonicalRemote(cwd)); const baseBranch = options.baseBranch ?? (await defaultBranchOf(remote, cwd)); await $({ cwd })`git fetch --no-tags ${remote} ${baseBranch}`; diff --git a/packages/code-infra/src/utils/git.test.mjs b/packages/code-infra/src/utils/git.test.mjs index 95f21fbe2f..e9380e8b48 100644 --- a/packages/code-infra/src/utils/git.test.mjs +++ b/packages/code-infra/src/utils/git.test.mjs @@ -135,28 +135,24 @@ describe('resolveBaseline: the canonical remote', () => { expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[1]); }); - it('uses the remote code-infra.canonicalRemote names', async () => { + it('uses the remote it is given', async () => { const { remote, work } = await makeForkClone('mui'); - await work.git('config', 'code-infra.canonicalRemote', 'mui'); - expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[1]); + expect(await resolveBaseline({ cwd: work.dir, remote: 'mui' })).toBe(remote.shas[1]); }); it('goes by name, not by what a remote points at', async () => { - // Without the setting, `mui` is just another remote and the fork's `origin` is canonical. + // Unless it is named, `mui` is just another remote and the fork's `origin` is canonical. const { remote, work } = await makeForkClone('mui'); expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[0]); }); - it('fails when code-infra.canonicalRemote names a missing remote', async () => { + it('fails for a remote the repository does not have', async () => { const remote = await makeRemote(); const work = await clone(remote.dir); - await work.git('config', 'code-infra.canonicalRemote', 'mui'); - await expect(resolveBaseline({ cwd: work.dir })).rejects.toThrow( - /"mui", which is not a remote/, - ); + await expect(resolveBaseline({ cwd: work.dir, remote: 'mui' })).rejects.toThrow(/'mui'/); }); it('fails without upstream or origin', async () => { From 9878c9d3495c1dbc166c4aaa99efb5155aa858c8 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 30 Sep 2026 16:46:01 +0200 Subject: [PATCH 105/173] [benchmark] Throw case failures from measureIteration; keep runCase private A render error and a recording window that captured no renders are thrown where the iteration runs, so both drivers fail on them without checking flags of their own. benchmark() sets its task meta in a finally, so the reporter still shows what was measured before a failure. runCase is no longer exported: benchmark() is its only caller, and the interleaved runner uses measureIteration. --- packages/benchmark/README.md | 23 ---- packages/benchmark/src/caseRuntime.tsx | 29 +++-- packages/benchmark/src/index.tsx | 107 +++++------------- .../benchmark/src/page/reactBenchmark.tsx | 7 -- 4 files changed, 42 insertions(+), 124 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 83633c9f69..379c53cc3d 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -265,28 +265,6 @@ benchmark('name', renderFn, interaction, { }); ``` -### Custom drivers - -`benchmark()` loops over `runCase()`, which mounts, interacts with and unmounts a case exactly once. Call it directly to control the iterations yourself — e.g. to alternate two variants so machine drift affects both equally, warming each one up right before it is measured: - -```tsx -import { runCase } from '@mui/internal-benchmark'; - -it('A/B', async () => { - for (let i = 0; i < 20; i += 1) { - await runCase(renderA, interaction, { warmup: true }); - const a = await runCase(renderA, interaction); - - await runCase(renderB, interaction, { warmup: true }); - const b = await runCase(renderB, interaction); - - // compare a.renders with b.renders, check a.renderError, ... - } -}); -``` - -`runCase()` takes the same `renderFn`, interaction and `afterEach`/`reactRecordingPaused` options as `benchmark()`, plus `warmup`, which skips `bench:paint` and drops custom metrics for that iteration. It registers and asserts nothing. Measured iterations record metrics against the running Vitest test, so call it from inside one. - ### Running ```bash @@ -525,7 +503,6 @@ none — so nothing else should build against the same checkout meanwhile. - `benchmark` — define a benchmark test case - `compare` — compare implementations, each a lazily loaded module of `benchmark()` cases -- `runCase` — run a single iteration of a case, for custom drivers - `@mui/internal-benchmark/vitePlugin` — `benchmarkPlugin()`, the harness's vite plugin for `benchmark run` - `@mui/internal-benchmark/runReport` — the `benchmark run` report schema and `analyzeRun` - `ElementTiming` — invisible marker component for paint timing (renders a `` tracked by the Element Timing API) diff --git a/packages/benchmark/src/caseRuntime.tsx b/packages/benchmark/src/caseRuntime.tsx index bb3636542c..dbe0e0e6f2 100644 --- a/packages/benchmark/src/caseRuntime.tsx +++ b/packages/benchmark/src/caseRuntime.tsx @@ -38,7 +38,7 @@ export const MILLISECONDS: Intl.NumberFormatOptions = { }; /** Why a case failed when React recording was active yet captured no renders. */ -export const EMPTY_RECORDING_MESSAGE = +const EMPTY_RECORDING_MESSAGE = 'React recording was active but captured no renders. If you only measure imperative DOM ' + 'updates or custom metrics, keep recording paused (reactRecordingPaused) instead of resuming.'; @@ -280,15 +280,13 @@ export interface MeasuredIteration { renders: RenderEvent[]; /** Element paints that happened while React recording was active. */ paints: PaintTiming[]; - /** The error a render threw, if one did. The iteration stops at mount when it does. */ - renderError: unknown; - /** Whether a recording window was active yet captured no renders. */ - hadEmptyActiveWindow: boolean; } /** * Mounts, interacts with and unmounts a case once. Records nothing itself: renders and paints are * returned, and the driver decides whether they count (they don't during warmup) and where they go. + * + * Throws what a render throws, and throws when React recording was active but captured no renders. */ export async function measureIteration( renderFn: () => React.ReactElement, @@ -306,7 +304,8 @@ export async function measureIteration( const captures: RenderEvent[] = []; const timing = createElementTimingWaiter(); - let renderError: unknown = null; + // Asserted rather than annotated: only the callback assigns it, which narrowing cannot see. + let renderFailure = null as { error: unknown } | null; const runtime = createCaseRuntime({ // Wrap the case in BenchProfiler so its renders are captured; the runtime mounts whatever @@ -324,7 +323,7 @@ export async function measureIteration( input, }, onUncaughtError: (error) => { - renderError = error; + renderFailure = { error }; }, }); @@ -332,10 +331,10 @@ export async function measureIteration( runtime.mount(); - if (renderError) { + if (renderFailure) { timing.disconnect(); runtime.unmount(); - return { renders: captures, paints: [], renderError, hadEmptyActiveWindow: false }; + throw renderFailure.error; } await runtime.interact?.(); @@ -366,10 +365,10 @@ export async function measureIteration( await options.afterEach(); } - return { - renders: captures, - paints, - renderError, - hadEmptyActiveWindow: recording.hadEmptyActiveWindow, - }; + // Windows where recording never ran (e.g. a fully paused, metric-only case) are not checked. + if (recording.hadEmptyActiveWindow) { + throw new Error(EMPTY_RECORDING_MESSAGE); + } + + return { renders: captures, paints }; } diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index e3efa643a7..b3598e7e94 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -1,5 +1,5 @@ import type * as React from 'react'; -import { describe, expect, it, TestRunner } from 'vitest'; +import { describe, expect, it } from 'vitest'; import type { RunnerTestCase } from 'vitest'; import { cdp } from 'vitest/browser'; import type { RenderEvent, IterationData } from './types'; @@ -10,7 +10,6 @@ import { createInput } from './input'; import { createCaseRuntime, createElementTimingWaiter, - EMPTY_RECORDING_MESSAGE, measureIteration, MILLISECONDS, PAINT_METRIC_NAME, @@ -42,23 +41,6 @@ type CdpMethod = Parameters['send']>[0]; // leaves unknown methods for Chrome to reject. const input = createInput((method, params) => cdp().send(method as CdpMethod, params)); -export interface RunCaseOptions extends CaseOptions { - /** - * Run the iteration without recording anything: no `bench:paint`, and custom metrics recorded - * inside the case are dropped. Defaults to `false`. - */ - warmup?: boolean; -} - -export interface RunCaseResult { - /** Renders captured while React recording was active. */ - renders: RenderEvent[]; - /** The error a render threw, if one did. The iteration stops at mount when it does. */ - renderError: unknown; - /** Whether a recording window was active yet captured no renders. */ - hadEmptyActiveWindow: boolean; -} - // Paint timings are recorded as one harness-owned `bench:paint` metric: the default sentinel // is the base series (`bench:paint`) and named `elementtiming` markers are sub-series // (`bench:paint#grid-header`, …), all sharing a single definition. Paint is informational (no @@ -70,47 +52,30 @@ const paint = new ScalarMetric({ }); /** - * Mounts, interacts with and unmounts a case exactly once, recording its renders and paint. - * - * This is the primitive `benchmark()` loops over. How many iterations to run, in which order, and - * how to aggregate them is up to the caller, so a driver can run cases differently — e.g. an A/B - * runner alternating two builds iteration by iteration instead of in blocks. It registers and - * asserts nothing; the caller decides what a render error or empty recording window means. - * - * Metrics resolve the running Vitest test when they record, so measured (non-warmup) iterations - * must run inside one. + * Runs one iteration of a case inside the running Vitest test, returning its renders. A warmup + * iteration records nothing: no `bench:paint`, and custom metrics recorded inside the case are + * dropped. */ -export async function runCase( +async function runIteration( + test: RunnerTestCase, renderFn: () => React.ReactElement, - interactionOrOptions?: BenchmarkInteraction | RunCaseOptions, - maybeOptions?: RunCaseOptions, -): Promise { - const { interaction, options } = splitCaseArgs(interactionOrOptions, maybeOptions); - + interaction: BenchmarkInteraction | undefined, + options: CaseOptions | undefined, + warmup: boolean, +): Promise { // Custom metrics recorded inside the case honor warmup exclusion through the gate, the same way - // renders and `bench:paint` are excluded during warmup. The gate is keyed on the running test, - // and is re-enabled afterwards so metrics the driver records between iterations are kept. - const test = TestRunner.getCurrentTest(); - if (test) { - metricsGate.setRecordingEnabled(test, !options?.warmup); - } + // renders and `bench:paint` are excluded during warmup. + metricsGate.setRecordingEnabled(test, !warmup); try { - const { renders, paints, renderError, hadEmptyActiveWindow } = await measureIteration( - renderFn, - interaction, - options, - input, - ); - if (!options?.warmup) { + const { renders, paints } = await measureIteration(renderFn, interaction, options, input); + if (!warmup) { for (const { id, start, end } of paints) { paint.record(end - start, id !== undefined ? { id } : undefined); } } - return { renders, renderError, hadEmptyActiveWindow }; + return renders; } finally { - if (test) { - metricsGate.setRecordingEnabled(test, true); - } + metricsGate.setRecordingEnabled(test, true); } } @@ -180,37 +145,21 @@ export function benchmark( warnIfNoGc(); const iterations: IterationData[] = []; - let renderError: unknown = null; - // Set if any iteration had a recording window that was active yet captured no renders. - let sawEmptyActiveWindow = false; - - for (let i = 0; i < warmupRuns + runs; i += 1) { - const warmup = i < warmupRuns; - // eslint-disable-next-line no-await-in-loop - const result = await runCase(renderFn, interaction, { ...options, warmup }); - if (result.renderError) { - renderError = result.renderError; - break; - } - if (result.hadEmptyActiveWindow) { - sawEmptyActiveWindow = true; - } - if (!warmup) { - iterations.push({ renders: result.renders }); + try { + for (let i = 0; i < warmupRuns + runs; i += 1) { + const warmup = i < warmupRuns; + // eslint-disable-next-line no-await-in-loop + const renders = await runIteration(task, renderFn, interaction, options, warmup); + if (!warmup) { + iterations.push({ renders }); + } } + } finally { + // Set even when an iteration throws, so the reporter can show what was measured. + task.meta.benchmarkIterations = iterations; + task.meta.benchmarkName = name; } - task.meta.benchmarkIterations = iterations; - task.meta.benchmarkName = name; - - if (renderError) { - throw renderError; - } - - // Every active recording window must capture at least one render. Windows where recording was - // never running (e.g. a fully-paused, metric-only benchmark) are not checked. - expect(sawEmptyActiveWindow, EMPTY_RECORDING_MESSAGE).toBe(false); - // Validate all iterations produced the same render events (count + order). // This runs after meta is set so the reporter can still display results on failure. if (iterations.length > 1) { diff --git a/packages/benchmark/src/page/reactBenchmark.tsx b/packages/benchmark/src/page/reactBenchmark.tsx index 7c448d9649..3bf8914632 100644 --- a/packages/benchmark/src/page/reactBenchmark.tsx +++ b/packages/benchmark/src/page/reactBenchmark.tsx @@ -1,6 +1,5 @@ import type * as React from 'react'; import { - EMPTY_RECORDING_MESSAGE, measureIteration, MILLISECONDS, PAINT_METRIC_NAME, @@ -54,12 +53,6 @@ export function reactBenchmark( const run = async ({ input }: BenchmarkContext) => { const result = await measureIteration(renderFn, interaction, options, input); - if (result.renderError) { - throw result.renderError; - } - if (result.hadEmptyActiveWindow) { - throw new Error(EMPTY_RECORDING_MESSAGE); - } // Render time is reported as totals rather than per render: an interaction's render count // follows whatever the browser coalesced, so per-render values would not line up across From bda6508a6d419567cbb852287ad1ba7a3546315a Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:44:05 +0200 Subject: [PATCH 106/173] [code-infra] Take upstream, then origin, only if it points at mui The canonical remote is still chosen by name, but a remote that does not point at a mui repository is skipped, so a fork's clone without an upstream fails instead of using the fork. getRepositoryInfo is that lookup; baseline's --remote still names any remote explicitly. --- packages/code-infra/src/cli/cmdBaseline.mjs | 3 +- packages/code-infra/src/utils/git.mjs | 61 ++++++++++----- packages/code-infra/src/utils/git.test.mjs | 83 +++++++++++++++------ 3 files changed, 102 insertions(+), 45 deletions(-) diff --git a/packages/code-infra/src/cli/cmdBaseline.mjs b/packages/code-infra/src/cli/cmdBaseline.mjs index c01fef0fc2..74e544239d 100644 --- a/packages/code-infra/src/cli/cmdBaseline.mjs +++ b/packages/code-infra/src/cli/cmdBaseline.mjs @@ -21,7 +21,8 @@ export default /** @type {import('yargs').CommandModule<{}, Args>} */ ({ }) .option('remote', { type: 'string', - description: 'Remote the base branch is fetched from. Default: upstream, else origin', + description: + 'Remote the base branch is fetched from. Default: upstream, else origin, whichever points at a mui repository', }); }, handler: async (argv) => { diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index c11b542d0a..382ed054d4 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -9,34 +9,54 @@ import gitUrlParse from 'git-url-parse'; */ /** - * The remote of the canonical repository, as opposed to a fork: `upstream`, else `origin`. - * @param {string} [cwd=process.cwd()] - * @returns {Promise} + * The repository a remote URL points at, if it is a GitHub repository under `mui`. + * @param {string} url + * @returns {{ owner: string, repo: string } | null} */ -async function getCanonicalRemote(cwd = process.cwd()) { - const { stdout } = await $({ cwd })`git remote`; - const remotes = stdout.split('\n'); - const remote = ['upstream', 'origin'].find((name) => remotes.includes(name)); - if (!remote) { - throw new Error('No canonical remote: this repository has neither `upstream` nor `origin`.'); +function muiRepositoryOf(url) { + try { + const parsed = gitUrlParse(url); + return parsed.source === 'github.com' && parsed.owner === 'mui' + ? { owner: parsed.owner, repo: parsed.name } + : null; + } catch { + // Not a URL git-url-parse understands, such as a local path: not a mui repository either. + return null; } - return remote; } /** - * The GitHub repository of the canonical remote (see {@link getCanonicalRemote}). + * The canonical repository's remote, as opposed to a fork's: `upstream`, else `origin`, each only if + * it points at a mui repository. Reads the configured URLs, which `insteadOf` does not rewrite. * @param {string} [cwd=process.cwd()] * @returns {Promise} */ export async function getRepositoryInfo(cwd = process.cwd()) { - const remoteName = await getCanonicalRemote(cwd); - const { stdout } = await $({ cwd })`git remote get-url ${remoteName}`; - const url = stdout.trim(); - const parsed = gitUrlParse(url); - if (parsed.source !== 'github.com') { - throw new Error(`Remote "${remoteName}" is not a GitHub repository: ${url}`); + // Exits 1 when neither remote is configured; the error below reports that. + const { stdout } = await $({ + cwd, + reject: false, + })`git config --get-regexp ${'^remote\\.(upstream|origin)\\.url$'}`; + /** @type {Map} */ + const urls = new Map(); + for (const line of stdout.split('\n').filter(Boolean)) { + const [key, url] = line.split(' ', 2); + urls.set(key.slice('remote.'.length, -'.url'.length), url); } - return { owner: parsed.owner, repo: parsed.name, remoteName }; + + for (const remoteName of ['upstream', 'origin']) { + const url = urls.get(remoteName); + const repository = url ? muiRepositoryOf(url) : null; + if (repository) { + return { ...repository, remoteName }; + } + } + + const found = [...urls].map(([name, url]) => `${name}: ${url}`).join(', ') || 'none'; + throw new Error( + `No canonical remote: neither \`upstream\` nor \`origin\` points at a mui repository (${found}). ` + + 'Add an `upstream` remote for it, or name the remote explicitly where the command allows it.', + ); } /** @@ -86,7 +106,7 @@ async function defaultBranchOf(remote, cwd) { */ export async function resolveBaseline(options = {}) { const cwd = options.cwd ?? process.cwd(); - const remote = options.remote ?? (await getCanonicalRemote(cwd)); + const remote = options.remote ?? (await getRepositoryInfo(cwd)).remoteName; const baseBranch = options.baseBranch ?? (await defaultBranchOf(remote, cwd)); await $({ cwd })`git fetch --no-tags ${remote} ${baseBranch}`; @@ -119,7 +139,8 @@ export async function findLatestTaggedVersion(opts) { const fetchAll = opts.fetchAll ?? true; if (fetchAll) { // --force to update any existing tags that may have changed to avoid the clobering error. - await $$`git fetch --tags --force ${await getCanonicalRemote(opts.cwd)}`; + const { remoteName } = await getRepositoryInfo(opts.cwd); + await $$`git fetch --tags --force ${remoteName}`; } const { stdout } = await $$`git describe --tags --abbrev=0 --match ${'v*'}`; // only include "version-tags" return stdout.trim(); diff --git a/packages/code-infra/src/utils/git.test.mjs b/packages/code-infra/src/utils/git.test.mjs index e9380e8b48..ba4378e914 100644 --- a/packages/code-infra/src/utils/git.test.mjs +++ b/packages/code-infra/src/utils/git.test.mjs @@ -44,27 +44,45 @@ async function makeRemote(commits = ['first', 'second', 'third']) { } /** - * Clones `remoteDir` as `origin`. + * Adds remote `name`, named by a GitHub URL under `owner` that `insteadOf` sends to the local + * `dir`: the code under test sees the GitHub URL, while git fetches locally. + * @param {(...args: string[]) => Promise} git + * @param {string} name + * @param {string} owner + * @param {string} dir + * @param {'add' | 'set-url'} [verb] + */ +async function setRemote(git, name, owner, dir, verb = 'add') { + const url = `https://github.com/${owner}/${path.basename(dir)}.git`; + await git('remote', verb, name, url); + await git('config', `url.${dir}.insteadOf`, url); +} + +/** + * Clones `remoteDir`, its `origin` named as a repository of `owner`. * @param {string} remoteDir - * @param {string[]} [cloneArgs] + * @param {{ owner?: string, cloneArgs?: string[] }} [options] */ -async function clone(remoteDir, cloneArgs = []) { +async function clone(remoteDir, { owner = 'mui', cloneArgs = [] } = {}) { const dir = await tempDir('baseline-clone-'); await execa('git', ['clone', ...cloneArgs, remoteDir, dir], { env: GIT_ENV }); - return { dir, git: gitIn(dir) }; + const git = gitIn(dir); + await setRemote(git, 'origin', owner, remoteDir, 'set-url'); + return { dir, git }; } /** - * A clone whose `origin` is a fork stuck at the first commit, with `remote` reachable under - * `remoteName`, and a `feature` branch forked from the remote's second commit. + * A clone whose `origin` is a copy stuck at the first commit, by default someone's fork, with the + * mui repository added as `remoteName`, and a `feature` branch forked from its second commit. * @param {string} remoteName + * @param {string} [originOwner] */ -async function makeForkClone(remoteName) { +async function makeForkClone(remoteName, originOwner = 'someone') { const remote = await makeRemote(); const fork = await clone(remote.dir); await fork.git('reset', '--hard', remote.shas[0]); - const work = await clone(fork.dir); - await work.git('remote', 'add', remoteName, remote.dir); + const work = await clone(fork.dir, { owner: originOwner }); + await setRemote(work.git, remoteName, 'mui', remote.dir); await work.git('fetch', remoteName); await work.git('checkout', '-b', 'feature', remote.shas[1]); await work.git('commit', '--allow-empty', '-m', 'work'); @@ -93,7 +111,9 @@ describe('resolveBaseline', () => { await remote.git('checkout', '-b', 'feature', remote.shas[0]); await remote.git('commit', '--allow-empty', '-m', 'work'); await remote.git('checkout', 'main'); - const work = await clone(remote.dir, ['--single-branch', '--branch', 'feature']); + const work = await clone(remote.dir, { + cloneArgs: ['--single-branch', '--branch', 'feature'], + }); expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[0]); }); @@ -129,37 +149,52 @@ describe('resolveBaseline', () => { }); describe('resolveBaseline: the canonical remote', () => { - it('prefers upstream over origin', async () => { + it('prefers upstream over a fork as origin', async () => { const { remote, work } = await makeForkClone('upstream'); expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[1]); }); - it('uses the remote it is given', async () => { - const { remote, work } = await makeForkClone('mui'); + it('prefers upstream when origin is a mui repository too', async () => { + const { remote, work } = await makeForkClone('upstream', 'mui'); - expect(await resolveBaseline({ cwd: work.dir, remote: 'mui' })).toBe(remote.shas[1]); + expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[1]); }); - it('goes by name, not by what a remote points at', async () => { - // Unless it is named, `mui` is just another remote and the fork's `origin` is canonical. - const { remote, work } = await makeForkClone('mui'); + it('skips an upstream that is not a mui repository', async () => { + const remote = await makeRemote(); + const unrelated = await makeRemote(['other']); + const work = await clone(remote.dir); + await setRemote(work.git, 'upstream', 'someone', unrelated.dir); - expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[0]); + expect(await resolveBaseline({ cwd: work.dir })).toBe(remote.shas[1]); }); - it('fails for a remote the repository does not have', async () => { + it('fails when origin is a fork and there is no upstream', async () => { const remote = await makeRemote(); - const work = await clone(remote.dir); + const work = await clone(remote.dir, { owner: 'someone' }); - await expect(resolveBaseline({ cwd: work.dir, remote: 'mui' })).rejects.toThrow(/'mui'/); + await expect(resolveBaseline({ cwd: work.dir })).rejects.toThrow( + /No canonical remote.*github\.com\/someone\//, + ); + }); + + it('does not fall back to a mui remote under another name', async () => { + const { work } = await makeForkClone('mui'); + + await expect(resolveBaseline({ cwd: work.dir })).rejects.toThrow(/No canonical remote/); + }); + + it('uses the remote it is given, whatever it points at', async () => { + const { remote, work } = await makeForkClone('mui'); + + expect(await resolveBaseline({ cwd: work.dir, remote: 'mui' })).toBe(remote.shas[1]); }); - it('fails without upstream or origin', async () => { + it('fails for a remote the repository does not have', async () => { const remote = await makeRemote(); const work = await clone(remote.dir); - await work.git('remote', 'rename', 'origin', 'mui'); - await expect(resolveBaseline({ cwd: work.dir })).rejects.toThrow(/No canonical remote/); + await expect(resolveBaseline({ cwd: work.dir, remote: 'mui' })).rejects.toThrow(/'mui'/); }); }); From 53f837104fc32ae298c55311e49ba052c1300763 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:45:23 +0200 Subject: [PATCH 107/173] [code-infra] Rename defaultBranchOf to getDefaultBranchOf --- packages/code-infra/src/utils/git.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/code-infra/src/utils/git.mjs b/packages/code-infra/src/utils/git.mjs index 382ed054d4..9a86d1ad67 100644 --- a/packages/code-infra/src/utils/git.mjs +++ b/packages/code-infra/src/utils/git.mjs @@ -86,7 +86,7 @@ export async function remoteGitTagExists(tagName, cwd = process.cwd()) { * @param {string} cwd * @returns {Promise} */ -async function defaultBranchOf(remote, cwd) { +async function getDefaultBranchOf(remote, cwd) { const { stdout } = await $({ cwd })`git ls-remote --symref ${remote} HEAD`; const match = /^ref: refs\/heads\/(\S+)\tHEAD$/m.exec(stdout); if (!match) { @@ -107,7 +107,7 @@ async function defaultBranchOf(remote, cwd) { export async function resolveBaseline(options = {}) { const cwd = options.cwd ?? process.cwd(); const remote = options.remote ?? (await getRepositoryInfo(cwd)).remoteName; - const baseBranch = options.baseBranch ?? (await defaultBranchOf(remote, cwd)); + const baseBranch = options.baseBranch ?? (await getDefaultBranchOf(remote, cwd)); await $({ cwd })`git fetch --no-tags ${remote} ${baseBranch}`; const [forkPoint, head] = await Promise.all([ From 32726efbeeb0e18c9bafe062ca30be405d08378a Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:19:48 +0200 Subject: [PATCH 108/173] [benchmark] Simplify the interleaved runner and its reports - Drop the Vitest compare(), which nothing used and the README described wrongly; compare() lives on the page runtime. - One benchmarkTable() for the terminal, the PR comment and the dashboard, which had drifted (the comment lacked Rounds); one cached number formatter. - The stopping check compares variants without summarising them, and the unused mean interval and rounds fields go. - A ref is its variant; the reserved github:/preview: baseline schemes go. - One install helper, one .gitignore marker, one viewport constant, baseMetricName next to seriesName; the report's branch comes from getCiMetadata, as the upload's does. - The page registry keeps sampling as given, and finds a case's comparison from the comparisons themselves. - Shorter comments and READMEs. --- .../src/components/BenchmarkRunHistory.tsx | 11 ++- .../benchmarkRun/buildMarkdownReport.test.ts | 36 ++++----- .../lib/benchmarkRun/buildMarkdownReport.ts | 50 ++++-------- .../src/views/BenchmarkRunDetails.tsx | 56 ++++++------- packages/benchmark/README.md | 59 +++----------- packages/benchmark/src/index.tsx | 33 +------- packages/benchmark/src/launchArgs.ts | 12 ++- packages/benchmark/src/metricCore.ts | 11 ++- packages/benchmark/src/page/page.tsx | 31 +++----- packages/benchmark/src/reporter.ts | 7 +- .../src/runReport/analyzeRun.test.ts | 1 - .../benchmark/src/runReport/analyzeRun.ts | 79 +++++++++++-------- packages/benchmark/src/runReport/format.ts | 55 ++++++++++++- packages/benchmark/src/runner/benchFiles.ts | 3 +- packages/benchmark/src/runner/buildPages.ts | 61 +++++--------- packages/benchmark/src/runner/outputDir.ts | 34 +++----- packages/benchmark/src/runner/plugin.ts | 36 +++------ packages/benchmark/src/runner/printReport.ts | 36 +++------ packages/benchmark/src/runner/refs.test.ts | 21 ----- packages/benchmark/src/runner/refs.ts | 61 +++----------- .../benchmark/src/runner/runBenchmarks.ts | 31 +++----- .../benchmark/src/runner/runInterleaved.ts | 14 ++-- packages/benchmark/src/sampling.ts | 7 +- packages/benchmark/src/vitest.ts | 7 +- test/benchmark/README.md | 10 +-- test/benchmark/scripts/runBenchmark.mjs | 13 +-- 26 files changed, 301 insertions(+), 474 deletions(-) diff --git a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx index 6f3c0e58e6..8ea3fa14bf 100644 --- a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx +++ b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx @@ -104,13 +104,18 @@ export default function BenchmarkRunHistory({ repo }: BenchmarkRunHistoryProps) [points, benchmark], ); const [userMetric, setUserMetric] = React.useState(null); - const metric = userMetric !== null && metricNames.includes(userMetric) ? userMetric : 'render'; + const metric = + [userMetric, 'render', metricNames[0]].find( + (name): name is string => name != null && metricNames.includes(name), + ) ?? null; // One series per kind of change, so each bar takes its change's colour. A commit's bar is the // midpoint of its confidence interval; the tooltip gives the interval itself. const series = React.useMemo(() => { const selected = points.map((point) => - benchmark === null ? undefined : point.comparisons.get(benchmark)?.get(metric), + benchmark === null || metric === null + ? undefined + : point.comparisons.get(benchmark)?.get(metric), ); return (['worse', 'better', 'unsure'] as const satisfies Change[]).map((change) => ({ type: 'bar' as const, @@ -162,7 +167,7 @@ export default function BenchmarkRunHistory({ repo }: BenchmarkRunHistoryProps) /> setUserMetric(value)} size="small" sx={{ minWidth: 200 }} diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index e8ec8eb50a..b04db91957 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -77,36 +77,36 @@ describe('buildBenchmarkRunMarkdownReport', () => { title: 'Performance', }), ).toMatchInlineSnapshot(` - "## Performance ⚠️ + "## Performance ⚠️ - 🔴 **Grid scroll** · render · \`worse +9.3% – +9.3%\` + 🔴 **Grid scroll** · render · \`worse +9.3% – +9.3%\` - 2 benchmarks measured · 1 with regressions · 1 failed + 2 benchmarks measured · 1 with regressions · 1 failed - ❌ **Chart zoom**: Render failed + ❌ **Chart zoom**: Render failed -

    - Full results +
    + Full results - **Grid scroll** + **Grid scroll** - | Metric | current | baseline | Δ vs baseline | - |:----------|----------:|----------:|:----------| - | render | 23.5 ms | 21.5 ms | \`worse +9.3% – +9.3%\` | - | bench:paint | 30.5 ms | 30.5 ms | \`unsure +0.0% – +0.0%\` | + | Metric | current | baseline | Δ vs baseline | Rounds | + |:----------|----------:|----------:|:----------|----------:| + | render | 23.5 ms | 21.5 ms | \`worse +9.3% – +9.3%\` | 4 | + | bench:paint | 30.5 ms | 30.5 ms | \`unsure +0.0% – +0.0%\` | 4 | - **libs / mount** + **libs / mount** - | Metric | ours | theirs | theirs vs ours | - |:----------|----------:|----------:|:----------| - | render | 10 ms | 20.5 ms | \`worse +95.8% – +114.2%\` | + | Metric | ours | theirs | theirs vs ours | Rounds | + |:----------|----------:|----------:|:----------|----------:| + | render | 10 ms | 20.5 ms | \`worse +95.8% – +114.2%\` | 4 | - _Each value is a median. Each Δ is a 95% confidence interval on the paired per-round difference, relative to the variant it is measured against; "unsure" means it straddles zero — the expected result for two equivalent builds._ + _Each value is a median. Each Δ is a 95% confidence interval on the paired per-round difference, relative to the variant it is measured against; "unsure" means it straddles zero — the expected result for two equivalent builds._ -
    " - `); +
    " + `); }); it('does not count a difference between libraries as a regression', () => { diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index a93a5929d2..037c108b36 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -1,11 +1,10 @@ import { formatMarkdownTable } from '@/utils/formatters'; import { analyzeRun, + benchmarkTable, findRegressions, formatComparison, - formatComparisonLabel, formatRunSummary, - formatValue, RUN_REPORT_FOOTNOTE, } from '@mui/internal-benchmark/runReport'; import type { BenchmarkAnalysis, BenchmarkRunReport } from '@mui/internal-benchmark/runReport'; @@ -18,39 +17,24 @@ interface BuildOptions { } /** One table per benchmark: every metric, the median of every variant, every comparison. */ -function renderBenchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): string { - const comparisons = metrics[0]?.comparisons ?? []; - - const columns = [ - { field: 'metric', header: 'Metric', align: 'left' as const }, - ...benchmark.variants.map((variant) => ({ - field: `variant:${variant}`, - header: variant, - align: 'right' as const, - })), - ...comparisons.map((comparison) => ({ - field: `comparison:${comparison.subject}:${comparison.against}`, - header: formatComparisonLabel(benchmark, comparison), - align: 'left' as const, +function renderBenchmarkTable(analysis: BenchmarkAnalysis): string { + const { columns, rows } = benchmarkTable(analysis); + const table = formatMarkdownTable( + columns.map(({ header, kind }, column) => ({ + field: String(column), + header, + align: kind === 'value' ? ('right' as const) : ('left' as const), })), - ]; - const rows = metrics.map(({ metric, definition, variants, comparisons: metricComparisons }) => ({ - metric, - ...Object.fromEntries( - benchmark.variants.map((variant) => [ - `variant:${variant}`, - formatValue(variants[variant].median, definition), - ]), + rows.map(({ cells }) => + Object.fromEntries( + cells.map((cell, column) => [ + String(column), + columns[column].kind === 'comparison' ? `\`${cell}\`` : cell, + ]), + ), ), - ...Object.fromEntries( - metricComparisons.map((comparison) => [ - `comparison:${comparison.subject}:${comparison.against}`, - `\`${formatComparison(comparison)}\``, - ]), - ), - })); - - return `**${benchmark.name}**\n\n${formatMarkdownTable(columns, rows)}`; + ); + return `**${analysis.benchmark.name}**\n\n${table}`; } /** diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index b86b944419..e49d523446 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -15,13 +15,7 @@ import TableHead from '@mui/material/TableHead'; import TableRow from '@mui/material/TableRow'; import Typography from '@mui/material/Typography'; import { fetchCiReport, isBenchmarkRunUpload } from '@/utils/fetchCiReport'; -import { - analyzeRun, - formatComparison, - formatComparisonLabel, - formatValue, - RUN_REPORT_FOOTNOTE, -} from '@mui/internal-benchmark/runReport'; +import { analyzeRun, benchmarkTable, RUN_REPORT_FOOTNOTE } from '@mui/internal-benchmark/runReport'; import type { BenchmarkAnalysis, BenchmarkRunReport, @@ -43,8 +37,10 @@ function comparisonColor(comparison: MetricComparison): string { } function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { - const { benchmark, metrics } = analysis; - const comparisons = metrics[0]?.comparisons ?? []; + const { benchmark } = analysis; + const { columns, rows } = benchmarkTable(analysis); + const firstComparison = columns.findIndex((column) => column.kind === 'comparison'); + const alignOf = (column: number) => (columns[column].kind === 'value' ? 'right' : undefined); return ( @@ -57,37 +53,31 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { - Metric - {benchmark.variants.map((variant) => ( - - {variant} - - ))} - {comparisons.map((comparison) => ( - - {formatComparisonLabel(benchmark, comparison)} + {columns.map(({ header }, column) => ( + + {header} ))} - Rounds - {metrics.map(({ metric, definition, variants, comparisons: metricComparisons }) => ( - - {metric} - {benchmark.variants.map((variant) => ( - - {formatValue(variants[variant].median, definition)} - - ))} - {metricComparisons.map((comparison) => ( - - - {formatComparison(comparison)} - + {rows.map((row) => ( + + {row.cells.map((cell, column) => ( + + {columns[column].kind === 'comparison' ? ( + + {cell} + + ) : ( + cell + )} ))} - {variants[benchmark.variants[0]].count} ))} diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 379c53cc3d..f6960f8598 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -89,41 +89,6 @@ iteration to the next. Wait for the mount to be painted — `await waitForElemen before starting a gesture on something the case just rendered; until then, the browser may not route the gesture to it. -### Comparing implementations - -`compare()` measures implementations against each other — one library against another — instead of -one build against another. Each variant is a module of ordinary `benchmark()` calls, loaded lazily; -cases are paired across variants by name, and the first variant is the reference: - -```tsx -// scatter.bench.tsx -import { compare } from '@mui/internal-benchmark'; - -compare('scatter', { - ours: () => import('./scatter.ours'), - recharts: () => import('./scatter.recharts'), -}); -``` - -```tsx -// scatter.recharts.tsx -import { benchmark } from '@mui/internal-benchmark'; - -benchmark('mount', () => ); -benchmark( - 'zoom', - () => , - async ({ input }) => { - await input.pinch({ x: 400, y: 300, scaleFactor: 2 }); - }, -); -``` - -Under `benchmark run` every variant gets a page of its own that loads only its own module, so -no variant's code, styles or module state is present while another is measured; all variants come -from the working tree. Under Vitest, which has no notion of variants, each variant's cases simply run -as tests of their own, named `mount [recharts]`. - ### Scoping which renders are measured By default a benchmark records every React render and paint, from the mount through the whole interaction. To measure only part of an interaction — or to exclude the mount — pause and resume recording from the interaction callback: @@ -453,16 +418,16 @@ its timeout; a horizon such as `'5%'` settles once a difference is known to be s ### The report The run writes `.benchmark/results/report.json`, prints it as tables, and with `--upload` sends it to -the dashboard as version 2 of the `benchmark` report — the commit's `benchmark.json`, which the Vitest -reporter writes as version 1, so a repository switches from one to the other rather than running -both. The dashboard renders the pull request comment's Performance section and the repository's -history from it. The report holds **raw samples**, round-aligned across variants, plus what each metric is — its kind, -format and alarm — the builds, and the environment. Conclusions are computed from it, the same way -everywhere, by `analyzeRun` from `@mui/internal-benchmark/runReport`: a 95% confidence interval on the -paired difference per metric, a change (`better`, `worse`, `unsure`) that respects the metric's -direction, and a severity from its alarm. `reactBenchmark()`'s `render` and `render:count` alarm on -any resolved change for the worse, and `bench:paint` is informational; every other metric brings its -own alarm. +the dashboard, which renders the pull request comment's Performance section and the repository's +history from it. + +- The report holds **raw samples**, round-aligned across variants, plus each metric's kind, format + and alarm, the builds, and the environment. +- `analyzeRun` from `@mui/internal-benchmark/runReport` draws every conclusion from it: a 95% + confidence interval on the paired difference per metric, a change (`better`, `worse`, `unsure`), + and a severity from the metric's alarm. +- `reactBenchmark()`'s `render` and `render:count` alarm on any resolved change for the worse; + `bench:paint` is informational; every other metric brings its own alarm. ### Choosing the baseline @@ -475,9 +440,6 @@ feature branch the fork point, on the base branch the previous commit. On master is measured against its parent, and the history the dashboard draws is each commit's own paired change — which commit moved a number reads off the chart rather than out of a noisy trend. -`github:/#` and `preview:` are reserved for a baseline that is not a commit in -this repository. Neither is implemented, and both are rejected by name rather than handed to git. - ### Building and installing each side Both sides are built the same way: packed to tarballs, in parallel, and installed the way a consumer @@ -502,7 +464,6 @@ none — so nothing else should build against the same checkout meanwhile. ## API - `benchmark` — define a benchmark test case -- `compare` — compare implementations, each a lazily loaded module of `benchmark()` cases - `@mui/internal-benchmark/vitePlugin` — `benchmarkPlugin()`, the harness's vite plugin for `benchmark run` - `@mui/internal-benchmark/runReport` — the `benchmark run` report schema and `analyzeRun` - `ElementTiming` — invisible marker component for paint timing (renders a `` tracked by the Element Timing API) diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index b3598e7e94..cb985d4fe2 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -1,5 +1,5 @@ import type * as React from 'react'; -import { describe, expect, it } from 'vitest'; +import { expect, it } from 'vitest'; import type { RunnerTestCase } from 'vitest'; import { cdp } from 'vitest/browser'; import type { RenderEvent, IterationData } from './types'; @@ -25,9 +25,6 @@ import './taskMetaAugmentation'; export * from './publicApi'; export type { BenchmarkOptions }; -/** Loads a `compare()` variant's module, whose `benchmark()` calls define that variant's cases. */ -export type VariantLoader = () => Promise; - // When true, `benchmark()` opens an interactive profiling session in a headed // browser instead of running the automated measurement loop. Enabled by // `createBenchmarkVitestConfig({ profile: true })` or `BENCHMARK_PROFILE=true`, @@ -79,39 +76,13 @@ async function runIteration( } } -// The `compare()` variant whose module is loading, so its `benchmark()` calls can say whose they are. -let loadingVariant: string | undefined; - -/** - * Compares implementations against each other — one library against another, say. Each variant is - * a module of ordinary `benchmark()` calls; cases are paired across variants by name, and the first - * variant is the reference. - * - * Vitest has no notion of variants, so here every variant's cases simply run as tests of their own, - * suffixed with the variant: `mount [recharts]`. The interleaved engine is what compares them. - */ -export function compare(name: string, variants: Record): void { - describe(name, async () => { - for (const [key, load] of Object.entries(variants)) { - loadingVariant = key; - try { - // eslint-disable-next-line no-await-in-loop - await load(); - } finally { - loadingVariant = undefined; - } - } - }); -} - export function benchmark( - caseName: string, + name: string, renderFn: () => React.ReactElement, interactionOrOptions?: BenchmarkInteraction | BenchmarkOptions, maybeOptions?: BenchmarkOptions, ) { const { interaction, options } = splitCaseArgs(interactionOrOptions, maybeOptions); - const name = loadingVariant === undefined ? caseName : `${caseName} [${loadingVariant}]`; // In profile mode, skip the automated measurement loop entirely: build a bare case runtime (no // BenchProfiler wrapper, no-op recording since the user drives DevTools by hand) and hand it to diff --git a/packages/benchmark/src/launchArgs.ts b/packages/benchmark/src/launchArgs.ts index a129b683a8..c4ab69ea72 100644 --- a/packages/benchmark/src/launchArgs.ts +++ b/packages/benchmark/src/launchArgs.ts @@ -1,13 +1,11 @@ -// Chromium/V8 launch args shared by every runner, kept intentionally minimal. `--expose-gc` is -// required: the harness forces GC between iterations for clean, comparable timings. The -// backgrounding flags stop Chrome from throttling the (headless or occluded) benchmark tab, which -// would otherwise add large variance. Heavier "determinism" flags (`--no-opt`, `--predictable`, -// `--hash-seed`/`--random-seed`, `--disable-gpu`, `--enable-benchmarking`) were measured to slow -// renders ~40% and distort paint timing without reducing variance, so they are omitted — add them -// per project if a specific workload needs them. +// Shared by every runner. `--expose-gc` lets the harness force GC between iterations; the +// backgrounding flags stop Chrome throttling the benchmark tab. export const BENCHMARK_LAUNCH_ARGS = [ '--js-flags=--expose-gc', '--disable-background-timer-throttling', '--disable-backgrounding-occluded-windows', '--disable-renderer-backgrounding', ]; + +/** The viewport every runner measures at. */ +export const BENCHMARK_VIEWPORT = { width: 1920, height: 1080 }; diff --git a/packages/benchmark/src/metricCore.ts b/packages/benchmark/src/metricCore.ts index b9ebe1546b..8cf93c0587 100644 --- a/packages/benchmark/src/metricCore.ts +++ b/packages/benchmark/src/metricCore.ts @@ -10,10 +10,13 @@ export function seriesName(name: string, id: string | undefined): string { return id === undefined ? name : `${name}#${id}`; } -/** - * Where recorded values go. The runtime a benchmark runs under installs one: the Vitest harness - * accumulates them per running test, the page runtime emits them as `performance.measure` entries. - */ +/** The metric a series belongs to: `name#id` without its `#id`. */ +export function baseMetricName(series: string): string { + const hashIndex = series.indexOf('#'); + return hashIndex === -1 ? series : series.slice(0, hashIndex); +} + +/** Where recorded values go: the runtime a benchmark runs under installs one. */ export type MetricRecorder = ( metric: Metric, value: number, diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 3d42f0c0b9..9d22952ee0 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -27,11 +27,8 @@ export interface BenchmarkCase { readonly name: string; } -/** Every case the file registered, by name. */ -const cases = new Map< - string, - { run: BenchmarkRun; sampling: Required; ownSampling: boolean } ->(); +/** Every case the file registered, by name, with its sampling options as given. */ +const cases = new Map(); /** * Registers a benchmark case. The runner calls `run` once per sample, warmup included, for as many @@ -46,16 +43,12 @@ export function benchmark( if (cases.has(name)) { throw new Error(`Two benchmarks share the name "${name}". Benchmark names must be unique.`); } - cases.set(name, { - run, - sampling: resolveSampling(sampling), - ownSampling: hasSampling(sampling), - }); + // Resolved here only to reject invalid options where they were written. + resolveSampling(sampling); + cases.set(name, { run, sampling }); return { name }; } -/** The `compare()` each compared case belongs to, by case name. */ -const comparedIn = new Map(); /** Each `compare()`, with its cases' names, the reference first. */ const comparisons: Array<{ name: string; @@ -84,17 +77,17 @@ export function compare( } const resolved = resolveSampling(sampling); for (const benchCase of compared) { - if (cases.get(benchCase.name)?.ownSampling) { + const own = cases.get(benchCase.name)?.sampling; + if (own && hasSampling(own)) { throw new Error( `"${benchCase.name}" sets its own sampling, but a compared case is sampled as its ` + `compare() asks: pass the options to compare("${name}", …) instead.`, ); } - const owner = comparedIn.get(benchCase.name); - if (owner !== undefined) { - throw new Error(`"${benchCase.name}" is already compared in "${owner}".`); + const owner = comparisons.find((comparison) => comparison.cases.includes(benchCase.name)); + if (owner) { + throw new Error(`"${benchCase.name}" is already compared in "${owner.name}".`); } - comparedIn.set(benchCase.name, name); } comparisons.push({ name, @@ -188,8 +181,8 @@ async function sample(name: string): Promise> { export function markPageReady(): void { window.benchmarkPage = { cases: [...cases] - .filter(([name]) => !comparedIn.has(name)) - .map(([name, { sampling }]) => ({ name, sampling })), + .filter(([name]) => !comparisons.some((comparison) => comparison.cases.includes(name))) + .map(([name, { sampling }]) => ({ name, sampling: resolveSampling(sampling) })), comparisons, metricDefinitions: () => Object.fromEntries(metricDefinitions), sample, diff --git a/packages/benchmark/src/reporter.ts b/packages/benchmark/src/reporter.ts index 4060b1cab5..bd17dcbfb3 100644 --- a/packages/benchmark/src/reporter.ts +++ b/packages/benchmark/src/reporter.ts @@ -5,6 +5,7 @@ import type { RenderEvent, IterationData, MetricReport, MetricDefinition } from import type { BenchmarkBaseUpload, BenchmarkReportEntry } from './ciReport'; import { benchmarkUploadSchema, getCiMetadata } from './ciReport'; import { calculateMean, aggregateSamples } from './stats'; +import { baseMetricName } from './metricCore'; import { dim, red, green, yellow, cyan, printTable, fileUrl } from './format'; import { uploadCiReport } from './upload'; import { syncPrComment } from './syncPrComment'; @@ -173,12 +174,6 @@ function printDurationMatrix(name: string, report: BenchmarkReportEntry, footer: ); } -/** Strips a `#sub-series` suffix to recover the metric name used to look up its definition. */ -function baseMetricName(key: string): string { - const hashIndex = key.indexOf('#'); - return hashIndex === -1 ? key : key.slice(0, hashIndex); -} - function formatMetricValue(value: number, definition?: MetricDefinition): string { if (definition?.format) { return new Intl.NumberFormat(undefined, definition.format).format(value); diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index 81dde64c7a..e290c20f7b 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -75,7 +75,6 @@ describe('compareSamples', () => { expect(comparison.absolute).toEqual({ low: 2, high: 2 }); expect(comparison.change).toBe('worse'); - expect(comparison.rounds).toBe(10); }); it('reads the direction from the alarm', () => { diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index 195cc9ad29..6559b43865 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -1,4 +1,5 @@ import { calculateMean, quantile } from '../stats'; +import { baseMetricName } from '../metricCore'; import type { BenchmarkRunReport, RunBenchmark, RunMetricDefinition } from './schema'; /** @@ -18,8 +19,6 @@ export interface Interval { export interface SampleSummary { count: number; median: number; - /** 95% confidence interval of the mean. */ - mean: Interval; } /** Which way a resolved change went, given the metric's direction; `unsure` if it did not resolve. */ @@ -39,8 +38,6 @@ export interface MetricComparison { relative: Interval; change: Change; severity: Severity; - /** Rounds both variants were measured in. */ - rounds: number; } export interface MetricAnalysis { @@ -93,7 +90,7 @@ export function median(values: number[]): number { } function summarizeSamples(values: number[]): SampleSummary { - return { count: values.length, median: median(values), mean: meanInterval(values) }; + return { count: values.length, median: median(values) }; } const DEFAULT_DEFINITION: RunMetricDefinition = { kind: 'scalar' }; @@ -103,8 +100,7 @@ function definitionOf( metrics: Record, metric: string, ): RunMetricDefinition { - const [base] = metric.split('#'); - return metrics[metric] ?? metrics[base] ?? DEFAULT_DEFINITION; + return metrics[metric] ?? metrics[baseMetricName(metric)] ?? DEFAULT_DEFINITION; } /** @@ -168,7 +164,7 @@ export function compareSamples( } const comparison = { subject: subject.name, against: against.name, absolute, relative, change }; - return { ...comparison, severity: severityOf(comparison, definition), rounds }; + return { ...comparison, severity: severityOf(comparison, definition) }; } /** The pairs a benchmark's kind compares: the current build against the baseline, or every variant against the reference. */ @@ -179,37 +175,58 @@ function pairsOf(benchmark: RunBenchmark): Array<{ subject: string; against: str : others.map((other) => ({ subject: other, against: reference })); } -/** Analyses one benchmark, its metrics described by `metrics` (a report's, or a run's so far). */ -export function analyzeBenchmark( +export type MetricComparisons = Omit; + +/** + * Compares one benchmark's variants, its metrics described by `metrics` (a report's, or a run's so + * far). The comparisons alone, which is all deciding whether to keep sampling needs. + */ +export function compareBenchmark( metrics: Record, benchmark: RunBenchmark, -): BenchmarkAnalysis { +): { metrics: MetricComparisons[] } { const { samples } = benchmark; if (!samples) { - return { benchmark, metrics: [] }; + return { metrics: [] }; } const [reference] = benchmark.variants; - const metricNames = Object.keys(samples[reference] ?? {}); - - const analyses = metricNames.map((metric): MetricAnalysis => { - const definition = definitionOf(metrics, metric); - const valuesOf = (variant: string) => samples[variant]?.[metric] ?? []; - return { - metric, - definition, - variants: Object.fromEntries( - benchmark.variants.map((variant) => [variant, summarizeSamples(valuesOf(variant))]), - ), - comparisons: pairsOf(benchmark).map(({ subject, against }) => - compareSamples( - { name: subject, values: valuesOf(subject) }, - { name: against, values: valuesOf(against) }, - definition, + return { + metrics: Object.keys(samples[reference] ?? {}).map((metric) => { + const definition = definitionOf(metrics, metric); + const valuesOf = (variant: string) => samples[variant]?.[metric] ?? []; + return { + metric, + definition, + comparisons: pairsOf(benchmark).map(({ subject, against }) => + compareSamples( + { name: subject, values: valuesOf(subject) }, + { name: against, values: valuesOf(against) }, + definition, + ), ), + }; + }), + }; +} + +/** Analyses one benchmark: its comparisons, and a summary of each variant's samples. */ +export function analyzeBenchmark( + metrics: Record, + benchmark: RunBenchmark, +): BenchmarkAnalysis { + const { samples } = benchmark; + return { + benchmark, + metrics: compareBenchmark(metrics, benchmark).metrics.map((comparison) => ({ + ...comparison, + variants: Object.fromEntries( + benchmark.variants.map((variant) => [ + variant, + summarizeSamples(samples?.[variant]?.[comparison.metric] ?? []), + ]), ), - }; - }); - return { benchmark, metrics: analyses }; + })), + }; } export function analyzeRun(report: BenchmarkRunReport): BenchmarkAnalysis[] { diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index 04a6e79f18..ede64966a7 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -9,12 +9,21 @@ import type { RunBenchmark, RunMetricDefinition } from './schema'; const DEFAULT_SCALAR_FORMAT: Intl.NumberFormatOptions = { maximumFractionDigits: 2 }; const DEFAULT_DISCRETE_FORMAT: Intl.NumberFormatOptions = { maximumFractionDigits: 1 }; +// Constructing a formatter is the expensive part, and a table formats every cell. +const formatters = new Map(); + /** A value in its metric's own format — `12.3 ms`, `4`. */ export function formatValue(value: number, definition: RunMetricDefinition): string { const options = definition.format ?? (definition.kind === 'discrete' ? DEFAULT_DISCRETE_FORMAT : DEFAULT_SCALAR_FORMAT); - return new Intl.NumberFormat('en-US', options).format(value); + const key = JSON.stringify(options); + let formatter = formatters.get(key); + if (!formatter) { + formatter = new Intl.NumberFormat('en-US', options); + formatters.set(key, formatter); + } + return formatter.format(value); } /** Both bounds signed, so the direction of a difference reads without the verdict beside it. */ @@ -36,6 +45,50 @@ export function formatComparisonLabel( return benchmark.kind === 'baseline' ? `Δ vs ${against}` : `${subject} vs ${against}`; } +export interface BenchmarkTableColumn { + header: string; + /** `value` columns hold numbers, `comparison` columns a formatted comparison. */ + kind: 'label' | 'value' | 'comparison'; +} + +export interface BenchmarkTableRow { + metric: string; + cells: string[]; + /** The comparisons behind the row's `comparison` cells, in column order. */ + comparisons: MetricComparison[]; +} + +/** + * A benchmark's results as one table, for every renderer of it: per metric, each variant's median, + * each comparison, and the rounds measured. + */ +export function benchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): { + columns: BenchmarkTableColumn[]; + rows: BenchmarkTableRow[]; +} { + const [reference] = benchmark.variants; + const columns: BenchmarkTableColumn[] = [ + { header: 'Metric', kind: 'label' }, + ...benchmark.variants.map((variant) => ({ header: variant, kind: 'value' as const })), + ...(metrics[0]?.comparisons ?? []).map((comparison) => ({ + header: formatComparisonLabel(benchmark, comparison), + kind: 'comparison' as const, + })), + { header: 'Rounds', kind: 'value' }, + ]; + const rows = metrics.map(({ metric, definition, variants, comparisons }) => ({ + metric, + comparisons, + cells: [ + metric, + ...benchmark.variants.map((variant) => formatValue(variants[variant].median, definition)), + ...comparisons.map(formatComparison), + String(variants[reference].count), + ], + })); + return { columns, rows }; +} + /** The line a run is summed up in: how many benchmarks measured, regressed and failed. */ export function formatRunSummary(analyses: BenchmarkAnalysis[], regressions: Regression[]): string { const measured = analyses.filter((analysis) => !analysis.benchmark.error).length; diff --git a/packages/benchmark/src/runner/benchFiles.ts b/packages/benchmark/src/runner/benchFiles.ts index b598113791..31a91e7210 100644 --- a/packages/benchmark/src/runner/benchFiles.ts +++ b/packages/benchmark/src/runner/benchFiles.ts @@ -2,6 +2,7 @@ import * as path from 'node:path'; import { mkdir, rm, writeFile } from 'node:fs/promises'; import { globby } from 'globby'; import { escapeHtml } from '../utils/html'; +import { markIgnored } from './outputDir'; /** * `*.bench.tsx` files: cases defined with `@mui/internal-benchmark/page`, measured by the interleaved @@ -95,7 +96,7 @@ export async function writeBenchPages(harnessDir: string, benchFiles: BenchFile[ } await mkdir(pagesDir, { recursive: true }); await Promise.all([ - writeFile(path.join(pagesDir, '.gitignore'), '*\n'), + markIgnored(pagesDir), ...benchFiles.flatMap((benchFile) => { const html = path.join(harnessDir, 'src', benchFile.page); const entry = `${path.basename(benchFile.page, '.html')}.entry.ts`; diff --git a/packages/benchmark/src/runner/buildPages.ts b/packages/benchmark/src/runner/buildPages.ts index 7f76e62940..43a4515088 100644 --- a/packages/benchmark/src/runner/buildPages.ts +++ b/packages/benchmark/src/runner/buildPages.ts @@ -21,13 +21,7 @@ import type { ResolvedRef } from './refs'; */ const RUNNER_ONLY_DEPS = ['@playwright/test', pkgJson.name]; -/** - * The packed pins a run adds to the repository's overrides. - * - * Everything the run itself resolves is left alone, this package above all: pinning it would hand - * the harness's vite config the ref's copy of the very tool doing the building, and a ref old enough - * not to export the plugin then fails to configure at all. - */ +/** The packed pins a run adds to the repository's overrides, leaving what the run resolves itself. */ function packedPins(packages: PackedPackage[]): Record { return Object.fromEntries( packages @@ -42,12 +36,8 @@ function backupPathOf(outputDir: string): string { } /** - * Points the repository's own install at this ref's packed build, through its overrides. - * - * The manifest is copied aside first and rewritten from that copy each time, so a second ref pins - * over the original rather than over the first ref's pins. A run that dies leaves both the pins and - * the copy: the manifest then names tarballs under the output directory, which fails the next - * install rather than resolving something else and saying nothing. + * Points the repository's own install at this ref's packed build, through its overrides. The + * manifest is rewritten from a copy set aside first, so a second ref pins over the original. */ async function pinPackedPackages( repoRoot: string, @@ -73,14 +63,17 @@ async function pinPackedPackages( } /** - * Puts the repository's manifest back, and installs from it. - * - * The install is the point: without it the tree on disk still resolves the library to a tarball - * under the output directory, and that surfaces at the next unrelated command instead of here. - * - * `--no-frozen-lockfile` because pnpm turns a frozen install on by itself in CI, and the lockfile - * here still carries the pins this is undoing. + * Installs the repository after its overrides changed. Not frozen: pnpm freezes the lockfile by + * itself in CI, and changing it is the point. */ +async function installWorkspace(repoRoot: string): Promise { + await execa('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], { + cwd: repoRoot, + verbose: 'short', + }); +} + +/** Puts the repository's manifest back, and installs from it. */ export async function restoreWorkspace(repoRoot: string, outputDir: string): Promise { const backupPath = backupPathOf(outputDir); let original: string; @@ -96,23 +89,13 @@ export async function restoreWorkspace(repoRoot: string, outputDir: string): Pro console.log(chalk.cyan('\nRestoring the repository install…')); await writeFile(path.join(repoRoot, 'pnpm-workspace.yaml'), original); await rm(backupPath, { force: true }); - await execa('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], { - cwd: repoRoot, - verbose: 'short', - }); + await installWorkspace(repoRoot); } /** - * Builds the benchmark pages against one ref's packed build. - * - * The pages are built **where they live**, from the harness's own directory and its own vite config, - * so a `benchmark run` build sees exactly what a plain `vite build` sees — the same tsconfig, the same - * postcss config, the same everything. Only the library differs: the repository's install is pinned - * to that ref's packed build first, so both sides resolve it the same way. Building one side through a - * workspace link instead would compare two different resolution paths, and that difference would land - * in the measurement rather than in the library. - * - * The pins stay until `restoreWorkspace` puts the repository back. + * Builds the benchmark pages against one ref's packed build, in place with the harness's own vite + * config, so only the library differs from a plain `vite build`. The pins stay until + * `restoreWorkspace` puts the repository back. */ export async function buildRefPages(options: { /** The harness package directory, built in place. */ @@ -132,15 +115,9 @@ export async function buildRefPages(options: { console.log(chalk.cyan(`\nBuilding benchmark pages for "${refLabel(ref)}"…`)); await pinPackedPackages(repoRoot, outputDir, packages); - // `--no-frozen-lockfile` because changing the overrides is the point, and pnpm turns a frozen - // install on by itself in CI — where it would refuse the very change being made. - await execa('pnpm', ['install', '--prefer-offline', '--no-frozen-lockfile'], { - cwd: repoRoot, - verbose: 'short', - }); + await installWorkspace(repoRoot); - // vite is the harness's own, not this package's: the harness declares the version its pages are - // built with, and the config being run is the harness's. + // The harness's own vite, the version its pages are built with. const requireFromHarness = createRequire(path.join(harnessDir, 'package.json')); const vite: typeof Vite = await import(pathToFileURL(requireFromHarness.resolve('vite')).href); diff --git a/packages/benchmark/src/runner/outputDir.ts b/packages/benchmark/src/runner/outputDir.ts index 67cec3362c..fc9b774a5b 100644 --- a/packages/benchmark/src/runner/outputDir.ts +++ b/packages/benchmark/src/runner/outputDir.ts @@ -1,41 +1,31 @@ import * as path from 'node:path'; import { mkdir, writeFile } from 'node:fs/promises'; -/** - * Everything a run writes — packed tarballs, built pages, the report — lives - * under this one directory inside the harness, so a repository has a single thing to ignore and - * deleting it is the whole reset story. - */ +/** Everything a run writes lives here, so there is one thing to ignore and deleting it resets. */ export const OUTPUT_DIR = '.benchmark'; /** - * Creates the output directory, marking it ignored from the inside. - * - * A `.gitignore` holding `*` ignores the directory's contents and itself, the way pytest marks its - * cache, so a harness that forgets to list it does not end up offering built pages and tarballs for - * commit. It does not replace an entry in the repository's own ignore file: tools that read only - * the root one — this repository's ESLint config among them — never see a nested `.gitignore`. + * Marks a generated directory ignored from the inside: a `.gitignore` holding `*` ignores its + * contents and itself. Tools that read only the root ignore file still need an entry there. */ +export async function markIgnored(dir: string): Promise { + await writeFile(path.join(dir, '.gitignore'), '*\n'); +} + +/** Creates the output directory, marked ignored. */ export async function prepareOutputDir(harnessDir: string): Promise { const outputDir = path.join(harnessDir, OUTPUT_DIR); await mkdir(outputDir, { recursive: true }); - await writeFile(path.join(outputDir, '.gitignore'), '*\n'); + await markIgnored(outputDir); return outputDir; } -/** - * Where the pages built for `refId` go, or the root of every ref's pages when it is omitted. - * - * Named here rather than composed at each call site because the runner and the vite plugin have to - * agree on it: the runner serves every ref's pages from under this path, and a plain `vite build` - * has to land somewhere that does not collide. - */ /** Where a run writes its JSON report, unless told otherwise. */ export function resultsPathOf(harnessDir: string): string { return path.join(harnessDir, OUTPUT_DIR, 'results', 'report.json'); } -export function buildsDirOf(harnessDir: string, refId?: string): string { - const builds = path.join(harnessDir, OUTPUT_DIR, 'builds'); - return refId ? path.join(builds, refId) : builds; +/** Where the pages of every ref are built, one directory per variant. */ +export function buildsDirOf(harnessDir: string): string { + return path.join(harnessDir, OUTPUT_DIR, 'builds'); } diff --git a/packages/benchmark/src/runner/plugin.ts b/packages/benchmark/src/runner/plugin.ts index d676e2834c..66a2b2df14 100644 --- a/packages/benchmark/src/runner/plugin.ts +++ b/packages/benchmark/src/runner/plugin.ts @@ -4,7 +4,7 @@ import type { Plugin } from 'vite'; import { escapeHtml } from '../utils/html'; import { discoverBenchFiles, writeBenchPages } from './benchFiles'; import type { BenchFile } from './benchFiles'; -import { buildsDirOf, prepareOutputDir, OUTPUT_DIR } from './outputDir'; +import { buildsDirOf, prepareOutputDir } from './outputDir'; /** * Vite plugin for a benchmark harness. @@ -128,17 +128,9 @@ export function benchmarkPlugin(options: BenchmarkPluginOptions = {}): Plugin { ...pkg.devDependencies, }); - // The run output directory, next to the per-ref directories the runner writes — vite would - // otherwise default to `/dist`, i.e. inside `src/` next to the benchmark sources. - // - // Resolved for every command, not just `build`: `vite preview` runs as `serve` but serves - // `build.outDir`, so leaving it unset there sends preview looking for `/dist` and it - // exits rather than serving the pages that were just built. - // - // Only fill it in when the caller said nothing: a plugin's returned config is merged *over* - // the inline config, so unconditionally setting it here would silently override - // `vite build --outDir`, which is exactly how `benchmark run` directs each ref's build. - const outDir = userConfig.build?.outDir ?? buildsDirOf(harnessDir, 'manual'); + // Set for every command, since `vite preview` serves `build.outDir` too; only when the caller + // gave none, because returned config is merged over `vite build --outDir`. + const outDir = userConfig.build?.outDir ?? path.join(buildsDirOf(harnessDir), 'manual'); // Written for every command, so the dev server serves a benchmark file's page as well. benchFiles = await discoverBenchFiles({ harnessDir }); @@ -147,18 +139,18 @@ export function benchmarkPlugin(options: BenchmarkPluginOptions = {}): Plugin { } await writeBenchPages(harnessDir, benchFiles); // One React for the harness, its benchmark files and the page runtime they run under. - const resolve = { dedupe: ['react', 'react-dom'] }; + const base = { + root: srcDir, + appType: 'mpa' as const, + resolve: { dedupe: ['react', 'react-dom'] }, + }; if (env.command !== 'build') { - return { root: srcDir, appType: 'mpa', resolve, build: { outDir } }; + return { ...base, build: { outDir } }; } - // Mark the output directory ignored from the inside before the build writes into it, so a - // harness that does not list `.benchmark` is not offered its built pages for commit. Keyed on - // where the output actually lands rather than on who chose it: `benchmark run` passes its own - // `outDir` under the same directory and prepares it itself, and any other caller that writes - // there deserves the marker too. Vite resolves a relative `outDir` against `root`. - if (path.resolve(srcDir, outDir).startsWith(path.join(harnessDir, OUTPUT_DIR))) { + // A plain `vite build` into the output directory marks it ignored, as `benchmark run` does. + if (!userConfig.build?.outDir) { await prepareOutputDir(harnessDir); } @@ -170,9 +162,7 @@ export function benchmarkPlugin(options: BenchmarkPluginOptions = {}): Plugin { ); return { - root: srcDir, - appType: 'mpa', - resolve, + ...base, // Relative asset urls: the runner serves each ref's build under its own path, so absolute // "/assets/…" paths would 404. base: './', diff --git a/packages/benchmark/src/runner/printReport.ts b/packages/benchmark/src/runner/printReport.ts index 89a85da149..e8316ba367 100644 --- a/packages/benchmark/src/runner/printReport.ts +++ b/packages/benchmark/src/runner/printReport.ts @@ -3,41 +3,25 @@ import { dim, printTable, red, yellow } from '../format'; import { analyzeRun, + benchmarkTable, findRegressions, formatComparison, - formatComparisonLabel, formatRunSummary, - formatValue, RUN_REPORT_FOOTNOTE, } from '../runReport'; import type { BenchmarkAnalysis, BenchmarkRunReport } from '../runReport'; -function printBenchmark({ benchmark, metrics }: BenchmarkAnalysis): void { - const [reference] = benchmark.variants; - - const headers = [ - 'Metric', - ...benchmark.variants.map((variant) => `${variant} (median)`), - ...(metrics[0]?.comparisons ?? []).map((comparison) => - formatComparisonLabel(benchmark, comparison), - ), - 'Rounds', - ]; - const rows = metrics.map(({ metric, definition, variants, comparisons }) => [ - metric, - ...benchmark.variants.map((variant) => formatValue(variants[variant].median, definition)), - ...comparisons.map(formatComparison), - String(variants[reference].count), - ]); - const widths = headers.map((header, column) => - Math.max(header.length, ...rows.map((row) => row[column].length)), - ); - +function printBenchmark(analysis: BenchmarkAnalysis): void { + const { columns, rows } = benchmarkTable(analysis); + const cells = rows.map((row) => row.cells); printTable( - headers.map((header, column) => ({ header, width: widths[column] })), - rows, + columns.map(({ header }, column) => ({ + header, + width: Math.max(header.length, ...cells.map((row) => row[column].length)), + })), + cells, undefined, - `${benchmark.name} (${benchmark.file})`, + `${analysis.benchmark.name} (${analysis.benchmark.file})`, ); } diff --git a/packages/benchmark/src/runner/refs.test.ts b/packages/benchmark/src/runner/refs.test.ts index 4d55ec205c..6d90c7f64a 100644 --- a/packages/benchmark/src/runner/refs.test.ts +++ b/packages/benchmark/src/runner/refs.test.ts @@ -85,27 +85,6 @@ describe('resolveBaselineRef', () => { expect((await resolveBaselineRef('git:v1.0.0', repoRoot)).requested).toBe('v1.0.0'); }); - it('names a revision and a tag the same build, so the commit is built once', async () => { - const { repoRoot, shas } = await makeRepo(); - - const viaTag = await resolveBaselineRef('v1.0.0', repoRoot); - const viaSha = await resolveBaselineRef(shas[0], repoRoot); - - // `id` is the build directory: two ids for one commit means packing and building it twice. - expect(viaTag.id).toBe(viaSha.id); - }); - - it.each(['github:owner/repo#abc1234', 'preview:abc1234'])( - 'reports %s as recognised but not implemented', - async (token) => { - const { repoRoot } = await makeRepo(); - - await expect(() => resolveBaselineRef(token, repoRoot)).rejects.toThrow( - /recognised but not implemented/, - ); - }, - ); - it('rejects the git scheme with no revision after it', async () => { const { repoRoot } = await makeRepo(); diff --git a/packages/benchmark/src/runner/refs.ts b/packages/benchmark/src/runner/refs.ts index c8fc29df6f..8c466e3c10 100644 --- a/packages/benchmark/src/runner/refs.ts +++ b/packages/benchmark/src/runner/refs.ts @@ -1,58 +1,26 @@ import { resolveCommit } from '../utils/git'; /** - * The builds a run measures: the working tree, and the baseline it is compared against. - * - * Those are the only two. A case either compares the working tree against the baseline or compares - * the pages it names against each other, all built from the working tree — so the only build anyone - * chooses is the baseline, and `--baseline` is where it is named. - */ - -/** - * A build with its identity. The working tree has no commit behind it and a git revision always - * does, so the two carry different fields rather than sharing optional ones a reader has to pair - * with the right `kind` by hand. - * - * `id` is the build directory name — `current`, or `git-230342ee2`. `requested` is the revision as - * it was given, which is what a report shows. + * A build a run measures: the working tree (`current`), or the revision it is compared against + * (`baseline`). `variant` names its pages, its build directory and its entry in the report. + * `requested` is the revision as it was given, which is what a report shows. */ export type ResolvedRef = - | { kind: 'worktree'; id: string; sha?: undefined; requested?: undefined } - | { kind: 'git'; id: string; sha: string; requested: string }; + | { variant: 'current'; sha?: undefined; requested?: undefined } + | { variant: 'baseline'; sha: string; requested: string }; /** What a ref is called in a report: the working tree, or the revision the run was given. */ export function refLabel(ref: ResolvedRef): string { - return ref.kind === 'worktree' ? 'working tree' : ref.requested; + return ref.variant === 'current' ? 'working tree' : ref.requested; } -/** - * The variant name a ref's pages are reported under: `current` for the working tree, `baseline` for - * the revision it is compared against. The report's `builds` are keyed by the same names. - */ -export function variantOf(ref: ResolvedRef): 'current' | 'baseline' { - return ref.kind === 'worktree' ? 'current' : 'baseline'; -} +export const WORKTREE_REF: ResolvedRef = { variant: 'current' }; -/** The working tree, which every run builds. Never cached, having no immutable identity. */ -export const WORKTREE_REF: ResolvedRef = { kind: 'worktree', id: 'current' }; - -/** Schemes a baseline may eventually name, each rejected until it does something. */ -const RESERVED_SCHEMES = ['github', 'preview']; - -/** - * The revision a baseline token names. - * - * A bare value is a revision and `git:` says so explicitly, which leaves room for a baseline that is - * not a commit in this repository — a published preview, another repository — to arrive as its own - * scheme without the flag changing shape. Anything else goes to git, which is the authority on what - * a revision is. - */ +/** The revision a baseline token names: a bare revision, or one prefixed with `git:`. */ function committishOf(token: string | undefined): string { if (!token) { - // Which commit a pull request should be measured against is not decided here: `code-infra - // baseline` answers that for every job that compares against a base — bundle size as well as - // benchmarks — and the harness scripts pass the answer in. Without one, the previous commit is - // the only choice that needs no policy. + // `code-infra baseline` answers which commit a pull request compares against; without it, the + // previous commit is the only choice that needs no policy. return 'HEAD~1'; } if (token.startsWith('git:')) { @@ -62,13 +30,6 @@ function committishOf(token: string | undefined): string { } return revision; } - const reserved = RESERVED_SCHEMES.find((scheme) => token.startsWith(`${scheme}:`)); - if (reserved) { - throw new Error( - `Baseline scheme "${reserved}:" is recognised but not implemented yet (in "${token}"). ` + - `Pass a git revision, on its own or as "git:".`, - ); - } return token; } @@ -79,5 +40,5 @@ export async function resolveBaselineRef( ): Promise { const committish = committishOf(token); const sha = await resolveCommit(repoRoot, committish); - return { kind: 'git', id: `git-${sha.slice(0, 9)}`, sha, requested: committish }; + return { variant: 'baseline', sha, requested: committish }; } diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index ff3930c2aa..41fd4e3bb6 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -3,12 +3,11 @@ import * as path from 'node:path'; import { mkdir, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; -import { execa } from 'execa'; import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; import { packRef, packWorkingTree } from '../utils/packWorkspace'; -import { resolveCommit } from '../utils/git'; +import { getCiMetadata } from '../ciReport'; import type { BenchmarkRunReport } from '../runReport'; -import { refLabel, resolveBaselineRef, variantOf, WORKTREE_REF } from './refs'; +import { refLabel, resolveBaselineRef, WORKTREE_REF } from './refs'; import type { ResolvedRef } from './refs'; import { discoverBenchFiles } from './benchFiles'; import { runInterleaved } from './runInterleaved'; @@ -65,17 +64,13 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise { - if (ref.kind === 'worktree') { + if (ref.variant === 'current') { return packWorkingTree({ repoRoot, outRoot: path.join(packedDir, 'current'), buildCmd }); } // packRef caches a ref's tarballs by SHA; a hit skips the checkout, install, and build. @@ -126,7 +120,7 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise [variantOf(ref), { sha: ref.sha, label: refLabel(ref) }]), + refs.map((ref) => [ref.variant, { sha: ref.sha, label: refLabel(ref) }]), ), metrics: results.metrics, benchmarks: results.benchmarks, diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index 400377879c..7322dfb9c2 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -2,13 +2,12 @@ import chalk from 'chalk'; import type { Browser, BrowserContext, CDPSession, Page } from '@playwright/test'; -import { BENCHMARK_LAUNCH_ARGS } from '../launchArgs'; -import { analyzeBenchmark } from '../runReport'; +import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; +import { compareBenchmark } from '../runReport'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; import { differencesResolved, parseHorizons } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { BenchFile } from './benchFiles'; -import { variantOf } from './refs'; import type { ResolvedRef } from './refs'; import { serveDirectory } from './serveDirectory'; import type { BenchPage } from '../page/page'; @@ -40,7 +39,6 @@ export interface InterleavedResults { browserVersion: string; } -const VIEWPORT = { width: 1920, height: 1080 }; const SAMPLE_TIMEOUT_MS = 120_000; type CdpMethod = Parameters[0]; @@ -84,7 +82,7 @@ interface OpenedPage { } async function openBenchPage(browser: Browser, url: string): Promise { - const context = await browser.newContext({ viewport: VIEWPORT }); + const context = await browser.newContext({ viewport: BENCHMARK_VIEWPORT }); const page = await context.newPage(); page.setDefaultTimeout(SAMPLE_TIMEOUT_MS); page.on('pageerror', (error) => console.error(chalk.red(` page error: ${error.message}`))); @@ -208,7 +206,7 @@ async function runBenchmark( const metrics = await targets[0].page.evaluate(() => window.benchmarkPage!.metricDefinitions()); const benchmarkOf = (): RunBenchmark => ({ ...entry, samples: samplesOf(slots, rounds) }); - const isSettled = () => differencesResolved(analyzeBenchmark(metrics, benchmarkOf()), horizons); + const isSettled = () => differencesResolved(compareBenchmark(metrics, benchmarkOf()), horizons); const deadline = Date.now() + timeout * 60_000; let resolved = isSettled(); @@ -247,7 +245,7 @@ function samplesOf(slots: PageSlot[], rounds: Round[]): RunBenchmark['samples'] } function benchPageUrl(origin: string, ref: ResolvedRef, benchFile: BenchFile): string { - return new URL(`${ref.id}/${benchFile.page}`, `${origin}/`).href; + return new URL(`${ref.variant}/${benchFile.page}`, `${origin}/`).href; } /** @@ -261,7 +259,7 @@ async function runBenchFile( options: RunInterleavedOptions, ): Promise { const urls = options.benchRefs.map((ref) => benchPageUrl(origin, ref, benchFile)); - const variants = options.benchRefs.map(variantOf); + const variants = options.benchRefs.map((ref) => ref.variant); // The other builds only matter for cases measured across them; a file of `compare()`s alone // never loads them. const [reference] = await listBenchPages(browser, urls.slice(0, 1)); diff --git a/packages/benchmark/src/sampling.ts b/packages/benchmark/src/sampling.ts index f852b2927d..483366d613 100644 --- a/packages/benchmark/src/sampling.ts +++ b/packages/benchmark/src/sampling.ts @@ -1,4 +1,4 @@ -import type { BenchmarkAnalysis } from './runReport/analyzeRun'; +import type { MetricComparisons } from './runReport/analyzeRun'; /** * How many rounds a benchmark is sampled for: tachometer's knobs, under tachometer's names. After @@ -83,7 +83,10 @@ export function isResolved(interval: { low: number; high: number }, horizons: nu * Whether sampling a benchmark can stop: every comparison it reports has resolved against the * horizons, on the metrics that alarm, or on every metric when none does. */ -export function differencesResolved(analysis: BenchmarkAnalysis, horizons: number[]): boolean { +export function differencesResolved( + analysis: { metrics: MetricComparisons[] }, + horizons: number[], +): boolean { const alarmed = analysis.metrics.filter((metric) => metric.definition.alarm !== undefined); const judged = alarmed.length > 0 ? alarmed : analysis.metrics; return judged.every((metric) => diff --git a/packages/benchmark/src/vitest.ts b/packages/benchmark/src/vitest.ts index 85a7e6e8b3..8b02caae8a 100644 --- a/packages/benchmark/src/vitest.ts +++ b/packages/benchmark/src/vitest.ts @@ -1,7 +1,7 @@ import react from '@vitejs/plugin-react'; import { playwright } from '@vitest/browser-playwright'; import type { ViteUserConfig } from 'vitest/config'; -import { BENCHMARK_LAUNCH_ARGS } from './launchArgs'; +import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from './launchArgs'; export interface CreateBenchmarkVitestConfigOptions { /** @@ -37,7 +37,6 @@ export interface CreateBenchmarkVitestConfigOptions { // Default viewport for all benchmark runs (measurement and profiling alike) — a desktop size is // more representative for component benchmarks than Vitest's phone-sized 414x896 browser default. -const DEFAULT_VIEWPORT = { width: 1920, height: 1080 }; // Explicit viewport from the `viewport` option or the `BENCHMARK_VIEWPORT` env var // (`x`); undefined when neither is set, so the caller can apply the default. @@ -74,7 +73,7 @@ export function createBenchmarkVitestConfig( const { outputPath, baselinePath, launchArgs = [] } = options ?? {}; const profile = options?.profile ?? process.env.BENCHMARK_PROFILE === 'true'; - const viewport = resolveViewport(options?.viewport) ?? DEFAULT_VIEWPORT; + const viewport = resolveViewport(options?.viewport) ?? BENCHMARK_VIEWPORT; // Profiling adds DevTools on top of the shared args, plus a window sized to match the viewport — // Vitest's `viewport` only sizes the iframe, so otherwise it's cropped/scrolled in the headed @@ -102,7 +101,7 @@ export function createBenchmarkVitestConfig( // Profiling renders into a clean page: hide Vitest's browser runner UI // so the orchestrator chrome doesn't clutter what you're profiling. ui: profile ? false : undefined, - // Same viewport for both modes (DEFAULT_VIEWPORT unless overridden). + // Same viewport for both modes (BENCHMARK_VIEWPORT unless overridden). viewport, screenshotFailures: false, instances: [ diff --git a/test/benchmark/README.md b/test/benchmark/README.md index f460092e7d..c81f4998fd 100644 --- a/test/benchmark/README.md +++ b/test/benchmark/README.md @@ -2,14 +2,8 @@ A working harness for `benchmark run`, used to exercise that tooling end to end. -This is **not** a meaningful benchmark, and it is not trying to be. This repository ships build and -test tooling, not a browser library, so there is nothing here whose render time is worth tracking. -The benchmarks run deterministic workloads purely so the pipeline has something real to drive: -discovering benchmark files, resolving the baseline, packing a commit's workspace to tarballs, -building each ref's pages, and sampling them in Chromium. - -For the real thing, see the consumers this tooling was extracted from — `base-ui-mosaic` and -`base-ui-charts` benchmark actual data grids and charts. +Its benchmarks are deterministic workloads, not numbers worth tracking: they give the pipeline something +real to drive. For real benchmarks, see `base-ui-mosaic` and `base-ui-charts`. ## What it covers diff --git a/test/benchmark/scripts/runBenchmark.mjs b/test/benchmark/scripts/runBenchmark.mjs index 63da240419..02b66746bd 100644 --- a/test/benchmark/scripts/runBenchmark.mjs +++ b/test/benchmark/scripts/runBenchmark.mjs @@ -1,12 +1,3 @@ -// Thin wrapper that runs the `@mui/internal-benchmark` CLI from the workspace package's build -// output. Importing the package by name (rather than relying on its `benchmark` bin) avoids -// depending on pnpm linking the bin shim: the package publishes from `build/`, so the workspace -// link points at a directory that does not exist until the first build — and a later build cannot -// create a shim that install already skipped. The same wrapper exists in `docs/scripts` for the -// same reason. -// -// A consumer installing the published package gets a self-contained directory and can use the bin -// directly; only this repository, where the dependency is a workspace link, needs this. -// -// The CLI parses `process.argv` itself on import. +// Runs the `@mui/internal-benchmark` CLI by importing it: the workspace link points at the +// package's `build/`, which has no bin shim until it is built. The CLI parses `process.argv` itself. import '@mui/internal-benchmark/cli'; From 4b5bf7fa5d09de7cd0cb550f4840c1e33b096230 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:03:56 +0200 Subject: [PATCH 109/173] [benchmark] Collect garbage before every sample; report how sampling went MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Plain benchmark() cases now settle and collect garbage before each sample, as React cases did, so no sample pays for another's allocations. The step moves out of measureIteration into a React-free gc.ts that both drivers call. Each benchmark records its sampleSize and whether it timed out. The terminal, the PR comment and the dashboard show it next to the benchmark's name — 50 rounds, 50 + 23 rounds, · timed out — instead of a Rounds column, and the run summary counts timed-out benchmarks. printTable widens its last column to fit a title instead of cutting it. --- .../benchmarkRun/buildMarkdownReport.test.ts | 21 +++++------ .../lib/benchmarkRun/buildMarkdownReport.ts | 3 +- .../src/views/BenchmarkRunDetails.tsx | 25 +++++++++++-- packages/benchmark/README.md | 2 +- packages/benchmark/src/caseRuntime.tsx | 35 ------------------- packages/benchmark/src/format.test.ts | 19 ++++------ packages/benchmark/src/format.ts | 19 +++++----- packages/benchmark/src/gc.ts | 27 ++++++++++++++ packages/benchmark/src/index.tsx | 5 ++- packages/benchmark/src/page/page.tsx | 2 ++ .../benchmark/src/page/reactBenchmark.tsx | 15 +------- packages/benchmark/src/runReport/format.ts | 27 ++++++++++---- packages/benchmark/src/runReport/schema.ts | 5 +++ packages/benchmark/src/runner/printReport.ts | 3 +- .../benchmark/src/runner/runInterleaved.ts | 5 ++- 15 files changed, 117 insertions(+), 96 deletions(-) create mode 100644 packages/benchmark/src/gc.ts diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index b04db91957..93d9b3ac88 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -40,6 +40,7 @@ const slower: RunBenchmark = { current: { render: [22, 24, 23, 25], 'bench:paint': [30, 31, 30, 31] }, baseline: { render: [20, 22, 21, 23], 'bench:paint': [30, 31, 30, 31] }, }, + sampling: { sampleSize: 3, timedOut: true }, }; const libraries: RunBenchmark = { @@ -81,26 +82,26 @@ describe('buildBenchmarkRunMarkdownReport', () => { 🔴 **Grid scroll** · render · \`worse +9.3% – +9.3%\` - 2 benchmarks measured · 1 with regressions · 1 failed + 2 benchmarks measured · 1 with regressions · 1 timed out · 1 failed ❌ **Chart zoom**: Render failed
    Full results - **Grid scroll** + **Grid scroll** · 3 + 1 rounds · timed out - | Metric | current | baseline | Δ vs baseline | Rounds | - |:----------|----------:|----------:|:----------|----------:| - | render | 23.5 ms | 21.5 ms | \`worse +9.3% – +9.3%\` | 4 | - | bench:paint | 30.5 ms | 30.5 ms | \`unsure +0.0% – +0.0%\` | 4 | + | Metric | current | baseline | Δ vs baseline | + |:----------|----------:|----------:|:----------| + | render | 23.5 ms | 21.5 ms | \`worse +9.3% – +9.3%\` | + | bench:paint | 30.5 ms | 30.5 ms | \`unsure +0.0% – +0.0%\` | - **libs / mount** + **libs / mount** · 4 rounds - | Metric | ours | theirs | theirs vs ours | Rounds | - |:----------|----------:|----------:|:----------|----------:| - | render | 10 ms | 20.5 ms | \`worse +95.8% – +114.2%\` | 4 | + | Metric | ours | theirs | theirs vs ours | + |:----------|----------:|----------:|:----------| + | render | 10 ms | 20.5 ms | \`worse +95.8% – +114.2%\` | _Each value is a median. Each Δ is a 95% confidence interval on the paired per-round difference, relative to the variant it is measured against; "unsure" means it straddles zero — the expected result for two equivalent builds._ diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 037c108b36..38e4d2ec2d 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -3,6 +3,7 @@ import { analyzeRun, benchmarkTable, findRegressions, + formatRounds, formatComparison, formatRunSummary, RUN_REPORT_FOOTNOTE, @@ -34,7 +35,7 @@ function renderBenchmarkTable(analysis: BenchmarkAnalysis): string { ), ), ); - return `**${analysis.benchmark.name}**\n\n${table}`; + return `**${analysis.benchmark.name}** · ${formatRounds(analysis.benchmark)}\n\n${table}`; } /** diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index e49d523446..5d61cae263 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -13,9 +13,15 @@ import TableCell from '@mui/material/TableCell'; import TableContainer from '@mui/material/TableContainer'; import TableHead from '@mui/material/TableHead'; import TableRow from '@mui/material/TableRow'; +import Tooltip from '@mui/material/Tooltip'; import Typography from '@mui/material/Typography'; import { fetchCiReport, isBenchmarkRunUpload } from '@/utils/fetchCiReport'; -import { analyzeRun, benchmarkTable, RUN_REPORT_FOOTNOTE } from '@mui/internal-benchmark/runReport'; +import { + analyzeRun, + benchmarkTable, + formatRounds, + RUN_REPORT_FOOTNOTE, +} from '@mui/internal-benchmark/runReport'; import type { BenchmarkAnalysis, BenchmarkRunReport, @@ -47,7 +53,22 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { {benchmark.name}{' '} - {benchmark.file} + {benchmark.file} ·{' '} + + + {formatRounds(benchmark)} + +
    diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index f6960f8598..2e8ff44a9a 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -422,7 +422,7 @@ the dashboard, which renders the pull request comment's Performance section and history from it. - The report holds **raw samples**, round-aligned across variants, plus each metric's kind, format - and alarm, the builds, and the environment. + and alarm, how each benchmark's sampling went, the builds, and the environment. - `analyzeRun` from `@mui/internal-benchmark/runReport` draws every conclusion from it: a 95% confidence interval on the paired difference per metric, a change (`better`, `worse`, `unsure`), and a severity from the metric's alarm. diff --git a/packages/benchmark/src/caseRuntime.tsx b/packages/benchmark/src/caseRuntime.tsx index dbe0e0e6f2..d9a4b0b829 100644 --- a/packages/benchmark/src/caseRuntime.tsx +++ b/packages/benchmark/src/caseRuntime.tsx @@ -68,14 +68,6 @@ export function splitCaseArgs( : { interaction: undefined, options: interactionOrOptions ?? maybeOptions }; } -export function warnIfNoGc(): void { - if (typeof window.gc !== 'function') { - console.warn( - 'window.gc is not available. Run with --js-flags=--expose-gc for consistent GC between iterations.', - ); - } -} - function BenchProfiler({ captures, recording, @@ -104,29 +96,6 @@ function BenchProfiler({ ); } -// Double GC: the first pass collects garbage, the second catches weak refs -// and prevent leaking into the next iteration. -function forceGC() { - if (typeof window.gc === 'function') { - window.gc(); - window.gc(); - } -} - -declare global { - interface Window { - gc?: () => void; - } -} - -// Flush pending microtasks and React cleanup effects (e.g. from a previous unmount) -// so they don't interfere with the next iteration's timing. -function settle(): Promise { - return new Promise((resolve) => { - setTimeout(resolve, 0); - }); -} - function supportsElementTiming(): boolean { return PerformanceObserver.supportedEntryTypes.includes('element'); } @@ -298,10 +267,6 @@ export async function measureIteration( // `reactRecordingPaused` is set; the interaction callback drives it from there. const recording = createReactRecordingControls(!(options?.reactRecordingPaused ?? false)); - // Drain event loop from previous unmount, then double GC for thorough cleanup - await settle(); - forceGC(); - const captures: RenderEvent[] = []; const timing = createElementTimingWaiter(); // Asserted rather than annotated: only the callback assigns it, which narrowing cannot see. diff --git a/packages/benchmark/src/format.test.ts b/packages/benchmark/src/format.test.ts index 6df341c019..317a736ccb 100644 --- a/packages/benchmark/src/format.test.ts +++ b/packages/benchmark/src/format.test.ts @@ -98,7 +98,7 @@ describe('printTable', () => { consoleSpy.mockRestore(); }); - it('truncates long titles with ellipsis', () => { + it('widens the table to fit a long title', () => { const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); const longTitle = 'A'.repeat(200); @@ -115,18 +115,11 @@ describe('printTable', () => { const calls = consoleSpy.mock.calls.map((call) => call[0] as string); - // Title top border determines the table width - const topBorder = calls[0]; - const tableWidth = stripAnsi(topBorder).length; - - // Title line should not exceed the table width - const titleLine = calls[1]; - expect(stripAnsi(titleLine).length).toBe(tableWidth); - - // Title should end with ellipsis and a trailing space before the closing border - const strippedTitle = stripAnsi(titleLine); - expect(strippedTitle).toContain('…'); - expect(strippedTitle).toMatch(/… │$/); + const tableWidth = stripAnsi(calls[0]).length; + const titleLine = stripAnsi(calls[1]); + expect(titleLine).toBe(`│ ${longTitle} │`); + // Every line of the table is as wide as the title line. + expect(calls.every((line) => stripAnsi(line).length === tableWidth)).toBe(true); consoleSpy.mockRestore(); }); diff --git a/packages/benchmark/src/format.ts b/packages/benchmark/src/format.ts index f7a069b553..2cea55422b 100644 --- a/packages/benchmark/src/format.ts +++ b/packages/benchmark/src/format.ts @@ -16,13 +16,6 @@ interface Column { width: number; } -function truncate(str: string, maxLength: number): string { - if (str.length <= maxLength) { - return str; - } - return `${str.slice(0, maxLength - 1)}…`; -} - export function printTable( columns: Column[], rows: string[][], @@ -30,11 +23,19 @@ export function printTable( title?: string, ): void { const colWidths = columns.map((col) => col.width); + // Widen the last column rather than cut the title short. + const shortBy = + (title?.length ?? 0) + + 2 - + (colWidths.reduce((sum, w) => sum + w + 2, 0) + colWidths.length - 1); + if (shortBy > 0) { + colWidths[colWidths.length - 1] += shortBy; + } const totalInner = colWidths.reduce((sum, w) => sum + w + 2, 0) + colWidths.length - 1; if (title) { const titleTop = dim(`┌${'─'.repeat(totalInner)}┐`); - const titleContent = ` ${truncate(title, totalInner - 2)}`; + const titleContent = ` ${title}`; const titlePadding = totalInner - titleContent.length; const titleLine = dim('│') + titleContent + ' '.repeat(Math.max(0, titlePadding)) + dim('│'); const titleSep = dim(`├${colWidths.map((w) => '─'.repeat(w + 2)).join('┬')}┤`); @@ -52,7 +53,7 @@ export function printTable( } const headerSep = dim(`├${colWidths.map((w) => '─'.repeat(w + 2)).join('┼')}┤`); - const headerCells = columns.map((col) => ` ${col.header.padStart(col.width)} `); + const headerCells = columns.map((col, i) => ` ${col.header.padStart(colWidths[i])} `); const headerLine = dim('│') + headerCells.join(dim('│')) + dim('│'); // eslint-disable-next-line no-console diff --git a/packages/benchmark/src/gc.ts b/packages/benchmark/src/gc.ts new file mode 100644 index 0000000000..7ffbde4a30 --- /dev/null +++ b/packages/benchmark/src/gc.ts @@ -0,0 +1,27 @@ +declare global { + interface Window { + gc?: () => void; + } +} + +let warned = false; + +/** + * Lets pending tasks finish — a previous iteration's cleanup — then collects garbage twice, the + * second pass catching what the first freed weak references to, so no iteration pays for another's + * allocations. Needs `--js-flags=--expose-gc`, and warns once without it. + */ +export async function collectGarbage(): Promise { + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); + if (typeof window.gc === 'function') { + window.gc(); + window.gc(); + } else if (!warned) { + warned = true; + console.warn( + 'window.gc is not available. Run with --js-flags=--expose-gc for consistent GC between iterations.', + ); + } +} diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index cb985d4fe2..dd8317a49f 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -7,6 +7,7 @@ import { ScalarMetric } from './ScalarMetric'; import { metricsGate } from './metricsGate'; import { runProfileSession } from './profileSession'; import { createInput } from './input'; +import { collectGarbage } from './gc'; import { createCaseRuntime, createElementTimingWaiter, @@ -14,7 +15,6 @@ import { MILLISECONDS, PAINT_METRIC_NAME, splitCaseArgs, - warnIfNoGc, } from './caseRuntime'; import type { BenchmarkInteraction, BenchmarkOptions, CaseOptions } from './caseRuntime'; // Installs the Vitest metric recorder. @@ -62,6 +62,7 @@ async function runIteration( ): Promise { // Custom metrics recorded inside the case honor warmup exclusion through the gate, the same way // renders and `bench:paint` are excluded during warmup. + await collectGarbage(); metricsGate.setRecordingEnabled(test, !warmup); try { const { renders, paints } = await measureIteration(renderFn, interaction, options, input); @@ -113,8 +114,6 @@ export function benchmark( const runs = options?.runs ?? 20; const warmupRuns = options?.warmupRuns ?? 10; - warnIfNoGc(); - const iterations: IterationData[] = []; try { for (let i = 0; i < warmupRuns + runs; i += 1) { diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 9d22952ee0..5750d2416d 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -1,3 +1,4 @@ +import { collectGarbage } from '../gc'; import { createInput } from '../input'; import type { BenchmarkInput } from '../input'; import { seriesName, setMetricRecorder } from '../metricCore'; @@ -150,6 +151,7 @@ async function sample(name: string): Promise> { if (!run) { throw new Error(`No benchmark named "${name}". Known: ${[...cases.keys()].join(', ')}`); } + await collectGarbage(); performance.clearMarks(); performance.clearMeasures(); diff --git a/packages/benchmark/src/page/reactBenchmark.tsx b/packages/benchmark/src/page/reactBenchmark.tsx index 3bf8914632..5e30c5e837 100644 --- a/packages/benchmark/src/page/reactBenchmark.tsx +++ b/packages/benchmark/src/page/reactBenchmark.tsx @@ -1,11 +1,5 @@ import type * as React from 'react'; -import { - measureIteration, - MILLISECONDS, - PAINT_METRIC_NAME, - splitCaseArgs, - warnIfNoGc, -} from '../caseRuntime'; +import { measureIteration, MILLISECONDS, PAINT_METRIC_NAME, splitCaseArgs } from '../caseRuntime'; import type { BenchmarkInteraction, CaseOptions } from '../caseRuntime'; import { DiscreteMetric } from '../DiscreteMetric'; import { ScalarMetric } from '../ScalarMetric'; @@ -31,9 +25,6 @@ function phaseMetricOf(phase: string): ScalarMetric { return metric; } -// Every iteration forces a GC before mounting, which needs `--expose-gc`; warned about once. -let checkedGc = false; - /** * Registers a React benchmark case: every sample mounts `renderFn()`, runs the interaction, waits * for the paint and unmounts. It records the renders React's profiler captured and the time to @@ -45,10 +36,6 @@ export function reactBenchmark( interactionOrOptions?: BenchmarkInteraction | ReactBenchmarkOptions, maybeOptions?: ReactBenchmarkOptions, ): BenchmarkCase { - if (!checkedGc) { - checkedGc = true; - warnIfNoGc(); - } const { interaction, options } = splitCaseArgs(interactionOrOptions, maybeOptions); const run = async ({ input }: BenchmarkContext) => { diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index ede64966a7..55512bc69f 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -59,14 +59,13 @@ export interface BenchmarkTableRow { } /** - * A benchmark's results as one table, for every renderer of it: per metric, each variant's median, - * each comparison, and the rounds measured. + * A benchmark's results as one table, for every renderer of it: per metric, each variant's median + * and each comparison. */ export function benchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): { columns: BenchmarkTableColumn[]; rows: BenchmarkTableRow[]; } { - const [reference] = benchmark.variants; const columns: BenchmarkTableColumn[] = [ { header: 'Metric', kind: 'label' }, ...benchmark.variants.map((variant) => ({ header: variant, kind: 'value' as const })), @@ -74,7 +73,6 @@ export function benchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): { header: formatComparisonLabel(benchmark, comparison), kind: 'comparison' as const, })), - { header: 'Rounds', kind: 'value' }, ]; const rows = metrics.map(({ metric, definition, variants, comparisons }) => ({ metric, @@ -83,20 +81,37 @@ export function benchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): { metric, ...benchmark.variants.map((variant) => formatValue(variants[variant].median, definition)), ...comparisons.map(formatComparison), - String(variants[reference].count), ], })); return { columns, rows }; } -/** The line a run is summed up in: how many benchmarks measured, regressed and failed. */ +/** + * How many rounds a benchmark ran, and whether sampling resolved: `50 rounds`, `50 + 23 rounds` past + * the sample size, `50 + 312 rounds · timed out`. `null` for a benchmark that measured nothing. + */ +export function formatRounds(benchmark: RunBenchmark): string | null { + const [reference] = benchmark.variants; + const rounds = Object.values(benchmark.samples?.[reference] ?? {})[0]?.length; + if (rounds === undefined) { + return null; + } + const sampleSize = benchmark.sampling?.sampleSize ?? rounds; + const counted = + rounds > sampleSize ? `${sampleSize} + ${rounds - sampleSize} rounds` : `${rounds} rounds`; + return benchmark.sampling?.timedOut ? `${counted} · timed out` : counted; +} + +/** The line a run is summed up in: how many benchmarks measured, regressed, timed out and failed. */ export function formatRunSummary(analyses: BenchmarkAnalysis[], regressions: Regression[]): string { const measured = analyses.filter((analysis) => !analysis.benchmark.error).length; const failed = analyses.length - measured; const regressed = new Set(regressions.map((regression) => regression.benchmark)).size; + const timedOut = analyses.filter((analysis) => analysis.benchmark.sampling?.timedOut).length; return [ `${measured} benchmark${measured === 1 ? '' : 's'} measured`, regressed > 0 ? `${regressed} with regressions` : 'no regressions', + timedOut > 0 ? `${timedOut} timed out` : null, failed > 0 ? `${failed} failed` : null, ] .filter(Boolean) diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts index e708fcfdcc..eef2a04f46 100644 --- a/packages/benchmark/src/runReport/schema.ts +++ b/packages/benchmark/src/runReport/schema.ts @@ -56,6 +56,11 @@ const benchmarkSchema = z.object({ variants: z.array(z.string()), /** Per variant, per metric: one value per round, round-aligned across variants and metrics. */ samples: z.record(z.string(), z.record(z.string(), z.array(z.number()))).optional(), + /** + * How sampling went: rounds past `sampleSize` were added while a difference was unresolved, and + * `timedOut` says it was still unresolved when the timeout stopped it. + */ + sampling: z.object({ sampleSize: z.number(), timedOut: z.boolean() }).optional(), /** Why the benchmark produced no samples. */ error: z.string().optional(), }); diff --git a/packages/benchmark/src/runner/printReport.ts b/packages/benchmark/src/runner/printReport.ts index e8316ba367..01e629a05a 100644 --- a/packages/benchmark/src/runner/printReport.ts +++ b/packages/benchmark/src/runner/printReport.ts @@ -4,6 +4,7 @@ import { dim, printTable, red, yellow } from '../format'; import { analyzeRun, benchmarkTable, + formatRounds, findRegressions, formatComparison, formatRunSummary, @@ -21,7 +22,7 @@ function printBenchmark(analysis: BenchmarkAnalysis): void { })), cells, undefined, - `${analysis.benchmark.name} (${analysis.benchmark.file})`, + `${analysis.benchmark.name} (${analysis.benchmark.file} · ${formatRounds(analysis.benchmark)})`, ); } diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index 7322dfb9c2..193d0c3714 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -222,7 +222,10 @@ async function runBenchmark( ), ); } - return { benchmark: benchmarkOf(), metrics }; + return { + benchmark: { ...benchmarkOf(), sampling: { sampleSize, timedOut: !resolved } }, + metrics, + }; } catch (error) { console.error(chalk.red(` ${errorMessage(error)}`)); return { benchmark: { ...entry, error: errorMessage(error) }, metrics: {} }; From 15ce6f2c63534094a2af18cf356f00755c977b9e Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:28:12 +0200 Subject: [PATCH 110/173] [benchmark] Run benchmark run on 3 vCPUs; warn below 3 logical CPUs On 2 logical CPUs V8's background compilers are starved, so a page can settle into a slower tier of the same code for a whole benchmark, which reads as a difference between identical builds: 17% of comparisons on untouched code resolved across this PR's CI runs. An isolated repro on 2 pinned CPUs showed identical pages at two persistent speeds, gone on 3 or 4. test_benchmark_run moves to medium+, and benchmark run prints the logical CPU count and warns below 3. --- .circleci/config.yml | 3 +++ packages/benchmark/README.md | 4 ++++ packages/benchmark/src/runner/runBenchmarks.ts | 17 +++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index b124bdf2cf..61b7ac8c30 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,6 +120,9 @@ jobs: executor: name: code-infra/mui-node-browser playwright-img-version: 'v1.62.1-noble' + # 2 vCPUs starve V8's background compilers, and pages of identical code then settle at different + # speeds; see the benchmark package's README. + resource_class: medium+ steps: - checkout - code-infra/install-deps diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 2e8ff44a9a..2a452c9621 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -387,6 +387,10 @@ per-round differences rather than on two independent sets of samples. Whatever t during a round — thermal throttling, a background process — then affects both sides of it and cancels out. 10 warmup rounds are discarded once per benchmark before any is measured. +Run it with at least 3 logical CPUs (vCPUs in CI); it warns with fewer. On 2, V8's background +compilers are starved, and a page can settle into a slower tier of the same code for the whole +benchmark, which shows up as a difference between identical builds. + ### Sampling How many rounds a benchmark is measured for adapts to its results, the way tachometer's diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index 41fd4e3bb6..d8fba1be6c 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -1,5 +1,6 @@ /* eslint-disable no-console */ +import * as os from 'node:os'; import * as path from 'node:path'; import { mkdir, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; @@ -17,6 +18,9 @@ import { printRunReport } from './printReport'; import { publishRunReport } from './upload'; import { buildsDirOf, prepareOutputDir, resultsPathOf } from './outputDir'; +/** Fewer logical CPUs than this let pages of identical code settle at different speeds. */ +const MIN_CPUS = 3; + export interface RunBenchmarksOptions { /** The harness package directory (where the command was run). */ harnessDir: string; @@ -96,6 +100,19 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise benchFile.file).join(', ')}`)); console.log(chalk.cyan(`Refs: ${refs.map(describeRef).join(', ')}`)); console.log(chalk.cyan(`Browser: ${browserBinary} ${launchArgs.join(' ')}`)); + // Logical CPUs (hardware threads, vCPUs in CI), not physical cores; it honours CPU affinity, and + // in practice a container's CPU quota. + const cpus = os.availableParallelism(); + console.log(chalk.cyan(`CPUs: ${cpus}`)); + if (cpus < MIN_CPUS) { + console.warn( + chalk.yellow( + `Only ${cpus} logical CPUs are available; use at least ${MIN_CPUS}. With fewer, V8's background ` + + 'compilers are starved, so a page can settle into a slower tier of the same code and stay ' + + 'there — which a comparison reads as a difference between builds.', + ), + ); + } try { // Pack both sides at once: each builds in its own checkout, and nothing is measured yet. From 838e9950d9d8e5eccc3267cd91d3999a995c6531 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:56:14 +0200 Subject: [PATCH 111/173] [benchmark] Sample every variant in a freshly loaded page of one shared tab Each benchmark now runs in one tab that loads the variant to be measured before every sample, with garbage collected right before it. A page or process that settles into a slower JIT tier can no longer favour one variant for the whole benchmark, which paired rounds read as a real difference. A/A runs under steady CPU load: var(z) 1.13 against 4.13 for a page per variant. There is no warmup any more, so the report drops sampling.warmup. --- .circleci/config.yml | 4 +- .../benchmarkRun/buildMarkdownReport.test.ts | 1 - .../src/views/BenchmarkRunDetails.tsx | 4 +- packages/benchmark/README.md | 21 ++- packages/benchmark/src/page/page.tsx | 9 +- .../src/runReport/analyzeRun.test.ts | 1 - packages/benchmark/src/runReport/schema.ts | 2 - .../benchmark/src/runner/runBenchmarks.ts | 11 +- .../benchmark/src/runner/runInterleaved.ts | 155 +++++++++--------- .../benchmark/src/runner/serveDirectory.ts | 4 +- test/benchmark/src/list/list.bench.tsx | 4 +- 11 files changed, 109 insertions(+), 107 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 61b7ac8c30..3d6c61e3fe 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -120,8 +120,8 @@ jobs: executor: name: code-infra/mui-node-browser playwright-img-version: 'v1.62.1-noble' - # 2 vCPUs starve V8's background compilers, and pages of identical code then settle at different - # speeds; see the benchmark package's README. + # 2 vCPUs starve V8's background compilers, which makes the same code's speed vary from sample to + # sample; see the benchmark package's README. resource_class: medium+ steps: - checkout diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index 93d9b3ac88..49d60b1ad9 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -10,7 +10,6 @@ function reportOf(benchmarks: RunBenchmark[]): BenchmarkRunReport { generatedAt: '2026-01-01T00:00:00.000Z', head: { sha: 'abc' }, environment: { browser: 'Chromium 151', platform: 'linux', arch: 'x64', launchArgs: [] }, - sampling: { warmup: 2 }, builds: { current: { label: 'working tree' }, baseline: { sha: 'def', label: 'HEAD~1' } }, metrics: { render: { kind: 'scalar', format: MS, alarm: {} }, diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index 5d61cae263..b31cd3a8ba 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -196,8 +196,8 @@ export default function BenchmarkRunDetails() { {RUN_REPORT_FOOTNOTE} - {report.sampling.warmup} warmup rounds per benchmark · {report.environment.browser} on{' '} - {report.environment.platform}/{report.environment.arch} + {report.environment.browser} on {report.environment.platform}/ + {report.environment.arch} {report.environment.launchArgs.length > 0 && ` · ${report.environment.launchArgs.join(' ')}`} diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 2a452c9621..7630cdd042 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -370,26 +370,29 @@ compare('scatter', [ ]); ``` -Every case is measured in a page of its own, but a page loads the whole benchmark file, so a static -import reaches every case's page. Import what only one case needs inside that case: the first -iteration is a warmup, so the `await import()` is never measured. +Every sample loads the whole benchmark file in a fresh page, so a static import reaches every case. +Import what only one case needs inside that case, before the part it times: it then loads in that +case's samples only. ### How it measures The plugin generates a page per benchmark file under `src/__bench__/`. A benchmark file that imports `vitest`, or the Vitest entry `@mui/internal-benchmark`, fails the build. The runner drives Chromium -through Playwright and gives every variant — the current and the baseline build, or each case of a -`compare()` — a page of its own, in a browser context of its own, that stays open for the whole -benchmark: every sample is one warm iteration, and module-scope data is built once. +through Playwright and runs each benchmark in one tab, which loads the page of the variant to be +measured — the current or the baseline build, or a case of a `compare()` — before every sample. +Every sample is the first iteration of a freshly loaded page, with garbage collected right before +it, and every variant runs in the same renderer process. Whatever a page or a process holds on to — +the JIT tier its code settled into, the heap it grew — then never favours one variant for the whole +benchmark, which the paired rounds below could not tell from a real difference. Every variant is sampled once per round, in a shuffled order, and a difference is judged on the per-round differences rather than on two independent sets of samples. Whatever the machine was doing during a round — thermal throttling, a background process — then affects both sides of it and cancels -out. 10 warmup rounds are discarded once per benchmark before any is measured. +out. Run it with at least 3 logical CPUs (vCPUs in CI); it warns with fewer. On 2, V8's background -compilers are starved, and a page can settle into a slower tier of the same code for the whole -benchmark, which shows up as a difference between identical builds. +compilers are starved, so how fast the same code runs depends on how its compilation happened to go, +which costs the results precision. ### Sampling diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 5750d2416d..7bb767efe4 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -32,8 +32,8 @@ export interface BenchmarkCase { const cases = new Map(); /** - * Registers a benchmark case. The runner calls `run` once per sample, warmup included, for as many - * rounds as `sampling` asks. A case on its own is measured across the builds; one passed to + * Registers a benchmark case. The runner calls `run` once per sample, each time in a freshly loaded + * page, for as many rounds as `sampling` asks. A case on its own is measured across the builds; one passed to * `compare()` is measured against the other cases there instead, sampled as the `compare()` asks. */ export function benchmark( @@ -61,9 +61,8 @@ const comparisons: Array<{ * Compares cases with each other — one library's implementation against another's, say — on the * working tree's build, rather than each against its baseline. The first case is the reference. * - * Every case is measured in a page of its own, but a page loads the whole file. Import what only one - * case needs inside that case (`await import()` resolves during warmup) to keep it out of the others' - * pages. + * Every sample loads the whole file in a fresh page. Import what only one case needs inside that + * case, before the part it times, to keep it out of the others' samples. */ export function compare( name: string, diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index e290c20f7b..005023f476 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -26,7 +26,6 @@ function reportOf(benchmarks: RunBenchmark[], metrics: BenchmarkRunReport['metri generatedAt: '2026-01-01T00:00:00.000Z', head: { sha: 'abc' }, environment: { browser: 'Chromium', platform: 'linux', arch: 'x64', launchArgs: [] }, - sampling: { warmup: 2 }, builds: { current: { label: 'working tree' }, baseline: { sha: 'def', label: 'HEAD~1' } }, metrics, benchmarks, diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts index eef2a04f46..ea1bff3589 100644 --- a/packages/benchmark/src/runReport/schema.ts +++ b/packages/benchmark/src/runReport/schema.ts @@ -75,8 +75,6 @@ export const benchmarkRunReportSchema = z.object({ arch: z.string(), launchArgs: z.array(z.string()), }), - /** Rounds per benchmark vary with its sampling options; warmup is the same for every one. */ - sampling: z.object({ warmup: z.number() }), /** What the `baseline`-kind variant names refer to. */ builds: z.record(z.string(), buildSchema), /** Every metric the benchmarks report, keyed by name; `name#id` sub-series share `name`'s entry. */ diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index d8fba1be6c..e42e410e5b 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -18,7 +18,7 @@ import { printRunReport } from './printReport'; import { publishRunReport } from './upload'; import { buildsDirOf, prepareOutputDir, resultsPathOf } from './outputDir'; -/** Fewer logical CPUs than this let pages of identical code settle at different speeds. */ +/** Fewer logical CPUs than this starve V8's background compilers. */ const MIN_CPUS = 3; export interface RunBenchmarksOptions { @@ -43,9 +43,6 @@ export interface RunBenchmarksOptions { upload?: boolean; } -/** Discarded rounds before measuring, once per benchmark. */ -const WARMUP = 10; - /** * Benchmarks a harness's `*.bench.tsx` files across two builds of the workspace — the working tree * and a baseline — and writes, prints and optionally uploads the run report. @@ -108,8 +105,8 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise [ref.variant, { sha: ref.sha, label: refLabel(ref) }]), ), diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index 193d0c3714..a30ed8ba09 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -13,11 +13,14 @@ import { serveDirectory } from './serveDirectory'; import type { BenchPage } from '../page/page'; /** - * Runs the cases of `*.bench.tsx` files in Playwright. Every variant gets a page of its own, in a - * browser context of its own, that stays open for the whole benchmark: every sample is one - * iteration, so module-scope data and the JIT stay warm, and interactions get trusted input through - * a CDP session the page is bridged to. Variants are sampled once per round in a shuffled order, so - * the samples are round-aligned and can be compared on paired differences. + * Runs the cases of `*.bench.tsx` files in Playwright. A benchmark runs in one tab, which loads the + * variant to be measured before every sample: every sample is the first iteration of a freshly + * loaded page, and every variant runs in the same renderer process. Whatever a page or a process + * holds on to — the JIT tier its code settled into, the heap it grew — then never favours one + * variant for the whole benchmark, which paired rounds could not tell from a real difference. + * Interactions get trusted input through a CDP session the tab is bridged to. Variants are sampled + * once per round in a shuffled order, so the samples are round-aligned and can be compared on paired + * differences. */ export interface RunInterleavedOptions { @@ -28,8 +31,6 @@ export interface RunInterleavedOptions { benchRefs: ResolvedRef[]; browserBinary: string; launchArgs: string[]; - /** Discarded rounds before measuring, once per benchmark. */ - warmup: number; } export interface InterleavedResults { @@ -75,25 +76,39 @@ function stableMetrics(rounds: Round[]): string[] { /** What a benchmark file's page defines. */ type PageListing = Pick; -interface OpenedPage { +interface BenchTab { context: BrowserContext; page: Page; - listing: PageListing; } -async function openBenchPage(browser: Browser, url: string): Promise { +/** Opens a tab, in a browser context of its own, bridged to a CDP session for trusted input. */ +async function openBenchTab(browser: Browser): Promise { const context = await browser.newContext({ viewport: BENCHMARK_VIEWPORT }); - const page = await context.newPage(); - page.setDefaultTimeout(SAMPLE_TIMEOUT_MS); - page.on('pageerror', (error) => console.error(chalk.red(` page error: ${error.message}`))); - const session = await context.newCDPSession(page); - // Interactions ask for trusted input through this bridge. `send` only accepts the protocol - // methods Playwright knows by name; the page may ask for any, and Chrome rejects unknown ones. - await page.exposeBinding( - 'benchmarkCdp', - (_source, method: string, params?: Record) => - session.send(method as CdpMethod, params), - ); + try { + const page = await context.newPage(); + page.setDefaultTimeout(SAMPLE_TIMEOUT_MS); + page.on('pageerror', (error) => console.error(chalk.red(` page error: ${error.message}`))); + const session = await context.newCDPSession(page); + // Interactions ask for trusted input through this bridge, which outlives navigations, so it + // serves every page the tab loads. `send` only accepts the protocol methods Playwright knows by + // name; the page may ask for any, and Chrome rejects unknown ones. + await page.exposeBinding( + 'benchmarkCdp', + (_source, method: string, params?: Record) => + session.send(method as CdpMethod, params), + ); + return { context, page }; + } catch (error) { + await context.close(); + throw error; + } +} + +/** Loads a benchmark page into the tab, and waits until it has registered its cases. */ +async function loadBenchPage( + page: Page, + url: string, +): Promise { await page.goto(url); const ready = await page.waitForFunction(() => { if (window.benchmarkPage === undefined) { @@ -107,35 +122,28 @@ async function openBenchPage(browser: Browser, url: string): Promise }); const state = await ready.jsonValue(); if (!state) { - await context.close(); throw new Error(`${url} never got ready.`); } - if (state.visibility !== 'visible') { - console.warn(chalk.yellow(` ${url} is ${state.visibility}; rAF-based waits may stall.`)); - } - return { context, page, listing: state }; -} - -async function closeBenchPages(targets: OpenedPage[]): Promise { - await Promise.all(targets.map((target) => target.context.close())); -} - -async function openBenchPages(browser: Browser, urls: string[]): Promise { - const settled = await Promise.allSettled(urls.map((url) => openBenchPage(browser, url))); - const opened = settled.flatMap((result) => (result.status === 'fulfilled' ? [result.value] : [])); - const failed = settled.find((result) => result.status === 'rejected'); - if (failed) { - await closeBenchPages(opened); - throw failed.reason; - } - return opened; + return state; } -/** Opens pages just to learn what they define; every benchmark then measures in fresh ones. */ +/** Loads pages just to learn what they define. */ async function listBenchPages(browser: Browser, urls: string[]): Promise { - const opened = await openBenchPages(browser, urls); - await closeBenchPages(opened); - return opened.map((target) => target.listing); + const tab = await openBenchTab(browser); + try { + const listings: PageListing[] = []; + for (const url of urls) { + // eslint-disable-next-line no-await-in-loop + const { visibility, ...listing } = await loadBenchPage(tab.page, url); + if (visibility !== 'visible') { + console.warn(chalk.yellow(` ${url} is ${visibility}; rAF-based waits may stall.`)); + } + listings.push(listing); + } + return listings; + } finally { + await tab.context.close(); + } } /** Runs one iteration of a case, and returns what it recorded, by series. */ @@ -163,47 +171,48 @@ interface CaseResult { type BenchmarkEntry = Pick; /** - * Measures one benchmark in fresh pages that stay open for all of it, so iterations stay warm and - * module-scope data is built once. Warmup and measured rounds alike run every slot once, in a - * shuffled order. After `sampleSize` rounds it keeps adding rounds while a difference is unresolved - * against the horizons, until the timeout. A failure is reported as the benchmark's error. + * Measures one benchmark in a tab of its own, loading each slot's page before sampling it. Every + * round samples every slot once, in a shuffled order. After `sampleSize` rounds it keeps adding + * rounds while a difference is unresolved against the horizons, until the timeout. A failure is + * reported as the benchmark's error. */ async function runBenchmark( browser: Browser, entry: BenchmarkEntry, slots: PageSlot[], sampling: Required, - options: RunInterleavedOptions, ): Promise { console.log(chalk.cyan(`\nRunning "${entry.name}" (${entry.file})…`)); const { sampleSize, timeout, autoSampleConditions } = sampling; const horizons = parseHorizons(autoSampleConditions); - let targets: OpenedPage[] = []; - const runRound = async (): Promise => { - const round: Round = []; - for (const index of shuffledIndices(targets.length)) { - // eslint-disable-next-line no-await-in-loop - round[index] = await sampleBenchCase(targets[index].page, slots[index].caseName); - } - return round; - }; + let tab: BenchTab | undefined; try { - targets = await openBenchPages( - browser, - slots.map((slot) => slot.url), - ); - for (let roundIndex = 0; roundIndex < options.warmup; roundIndex += 1) { - // eslint-disable-next-line no-await-in-loop - await runRound(); - } + tab = await openBenchTab(browser); + const { page } = tab; + let metrics: Record = {}; + const runRound = async (): Promise => { + const round: Round = []; + for (const index of shuffledIndices(slots.length)) { + // eslint-disable-next-line no-await-in-loop + await loadBenchPage(page, slots[index].url); + // eslint-disable-next-line no-await-in-loop + round[index] = await sampleBenchCase(page, slots[index].caseName); + if (index === 0) { + // Read from the reference page: a metric whose config the change under test altered is + // described the way the change describes it. A freshly loaded page knows the metrics it + // recorded itself. + // eslint-disable-next-line no-await-in-loop + metrics = await page.evaluate(() => window.benchmarkPage!.metricDefinitions()); + } + } + return round; + }; + const rounds: Round[] = []; for (let roundIndex = 0; roundIndex < sampleSize; roundIndex += 1) { // eslint-disable-next-line no-await-in-loop rounds.push(await runRound()); } - // Read from the reference page: a metric whose config the change under test altered is - // described the way the change describes it. Every metric has been recorded by now. - const metrics = await targets[0].page.evaluate(() => window.benchmarkPage!.metricDefinitions()); const benchmarkOf = (): RunBenchmark => ({ ...entry, samples: samplesOf(slots, rounds) }); const isSettled = () => differencesResolved(compareBenchmark(metrics, benchmarkOf()), horizons); @@ -230,7 +239,7 @@ async function runBenchmark( console.error(chalk.red(` ${errorMessage(error)}`)); return { benchmark: { ...entry, error: errorMessage(error) }, metrics: {} }; } finally { - await closeBenchPages(targets); + await tab?.context.close(); } } @@ -292,14 +301,14 @@ async function runBenchFile( } const slots = urls.map((url, index) => ({ variant: variants[index], url, caseName })); // eslint-disable-next-line no-await-in-loop - results.push(await runBenchmark(browser, entry, slots, sampling, options)); + results.push(await runBenchmark(browser, entry, slots, sampling)); } for (const { name, cases, sampling } of reference.comparisons) { const entry: BenchmarkEntry = { name, file: benchFile.file, kind: 'compare', variants: cases }; const slots = cases.map((caseName) => ({ variant: caseName, url: urls[0], caseName })); // eslint-disable-next-line no-await-in-loop - results.push(await runBenchmark(browser, entry, slots, sampling, options)); + results.push(await runBenchmark(browser, entry, slots, sampling)); } return results; } diff --git a/packages/benchmark/src/runner/serveDirectory.ts b/packages/benchmark/src/runner/serveDirectory.ts index 5112f55196..ff41cc5010 100644 --- a/packages/benchmark/src/runner/serveDirectory.ts +++ b/packages/benchmark/src/runner/serveDirectory.ts @@ -25,8 +25,8 @@ export interface StaticServer { } /** - * Serves a directory of built pages on a free local port. Nothing is cached by the server; the - * browser's own cache is per context, so each side of a comparison warms its own. + * Serves a directory of built pages on a free local port. Nothing is cached by the server; each + * build is served under its own path, so each side of a comparison warms its own browser cache. */ export async function serveDirectory(root: string): Promise { const server = http.createServer(async (request, response) => { diff --git a/test/benchmark/src/list/list.bench.tsx b/test/benchmark/src/list/list.bench.tsx index 5bd36b503a..16e11c6f59 100644 --- a/test/benchmark/src/list/list.bench.tsx +++ b/test/benchmark/src/list/list.bench.tsx @@ -5,8 +5,8 @@ import { ScrollingList } from '../_shared/ScrollingList'; import { SMOKE_SAMPLING } from '../_shared/sampling'; /** - * `reactBenchmark()` cases. Under `benchmark run` each case runs one iteration at a time in a page - * that stays open per build, with the builds alternating round by round. + * `reactBenchmark()` cases. Under `benchmark run` each sample runs one iteration in a freshly loaded + * page, with the builds alternating round by round. * * Like the `workload` benchmark, it imports a workspace package so both builds resolve it from their * own packed tarball. From 82ea68c7da2f7adf836a8bb3203d3fcedcee2bb1 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:19:12 +0200 Subject: [PATCH 112/173] [benchmark] Share one 5% false-alarm chance across a run's alarmed comparisons MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each comparison that can raise an alarm — an alarmed metric of a baseline benchmark — now gets a Bonferroni-corrected interval, so a run of unchanged code raises a false alarm at most 5% of the time instead of 5% per comparison. Comparisons that never vary take no share. Other metrics and compare() stay at 95%. On the A/A runs collected so far, runs with any false flag drop from 12-22% to 0% in recycle and fresh modes. Sampling resolves at the same level, counting one alarmed comparison for every other baseline benchmark, which is why every file is now listed before any is measured. The footnote states the levels. --- .../lib/benchmarkRun/buildMarkdownReport.ts | 4 +- .../src/views/BenchmarkRunDetails.tsx | 4 +- packages/benchmark/README.md | 8 + .../src/runReport/analyzeRun.test.ts | 93 ++++++++- .../benchmark/src/runReport/analyzeRun.ts | 183 +++++++++++++++--- .../benchmark/src/runReport/format.test.ts | 41 ++++ packages/benchmark/src/runReport/format.ts | 28 ++- packages/benchmark/src/runner/printReport.ts | 4 +- .../benchmark/src/runner/runInterleaved.ts | 73 ++++--- 9 files changed, 368 insertions(+), 70 deletions(-) create mode 100644 packages/benchmark/src/runReport/format.test.ts diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 38e4d2ec2d..7294247236 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -6,7 +6,7 @@ import { formatRounds, formatComparison, formatRunSummary, - RUN_REPORT_FOOTNOTE, + runReportFootnote, } from '@mui/internal-benchmark/runReport'; import type { BenchmarkAnalysis, BenchmarkRunReport } from '@mui/internal-benchmark/runReport'; @@ -76,7 +76,7 @@ export function buildBenchmarkRunMarkdownReport( if (measured.length > 0) { lines.push('
    ', 'Full results', ''); lines.push(measured.map(renderBenchmarkTable).join('\n\n')); - lines.push('', `_${RUN_REPORT_FOOTNOTE}_`, '', '
    '); + lines.push('', `_${runReportFootnote(report)}_`, '', ''); } if (options.detailsUrl) { diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index b31cd3a8ba..6191775046 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -20,7 +20,7 @@ import { analyzeRun, benchmarkTable, formatRounds, - RUN_REPORT_FOOTNOTE, + runReportFootnote, } from '@mui/internal-benchmark/runReport'; import type { BenchmarkAnalysis, @@ -193,7 +193,7 @@ export default function BenchmarkRunDetails() { {report && ( - {RUN_REPORT_FOOTNOTE} + {runReportFootnote(report)} {report.environment.browser} on {report.environment.platform}/ diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 7630cdd042..a487472fe6 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -390,6 +390,14 @@ per-round differences rather than on two independent sets of samples. Whatever t during a round — thermal throttling, a background process — then affects both sides of it and cancels out. +Intervals are 95%, except where a change can raise an alarm — an alarmed metric of a benchmark +measured against the baseline. Every such comparison in the run shares one 5% chance of a false alarm +(Bonferroni): with 5 of them, each is a 99% interval, so a run of unchanged code raises a false alarm +at most 5% of the time rather than in about one run in five. A comparison whose per-round differences +never vary — a render count that is the same in every round — can't be flagged by chance, so it takes +no share. Sampling resolves at the same level, counting one alarmed comparison for every other +benchmark in the run; the report counts them exactly. + Run it with at least 3 logical CPUs (vCPUs in CI); it warns with fewer. On 2, V8's background compilers are starved, so how fast the same code runs depends on how its compilation happened to go, which costs the results precision. diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index 005023f476..a22a1fa09d 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -5,8 +5,9 @@ import { findRegressions, meanInterval, median, - tCritical95, + tCritical, } from './analyzeRun'; +import type { BenchmarkAnalysis } from './analyzeRun'; import type { BenchmarkRunReport, RunBenchmark, RunMetricDefinition } from './schema'; const SCALAR: RunMetricDefinition = { kind: 'scalar', alarm: {} }; @@ -33,14 +34,22 @@ function reportOf(benchmarks: RunBenchmark[], metrics: BenchmarkRunReport['metri return report; } -describe('tCritical95', () => { - it('reads small samples from the table', () => { - expect(tCritical95(1)).toBe(12.706); - expect(tCritical95(29)).toBe(2.045); +describe('tCritical', () => { + it('matches the published table at 95%', () => { + expect(tCritical(1)).toBeCloseTo(12.706, 3); + expect(tCritical(9)).toBeCloseTo(2.262, 3); + expect(tCritical(29)).toBeCloseTo(2.045, 3); + }); + + it('matches the published table at other levels', () => { + expect(tCritical(1, 0.99)).toBeCloseTo(63.657, 2); + expect(tCritical(9, 0.99)).toBeCloseTo(3.25, 3); + expect(tCritical(29, 0.999)).toBeCloseTo(3.659, 3); }); it('approaches the normal quantile for large samples', () => { - expect(tCritical95(10_000)).toBeCloseTo(1.96, 2); + expect(tCritical(100_000)).toBeCloseTo(1.96, 2); + expect(tCritical(100_000, 0.99)).toBeCloseTo(2.576, 2); }); }); @@ -206,6 +215,78 @@ describe('analyzeRun', () => { }); }); + describe('the run-wide chance of a false alarm', () => { + // Per-round differences of 1, -2, 3, 0, 2, -1, 4, 1: noisy, so the interval has a width. + const noisy = { + current: [101, 98, 103, 100, 102, 99, 104, 101], + baseline: [100, 100, 100, 100, 100, 100, 100, 100], + }; + + function benchmarkOf(name: string, samples: Record>) { + return { + name, + file: `${name}.bench.tsx`, + kind: 'baseline' as const, + variants: ['current', 'baseline'], + samples, + }; + } + + const widthOf = (analysis: BenchmarkAnalysis) => { + const [comparison] = analysis.metrics[0].comparisons; + return comparison.absolute.high - comparison.absolute.low; + }; + + const alone = benchmarkOf('a', { + current: { render: noisy.current }, + baseline: { render: noisy.baseline }, + }); + const other = benchmarkOf('b', { + current: { render: noisy.current }, + baseline: { render: noisy.baseline }, + }); + + it('is shared by the alarmed comparisons of the run', () => { + const [single] = analyzeRun(reportOf([alone], { render: SCALAR })); + const [shared] = analyzeRun(reportOf([alone, other], { render: SCALAR })); + + expect(widthOf(shared) / widthOf(single)).toBeCloseTo(tCritical(7, 0.975) / tCritical(7), 6); + }); + + it('leaves metrics without an alarm at 95%', () => { + const plain = { render: { kind: 'scalar' as const } }; + const [single] = analyzeRun(reportOf([alone], plain)); + const [shared] = analyzeRun(reportOf([alone, other], plain)); + + expect(widthOf(shared)).toBe(widthOf(single)); + }); + + it('leaves out comparisons that never varied', () => { + const steady = benchmarkOf('c', { + current: { render: [3, 3, 3, 3, 3, 3, 3, 3] }, + baseline: { render: [3, 3, 3, 3, 3, 3, 3, 3] }, + }); + const [single] = analyzeRun(reportOf([alone], { render: SCALAR })); + const [withSteady] = analyzeRun(reportOf([alone, steady], { render: SCALAR })); + + expect(widthOf(withSteady)).toBe(widthOf(single)); + }); + + it('can leave unsure what 95% would have flagged', () => { + const { base, shifted } = drifting(2); + const flagged = benchmarkOf('d', { + current: { render: shifted.map((value, round) => value + (round % 2 ? 2 : -2)) }, + baseline: { render: base }, + }); + const [single] = analyzeRun(reportOf([flagged], { render: SCALAR })); + const crowd = Array.from({ length: 20 }, (_, index) => ({ ...other, name: `b${index}` })); + const [crowded] = analyzeRun(reportOf([flagged, ...crowd], { render: SCALAR })); + + expect(single.metrics[0].comparisons[0].change).toBe('worse'); + expect(crowded.metrics[0].comparisons[0].change).toBe('unsure'); + }); + }); + it('keeps a failed benchmark, with no metrics', () => { const [analysis] = analyzeRun( reportOf([ diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index 6559b43865..e0eb129f8a 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -9,6 +9,10 @@ import type { BenchmarkRunReport, RunBenchmark, RunMetricDefinition } from './sc * Variants are compared on paired differences: each round measured every variant once, so the * per-round difference cancels whatever the machine was doing during that round, and its * confidence interval is far tighter than one computed from two independent sets of samples. + * + * Intervals are 95%, except where a change can raise an alarm: there, every such comparison in the + * run shares the 5% chance of a false one (Bonferroni), so a run of unchanged code raises one at + * most 5% of the time however many it checks, rather than 5% per comparison. */ export interface Interval { @@ -32,7 +36,7 @@ export interface MetricComparison { subject: string; /** The variant it is measured against. */ against: string; - /** `subject − against`, in the metric's unit: 95% confidence interval of the mean difference. */ + /** `subject − against`, in the metric's unit: confidence interval of the mean difference. */ absolute: Interval; /** The same, as a percentage of `against`'s mean. */ relative: Interval; @@ -53,35 +57,117 @@ export interface BenchmarkAnalysis { metrics: MetricAnalysis[]; } -// Two-sided 95% critical values of Student's t for 1–30 degrees of freedom. -const T_95 = [ - 12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145, - 2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048, - 2.045, 2.042, +/** The confidence level of an interval no alarm depends on. */ +export const CONFIDENCE = 0.95; + +const LANCZOS = [ + 0.99999999999980993, 676.5203681218851, -1259.1392167224028, 771.32342877765313, + -176.61502916214059, 12.507343278686905, -0.13857109526572012, 9.9843695780195716e-6, + 1.5056327351493116e-7, ]; -/** The two-sided 95% critical value of Student's t, approximated past the table. */ -export function tCritical95(degreesOfFreedom: number): number { - if (degreesOfFreedom <= T_95.length) { - return T_95[Math.max(degreesOfFreedom, 1) - 1]; +/** ln Γ(x) for x > 0, by the Lanczos approximation. */ +function logGamma(x: number): number { + const shifted = x - 1; + let sum = LANCZOS[0]; + for (let index = 1; index < LANCZOS.length; index += 1) { + sum += LANCZOS[index] / (shifted + index); + } + const base = shifted + 7.5; + return 0.5 * Math.log(2 * Math.PI) + (shifted + 0.5) * Math.log(base) - base + Math.log(sum); +} + +/** The continued fraction of the incomplete beta function, by Lentz's method. */ +function betaContinuedFraction(x: number, a: number, b: number): number { + const tiny = 1e-300; + const guard = (value: number) => (Math.abs(value) < tiny ? tiny : value); + let numerator = 1; + let denominator = 1 / guard(1 - ((a + b) * x) / (a + 1)); + let result = denominator; + for (let step = 1; step <= 300; step += 1) { + const twice = 2 * step; + const even = (step * (b - step) * x) / ((a + twice - 1) * (a + twice)); + const odd = (-(a + step) * (a + b + step) * x) / ((a + twice) * (a + twice + 1)); + let factor = 1; + for (const coefficient of [even, odd]) { + denominator = 1 / guard(1 + coefficient * denominator); + numerator = guard(1 + coefficient / numerator); + factor = denominator * numerator; + result *= factor; + } + if (Math.abs(factor - 1) < 1e-15) { + break; + } } - // Cornish–Fisher expansion around the normal quantile; well within 0.1% past 30 degrees. - const z = 1.959964; - return z + (z ** 3 + z) / (4 * degreesOfFreedom); + return result; } -/** The 95% confidence interval of the mean. A single value is its own degenerate interval. */ -export function meanInterval(values: number[]): Interval { +/** The regularized incomplete beta function Iₓ(a, b). */ +function incompleteBeta(x: number, a: number, b: number): number { + if (x <= 0) { + return 0; + } + if (x >= 1) { + return 1; + } + const front = Math.exp( + logGamma(a + b) - logGamma(a) - logGamma(b) + a * Math.log(x) + b * Math.log(1 - x), + ); + return x < (a + 1) / (a + b + 2) + ? (front * betaContinuedFraction(x, a, b)) / a + : 1 - (front * betaContinuedFraction(1 - x, b, a)) / b; +} + +const criticalValues = new Map(); + +/** The two-sided critical value of Student's t at a confidence level, 95% by default. */ +export function tCritical(degreesOfFreedom: number, confidence = CONFIDENCE): number { + const key = `${degreesOfFreedom}:${confidence}`; + const cached = criticalValues.get(key); + if (cached !== undefined) { + return cached; + } + const df = Math.max(degreesOfFreedom, 1); + // The chance of |t| beyond `bound`, which only falls as `bound` grows, so bisect on it. + const tail = (bound: number) => incompleteBeta(df / (df + bound * bound), df / 2, 0.5); + const alpha = 1 - confidence; + let low = 0; + let high = 1; + while (tail(high) > alpha) { + high *= 2; + } + for (let step = 0; step < 100; step += 1) { + const middle = (low + high) / 2; + if (tail(middle) > alpha) { + low = middle; + } else { + high = middle; + } + } + criticalValues.set(key, high); + return high; +} + +/** The confidence interval of the mean. A single value is its own degenerate interval. */ +export function meanInterval(values: number[], confidence = CONFIDENCE): Interval { const mean = calculateMean(values); if (values.length < 2) { return { low: mean, high: mean }; } const variance = values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (values.length - 1); - const halfWidth = tCritical95(values.length - 1) * Math.sqrt(variance / values.length); + const halfWidth = tCritical(values.length - 1, confidence) * Math.sqrt(variance / values.length); return { low: mean - halfWidth, high: mean + halfWidth }; } +/** + * The confidence level of each comparison that can raise an alarm, when `familySize` of them share + * the run's 5% chance of a false alarm. + */ +export function alarmedConfidence(familySize: number): number { + return 1 - (1 - CONFIDENCE) / Math.max(familySize, 1); +} + export function median(values: number[]): number { return quantile( [...values].sort((left, right) => left - right), @@ -139,17 +225,21 @@ function severityOf( return 'none'; } +/** The per-round differences `subject − against`, over the rounds both have. */ +function differencesOf(subject: number[], against: number[]): number[] { + const rounds = Math.min(subject.length, against.length); + return Array.from({ length: rounds }, (_, round) => subject[round] - against[round]); +} + export function compareSamples( subject: { name: string; values: number[] }, against: { name: string; values: number[] }, definition: RunMetricDefinition, + confidence = CONFIDENCE, ): MetricComparison { - const rounds = Math.min(subject.values.length, against.values.length); - const differences = Array.from( - { length: rounds }, - (_, round) => subject.values[round] - against.values[round], - ); - const absolute = meanInterval(differences); + const differences = differencesOf(subject.values, against.values); + const rounds = differences.length; + const absolute = meanInterval(differences, confidence); const reference = calculateMean(against.values.slice(0, rounds)); const relative = { low: (absolute.low / reference) * 100, @@ -177,13 +267,48 @@ function pairsOf(benchmark: RunBenchmark): Array<{ subject: string; against: str export type MetricComparisons = Omit; +/** Whether a metric's change can raise an alarm: an alarmed metric of a `baseline` benchmark. */ +function canAlarm(benchmark: RunBenchmark, definition: RunMetricDefinition): boolean { + return benchmark.kind === 'baseline' && definition.alarm !== undefined; +} + +/** + * How many comparisons share the run's 5% chance of a false alarm: those that can raise an alarm, + * leaving out any whose per-round differences never varied. Those can't be flagged by chance — a + * render count that is the same in every round — so they would only make the others stricter. + */ +export function countAlarmedComparisons( + metrics: Record, + benchmarks: RunBenchmark[], +): number { + let count = 0; + for (const benchmark of benchmarks) { + const { samples } = benchmark; + const [reference] = benchmark.variants; + const valuesOf = (variant: string, metric: string) => samples?.[variant]?.[metric] ?? []; + for (const metric of Object.keys(samples?.[reference] ?? {})) { + if (canAlarm(benchmark, definitionOf(metrics, metric))) { + for (const { subject, against } of pairsOf(benchmark)) { + const differences = differencesOf(valuesOf(subject, metric), valuesOf(against, metric)); + if (differences.some((difference) => difference !== differences[0])) { + count += 1; + } + } + } + } + } + return count; +} + /** * Compares one benchmark's variants, its metrics described by `metrics` (a report's, or a run's so - * far). The comparisons alone, which is all deciding whether to keep sampling needs. + * far). The comparisons alone, which is all deciding whether to keep sampling needs. A comparison + * that can raise an alarm is one of `familySize` sharing the run's 5% chance of a false one. */ export function compareBenchmark( metrics: Record, benchmark: RunBenchmark, + familySize = 1, ): { metrics: MetricComparisons[] } { const { samples } = benchmark; if (!samples) { @@ -193,6 +318,9 @@ export function compareBenchmark( return { metrics: Object.keys(samples[reference] ?? {}).map((metric) => { const definition = definitionOf(metrics, metric); + const confidence = canAlarm(benchmark, definition) + ? alarmedConfidence(familySize) + : CONFIDENCE; const valuesOf = (variant: string) => samples[variant]?.[metric] ?? []; return { metric, @@ -202,6 +330,7 @@ export function compareBenchmark( { name: subject, values: valuesOf(subject) }, { name: against, values: valuesOf(against) }, definition, + confidence, ), ), }; @@ -213,11 +342,12 @@ export function compareBenchmark( export function analyzeBenchmark( metrics: Record, benchmark: RunBenchmark, + familySize = 1, ): BenchmarkAnalysis { const { samples } = benchmark; return { benchmark, - metrics: compareBenchmark(metrics, benchmark).metrics.map((comparison) => ({ + metrics: compareBenchmark(metrics, benchmark, familySize).metrics.map((comparison) => ({ ...comparison, variants: Object.fromEntries( benchmark.variants.map((variant) => [ @@ -230,7 +360,10 @@ export function analyzeBenchmark( } export function analyzeRun(report: BenchmarkRunReport): BenchmarkAnalysis[] { - return report.benchmarks.map((benchmark) => analyzeBenchmark(report.metrics, benchmark)); + const familySize = countAlarmedComparisons(report.metrics, report.benchmarks); + return report.benchmarks.map((benchmark) => + analyzeBenchmark(report.metrics, benchmark, familySize), + ); } export interface Regression { diff --git a/packages/benchmark/src/runReport/format.test.ts b/packages/benchmark/src/runReport/format.test.ts new file mode 100644 index 0000000000..dde4f87e57 --- /dev/null +++ b/packages/benchmark/src/runReport/format.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it } from 'vitest'; +import { runReportFootnote } from './format'; +import type { BenchmarkRunReport, RunBenchmark } from './schema'; + +function reportOf(benchmarks: RunBenchmark[]): BenchmarkRunReport { + return { + version: 2, + generatedAt: '2026-01-01T00:00:00.000Z', + head: { sha: 'abc' }, + environment: { browser: 'Chromium', platform: 'linux', arch: 'x64', launchArgs: [] }, + builds: { current: { label: 'working tree' }, baseline: { sha: 'def', label: 'HEAD~1' } }, + metrics: { render: { kind: 'scalar', alarm: {} } }, + benchmarks, + }; +} + +function benchmarkOf(name: string): RunBenchmark { + return { + name, + file: `${name}.bench.tsx`, + kind: 'baseline', + variants: ['current', 'baseline'], + samples: { current: { render: [1, 3, 2, 4] }, baseline: { render: [2, 2, 2, 2] } }, + }; +} + +describe('runReportFootnote', () => { + it('states one level when at most one comparison can raise an alarm', () => { + expect(runReportFootnote(reportOf([benchmarkOf('a')]))).toContain( + 'Each Δ is a 95% confidence interval', + ); + }); + + it('states the level the alarmed comparisons share', () => { + const footnote = runReportFootnote( + reportOf([benchmarkOf('a'), benchmarkOf('b'), benchmarkOf('c'), benchmarkOf('d')]), + ); + expect(footnote).toContain('98.75% for the 4 comparisons that can raise an alarm'); + expect(footnote).toContain('95% for the rest'); + }); +}); diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index 55512bc69f..875460c3d0 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -1,5 +1,6 @@ +import { alarmedConfidence, CONFIDENCE, countAlarmedComparisons } from './analyzeRun'; import type { BenchmarkAnalysis, Interval, MetricComparison, Regression } from './analyzeRun'; -import type { RunBenchmark, RunMetricDefinition } from './schema'; +import type { BenchmarkRunReport, RunBenchmark, RunMetricDefinition } from './schema'; /** * How a run report's numbers read, for every renderer of one: the terminal table, the pull request @@ -118,8 +119,23 @@ export function formatRunSummary(analyses: BenchmarkAnalysis[], regressions: Reg .join(' · '); } -/** How to read the numbers, under every rendering of a run. */ -export const RUN_REPORT_FOOTNOTE = - 'Each value is a median. Each Δ is a 95% confidence interval on the paired per-round ' + - 'difference, relative to the variant it is measured against; "unsure" means it straddles ' + - 'zero — the expected result for two equivalent builds.'; +/** How to read a run's numbers, under every rendering of it. */ +export function runReportFootnote(report: BenchmarkRunReport): string { + const familySize = countAlarmedComparisons(report.metrics, report.benchmarks); + const interval = + familySize > 1 + ? 'Each Δ is a confidence interval on the paired per-round difference, relative to the ' + + `variant it is measured against: ${formatConfidence(alarmedConfidence(familySize))} for ` + + `the ${familySize} comparisons that can raise an alarm, which together raise a false one ` + + `at most 5% of the time, and ${formatConfidence(CONFIDENCE)} for the rest` + : `Each Δ is a ${formatConfidence(CONFIDENCE)} confidence interval on the paired per-round ` + + 'difference, relative to the variant it is measured against'; + return ( + `Each value is a median. ${interval}; "unsure" means it straddles zero — the expected ` + + 'result for two equivalent builds.' + ); +} + +function formatConfidence(fraction: number): string { + return `${new Intl.NumberFormat('en-US', { maximumFractionDigits: 2 }).format(fraction * 100)}%`; +} diff --git a/packages/benchmark/src/runner/printReport.ts b/packages/benchmark/src/runner/printReport.ts index 01e629a05a..2e3fc69a85 100644 --- a/packages/benchmark/src/runner/printReport.ts +++ b/packages/benchmark/src/runner/printReport.ts @@ -8,7 +8,7 @@ import { findRegressions, formatComparison, formatRunSummary, - RUN_REPORT_FOOTNOTE, + runReportFootnote, } from '../runReport'; import type { BenchmarkAnalysis, BenchmarkRunReport } from '../runReport'; @@ -48,5 +48,5 @@ export function printRunReport(report: BenchmarkRunReport): void { console.log(color(`⚠ ${benchmark} · ${metric} · ${formatComparison(comparison)}`)); } console.log(formatRunSummary(analyses, regressions)); - console.log(dim(RUN_REPORT_FOOTNOTE)); + console.log(dim(runReportFootnote(report))); } diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index a30ed8ba09..2b4c958121 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -3,7 +3,7 @@ import chalk from 'chalk'; import type { Browser, BrowserContext, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; -import { compareBenchmark } from '../runReport'; +import { compareBenchmark, countAlarmedComparisons } from '../runReport'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; import { differencesResolved, parseHorizons } from '../sampling'; import type { SamplingOptions } from '../sampling'; @@ -170,17 +170,25 @@ interface CaseResult { type BenchmarkEntry = Pick; +/** A benchmark to measure, or why it can't be. */ +type PlannedBenchmark = + | { entry: BenchmarkEntry; slots: PageSlot[]; sampling: Required } + | { entry: BenchmarkEntry; error: string }; + /** * Measures one benchmark in a tab of its own, loading each slot's page before sampling it. Every * round samples every slot once, in a shuffled order. After `sampleSize` rounds it keeps adding * rounds while a difference is unresolved against the horizons, until the timeout. A failure is * reported as the benchmark's error. + * + * Its alarmed comparisons share the run's 5% chance of a false alarm with those of the other + * `baseline` benchmarks, which haven't been measured yet, or not by this call: each is taken to have + * one, the common case, and the report counts them exactly afterwards. */ async function runBenchmark( browser: Browser, - entry: BenchmarkEntry, - slots: PageSlot[], - sampling: Required, + { entry, slots, sampling }: Extract, + otherBaselineBenchmarks: number, ): Promise { console.log(chalk.cyan(`\nRunning "${entry.name}" (${entry.file})…`)); const { sampleSize, timeout, autoSampleConditions } = sampling; @@ -215,7 +223,11 @@ async function runBenchmark( } const benchmarkOf = (): RunBenchmark => ({ ...entry, samples: samplesOf(slots, rounds) }); - const isSettled = () => differencesResolved(compareBenchmark(metrics, benchmarkOf()), horizons); + const isSettled = () => { + const benchmark = benchmarkOf(); + const familySize = countAlarmedComparisons(metrics, [benchmark]) + otherBaselineBenchmarks; + return differencesResolved(compareBenchmark(metrics, benchmark, familySize), horizons); + }; const deadline = Date.now() + timeout * 60_000; let resolved = isSettled(); @@ -261,15 +273,16 @@ function benchPageUrl(origin: string, ref: ResolvedRef, benchFile: BenchFile): s } /** - * Runs a benchmark file: each case on its own across the builds, paired by name with the working - * tree as the reference, and each `compare()` across its cases on the working tree's build. + * Lists a benchmark file's benchmarks: each case on its own across the builds, paired by name with + * the working tree as the reference, and each `compare()` across its cases on the working tree's + * build. */ -async function runBenchFile( +async function planBenchFile( browser: Browser, origin: string, benchFile: BenchFile, options: RunInterleavedOptions, -): Promise { +): Promise { const urls = options.benchRefs.map((ref) => benchPageUrl(origin, ref, benchFile)); const variants = options.benchRefs.map((ref) => ref.variant); // The other builds only matter for cases measured across them; a file of `compare()`s alone @@ -277,8 +290,7 @@ async function runBenchFile( const [reference] = await listBenchPages(browser, urls.slice(0, 1)); const others = reference.cases.length > 0 ? await listBenchPages(browser, urls.slice(1)) : []; - const results: CaseResult[] = []; - for (const { name: caseName, sampling } of reference.cases) { + const planned: PlannedBenchmark[] = reference.cases.map(({ name: caseName, sampling }) => { const entry: BenchmarkEntry = { name: caseName, file: benchFile.file, @@ -290,27 +302,18 @@ async function runBenchFile( ); if (missing !== -1) { // A case added by the change under test has nothing to compare with. - results.push({ - benchmark: { - ...entry, - error: `"${caseName}" does not exist in [${variants[missing + 1]}].`, - }, - metrics: {}, - }); - continue; + return { entry, error: `"${caseName}" does not exist in [${variants[missing + 1]}].` }; } const slots = urls.map((url, index) => ({ variant: variants[index], url, caseName })); - // eslint-disable-next-line no-await-in-loop - results.push(await runBenchmark(browser, entry, slots, sampling)); - } + return { entry, slots, sampling }; + }); for (const { name, cases, sampling } of reference.comparisons) { const entry: BenchmarkEntry = { name, file: benchFile.file, kind: 'compare', variants: cases }; const slots = cases.map((caseName) => ({ variant: caseName, url: urls[0], caseName })); - // eslint-disable-next-line no-await-in-loop - results.push(await runBenchmark(browser, entry, slots, sampling)); + planned.push({ entry, slots, sampling }); } - return results; + return planned; } export async function runInterleaved(options: RunInterleavedOptions): Promise { @@ -325,11 +328,27 @@ export async function runInterleaved(options: RunInterleavedOptions): Promise 'slots' in plan && plan.entry.kind === 'baseline', + ).length; + const results: CaseResult[] = []; // Sequential on purpose: concurrent cases would contend for the same machine. - for (const benchFile of options.benchFiles) { + for (const plan of planned) { + if ('error' in plan) { + results.push({ benchmark: { ...plan.entry, error: plan.error }, metrics: {} }); + continue; + } + const others = plan.entry.kind === 'baseline' ? baselineBenchmarks - 1 : 0; // eslint-disable-next-line no-await-in-loop - results.push(...(await runBenchFile(browser, server.origin, benchFile, options))); + results.push(await runBenchmark(browser, plan, others)); } return { benchmarks: results.map((result) => result.benchmark), From 4c1fb343f9aa8a4414f1210f4f4ca2f78d7fea16 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 13:24:56 +0200 Subject: [PATCH 113/173] [benchmark] Simplify the fresh-load runner and the false-alarm correction The footnote reads the confidence levels from the analysis, which now records each comparison's, instead of counting the comparisons again. Counting and comparing share one walk over a benchmark's metric pairs. Metric definitions are read once per benchmark, bench files are listed in parallel, and the tab is passed around as a plain Page. --- .../lib/benchmarkRun/buildMarkdownReport.ts | 2 +- .../src/views/BenchmarkRunDetails.tsx | 2 +- .../benchmark/src/runReport/analyzeRun.ts | 98 +++++++++++-------- .../benchmark/src/runReport/format.test.ts | 9 +- packages/benchmark/src/runReport/format.ts | 39 +++++--- packages/benchmark/src/runner/printReport.ts | 2 +- .../benchmark/src/runner/runInterleaved.ts | 73 +++++++------- 7 files changed, 129 insertions(+), 96 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 7294247236..afe5c026a1 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -76,7 +76,7 @@ export function buildBenchmarkRunMarkdownReport( if (measured.length > 0) { lines.push('
    ', 'Full results', ''); lines.push(measured.map(renderBenchmarkTable).join('\n\n')); - lines.push('', `_${runReportFootnote(report)}_`, '', '
    '); + lines.push('', `_${runReportFootnote(analyses)}_`, '', ''); } if (options.detailsUrl) { diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index 6191775046..b72dd3fb97 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -193,7 +193,7 @@ export default function BenchmarkRunDetails() { {report && ( - {runReportFootnote(report)} + {runReportFootnote(analyses)} {report.environment.browser} on {report.environment.platform}/ diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index e0eb129f8a..c90485641e 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -36,6 +36,8 @@ export interface MetricComparison { subject: string; /** The variant it is measured against. */ against: string; + /** The level of the intervals: 95%, or stricter where a change can raise an alarm. */ + confidence: number; /** `subject − against`, in the metric's unit: confidence interval of the mean difference. */ absolute: Interval; /** The same, as a percentage of `against`'s mean. */ @@ -253,7 +255,14 @@ export function compareSamples( change = increased === lowerIsBetter ? 'worse' : 'better'; } - const comparison = { subject: subject.name, against: against.name, absolute, relative, change }; + const comparison = { + subject: subject.name, + against: against.name, + confidence, + absolute, + relative, + change, + }; return { ...comparison, severity: severityOf(comparison, definition) }; } @@ -267,37 +276,56 @@ function pairsOf(benchmark: RunBenchmark): Array<{ subject: string; against: str export type MetricComparisons = Omit; -/** Whether a metric's change can raise an alarm: an alarmed metric of a `baseline` benchmark. */ -function canAlarm(benchmark: RunBenchmark, definition: RunMetricDefinition): boolean { - return benchmark.kind === 'baseline' && definition.alarm !== undefined; +interface PairedMetric { + metric: string; + definition: RunMetricDefinition; + canAlarm: boolean; + pairs: Array<{ + subject: { name: string; values: number[] }; + against: { name: string; values: number[] }; + }>; +} + +/** Every metric a benchmark reported, with the pairs of variants its kind compares on it. */ +function pairedMetricsOf( + metrics: Record, + benchmark: RunBenchmark, +): PairedMetric[] { + const { samples } = benchmark; + const [reference] = benchmark.variants; + return Object.keys(samples?.[reference] ?? {}).map((metric) => { + const definition = definitionOf(metrics, metric); + const sideOf = (name: string) => ({ name, values: samples?.[name]?.[metric] ?? [] }); + return { + metric, + definition, + canAlarm: benchmark.kind === 'baseline' && definition.alarm !== undefined, + pairs: pairsOf(benchmark).map(({ subject, against }) => ({ + subject: sideOf(subject), + against: sideOf(against), + })), + }; + }); } /** - * How many comparisons share the run's 5% chance of a false alarm: those that can raise an alarm, - * leaving out any whose per-round differences never varied. Those can't be flagged by chance — a - * render count that is the same in every round — so they would only make the others stricter. + * How many comparisons share the run's 5% chance of a false alarm: those that can raise an alarm — + * alarmed metrics of `baseline` benchmarks — leaving out any whose per-round differences never + * varied. Those can't be flagged by chance — a render count that is the same in every round — so + * they would only make the others stricter. */ export function countAlarmedComparisons( metrics: Record, benchmarks: RunBenchmark[], ): number { - let count = 0; - for (const benchmark of benchmarks) { - const { samples } = benchmark; - const [reference] = benchmark.variants; - const valuesOf = (variant: string, metric: string) => samples?.[variant]?.[metric] ?? []; - for (const metric of Object.keys(samples?.[reference] ?? {})) { - if (canAlarm(benchmark, definitionOf(metrics, metric))) { - for (const { subject, against } of pairsOf(benchmark)) { - const differences = differencesOf(valuesOf(subject, metric), valuesOf(against, metric)); - if (differences.some((difference) => difference !== differences[0])) { - count += 1; - } - } - } - } - } - return count; + return benchmarks + .flatMap((benchmark) => pairedMetricsOf(metrics, benchmark)) + .filter((paired) => paired.canAlarm) + .flatMap((paired) => paired.pairs) + .filter(({ subject, against }) => { + const differences = differencesOf(subject.values, against.values); + return differences.some((difference) => difference !== differences[0]); + }).length; } /** @@ -310,28 +338,14 @@ export function compareBenchmark( benchmark: RunBenchmark, familySize = 1, ): { metrics: MetricComparisons[] } { - const { samples } = benchmark; - if (!samples) { - return { metrics: [] }; - } - const [reference] = benchmark.variants; return { - metrics: Object.keys(samples[reference] ?? {}).map((metric) => { - const definition = definitionOf(metrics, metric); - const confidence = canAlarm(benchmark, definition) - ? alarmedConfidence(familySize) - : CONFIDENCE; - const valuesOf = (variant: string) => samples[variant]?.[metric] ?? []; + metrics: pairedMetricsOf(metrics, benchmark).map(({ metric, definition, canAlarm, pairs }) => { + const confidence = canAlarm ? alarmedConfidence(familySize) : CONFIDENCE; return { metric, definition, - comparisons: pairsOf(benchmark).map(({ subject, against }) => - compareSamples( - { name: subject, values: valuesOf(subject) }, - { name: against, values: valuesOf(against) }, - definition, - confidence, - ), + comparisons: pairs.map(({ subject, against }) => + compareSamples(subject, against, definition, confidence), ), }; }), diff --git a/packages/benchmark/src/runReport/format.test.ts b/packages/benchmark/src/runReport/format.test.ts index dde4f87e57..c9c89f9625 100644 --- a/packages/benchmark/src/runReport/format.test.ts +++ b/packages/benchmark/src/runReport/format.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest'; +import { analyzeRun } from './analyzeRun'; import { runReportFootnote } from './format'; import type { BenchmarkRunReport, RunBenchmark } from './schema'; @@ -26,16 +27,18 @@ function benchmarkOf(name: string): RunBenchmark { describe('runReportFootnote', () => { it('states one level when at most one comparison can raise an alarm', () => { - expect(runReportFootnote(reportOf([benchmarkOf('a')]))).toContain( + expect(runReportFootnote(analyzeRun(reportOf([benchmarkOf('a')])))).toContain( 'Each Δ is a 95% confidence interval', ); }); it('states the level the alarmed comparisons share', () => { const footnote = runReportFootnote( - reportOf([benchmarkOf('a'), benchmarkOf('b'), benchmarkOf('c'), benchmarkOf('d')]), + analyzeRun( + reportOf([benchmarkOf('a'), benchmarkOf('b'), benchmarkOf('c'), benchmarkOf('d')]), + ), ); - expect(footnote).toContain('98.75% for the 4 comparisons that can raise an alarm'); + expect(footnote).toContain('98.75% where a change can raise an alarm'); expect(footnote).toContain('95% for the rest'); }); }); diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index 875460c3d0..aa8bdaccac 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -1,6 +1,6 @@ -import { alarmedConfidence, CONFIDENCE, countAlarmedComparisons } from './analyzeRun'; +import { CONFIDENCE } from './analyzeRun'; import type { BenchmarkAnalysis, Interval, MetricComparison, Regression } from './analyzeRun'; -import type { BenchmarkRunReport, RunBenchmark, RunMetricDefinition } from './schema'; +import type { RunBenchmark, RunMetricDefinition } from './schema'; /** * How a run report's numbers read, for every renderer of one: the terminal table, the pull request @@ -13,18 +13,22 @@ const DEFAULT_DISCRETE_FORMAT: Intl.NumberFormatOptions = { maximumFractionDigit // Constructing a formatter is the expensive part, and a table formats every cell. const formatters = new Map(); -/** A value in its metric's own format — `12.3 ms`, `4`. */ -export function formatValue(value: number, definition: RunMetricDefinition): string { - const options = - definition.format ?? - (definition.kind === 'discrete' ? DEFAULT_DISCRETE_FORMAT : DEFAULT_SCALAR_FORMAT); +function formatterFor(options: Intl.NumberFormatOptions): Intl.NumberFormat { const key = JSON.stringify(options); let formatter = formatters.get(key); if (!formatter) { formatter = new Intl.NumberFormat('en-US', options); formatters.set(key, formatter); } - return formatter.format(value); + return formatter; +} + +/** A value in its metric's own format — `12.3 ms`, `4`. */ +export function formatValue(value: number, definition: RunMetricDefinition): string { + const options = + definition.format ?? + (definition.kind === 'discrete' ? DEFAULT_DISCRETE_FORMAT : DEFAULT_SCALAR_FORMAT); + return formatterFor(options).format(value); } /** Both bounds signed, so the direction of a difference reads without the verdict beside it. */ @@ -120,14 +124,19 @@ export function formatRunSummary(analyses: BenchmarkAnalysis[], regressions: Reg } /** How to read a run's numbers, under every rendering of it. */ -export function runReportFootnote(report: BenchmarkRunReport): string { - const familySize = countAlarmedComparisons(report.metrics, report.benchmarks); +export function runReportFootnote(analyses: BenchmarkAnalysis[]): string { + const alarmed = Math.max( + CONFIDENCE, + ...analyses.flatMap(({ metrics }) => + metrics.flatMap(({ comparisons }) => comparisons.map((comparison) => comparison.confidence)), + ), + ); const interval = - familySize > 1 + alarmed > CONFIDENCE ? 'Each Δ is a confidence interval on the paired per-round difference, relative to the ' + - `variant it is measured against: ${formatConfidence(alarmedConfidence(familySize))} for ` + - `the ${familySize} comparisons that can raise an alarm, which together raise a false one ` + - `at most 5% of the time, and ${formatConfidence(CONFIDENCE)} for the rest` + `variant it is measured against: ${formatConfidence(alarmed)} where a change can raise ` + + 'an alarm, so that together they raise a false one at most 5% of the time, and ' + + `${formatConfidence(CONFIDENCE)} for the rest` : `Each Δ is a ${formatConfidence(CONFIDENCE)} confidence interval on the paired per-round ` + 'difference, relative to the variant it is measured against'; return ( @@ -137,5 +146,5 @@ export function runReportFootnote(report: BenchmarkRunReport): string { } function formatConfidence(fraction: number): string { - return `${new Intl.NumberFormat('en-US', { maximumFractionDigits: 2 }).format(fraction * 100)}%`; + return `${formatterFor(DEFAULT_SCALAR_FORMAT).format(fraction * 100)}%`; } diff --git a/packages/benchmark/src/runner/printReport.ts b/packages/benchmark/src/runner/printReport.ts index 2e3fc69a85..42381a947e 100644 --- a/packages/benchmark/src/runner/printReport.ts +++ b/packages/benchmark/src/runner/printReport.ts @@ -48,5 +48,5 @@ export function printRunReport(report: BenchmarkRunReport): void { console.log(color(`⚠ ${benchmark} · ${metric} · ${formatComparison(comparison)}`)); } console.log(formatRunSummary(analyses, regressions)); - console.log(dim(runReportFootnote(report))); + console.log(dim(runReportFootnote(analyses))); } diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index 2b4c958121..e45e00117b 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -1,7 +1,7 @@ /* eslint-disable no-console */ import chalk from 'chalk'; -import type { Browser, BrowserContext, CDPSession, Page } from '@playwright/test'; +import type { Browser, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; import { compareBenchmark, countAlarmedComparisons } from '../runReport'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; @@ -76,13 +76,11 @@ function stableMetrics(rounds: Round[]): string[] { /** What a benchmark file's page defines. */ type PageListing = Pick; -interface BenchTab { - context: BrowserContext; - page: Page; -} - -/** Opens a tab, in a browser context of its own, bridged to a CDP session for trusted input. */ -async function openBenchTab(browser: Browser): Promise { +/** + * Opens a tab, in a browser context of its own, bridged to a CDP session for trusted input. Closing + * its context closes it. + */ +async function openBenchTab(browser: Browser): Promise { const context = await browser.newContext({ viewport: BENCHMARK_VIEWPORT }); try { const page = await context.newPage(); @@ -97,7 +95,7 @@ async function openBenchTab(browser: Browser): Promise { (_source, method: string, params?: Record) => session.send(method as CdpMethod, params), ); - return { context, page }; + return page; } catch (error) { await context.close(); throw error; @@ -129,12 +127,12 @@ async function loadBenchPage( /** Loads pages just to learn what they define. */ async function listBenchPages(browser: Browser, urls: string[]): Promise { - const tab = await openBenchTab(browser); + const page = await openBenchTab(browser); try { const listings: PageListing[] = []; for (const url of urls) { // eslint-disable-next-line no-await-in-loop - const { visibility, ...listing } = await loadBenchPage(tab.page, url); + const { visibility, ...listing } = await loadBenchPage(page, url); if (visibility !== 'visible') { console.warn(chalk.yellow(` ${url} is ${visibility}; rAF-based waits may stall.`)); } @@ -142,7 +140,7 @@ async function listBenchPages(browser: Browser, urls: string[]): Promise; +/** A benchmark to measure: its variants' slots, sampled as its options ask. */ +interface MeasuredBenchmark { + entry: BenchmarkEntry; + slots: PageSlot[]; + sampling: Required; +} + /** A benchmark to measure, or why it can't be. */ -type PlannedBenchmark = - | { entry: BenchmarkEntry; slots: PageSlot[]; sampling: Required } - | { entry: BenchmarkEntry; error: string }; +type PlannedBenchmark = MeasuredBenchmark | { entry: BenchmarkEntry; error: string }; /** * Measures one benchmark in a tab of its own, loading each slot's page before sampling it. Every @@ -187,17 +190,17 @@ type PlannedBenchmark = */ async function runBenchmark( browser: Browser, - { entry, slots, sampling }: Extract, + { entry, slots, sampling }: MeasuredBenchmark, otherBaselineBenchmarks: number, ): Promise { console.log(chalk.cyan(`\nRunning "${entry.name}" (${entry.file})…`)); const { sampleSize, timeout, autoSampleConditions } = sampling; const horizons = parseHorizons(autoSampleConditions); - let tab: BenchTab | undefined; + let opened: Page | undefined; try { - tab = await openBenchTab(browser); - const { page } = tab; - let metrics: Record = {}; + const page = await openBenchTab(browser); + opened = page; + let metrics: Record | undefined; const runRound = async (): Promise => { const round: Round = []; for (const index of shuffledIndices(slots.length)) { @@ -205,10 +208,10 @@ async function runBenchmark( await loadBenchPage(page, slots[index].url); // eslint-disable-next-line no-await-in-loop round[index] = await sampleBenchCase(page, slots[index].caseName); - if (index === 0) { + if (index === 0 && !metrics) { // Read from the reference page: a metric whose config the change under test altered is - // described the way the change describes it. A freshly loaded page knows the metrics it - // recorded itself. + // described the way the change describes it. Its first sample recorded every metric the + // report keeps, as those are the ones recorded in every round. // eslint-disable-next-line no-await-in-loop metrics = await page.evaluate(() => window.benchmarkPage!.metricDefinitions()); } @@ -221,12 +224,14 @@ async function runBenchmark( // eslint-disable-next-line no-await-in-loop rounds.push(await runRound()); } + const definitions = metrics ?? {}; const benchmarkOf = (): RunBenchmark => ({ ...entry, samples: samplesOf(slots, rounds) }); const isSettled = () => { const benchmark = benchmarkOf(); - const familySize = countAlarmedComparisons(metrics, [benchmark]) + otherBaselineBenchmarks; - return differencesResolved(compareBenchmark(metrics, benchmark, familySize), horizons); + const familySize = + countAlarmedComparisons(definitions, [benchmark]) + otherBaselineBenchmarks; + return differencesResolved(compareBenchmark(definitions, benchmark, familySize), horizons); }; const deadline = Date.now() + timeout * 60_000; @@ -245,13 +250,13 @@ async function runBenchmark( } return { benchmark: { ...benchmarkOf(), sampling: { sampleSize, timedOut: !resolved } }, - metrics, + metrics: definitions, }; } catch (error) { console.error(chalk.red(` ${errorMessage(error)}`)); return { benchmark: { ...entry, error: errorMessage(error) }, metrics: {} }; } finally { - await tab?.context.close(); + await opened?.context().close(); } } @@ -329,12 +334,14 @@ export async function runInterleaved(options: RunInterleavedOptions): Promise + planBenchFile(browser, server.origin, benchFile, options), + ), + ) + ).flat(); const baselineBenchmarks = planned.filter( (plan) => 'slots' in plan && plan.entry.kind === 'baseline', ).length; From 9259b3a408676a0a82a282d024c7a1244de73833 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 14:10:43 +0200 Subject: [PATCH 114/173] [benchmark] Run the benchmarks from the root command, not `benchmark run` The package had no CLI before, so the subcommand only added a word. --- .../src/components/BenchmarkRunHistory.tsx | 2 +- .../src/utils/fetchCiReport.ts | 4 ++-- packages/benchmark/README.md | 16 ++++++++-------- packages/benchmark/src/cli/index.ts | 2 -- packages/benchmark/src/cli/run.ts | 2 +- packages/benchmark/src/page/index.ts | 2 +- packages/benchmark/src/page/page.tsx | 2 +- packages/benchmark/src/publicApi.ts | 2 +- packages/benchmark/src/runReport/schema.ts | 2 +- packages/benchmark/src/runner/plugin.test.ts | 2 +- packages/benchmark/src/runner/plugin.ts | 6 +++--- test/benchmark/README.md | 6 +++--- test/benchmark/package.json | 4 ++-- test/benchmark/src/list/list.bench.tsx | 2 +- test/benchmark/src/workload/workload.bench.tsx | 4 ++-- 15 files changed, 28 insertions(+), 30 deletions(-) diff --git a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx index 8ea3fa14bf..6a857ff301 100644 --- a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx +++ b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx @@ -20,7 +20,7 @@ import { useCiReports } from '../hooks/useCiReports'; import ErrorDisplay from './ErrorDisplay'; /** - * Master history of `benchmark run` reports. Every master commit is benchmarked against its parent + * Master history of the `benchmark` CLI reports. Every master commit is benchmarked against its parent * in the same run, so each bar is that commit's own paired change — which commit moved a number is * read off the chart directly, rather than inferred from a noisy trend. */ diff --git a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts index 90d3f5d542..77d4756532 100644 --- a/apps/code-infra-dashboard/src/utils/fetchCiReport.ts +++ b/apps/code-infra-dashboard/src/utils/fetchCiReport.ts @@ -5,7 +5,7 @@ import { migrateBenchmarkReport } from '@/lib/benchmark/migrateBenchmarkReport'; export interface CiReportTypes { /** - * Version 1 from the Vitest reporter, or version 2 from `benchmark run`: a repository uploads one or + * Version 1 from the Vitest reporter, or version 2 from the `benchmark` CLI: a repository uploads one or * the other while it moves over, so every reader narrows on `version`. */ 'benchmark.json': BenchmarkUpload | BenchmarkRunUpload; @@ -14,7 +14,7 @@ export interface CiReportTypes { export type CiReportName = keyof CiReportTypes; -/** Whether a `benchmark.json` is the version 2 report `benchmark run` uploads. */ +/** Whether a `benchmark.json` is the version 2 report the `benchmark` CLI uploads. */ export function isBenchmarkRunUpload( upload: BenchmarkUpload | BenchmarkRunUpload, ): upload is BenchmarkRunUpload { diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index a487472fe6..72f93664aa 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -4,9 +4,9 @@ Two ways to measure, while repositories move from the first to the second. **Under Vitest** — a React component benchmarking tool built on Vitest and Playwright, using React's profiling build to capture render durations against your source. Everything up to -[`benchmark run`](#benchmark-run) covers it. It uploads version 1 of the `benchmark` report. +[the `benchmark` CLI](#the-benchmark-cli) covers it. It uploads version 1 of the `benchmark` report. -**`benchmark run`** — `benchmark()` and `reactBenchmark()` files, the working tree against a baseline +**The `benchmark` CLI** — `benchmark()` and `reactBenchmark()` files, the working tree against a baseline commit, both **built** and installed the way a consumer gets them, sampled alternately in one browser run and compared on paired differences. It uploads version 2 of the `benchmark` report. A repository uploads one version or the other; the dashboard reads each by its version. @@ -304,9 +304,9 @@ BENCHMARK_BASELINE_PATH=/tmp/base-bench.json pnpm test:bench # head run, inlin The feature is opt-in — without `BENCHMARK_BASELINE_PATH` (or the `baselinePath` config option), the dashboard falls back to fetching the base from S3 by merge-base SHA as before. -## `benchmark run` +## The `benchmark` CLI -`benchmark run` measures a harness package's `*.bench.tsx` files across two builds of the workspace: +The `benchmark` CLI measures a harness package's `*.bench.tsx` files across two builds of the workspace: the working tree and a baseline. Run it from the harness, whose vite config uses the plugin: ```js @@ -318,7 +318,7 @@ export default defineConfig({ plugins: [benchmarkPlugin()] }); ``` ```bash -benchmark run --baseline "$(code-infra baseline)" +benchmark --baseline "$(code-infra baseline)" ``` ### Defining cases @@ -474,13 +474,13 @@ if one was killed outright. Two things follow from it. A tracked file names tarb the workspace for that time, because pnpm scopes an override by parent package name and a harness has none — so nothing else should build against the same checkout meanwhile. -`benchmark run --help` lists the remaining options. +`benchmark --help` lists the remaining options. ## API - `benchmark` — define a benchmark test case -- `@mui/internal-benchmark/vitePlugin` — `benchmarkPlugin()`, the harness's vite plugin for `benchmark run` -- `@mui/internal-benchmark/runReport` — the `benchmark run` report schema and `analyzeRun` +- `@mui/internal-benchmark/vitePlugin` — `benchmarkPlugin()`, the harness's vite plugin for the `benchmark` CLI +- `@mui/internal-benchmark/runReport` — the report schema of the `benchmark` CLI, and `analyzeRun` - `ElementTiming` — invisible marker component for paint timing (renders a `` tracked by the Element Timing API) - `ScalarMetric` — record a continuous custom measurement (with a `console.time`-style timing helper) - `DiscreteMetric` — record a discrete custom count diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index e77235372b..a1fa2f06c5 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -14,7 +14,6 @@ let globalArgv: { verbose?: boolean } = {}; await yargs(hideBin(process.argv)) .scriptName('benchmark') - .usage('$0 [args]') .option('verbose', { alias: 'v', type: 'boolean', @@ -38,7 +37,6 @@ await yargs(hideBin(process.argv)) } process.exit(1); }) - .demandCommand(1, 'You need at least one command before moving on') .strict() .help() .version(pkgJson.version) diff --git a/packages/benchmark/src/cli/run.ts b/packages/benchmark/src/cli/run.ts index f74a1e79b1..fec045a53d 100644 --- a/packages/benchmark/src/cli/run.ts +++ b/packages/benchmark/src/cli/run.ts @@ -5,7 +5,7 @@ import type { RunBenchmarksOptions } from '../runner/runBenchmarks'; type Args = Omit; const command: CommandModule<{}, Args> = { - command: 'run [filters...]', + command: '$0 [filters...]', describe: "Benchmark a harness's *.bench.tsx files, the working tree against a baseline build, and write a JSON report. Run from the harness package.", builder: (yargs) => { diff --git a/packages/benchmark/src/page/index.ts b/packages/benchmark/src/page/index.ts index 2d5138cd95..8567725861 100644 --- a/packages/benchmark/src/page/index.ts +++ b/packages/benchmark/src/page/index.ts @@ -1,4 +1,4 @@ -// `@mui/internal-benchmark/page`: what a benchmark file for `benchmark run` imports. +// `@mui/internal-benchmark/page`: what a benchmark file for the `benchmark` CLI imports. export * from '../publicApi'; export { benchmark, compare, markPageReady } from './page'; diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 7bb767efe4..47bdaf2728 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -6,7 +6,7 @@ import { hasSampling, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { MetricDefinition } from '../types'; -// The page runtime `benchmark run` measures. `benchmark()` registers a case, and the runner calls +// The page runtime the `benchmark` CLI measures. `benchmark()` registers a case, and the runner calls // `window.benchmarkPage.sample()` to run one iteration of it at a time. What a case measures is // whatever it records through a `Metric`; every recorded value leaves the page as a // `performance.measure` entry, so it also shows up in a DevTools trace. diff --git a/packages/benchmark/src/publicApi.ts b/packages/benchmark/src/publicApi.ts index 9ac9721d99..1ded0fd2de 100644 --- a/packages/benchmark/src/publicApi.ts +++ b/packages/benchmark/src/publicApi.ts @@ -1,5 +1,5 @@ // What both runtimes export: the Vitest harness (`@mui/internal-benchmark`) and the page runtime -// `benchmark run` measures (`@mui/internal-benchmark/page`). Each entry adds its own case API. +// the `benchmark` CLI measures (`@mui/internal-benchmark/page`). Each entry adds its own case API. export type { RenderEvent, IterationData, InteractionContext } from './types'; export type { diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts index ea1bff3589..62f3e1f572 100644 --- a/packages/benchmark/src/runReport/schema.ts +++ b/packages/benchmark/src/runReport/schema.ts @@ -2,7 +2,7 @@ import { z } from 'zod/v4'; import type { MetricDefinition } from '../types'; /** - * The report `benchmark run` produces, uploads and the dashboard reads: version 2 of the `benchmark` + * The report the `benchmark` CLI produces, uploads and the dashboard reads: version 2 of the `benchmark` * report, which the Vitest reporter uploads as version 1. A repository uploads one or the other, and * the dashboard reads each by its version. * diff --git a/packages/benchmark/src/runner/plugin.test.ts b/packages/benchmark/src/runner/plugin.test.ts index a1a4403244..48363271af 100644 --- a/packages/benchmark/src/runner/plugin.test.ts +++ b/packages/benchmark/src/runner/plugin.test.ts @@ -82,7 +82,7 @@ describe('benchmarkPlugin', () => { it('leaves an output directory the caller chose alone', async () => { // A plugin's returned config is merged *over* the inline config, so defaulting this // unconditionally would silently override `vite build --outDir` — which is how - // `benchmark run` directs each ref's build into its own directory. + // the `benchmark` CLI directs each ref's build into its own directory. const harnessDir = await makeHarness(); const config = await callConfig(harnessDir, 'build', { diff --git a/packages/benchmark/src/runner/plugin.ts b/packages/benchmark/src/runner/plugin.ts index 66a2b2df14..7e0ef93a34 100644 --- a/packages/benchmark/src/runner/plugin.ts +++ b/packages/benchmark/src/runner/plugin.ts @@ -108,7 +108,7 @@ export function benchmarkPlugin(options: BenchmarkPluginOptions = {}): Plugin { if (source === '@mui/internal-benchmark') { this.error( `${importer ?? 'A benchmark page'} imports "@mui/internal-benchmark", the Vitest ` + - 'harness. Benchmark files for `benchmark run` import "@mui/internal-benchmark/page".', + 'harness. Benchmark files for the `benchmark` CLI import "@mui/internal-benchmark/page".', ); } if (source === 'vitest' || source.startsWith('vitest/')) { @@ -149,7 +149,7 @@ export function benchmarkPlugin(options: BenchmarkPluginOptions = {}): Plugin { return { ...base, build: { outDir } }; } - // A plain `vite build` into the output directory marks it ignored, as `benchmark run` does. + // A plain `vite build` into the output directory marks it ignored, as the `benchmark` CLI does. if (!userConfig.build?.outDir) { await prepareOutputDir(harnessDir); } @@ -184,7 +184,7 @@ export function benchmarkPlugin(options: BenchmarkPluginOptions = {}): Plugin { fileName: 'index.html', source: renderIndex( benchFiles, - 'These are production bundles — the same build `benchmark run` measures.', + 'These are production bundles — the same build the `benchmark` CLI measures.', ), }); }, diff --git a/test/benchmark/README.md b/test/benchmark/README.md index c81f4998fd..ac6dd21e7d 100644 --- a/test/benchmark/README.md +++ b/test/benchmark/README.md @@ -1,6 +1,6 @@ # Benchmark smoke test -A working harness for `benchmark run`, used to exercise that tooling end to end. +A working harness for the `benchmark` CLI, used to exercise that tooling end to end. Its benchmarks are deterministic workloads, not numbers worth tracking: they give the pipeline something real to drive. For real benchmarks, see `base-ui-mosaic` and `base-ui-charts`. @@ -54,7 +54,7 @@ update cannot move the numbers on its own. ### A note on the baseline -`benchmark run` does not work out which commit to compare against — `code-infra baseline` does, for +The `benchmark` CLI does not work out which commit to compare against — `code-infra baseline` does, for every job that compares a branch to its base, so bundle size and the benchmarks agree on the answer instead of each deriving it. Pass it in: @@ -64,7 +64,7 @@ pnpm test:benchmark-run --baseline "$(pnpm code-infra baseline)" Without `--baseline` the previous commit is used. -A baseline older than the introduction of `benchmark run` will fail to build its pages: this +A baseline older than the introduction of the `benchmark` CLI will fail to build its pages: this harness's vite config imports a plugin that did not exist at that commit. `--baseline HEAD` compares the working tree against the current commit while that is still true. diff --git a/test/benchmark/package.json b/test/benchmark/package.json index 56ac2c7083..0f419f37d4 100644 --- a/test/benchmark/package.json +++ b/test/benchmark/package.json @@ -1,12 +1,12 @@ { "private": true, - "description": "Smoke-test harness for `benchmark run` from @mui/internal-benchmark", + "description": "Smoke-test harness for the `benchmark` CLI from @mui/internal-benchmark", "scripts": { "dev": "vite", "build": "vite build", "preview": "vite preview", "typescript": "tsc -noEmit", - "benchmark:run": "node ./scripts/runBenchmark.mjs run" + "benchmark:run": "node ./scripts/runBenchmark.mjs" }, "dependencies": { "@mui/internal-test-utils": "workspace:*", diff --git a/test/benchmark/src/list/list.bench.tsx b/test/benchmark/src/list/list.bench.tsx index 16e11c6f59..092de49f8c 100644 --- a/test/benchmark/src/list/list.bench.tsx +++ b/test/benchmark/src/list/list.bench.tsx @@ -5,7 +5,7 @@ import { ScrollingList } from '../_shared/ScrollingList'; import { SMOKE_SAMPLING } from '../_shared/sampling'; /** - * `reactBenchmark()` cases. Under `benchmark run` each sample runs one iteration in a freshly loaded + * `reactBenchmark()` cases. Under the `benchmark` CLI each sample runs one iteration in a freshly loaded * page, with the builds alternating round by round. * * Like the `workload` benchmark, it imports a workspace package so both builds resolve it from their diff --git a/test/benchmark/src/workload/workload.bench.tsx b/test/benchmark/src/workload/workload.bench.tsx index b7879df731..cfa33cb4b5 100644 --- a/test/benchmark/src/workload/workload.bench.tsx +++ b/test/benchmark/src/workload/workload.bench.tsx @@ -8,10 +8,10 @@ import { SMOKE_SAMPLING } from '../_shared/sampling'; * * The point of this benchmark is not the number it produces — this repository ships build tooling, * not a browser library, so there is nothing here whose render time is worth tracking. It exists to - * give `benchmark run` something real to drive end to end. + * give the `benchmark` CLI something real to drive end to end. * * It does import a workspace package (`@mui/internal-test-utils`) and put its value on screen. That - * is the part that matters: under `benchmark run` every ref — the working tree included — resolves + * is the part that matters: under the `benchmark` CLI every ref — the working tree included — resolves * it from a packed tarball, so both sides of a comparison resolve identically. */ From 4cfbe22819522a20897aa0bf456e6f81c52e1cfb Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:12:41 +0200 Subject: [PATCH 115/173] [benchmark] Read tables old to new, keep the comment to changes, stop alarming on render count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Tables list the baseline before the current build, so a row reads like a diff; current-first made a 4 → 5 render count read as 5 → 4. - A comparison whose every round measured the same is `unchanged`, shown without an interval, rather than `unsure`. - The pull request comment keeps only metrics that got better or worse, and sums up a benchmark where none did as "no change detected"; the terminal and the dashboard still show every metric. - `render:count` is informational: an extra render matters through the time it adds, which `render` already measures. --- .../src/components/BenchmarkRunHistory.tsx | 33 ++++++------ .../benchmarkRun/buildMarkdownReport.test.ts | 18 +++++-- .../lib/benchmarkRun/buildMarkdownReport.ts | 19 +++++-- packages/benchmark/README.md | 13 +++-- .../benchmark/src/page/reactBenchmark.tsx | 8 +-- .../src/runReport/analyzeRun.test.ts | 10 ++++ .../benchmark/src/runReport/analyzeRun.ts | 11 ++-- .../benchmark/src/runReport/format.test.ts | 51 ++++++++++++++++++- packages/benchmark/src/runReport/format.ts | 18 ++++--- 9 files changed, 139 insertions(+), 42 deletions(-) diff --git a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx index 6a857ff301..cf91279f5c 100644 --- a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx +++ b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx @@ -29,6 +29,7 @@ const CHANGE_COLORS: Record = { worse: 'var(--mui-palette-error-main)', better: 'var(--mui-palette-success-main)', unsure: 'var(--mui-palette-grey-500)', + unchanged: 'var(--mui-palette-grey-500)', }; /** Per benchmark, per metric: a commit's change against its parent. */ @@ -117,21 +118,23 @@ export default function BenchmarkRunHistory({ repo }: BenchmarkRunHistoryProps) ? undefined : point.comparisons.get(benchmark)?.get(metric), ); - return (['worse', 'better', 'unsure'] as const satisfies Change[]).map((change) => ({ - type: 'bar' as const, - stack: 'change', - label: change, - color: CHANGE_COLORS[change], - data: selected.map((comparison) => - comparison?.change === change - ? (comparison.relative.low + comparison.relative.high) / 2 - : null, - ), - valueFormatter: (_value: number | null, { dataIndex }: { dataIndex: number }) => { - const comparison = selected[dataIndex]; - return comparison?.change === change ? formatComparison(comparison) : null; - }, - })); + return (['worse', 'better', 'unsure', 'unchanged'] as const satisfies Change[]).map( + (change) => ({ + type: 'bar' as const, + stack: 'change', + label: change, + color: CHANGE_COLORS[change], + data: selected.map((comparison) => + comparison?.change === change + ? (comparison.relative.low + comparison.relative.high) / 2 + : null, + ), + valueFormatter: (_value: number | null, { dataIndex }: { dataIndex: number }) => { + const comparison = selected[dataIndex]; + return comparison?.change === change ? formatComparison(comparison) : null; + }, + }), + ); }, [points, benchmark, metric]); const [selectedSha, setSelectedSha] = React.useState(null); diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index 49d60b1ad9..8a82e0f8ef 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -59,6 +59,15 @@ const broken: RunBenchmark = { }; describe('buildBenchmarkRunMarkdownReport', () => { + it('shows a benchmark where nothing changed as one line', () => { + const markdown = buildBenchmarkRunMarkdownReport(reportOf([unchanged]), { + title: 'Performance', + }); + + expect(markdown).toContain('**Button mount** · 4 rounds · no change detected'); + expect(markdown).not.toContain('| render |'); + }); + it('states nothing but the summary when no benchmark regressed', () => { const markdown = buildBenchmarkRunMarkdownReport(reportOf([unchanged]), { title: 'Performance', @@ -86,14 +95,13 @@ describe('buildBenchmarkRunMarkdownReport', () => { ❌ **Chart zoom**: Render failed
    - Full results + Changes **Grid scroll** · 3 + 1 rounds · timed out - | Metric | current | baseline | Δ vs baseline | + | Metric | baseline | current | Δ vs baseline | |:----------|----------:|----------:|:----------| - | render | 23.5 ms | 21.5 ms | \`worse +9.3% – +9.3%\` | - | bench:paint | 30.5 ms | 30.5 ms | \`unsure +0.0% – +0.0%\` | + | render | 21.5 ms | 23.5 ms | \`worse +9.3% – +9.3%\` | **libs / mount** · 4 rounds @@ -103,7 +111,7 @@ describe('buildBenchmarkRunMarkdownReport', () => { | render | 10 ms | 20.5 ms | \`worse +95.8% – +114.2%\` | - _Each value is a median. Each Δ is a 95% confidence interval on the paired per-round difference, relative to the variant it is measured against; "unsure" means it straddles zero — the expected result for two equivalent builds._ + _Each value is a median. Each Δ is a 95% confidence interval on the paired per-round difference, relative to the variant it is measured against; "unsure" means it straddles zero — the expected result for two equivalent builds — and "unchanged" that every round measured the same._
    " `); diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index afe5c026a1..94402df755 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -17,9 +17,20 @@ interface BuildOptions { detailsUrl?: string; } -/** One table per benchmark: every metric, the median of every variant, every comparison. */ +/** + * One table per benchmark, of the metrics that changed — better or worse — with the median of every + * variant and every comparison. A benchmark where nothing changed is one line; the full run has + * every metric. + */ function renderBenchmarkTable(analysis: BenchmarkAnalysis): string { - const { columns, rows } = benchmarkTable(analysis); + const heading = `**${analysis.benchmark.name}** · ${formatRounds(analysis.benchmark)}`; + const { columns, rows: allRows } = benchmarkTable(analysis); + const rows = allRows.filter((row) => + row.comparisons.some(({ change }) => change === 'better' || change === 'worse'), + ); + if (rows.length === 0) { + return `${heading} · no change detected`; + } const table = formatMarkdownTable( columns.map(({ header, kind }, column) => ({ field: String(column), @@ -35,7 +46,7 @@ function renderBenchmarkTable(analysis: BenchmarkAnalysis): string { ), ), ); - return `**${analysis.benchmark.name}** · ${formatRounds(analysis.benchmark)}\n\n${table}`; + return `${heading}\n\n${table}`; } /** @@ -74,7 +85,7 @@ export function buildBenchmarkRunMarkdownReport( } if (measured.length > 0) { - lines.push('
    ', 'Full results', ''); + lines.push('
    ', 'Changes', ''); lines.push(measured.map(renderBenchmarkTable).join('\n\n')); lines.push('', `_${runReportFootnote(analyses)}_`, '', '
    '); } diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 72f93664aa..e9caf88b39 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -438,11 +438,14 @@ history from it. - The report holds **raw samples**, round-aligned across variants, plus each metric's kind, format and alarm, how each benchmark's sampling went, the builds, and the environment. -- `analyzeRun` from `@mui/internal-benchmark/runReport` draws every conclusion from it: a 95% - confidence interval on the paired difference per metric, a change (`better`, `worse`, `unsure`), - and a severity from the metric's alarm. -- `reactBenchmark()`'s `render` and `render:count` alarm on any resolved change for the worse; - `bench:paint` is informational; every other metric brings its own alarm. +- `analyzeRun` from `@mui/internal-benchmark/runReport` draws every conclusion from it: a confidence + interval on the paired difference per metric, a change (`better`, `worse`, `unsure`, or + `unchanged` when every round measured the same), and a severity from the metric's alarm. +- `reactBenchmark()`'s `render` alarms on any resolved change for the worse; `render:count`, the + per-phase split and `bench:paint` are informational; every other metric brings its own alarm. +- Tables list the baseline before the current build, so a row reads old to new. The pull request + comment keeps only the metrics that got better or worse, and sums up a benchmark where none did + as "no change detected"; the terminal and the dashboard show every metric. ### Choosing the baseline diff --git a/packages/benchmark/src/page/reactBenchmark.tsx b/packages/benchmark/src/page/reactBenchmark.tsx index 5e30c5e837..d1354b7290 100644 --- a/packages/benchmark/src/page/reactBenchmark.tsx +++ b/packages/benchmark/src/page/reactBenchmark.tsx @@ -9,10 +9,12 @@ import type { BenchmarkCase, BenchmarkContext } from './page'; export interface ReactBenchmarkOptions extends CaseOptions, SamplingOptions {} -// Render time and render count alarm on any resolved change for the worse; paint dominates each -// case's duration and duplicates that signal, so it is informational, as is the per-phase split. +// Render time alarms on any resolved change for the worse. The render count is informational: an +// extra render matters only through the time it adds, which render time already measures. Paint +// dominates each case's duration and duplicates that signal, so it is informational too, as is the +// per-phase split. const renderMetric = new ScalarMetric({ name: 'render', format: MILLISECONDS, alarm: {} }); -const renderCountMetric = new DiscreteMetric({ name: 'render:count', alarm: {} }); +const renderCountMetric = new DiscreteMetric({ name: 'render:count' }); const paintMetric = new ScalarMetric({ name: PAINT_METRIC_NAME, format: MILLISECONDS }); const phaseMetrics = new Map(); diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index a22a1fa09d..d6740a4b59 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -97,6 +97,16 @@ describe('compareSamples', () => { expect(comparison.severity).toBe('none'); }); + it('is unchanged when every round measured the same on both sides', () => { + const comparison = compareSamples( + { name: 'current', values: [3, 4, 3] }, + { name: 'baseline', values: [3, 4, 3] }, + { kind: 'discrete' }, + ); + + expect(comparison.change).toBe('unchanged'); + }); + it('is unsure when the interval straddles zero', () => { const comparison = compareSamples( { name: 'current', values: [10, 12, 9, 11] }, diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index c90485641e..2ad84049b8 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -25,8 +25,11 @@ export interface SampleSummary { median: number; } -/** Which way a resolved change went, given the metric's direction; `unsure` if it did not resolve. */ -export type Change = 'better' | 'worse' | 'unsure'; +/** + * Which way a resolved change went, given the metric's direction; `unsure` if it did not resolve, + * and `unchanged` if every round measured the same on both sides — a render count, say. + */ +export type Change = 'better' | 'worse' | 'unsure' | 'unchanged'; /** How much a change for the worse matters, per the metric's alarm. */ export type Severity = 'error' | 'warning' | 'none'; @@ -249,7 +252,9 @@ export function compareSamples( }; let change: Change = 'unsure'; - if (absolute.low > 0 || absolute.high < 0) { + if (rounds > 0 && differences.every((difference) => difference === 0)) { + change = 'unchanged'; + } else if (absolute.low > 0 || absolute.high < 0) { const increased = absolute.low > 0; const lowerIsBetter = definition.alarm?.direction !== 'higherIsBetter'; change = increased === lowerIsBetter ? 'worse' : 'better'; diff --git a/packages/benchmark/src/runReport/format.test.ts b/packages/benchmark/src/runReport/format.test.ts index c9c89f9625..caf05c8c8f 100644 --- a/packages/benchmark/src/runReport/format.test.ts +++ b/packages/benchmark/src/runReport/format.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import { analyzeRun } from './analyzeRun'; -import { runReportFootnote } from './format'; +import { benchmarkTable, runReportFootnote } from './format'; import type { BenchmarkRunReport, RunBenchmark } from './schema'; function reportOf(benchmarks: RunBenchmark[]): BenchmarkRunReport { @@ -25,6 +25,55 @@ function benchmarkOf(name: string): RunBenchmark { }; } +describe('benchmarkTable', () => { + it('puts the baseline before the current build, so a row reads old to new', () => { + const [analysis] = analyzeRun(reportOf([benchmarkOf('a')])); + const { columns, rows } = benchmarkTable(analysis); + + expect(columns.map((column) => column.header)).toEqual([ + 'Metric', + 'baseline', + 'current', + 'Δ vs baseline', + ]); + expect(rows[0].cells.slice(1, 3)).toEqual(['2', '2.5']); + }); + + it('keeps the reference first for compared cases', () => { + const [analysis] = analyzeRun( + reportOf([ + { + name: 'libs', + file: 'libs.bench.tsx', + kind: 'compare', + variants: ['ours', 'theirs'], + samples: { ours: { render: [1, 1] }, theirs: { render: [2, 2] } }, + }, + ]), + ); + + expect(benchmarkTable(analysis).columns.map((column) => column.header)).toEqual([ + 'Metric', + 'ours', + 'theirs', + 'theirs vs ours', + ]); + }); + + it('shows an unchanged comparison without an interval', () => { + const [analysis] = analyzeRun( + reportOf([ + { + ...benchmarkOf('a'), + samples: { current: { render: [2, 2] }, baseline: { render: [2, 2] } }, + }, + ]), + ); + + expect(benchmarkTable(analysis).rows[0].cells.at(-1)).toBe('unchanged'); + }); +}); + describe('runReportFootnote', () => { it('states one level when at most one comparison can raise an alarm', () => { expect(runReportFootnote(analyzeRun(reportOf([benchmarkOf('a')])))).toContain( diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index aa8bdaccac..0d2d7adeea 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -37,9 +37,11 @@ export function formatPercent(interval: Interval): string { return `${signed(interval.low)} – ${signed(interval.high)}`; } -/** A comparison as one phrase: `worse +2.7% – +6.8%`, `unsure -1.2% – +0.9%`. */ +/** A comparison as one phrase: `worse +2.7% – +6.8%`, `unsure -1.2% – +0.9%`, `unchanged`. */ export function formatComparison(comparison: MetricComparison): string { - return `${comparison.change} ${formatPercent(comparison.relative)}`; + return comparison.change === 'unchanged' + ? comparison.change + : `${comparison.change} ${formatPercent(comparison.relative)}`; } /** What a comparison column is headed: `Δ vs baseline` for a build, `theirs vs ours` for a variant. */ @@ -71,20 +73,24 @@ export function benchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): { columns: BenchmarkTableColumn[]; rows: BenchmarkTableRow[]; } { + // Builds read old to new, the baseline first, like a diff; compared cases keep the reference + // first, as the comparison columns measure against it. + const variants = + benchmark.kind === 'baseline' ? [...benchmark.variants].reverse() : benchmark.variants; const columns: BenchmarkTableColumn[] = [ { header: 'Metric', kind: 'label' }, - ...benchmark.variants.map((variant) => ({ header: variant, kind: 'value' as const })), + ...variants.map((variant) => ({ header: variant, kind: 'value' as const })), ...(metrics[0]?.comparisons ?? []).map((comparison) => ({ header: formatComparisonLabel(benchmark, comparison), kind: 'comparison' as const, })), ]; - const rows = metrics.map(({ metric, definition, variants, comparisons }) => ({ + const rows = metrics.map(({ metric, definition, variants: medians, comparisons }) => ({ metric, comparisons, cells: [ metric, - ...benchmark.variants.map((variant) => formatValue(variants[variant].median, definition)), + ...variants.map((variant) => formatValue(medians[variant].median, definition)), ...comparisons.map(formatComparison), ], })); @@ -141,7 +147,7 @@ export function runReportFootnote(analyses: BenchmarkAnalysis[]): string { 'difference, relative to the variant it is measured against'; return ( `Each value is a median. ${interval}; "unsure" means it straddles zero — the expected ` + - 'result for two equivalent builds.' + 'result for two equivalent builds — and "unchanged" that every round measured the same.' ); } From 98fec277ca98421f5e73f8d776606dfc44ea137d Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:15:40 +0200 Subject: [PATCH 116/173] [benchmark] Fold the CLI's one command into its entry; tidy the table helpers The CLI's only command lives in its entry now. The table names its column order apart from each metric's variants, the comment renderer filters without renaming, and the history chart leaves out unchanged comparisons, which sit at zero and have no bar to draw. --- .../src/components/BenchmarkRunHistory.tsx | 36 ++++++------- .../lib/benchmarkRun/buildMarkdownReport.ts | 8 +-- packages/benchmark/src/cli/index.ts | 52 ++++++++++++++++++- packages/benchmark/src/cli/run.ts | 52 ------------------- packages/benchmark/src/runReport/format.ts | 10 ++-- 5 files changed, 76 insertions(+), 82 deletions(-) delete mode 100644 packages/benchmark/src/cli/run.ts diff --git a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx index cf91279f5c..32c44ad987 100644 --- a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx +++ b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx @@ -25,11 +25,11 @@ import ErrorDisplay from './ErrorDisplay'; * read off the chart directly, rather than inferred from a noisy trend. */ -const CHANGE_COLORS: Record = { +// An `unchanged` comparison sits at zero, so it has no bar to colour. +const CHANGE_COLORS: Record, string> = { worse: 'var(--mui-palette-error-main)', better: 'var(--mui-palette-success-main)', unsure: 'var(--mui-palette-grey-500)', - unchanged: 'var(--mui-palette-grey-500)', }; /** Per benchmark, per metric: a commit's change against its parent. */ @@ -118,23 +118,21 @@ export default function BenchmarkRunHistory({ repo }: BenchmarkRunHistoryProps) ? undefined : point.comparisons.get(benchmark)?.get(metric), ); - return (['worse', 'better', 'unsure', 'unchanged'] as const satisfies Change[]).map( - (change) => ({ - type: 'bar' as const, - stack: 'change', - label: change, - color: CHANGE_COLORS[change], - data: selected.map((comparison) => - comparison?.change === change - ? (comparison.relative.low + comparison.relative.high) / 2 - : null, - ), - valueFormatter: (_value: number | null, { dataIndex }: { dataIndex: number }) => { - const comparison = selected[dataIndex]; - return comparison?.change === change ? formatComparison(comparison) : null; - }, - }), - ); + return (['worse', 'better', 'unsure'] as const satisfies Change[]).map((change) => ({ + type: 'bar' as const, + stack: 'change', + label: change, + color: CHANGE_COLORS[change], + data: selected.map((comparison) => + comparison?.change === change + ? (comparison.relative.low + comparison.relative.high) / 2 + : null, + ), + valueFormatter: (_value: number | null, { dataIndex }: { dataIndex: number }) => { + const comparison = selected[dataIndex]; + return comparison?.change === change ? formatComparison(comparison) : null; + }, + })); }, [points, benchmark, metric]); const [selectedSha, setSelectedSha] = React.useState(null); diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 94402df755..7964f8b244 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -24,11 +24,11 @@ interface BuildOptions { */ function renderBenchmarkTable(analysis: BenchmarkAnalysis): string { const heading = `**${analysis.benchmark.name}** · ${formatRounds(analysis.benchmark)}`; - const { columns, rows: allRows } = benchmarkTable(analysis); - const rows = allRows.filter((row) => + const { columns, rows } = benchmarkTable(analysis); + const changed = rows.filter((row) => row.comparisons.some(({ change }) => change === 'better' || change === 'worse'), ); - if (rows.length === 0) { + if (changed.length === 0) { return `${heading} · no change detected`; } const table = formatMarkdownTable( @@ -37,7 +37,7 @@ function renderBenchmarkTable(analysis: BenchmarkAnalysis): string { header, align: kind === 'value' ? ('right' as const) : ('left' as const), })), - rows.map(({ cells }) => + changed.map(({ cells }) => Object.fromEntries( cells.map((cell, column) => [ String(column), diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index a1fa2f06c5..b75a664853 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -2,13 +2,61 @@ import chalk from 'chalk'; import yargs from 'yargs'; +import type { CommandModule } from 'yargs'; import { hideBin } from 'yargs/helpers'; // Self-referencing rather than `../../package.json`: this file is one directory deeper in the // repository than in the published package, which is built from `build/`, so a relative path // cannot reach the manifest from both — and the repository's workspace symlink hides the break. import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; -import run from './run'; +import type { RunBenchmarksOptions } from '../runner/runBenchmarks'; + +/** Everything `runBenchmarks` takes except the directory, which is where the command was run. */ +type Args = Omit; + +const runCommand: CommandModule<{}, Args> = { + command: '$0 [filters...]', + describe: + "Benchmark a harness's *.bench.tsx files, the working tree against a baseline build, and write a JSON report. Run from the harness package.", + builder: (args) => + args + .positional('filters', { + type: 'string', + array: true, + describe: 'Only run benchmark files whose path under src contains this substring', + }) + .option('baseline', { + type: 'string', + describe: + 'The build to compare against: a revision, on its own or as git:. Defaults to HEAD~1, the parent commit; `code-infra baseline` resolves a fork point to pass here', + }) + .option('build-cmd', { + type: 'string', + describe: + "Command that builds the publishable workspace packages, in each ref's checkout and in the working tree. Default: pnpm release:build", + }) + .option('upload', { + type: 'boolean', + default: process.env.BENCHMARK_UPLOAD === 'true', + describe: + 'Upload the report and refresh the PR comment. Defaults to BENCHMARK_UPLOAD=true, so CI can switch it on by environment. Requires CIRCLE_OIDC_TOKEN_V2', + }) + .option('out', { + type: 'string', + describe: 'Write the JSON report here. Default: .benchmark/results/report.json', + }), + handler: async (argv) => { + const { runBenchmarks } = await import('../runner/runBenchmarks'); + await runBenchmarks({ + harnessDir: process.cwd(), + filters: argv.filters ?? [], + baseline: argv.baseline, + buildCmd: argv.buildCmd, + out: argv.out, + upload: argv.upload, + }); + }, +}; let globalArgv: { verbose?: boolean } = {}; @@ -24,7 +72,7 @@ await yargs(hideBin(process.argv)) .middleware((argv) => { globalArgv = argv; }, true) - .command(run) + .command(runCommand) .fail((msg, err, yargsInstance) => { if (msg) { yargsInstance.showHelp(); diff --git a/packages/benchmark/src/cli/run.ts b/packages/benchmark/src/cli/run.ts deleted file mode 100644 index fec045a53d..0000000000 --- a/packages/benchmark/src/cli/run.ts +++ /dev/null @@ -1,52 +0,0 @@ -import type { CommandModule } from 'yargs'; -import type { RunBenchmarksOptions } from '../runner/runBenchmarks'; - -/** Everything `runBenchmarks` takes except the directory, which is where the command was run. */ -type Args = Omit; - -const command: CommandModule<{}, Args> = { - command: '$0 [filters...]', - describe: - "Benchmark a harness's *.bench.tsx files, the working tree against a baseline build, and write a JSON report. Run from the harness package.", - builder: (yargs) => { - return yargs - .positional('filters', { - type: 'string', - array: true, - describe: 'Only run benchmark files whose path under src contains this substring', - }) - .option('baseline', { - type: 'string', - describe: - 'The build to compare against: a revision, on its own or as git:. Defaults to HEAD~1, the parent commit; `code-infra baseline` resolves a fork point to pass here', - }) - .option('build-cmd', { - type: 'string', - describe: - "Command that builds the publishable workspace packages, in each ref's checkout and in the working tree. Default: pnpm release:build", - }) - .option('upload', { - type: 'boolean', - default: process.env.BENCHMARK_UPLOAD === 'true', - describe: - 'Upload the report and refresh the PR comment. Defaults to BENCHMARK_UPLOAD=true, so CI can switch it on by environment. Requires CIRCLE_OIDC_TOKEN_V2', - }) - .option('out', { - type: 'string', - describe: 'Write the JSON report here. Default: .benchmark/results/report.json', - }); - }, - handler: async (argv) => { - const { runBenchmarks } = await import('../runner/runBenchmarks'); - await runBenchmarks({ - harnessDir: process.cwd(), - filters: argv.filters ?? [], - baseline: argv.baseline, - buildCmd: argv.buildCmd, - out: argv.out, - upload: argv.upload, - }); - }, -}; - -export default command; diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index 0d2d7adeea..108c1350b3 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -40,7 +40,7 @@ export function formatPercent(interval: Interval): string { /** A comparison as one phrase: `worse +2.7% – +6.8%`, `unsure -1.2% – +0.9%`, `unchanged`. */ export function formatComparison(comparison: MetricComparison): string { return comparison.change === 'unchanged' - ? comparison.change + ? 'unchanged' : `${comparison.change} ${formatPercent(comparison.relative)}`; } @@ -75,22 +75,22 @@ export function benchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): { } { // Builds read old to new, the baseline first, like a diff; compared cases keep the reference // first, as the comparison columns measure against it. - const variants = + const columnOrder = benchmark.kind === 'baseline' ? [...benchmark.variants].reverse() : benchmark.variants; const columns: BenchmarkTableColumn[] = [ { header: 'Metric', kind: 'label' }, - ...variants.map((variant) => ({ header: variant, kind: 'value' as const })), + ...columnOrder.map((variant) => ({ header: variant, kind: 'value' as const })), ...(metrics[0]?.comparisons ?? []).map((comparison) => ({ header: formatComparisonLabel(benchmark, comparison), kind: 'comparison' as const, })), ]; - const rows = metrics.map(({ metric, definition, variants: medians, comparisons }) => ({ + const rows = metrics.map(({ metric, definition, variants, comparisons }) => ({ metric, comparisons, cells: [ metric, - ...variants.map((variant) => formatValue(medians[variant].median, definition)), + ...columnOrder.map((variant) => formatValue(variants[variant].median, definition)), ...comparisons.map(formatComparison), ], })); From a6905eb879d99a2de1b9b8f1100b67a481f91556 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:22:40 +0200 Subject: [PATCH 117/173] [benchmark] Say "no change detected" rather than "unsure" "Unsure" read as the tool failing; the interval straddling zero means no change was detected, which is what an equivalent build should show. --- .../src/components/BenchmarkRunHistory.tsx | 8 ++--- .../benchmarkRun/buildMarkdownReport.test.ts | 2 +- .../src/views/BenchmarkRunDetails.tsx | 2 +- packages/benchmark/README.md | 2 +- .../src/runReport/analyzeRun.test.ts | 8 ++--- .../benchmark/src/runReport/analyzeRun.ts | 9 +++--- packages/benchmark/src/runReport/format.ts | 32 ++++++++++++++++--- test/benchmark/README.md | 2 +- 8 files changed, 44 insertions(+), 21 deletions(-) diff --git a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx index 32c44ad987..c02f3334e2 100644 --- a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx +++ b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx @@ -8,7 +8,7 @@ import Paper from '@mui/material/Paper'; import TextField from '@mui/material/TextField'; import Typography from '@mui/material/Typography'; import { BarChartPro } from '@mui/x-charts-pro/BarChartPro'; -import { analyzeRun, formatComparison } from '@mui/internal-benchmark/runReport'; +import { analyzeRun, formatChange, formatComparison } from '@mui/internal-benchmark/runReport'; import type { BenchmarkRunReport, Change, @@ -29,7 +29,7 @@ import ErrorDisplay from './ErrorDisplay'; const CHANGE_COLORS: Record, string> = { worse: 'var(--mui-palette-error-main)', better: 'var(--mui-palette-success-main)', - unsure: 'var(--mui-palette-grey-500)', + undetected: 'var(--mui-palette-grey-500)', }; /** Per benchmark, per metric: a commit's change against its parent. */ @@ -118,10 +118,10 @@ export default function BenchmarkRunHistory({ repo }: BenchmarkRunHistoryProps) ? undefined : point.comparisons.get(benchmark)?.get(metric), ); - return (['worse', 'better', 'unsure'] as const satisfies Change[]).map((change) => ({ + return (['worse', 'better', 'undetected'] as const satisfies Change[]).map((change) => ({ type: 'bar' as const, stack: 'change', - label: change, + label: formatChange(change), color: CHANGE_COLORS[change], data: selected.map((comparison) => comparison?.change === change diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index 8a82e0f8ef..1fa94eb8a9 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -111,7 +111,7 @@ describe('buildBenchmarkRunMarkdownReport', () => { | render | 10 ms | 20.5 ms | \`worse +95.8% – +114.2%\` | - _Each value is a median. Each Δ is a 95% confidence interval on the paired per-round difference, relative to the variant it is measured against; "unsure" means it straddles zero — the expected result for two equivalent builds — and "unchanged" that every round measured the same._ + _Each value is a median. Each Δ is a 95% confidence interval on the paired per-round difference, relative to the variant it is measured against; "no change detected" means it straddles zero — the expected result for two equivalent builds — and "unchanged" that every round measured the same._
    " `); diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index b72dd3fb97..d644f8defb 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -31,7 +31,7 @@ import Heading from '../components/Heading'; import ReportHeader from '../components/ReportHeader'; import ErrorDisplay from '../components/ErrorDisplay'; -/** A regression is coloured by severity; `better` is green; `unsure` is the expected result. */ +/** A regression is coloured by severity; `better` is green; no change detected is the expected result. */ function comparisonColor(comparison: MetricComparison): string { if (comparison.severity === 'error') { return 'error'; diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index e9caf88b39..bcd3611a17 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -439,7 +439,7 @@ history from it. - The report holds **raw samples**, round-aligned across variants, plus each metric's kind, format and alarm, how each benchmark's sampling went, the builds, and the environment. - `analyzeRun` from `@mui/internal-benchmark/runReport` draws every conclusion from it: a confidence - interval on the paired difference per metric, a change (`better`, `worse`, `unsure`, or + interval on the paired difference per metric, a change (`better`, `worse`, `no change detected`, or `unchanged` when every round measured the same), and a severity from the metric's alarm. - `reactBenchmark()`'s `render` alarms on any resolved change for the worse; `render:count`, the per-phase split and `bench:paint` are informational; every other metric brings its own alarm. diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index d6740a4b59..9bfab90a81 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -107,14 +107,14 @@ describe('compareSamples', () => { expect(comparison.change).toBe('unchanged'); }); - it('is unsure when the interval straddles zero', () => { + it('detects no change when the interval straddles zero', () => { const comparison = compareSamples( { name: 'current', values: [10, 12, 9, 11] }, { name: 'baseline', values: [11, 10, 10, 12] }, SCALAR, ); - expect(comparison.change).toBe('unsure'); + expect(comparison.change).toBe('undetected'); }); describe('severity', () => { @@ -282,7 +282,7 @@ describe('analyzeRun', () => { expect(widthOf(withSteady)).toBe(widthOf(single)); }); - it('can leave unsure what 95% would have flagged', () => { + it('can leave undetected what 95% would have flagged', () => { const { base, shifted } = drifting(2); const flagged = benchmarkOf('d', { current: { render: shifted.map((value, round) => value + (round % 2 ? 2 : -2)) }, @@ -293,7 +293,7 @@ describe('analyzeRun', () => { const [crowded] = analyzeRun(reportOf([flagged, ...crowd], { render: SCALAR })); expect(single.metrics[0].comparisons[0].change).toBe('worse'); - expect(crowded.metrics[0].comparisons[0].change).toBe('unsure'); + expect(crowded.metrics[0].comparisons[0].change).toBe('undetected'); }); }); diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index 2ad84049b8..2f6becedb3 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -26,10 +26,11 @@ export interface SampleSummary { } /** - * Which way a resolved change went, given the metric's direction; `unsure` if it did not resolve, - * and `unchanged` if every round measured the same on both sides — a render count, say. + * Which way a resolved change went, given the metric's direction; `undetected` if it did not + * resolve — no change detected — and `unchanged` if every round measured the same on both sides, + * a render count, say. */ -export type Change = 'better' | 'worse' | 'unsure' | 'unchanged'; +export type Change = 'better' | 'worse' | 'undetected' | 'unchanged'; /** How much a change for the worse matters, per the metric's alarm. */ export type Severity = 'error' | 'warning' | 'none'; @@ -251,7 +252,7 @@ export function compareSamples( high: (absolute.high / reference) * 100, }; - let change: Change = 'unsure'; + let change: Change = 'undetected'; if (rounds > 0 && differences.every((difference) => difference === 0)) { change = 'unchanged'; } else if (absolute.low > 0 || absolute.high < 0) { diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index 108c1350b3..992c4fba0a 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -1,5 +1,11 @@ import { CONFIDENCE } from './analyzeRun'; -import type { BenchmarkAnalysis, Interval, MetricComparison, Regression } from './analyzeRun'; +import type { + BenchmarkAnalysis, + Change, + Interval, + MetricComparison, + Regression, +} from './analyzeRun'; import type { RunBenchmark, RunMetricDefinition } from './schema'; /** @@ -37,11 +43,27 @@ export function formatPercent(interval: Interval): string { return `${signed(interval.low)} – ${signed(interval.high)}`; } -/** A comparison as one phrase: `worse +2.7% – +6.8%`, `unsure -1.2% – +0.9%`, `unchanged`. */ +const CHANGE_LABELS: Record = { + better: 'better', + worse: 'worse', + undetected: 'no change detected', + unchanged: 'unchanged', +}; + +/** How a change reads: `better`, `worse`, `no change detected`, `unchanged`. */ +export function formatChange(change: Change): string { + return CHANGE_LABELS[change]; +} + +/** + * A comparison as one phrase: `worse +2.7% – +6.8%`, `no change detected -1.2% – +0.9%`, + * `unchanged`. + */ export function formatComparison(comparison: MetricComparison): string { + const label = formatChange(comparison.change); return comparison.change === 'unchanged' - ? 'unchanged' - : `${comparison.change} ${formatPercent(comparison.relative)}`; + ? label + : `${label} ${formatPercent(comparison.relative)}`; } /** What a comparison column is headed: `Δ vs baseline` for a build, `theirs vs ours` for a variant. */ @@ -146,7 +168,7 @@ export function runReportFootnote(analyses: BenchmarkAnalysis[]): string { : `Each Δ is a ${formatConfidence(CONFIDENCE)} confidence interval on the paired per-round ` + 'difference, relative to the variant it is measured against'; return ( - `Each value is a median. ${interval}; "unsure" means it straddles zero — the expected ` + + `Each value is a median. ${interval}; "no change detected" means it straddles zero — the expected ` + 'result for two equivalent builds — and "unchanged" that every round measured the same.' ); } diff --git a/test/benchmark/README.md b/test/benchmark/README.md index ac6dd21e7d..812b65fc18 100644 --- a/test/benchmark/README.md +++ b/test/benchmark/README.md @@ -38,7 +38,7 @@ pnpm -F ./test/benchmark benchmark:run --baseline HEAD ``` A filter is a case-insensitive substring of a benchmark file's path under `src/`, the way vitest -matches test files. Because both sides run identical code, the expected result is `unsure` — that is +matches test files. Because both sides run identical code, the expected result is `no change detected` — that is the "no change" outcome, not a failure. Everything a run writes goes under `.benchmark/`: the report in `results/`, the pages built per ref in From b87310fab1893e4b45d5f5787ad2d357127c49f0 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 16:25:29 +0200 Subject: [PATCH 118/173] [benchmark] Read the footnote's verdict names from formatChange The footnote quotes the same labels the tables show, so a relabel can't leave it behind; formatChange renames the one verdict that needs it. --- .../benchmark/src/runReport/analyzeRun.test.ts | 2 +- packages/benchmark/src/runReport/analyzeRun.ts | 6 +++--- packages/benchmark/src/runReport/format.ts | 14 ++++---------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index 9bfab90a81..d6ab4e4b94 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -282,7 +282,7 @@ describe('analyzeRun', () => { expect(widthOf(withSteady)).toBe(widthOf(single)); }); - it('can leave undetected what 95% would have flagged', () => { + it('can miss a change that 95% would have flagged', () => { const { base, shifted } = drifting(2); const flagged = benchmarkOf('d', { current: { render: shifted.map((value, round) => value + (round % 2 ? 2 : -2)) }, diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index 2f6becedb3..9907074124 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -26,9 +26,9 @@ export interface SampleSummary { } /** - * Which way a resolved change went, given the metric's direction; `undetected` if it did not - * resolve — no change detected — and `unchanged` if every round measured the same on both sides, - * a render count, say. + * Which way a resolved change went, given the metric's direction; `undetected` (shown as "no change + * detected") if it did not resolve; `unchanged` if every round measured the same on both sides, a + * render count, say. */ export type Change = 'better' | 'worse' | 'undetected' | 'unchanged'; diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index 992c4fba0a..35f9f46f94 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -43,16 +43,9 @@ export function formatPercent(interval: Interval): string { return `${signed(interval.low)} – ${signed(interval.high)}`; } -const CHANGE_LABELS: Record = { - better: 'better', - worse: 'worse', - undetected: 'no change detected', - unchanged: 'unchanged', -}; - /** How a change reads: `better`, `worse`, `no change detected`, `unchanged`. */ export function formatChange(change: Change): string { - return CHANGE_LABELS[change]; + return change === 'undetected' ? 'no change detected' : change; } /** @@ -168,8 +161,9 @@ export function runReportFootnote(analyses: BenchmarkAnalysis[]): string { : `Each Δ is a ${formatConfidence(CONFIDENCE)} confidence interval on the paired per-round ` + 'difference, relative to the variant it is measured against'; return ( - `Each value is a median. ${interval}; "no change detected" means it straddles zero — the expected ` + - 'result for two equivalent builds — and "unchanged" that every round measured the same.' + `Each value is a median. ${interval}; "${formatChange('undetected')}" means it straddles ` + + 'zero — the expected result for two equivalent builds — and ' + + `"${formatChange('unchanged')}" that every round measured the same.` ); } From 33271acb4358d43c81b14c6a1b18b3d75cbbeabf Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:32:14 +0200 Subject: [PATCH 119/173] [benchmark] Keep function names and emit sourcemaps in the measured pages CPU profiles and stack traces of the built pages now read with real names. Compression stays on, so the code is otherwise the production code. Measured against mangled pages, 3 runs of every benchmark: no alarmed metric moved (15 of 15 render and json:parse intervals include zero); the 3 paint intervals that excluded zero point both ways. --- packages/benchmark/src/runner/plugin.test.ts | 4 ++-- packages/benchmark/src/runner/plugin.ts | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/benchmark/src/runner/plugin.test.ts b/packages/benchmark/src/runner/plugin.test.ts index 48363271af..0c92da32bf 100644 --- a/packages/benchmark/src/runner/plugin.test.ts +++ b/packages/benchmark/src/runner/plugin.test.ts @@ -56,7 +56,7 @@ describe('benchmarkPlugin', () => { const config = await callConfig(harnessDir, 'serve'); - expect(config.build?.rollupOptions).toBeUndefined(); + expect(config.build?.rolldownOptions).toBeUndefined(); }); it('points serving at the output directory too, so preview finds the built pages', async () => { @@ -99,7 +99,7 @@ describe('benchmarkPlugin', () => { const config = await callConfig(harnessDir, 'build'); - expect(config.build.rollupOptions.input).toEqual({ + expect(config.build.rolldownOptions.input).toEqual({ '__bench__/button-button': path.join(harnessDir, 'src', '__bench__', 'button-button.html'), '__bench__/grid-grid': path.join(harnessDir, 'src', '__bench__', 'grid-grid.html'), }); diff --git a/packages/benchmark/src/runner/plugin.ts b/packages/benchmark/src/runner/plugin.ts index 7e0ef93a34..ee2b536263 100644 --- a/packages/benchmark/src/runner/plugin.ts +++ b/packages/benchmark/src/runner/plugin.ts @@ -171,7 +171,11 @@ export function benchmarkPlugin(options: BenchmarkPluginOptions = {}): Plugin { // The output directory sits outside `root`, which is `src/`; allow vite to clean it anyway. emptyOutDir: true, chunkSizeWarningLimit: 9999, - rollupOptions: { input }, + // Function names survive into CPU profiles and stack traces: a name's length costs a + // page nothing once it is parsed, and parsing happens on load, before a sample. + // Compression stays on, so the code is the production code otherwise. + sourcemap: true, + rolldownOptions: { input, output: { minify: { compress: true, mangle: false } } }, }, }; }, From 24d3951b4e07dc1071ca2f65b06d0149031cff4b Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:36:31 +0200 Subject: [PATCH 120/173] [benchmark] Colour changes, line up tables, report how long a run took MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Changes draw the eye: the dashboard tints a comparison red when it fails the check, orange for any other change for the worse and green for one for the better; the pull request comment marks each row 🔴/🟠/🟢. - Dashboard tables share fixed column widths, so their columns line up; the comment puts every baseline benchmark's changes in one table. - The report records how long the run and each benchmark took, shown on the summary line and next to each benchmark's rounds. - The footnote is one line. - A compare() comparison never carries a severity: variants differ on purpose. --- .../benchmarkRun/buildMarkdownReport.test.ts | 14 +- .../lib/benchmarkRun/buildMarkdownReport.ts | 126 ++++++++++++++---- .../src/views/BenchmarkRunDetails.tsx | 67 ++++++---- .../benchmark/src/runReport/analyzeRun.ts | 8 +- .../benchmark/src/runReport/format.test.ts | 38 +++++- packages/benchmark/src/runReport/format.ts | 76 ++++++++--- packages/benchmark/src/runReport/schema.ts | 4 + packages/benchmark/src/runner/printReport.ts | 2 +- .../benchmark/src/runner/runBenchmarks.ts | 2 + .../benchmark/src/runner/runInterleaved.ts | 12 +- 10 files changed, 262 insertions(+), 87 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index 1fa94eb8a9..c90aa2e781 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -64,7 +64,7 @@ describe('buildBenchmarkRunMarkdownReport', () => { title: 'Performance', }); - expect(markdown).toContain('**Button mount** · 4 rounds · no change detected'); + expect(markdown).toContain('No change detected: **Button mount**'); expect(markdown).not.toContain('| render |'); }); @@ -97,21 +97,19 @@ describe('buildBenchmarkRunMarkdownReport', () => {
    Changes - **Grid scroll** · 3 + 1 rounds · timed out - - | Metric | baseline | current | Δ vs baseline | - |:----------|----------:|----------:|:----------| - | render | 21.5 ms | 23.5 ms | \`worse +9.3% – +9.3%\` | + | Benchmark | Metric | baseline | current | Δ vs baseline | + |:----------|:----------|----------:|----------:|:----------| + | **Grid scroll** | render | 21.5 ms | 23.5 ms | 🔴 \`worse +9.3% – +9.3%\` | **libs / mount** · 4 rounds | Metric | ours | theirs | theirs vs ours | |:----------|----------:|----------:|:----------| - | render | 10 ms | 20.5 ms | \`worse +95.8% – +114.2%\` | + | render | 10 ms | 20.5 ms | 🟠 \`worse +95.8% – +114.2%\` | - _Each value is a median. Each Δ is a 95% confidence interval on the paired per-round difference, relative to the variant it is measured against; "no change detected" means it straddles zero — the expected result for two equivalent builds — and "unchanged" that every round measured the same._ + _Medians; Δ is the confidence interval of the paired per-round difference (95%)._
    " `); diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 7964f8b244..416ffb9503 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -2,13 +2,21 @@ import { formatMarkdownTable } from '@/utils/formatters'; import { analyzeRun, benchmarkTable, + changeTone, findRegressions, + formatDuration, formatRounds, formatComparison, formatRunSummary, runReportFootnote, } from '@mui/internal-benchmark/runReport'; -import type { BenchmarkAnalysis, BenchmarkRunReport } from '@mui/internal-benchmark/runReport'; +import type { + BenchmarkAnalysis, + BenchmarkRunReport, + BenchmarkTableColumn, + BenchmarkTableRow, + ChangeTone, +} from '@mui/internal-benchmark/runReport'; interface BuildOptions { /** The comment section's heading. */ @@ -17,34 +25,94 @@ interface BuildOptions { detailsUrl?: string; } -/** - * One table per benchmark, of the metrics that changed — better or worse — with the median of every - * variant and every comparison. A benchmark where nothing changed is one line; the full run has - * every metric. - */ -function renderBenchmarkTable(analysis: BenchmarkAnalysis): string { - const heading = `**${analysis.benchmark.name}** · ${formatRounds(analysis.benchmark)}`; +const TONE_MARKERS: Record = { + error: '🔴 ', + warning: '🟠 ', + success: '🟢 ', + none: '', +}; + +/** The rows of a benchmark's table that changed — better or worse — with their columns. */ +function changedRowsOf(analysis: BenchmarkAnalysis) { const { columns, rows } = benchmarkTable(analysis); const changed = rows.filter((row) => row.comparisons.some(({ change }) => change === 'better' || change === 'worse'), ); - if (changed.length === 0) { - return `${heading} · no change detected`; - } - const table = formatMarkdownTable( - columns.map(({ header, kind }, column) => ({ + return { columns, changed }; +} + +/** A row's cells for markdown: each comparison as code, behind a marker of which way it went. */ +function markdownCells(columns: BenchmarkTableColumn[], row: BenchmarkTableRow): string[] { + const firstComparison = columns.findIndex((column) => column.kind === 'comparison'); + return row.cells.map((cell, column) => { + if (columns[column].kind !== 'comparison') { + return cell; + } + return `${TONE_MARKERS[changeTone(row.comparisons[column - firstComparison])]}\`${cell}\``; + }); +} + +function markdownTable( + headers: string[], + kinds: Array, + rows: string[][], +) { + return formatMarkdownTable( + headers.map((header, column) => ({ field: String(column), header, - align: kind === 'value' ? ('right' as const) : ('left' as const), + align: kinds[column] === 'value' ? ('right' as const) : ('left' as const), })), - changed.map(({ cells }) => - Object.fromEntries( - cells.map((cell, column) => [ - String(column), - columns[column].kind === 'comparison' ? `\`${cell}\`` : cell, - ]), - ), - ), + rows.map((cells) => Object.fromEntries(cells.map((cell, column) => [String(column), cell]))), + ); +} + +/** + * The benchmarks measured against the baseline, as one table of the metrics that changed, so the + * columns line up across benchmarks; those where nothing changed are named below it. + */ +function renderBaselineChanges(analyses: BenchmarkAnalysis[]): string[] { + const lines: string[] = []; + const unchanged: string[] = []; + let header: { headers: string[]; kinds: Array } | undefined; + const rows: string[][] = []; + for (const analysis of analyses) { + const { name, durationMs } = analysis.benchmark; + const label = + durationMs === undefined ? `**${name}**` : `**${name}** (${formatDuration(durationMs)})`; + const { columns, changed } = changedRowsOf(analysis); + if (changed.length === 0) { + unchanged.push(label); + continue; + } + header ??= { + headers: ['Benchmark', ...columns.map((column) => column.header)], + kinds: ['label', ...columns.map((column) => column.kind)], + }; + changed.forEach((row, index) => { + rows.push([index === 0 ? label : '', ...markdownCells(columns, row)]); + }); + } + if (header) { + lines.push(markdownTable(header.headers, header.kinds, rows), ''); + } + if (unchanged.length > 0) { + lines.push(`No change detected: ${unchanged.join(', ')}`, ''); + } + return lines; +} + +/** A `compare()` benchmark: a table of the metrics that changed, or one line when none did. */ +function renderComparison(analysis: BenchmarkAnalysis): string { + const heading = `**${analysis.benchmark.name}** · ${formatRounds(analysis.benchmark)}`; + const { columns, changed } = changedRowsOf(analysis); + if (changed.length === 0) { + return `${heading} · no change detected`; + } + const table = markdownTable( + columns.map((column) => column.header), + columns.map((column) => column.kind), + changed.map((row) => markdownCells(columns, row)), ); return `${heading}\n\n${table}`; } @@ -75,7 +143,7 @@ export function buildBenchmarkRunMarkdownReport( lines.push(''); } - lines.push(formatRunSummary(analyses, regressions), ''); + lines.push(formatRunSummary(analyses, regressions, report.durationMs), ''); for (const { benchmark } of failed) { lines.push(`❌ **${benchmark.name}**: ${benchmark.error}`); @@ -86,8 +154,16 @@ export function buildBenchmarkRunMarkdownReport( if (measured.length > 0) { lines.push('
    ', 'Changes', ''); - lines.push(measured.map(renderBenchmarkTable).join('\n\n')); - lines.push('', `_${runReportFootnote(analyses)}_`, '', '
    '); + lines.push( + ...renderBaselineChanges( + measured.filter((analysis) => analysis.benchmark.kind === 'baseline'), + ), + ); + const comparisons = measured.filter((analysis) => analysis.benchmark.kind === 'compare'); + if (comparisons.length > 0) { + lines.push(comparisons.map(renderComparison).join('\n\n'), ''); + } + lines.push(`_${runReportFootnote(analyses)}_`, '', ''); } if (options.detailsUrl) { diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index d644f8defb..29937d93d4 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -15,38 +15,52 @@ import TableHead from '@mui/material/TableHead'; import TableRow from '@mui/material/TableRow'; import Tooltip from '@mui/material/Tooltip'; import Typography from '@mui/material/Typography'; +import { alpha } from '@mui/material/styles'; +import type { SxProps, Theme } from '@mui/material/styles'; import { fetchCiReport, isBenchmarkRunUpload } from '@/utils/fetchCiReport'; import { analyzeRun, benchmarkTable, + changeTone, + formatDuration, formatRounds, runReportFootnote, } from '@mui/internal-benchmark/runReport'; import type { BenchmarkAnalysis, BenchmarkRunReport, + BenchmarkTableColumn, MetricComparison, } from '@mui/internal-benchmark/runReport'; import Heading from '../components/Heading'; import ReportHeader from '../components/ReportHeader'; import ErrorDisplay from '../components/ErrorDisplay'; -/** A regression is coloured by severity; `better` is green; no change detected is the expected result. */ -function comparisonColor(comparison: MetricComparison): string { - if (comparison.severity === 'error') { - return 'error'; +/** A comparison cell's colours: a tinted background where something moved, so the eye finds it. */ +function comparisonSx(comparison: MetricComparison): SxProps { + const tone = changeTone(comparison); + if (tone === 'none') { + return { color: 'text.secondary' }; } - if (comparison.severity === 'warning') { - return 'warning.main'; - } - return comparison.change === 'better' ? 'success.main' : 'text.secondary'; + return (theme) => ({ + color: theme.palette[tone].main, + bgcolor: alpha(theme.palette[tone].main, 0.12), + }); } +// Fixed widths per kind of column, so the columns of every benchmark's table line up. +const COLUMN_WIDTHS: Record = { + label: 220, + value: 120, + comparison: 280, +}; + function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { const { benchmark } = analysis; const { columns, rows } = benchmarkTable(analysis); const firstComparison = columns.findIndex((column) => column.kind === 'comparison'); const alignOf = (column: number) => (columns[column].kind === 'value' ? 'right' : undefined); + const tableWidth = columns.reduce((sum, column) => sum + COLUMN_WIDTHS[column.kind], 0); return ( @@ -71,7 +85,12 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) {     -
    +
    + + {columns.map((column, index) => ( + + ))} + {columns.map(({ header }, column) => ( @@ -84,21 +103,20 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { {rows.map((row) => ( - {row.cells.map((cell, column) => ( - - {columns[column].kind === 'comparison' ? ( - - {cell} - - ) : ( - cell - )} - - ))} + {row.cells.map((cell, column) => + columns[column].kind === 'comparison' ? ( + + {cell} + + ) : ( + + {cell} + + ), + )} ))} @@ -196,6 +214,7 @@ export default function BenchmarkRunDetails() { {runReportFootnote(analyses)} + {report.durationMs !== undefined && `Ran ${formatDuration(report.durationMs)} · `} {report.environment.browser} on {report.environment.platform}/ {report.environment.arch} {report.environment.launchArgs.length > 0 && diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index 9907074124..1296e71f06 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -350,9 +350,11 @@ export function compareBenchmark( return { metric, definition, - comparisons: pairs.map(({ subject, against }) => - compareSamples(subject, against, definition, confidence), - ), + comparisons: pairs.map(({ subject, against }) => { + const comparison = compareSamples(subject, against, definition, confidence); + // Variants of a `compare()` differ on purpose: no difference between them is a regression. + return canAlarm ? comparison : { ...comparison, severity: 'none' as const }; + }), }; }), }; diff --git a/packages/benchmark/src/runReport/format.test.ts b/packages/benchmark/src/runReport/format.test.ts index caf05c8c8f..8310c6441d 100644 --- a/packages/benchmark/src/runReport/format.test.ts +++ b/packages/benchmark/src/runReport/format.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it } from 'vitest'; import { analyzeRun } from './analyzeRun'; -import { benchmarkTable, runReportFootnote } from './format'; +import { + benchmarkTable, + formatDuration, + formatRounds, + formatRunSummary, + runReportFootnote, +} from './format'; import type { BenchmarkRunReport, RunBenchmark } from './schema'; function reportOf(benchmarks: RunBenchmark[]): BenchmarkRunReport { @@ -74,10 +80,33 @@ describe('benchmarkTable', () => { }); }); +describe('formatDuration', () => { + it('reads at a glance from milliseconds to hours', () => { + expect(formatDuration(850)).toBe('850ms'); + expect(formatDuration(42_000)).toBe('42s'); + expect(formatDuration(185_000)).toBe('3m 05s'); + expect(formatDuration(3_720_000)).toBe('1h 02m'); + }); +}); + +describe('formatRounds', () => { + it('says how long a benchmark took next to its rounds', () => { + expect(formatRounds({ ...benchmarkOf('a'), durationMs: 72_000 })).toBe('4 rounds · 1m 12s'); + }); +}); + +describe('formatRunSummary', () => { + it('ends with how long the run took', () => { + expect(formatRunSummary(analyzeRun(reportOf([benchmarkOf('a')])), [], 845_000)).toBe( + '1 benchmark measured · no regressions · ran 14m 05s', + ); + }); +}); + describe('runReportFootnote', () => { it('states one level when at most one comparison can raise an alarm', () => { - expect(runReportFootnote(analyzeRun(reportOf([benchmarkOf('a')])))).toContain( - 'Each Δ is a 95% confidence interval', + expect(runReportFootnote(analyzeRun(reportOf([benchmarkOf('a')])))).toBe( + 'Medians; Δ is the confidence interval of the paired per-round difference (95%).', ); }); @@ -87,7 +116,6 @@ describe('runReportFootnote', () => { reportOf([benchmarkOf('a'), benchmarkOf('b'), benchmarkOf('c'), benchmarkOf('d')]), ), ); - expect(footnote).toContain('98.75% where a change can raise an alarm'); - expect(footnote).toContain('95% for the rest'); + expect(footnote).toContain('(98.75% for alarmed metrics, 95% otherwise)'); }); }); diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index 35f9f46f94..8a7cfff240 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -43,6 +43,23 @@ export function formatPercent(interval: Interval): string { return `${signed(interval.low)} – ${signed(interval.high)}`; } +/** + * The colour a comparison draws the eye with: `error` for a regression that fails the check, + * `warning` for any other change for the worse, `success` for one for the better, and `none` when + * nothing moved. + */ +export type ChangeTone = 'error' | 'warning' | 'success' | 'none'; + +export function changeTone({ change, severity }: MetricComparison): ChangeTone { + if (severity === 'error') { + return 'error'; + } + if (change === 'worse') { + return 'warning'; + } + return change === 'better' ? 'success' : 'none'; +} + /** How a change reads: `better`, `worse`, `no change detected`, `unchanged`. */ export function formatChange(change: Change): string { return change === 'undetected' ? 'no change detected' : change; @@ -112,9 +129,26 @@ export function benchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): { return { columns, rows }; } +/** A duration at a glance: `850ms`, `42s`, `3m 05s`, `1h 02m`. */ +export function formatDuration(durationMs: number): string { + if (durationMs < 1000) { + return `${Math.round(durationMs)}ms`; + } + const seconds = Math.round(durationMs / 1000); + if (seconds < 60) { + return `${seconds}s`; + } + const minutes = Math.floor(seconds / 60); + const twoDigits = (value: number) => String(value).padStart(2, '0'); + return minutes < 60 + ? `${minutes}m ${twoDigits(seconds % 60)}s` + : `${Math.floor(minutes / 60)}h ${twoDigits(minutes % 60)}m`; +} + /** - * How many rounds a benchmark ran, and whether sampling resolved: `50 rounds`, `50 + 23 rounds` past - * the sample size, `50 + 312 rounds · timed out`. `null` for a benchmark that measured nothing. + * How many rounds a benchmark ran, how long it took, and whether sampling resolved: `50 rounds`, + * `50 + 23 rounds · 1m 12s` past the sample size, `50 + 312 rounds · 3m 04s · timed out`. `null` + * for a benchmark that measured nothing. */ export function formatRounds(benchmark: RunBenchmark): string | null { const [reference] = benchmark.variants; @@ -123,13 +157,24 @@ export function formatRounds(benchmark: RunBenchmark): string | null { return null; } const sampleSize = benchmark.sampling?.sampleSize ?? rounds; - const counted = - rounds > sampleSize ? `${sampleSize} + ${rounds - sampleSize} rounds` : `${rounds} rounds`; - return benchmark.sampling?.timedOut ? `${counted} · timed out` : counted; + return [ + rounds > sampleSize ? `${sampleSize} + ${rounds - sampleSize} rounds` : `${rounds} rounds`, + benchmark.durationMs === undefined ? null : formatDuration(benchmark.durationMs), + benchmark.sampling?.timedOut ? 'timed out' : null, + ] + .filter(Boolean) + .join(' · '); } -/** The line a run is summed up in: how many benchmarks measured, regressed, timed out and failed. */ -export function formatRunSummary(analyses: BenchmarkAnalysis[], regressions: Regression[]): string { +/** + * The line a run is summed up in: how many benchmarks measured, regressed, timed out and failed, and + * how long the run took when the report says. + */ +export function formatRunSummary( + analyses: BenchmarkAnalysis[], + regressions: Regression[], + durationMs?: number, +): string { const measured = analyses.filter((analysis) => !analysis.benchmark.error).length; const failed = analyses.length - measured; const regressed = new Set(regressions.map((regression) => regression.benchmark)).size; @@ -139,6 +184,7 @@ export function formatRunSummary(analyses: BenchmarkAnalysis[], regressions: Reg regressed > 0 ? `${regressed} with regressions` : 'no regressions', timedOut > 0 ? `${timedOut} timed out` : null, failed > 0 ? `${failed} failed` : null, + durationMs === undefined ? null : `ran ${formatDuration(durationMs)}`, ] .filter(Boolean) .join(' · '); @@ -152,19 +198,11 @@ export function runReportFootnote(analyses: BenchmarkAnalysis[]): string { metrics.flatMap(({ comparisons }) => comparisons.map((comparison) => comparison.confidence)), ), ); - const interval = + const levels = alarmed > CONFIDENCE - ? 'Each Δ is a confidence interval on the paired per-round difference, relative to the ' + - `variant it is measured against: ${formatConfidence(alarmed)} where a change can raise ` + - 'an alarm, so that together they raise a false one at most 5% of the time, and ' + - `${formatConfidence(CONFIDENCE)} for the rest` - : `Each Δ is a ${formatConfidence(CONFIDENCE)} confidence interval on the paired per-round ` + - 'difference, relative to the variant it is measured against'; - return ( - `Each value is a median. ${interval}; "${formatChange('undetected')}" means it straddles ` + - 'zero — the expected result for two equivalent builds — and ' + - `"${formatChange('unchanged')}" that every round measured the same.` - ); + ? `${formatConfidence(alarmed)} for alarmed metrics, ${formatConfidence(CONFIDENCE)} otherwise` + : formatConfidence(CONFIDENCE); + return `Medians; Δ is the confidence interval of the paired per-round difference (${levels}).`; } function formatConfidence(fraction: number): string { diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts index 62f3e1f572..249e3c40b4 100644 --- a/packages/benchmark/src/runReport/schema.ts +++ b/packages/benchmark/src/runReport/schema.ts @@ -61,6 +61,8 @@ const benchmarkSchema = z.object({ * `timedOut` says it was still unresolved when the timeout stopped it. */ sampling: z.object({ sampleSize: z.number(), timedOut: z.boolean() }).optional(), + /** How long measuring it took, page loads included. */ + durationMs: z.number().optional(), /** Why the benchmark produced no samples. */ error: z.string().optional(), }); @@ -68,6 +70,8 @@ const benchmarkSchema = z.object({ export const benchmarkRunReportSchema = z.object({ version: z.literal(2), generatedAt: z.string(), + /** How long the whole run took, packing and building both refs included. */ + durationMs: z.number().optional(), head: z.object({ sha: z.string(), branch: z.string().optional() }), environment: z.object({ browser: z.string(), diff --git a/packages/benchmark/src/runner/printReport.ts b/packages/benchmark/src/runner/printReport.ts index 42381a947e..b88ccb940d 100644 --- a/packages/benchmark/src/runner/printReport.ts +++ b/packages/benchmark/src/runner/printReport.ts @@ -47,6 +47,6 @@ export function printRunReport(report: BenchmarkRunReport): void { const color = comparison.severity === 'error' ? red : yellow; console.log(color(`⚠ ${benchmark} · ${metric} · ${formatComparison(comparison)}`)); } - console.log(formatRunSummary(analyses, regressions)); + console.log(formatRunSummary(analyses, regressions, report.durationMs)); console.log(dim(runReportFootnote(analyses))); } diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index e42e410e5b..caac5567fc 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -51,6 +51,7 @@ export interface RunBenchmarksOptions { * neither resolves the library through a workspace link. */ export async function runBenchmarks(options: RunBenchmarksOptions): Promise { + const startedAt = Date.now(); const { harnessDir, filters = [], @@ -150,6 +151,7 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise { console.log(chalk.cyan(`\nRunning "${entry.name}" (${entry.file})…`)); + const startedAt = Date.now(); const { sampleSize, timeout, autoSampleConditions } = sampling; const horizons = parseHorizons(autoSampleConditions); let opened: Page | undefined; @@ -249,12 +250,19 @@ async function runBenchmark( ); } return { - benchmark: { ...benchmarkOf(), sampling: { sampleSize, timedOut: !resolved } }, + benchmark: { + ...benchmarkOf(), + sampling: { sampleSize, timedOut: !resolved }, + durationMs: Date.now() - startedAt, + }, metrics: definitions, }; } catch (error) { console.error(chalk.red(` ${errorMessage(error)}`)); - return { benchmark: { ...entry, error: errorMessage(error) }, metrics: {} }; + return { + benchmark: { ...entry, error: errorMessage(error), durationMs: Date.now() - startedAt }, + metrics: {}, + }; } finally { await opened?.context().close(); } From 119208c5c4e269b81d706c56f32c3fc456d65170 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:48:20 +0200 Subject: [PATCH 121/173] [benchmark] Leave the repository as it was: lockfile, checkouts, pnpm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The restore puts back the lockfile byte for byte along with the manifest, so the reinstall resolves nothing again; it used to drift unrelated dependencies to newer versions. - A baseline's temporary checkout is removed when the run is interrupted, and both refs settle before a packing failure is thrown, so neither side's cleanup is cut short. - When `pnpm` isn't on PATH — corepack without its shims — a shim in the output directory forwards to `corepack pnpm`, so the commands the run starts, and the scripts they run, find the version each checkout pins. --- packages/benchmark/src/runner/buildPages.ts | 78 +++++++++++++------ .../benchmark/src/runner/runBenchmarks.ts | 13 +++- packages/benchmark/src/utils/interrupt.ts | 39 ++++++++++ packages/benchmark/src/utils/packWorkspace.ts | 3 + packages/benchmark/src/utils/pnpm.ts | 44 +++++++++++ 5 files changed, 154 insertions(+), 23 deletions(-) create mode 100644 packages/benchmark/src/utils/interrupt.ts diff --git a/packages/benchmark/src/runner/buildPages.ts b/packages/benchmark/src/runner/buildPages.ts index 43a4515088..0ed7206566 100644 --- a/packages/benchmark/src/runner/buildPages.ts +++ b/packages/benchmark/src/runner/buildPages.ts @@ -2,7 +2,7 @@ import * as path from 'node:path'; import { createRequire } from 'node:module'; -import { constants, copyFile, readFile, rm, writeFile } from 'node:fs/promises'; +import { access, constants, copyFile, readFile, rm, writeFile } from 'node:fs/promises'; import { pathToFileURL } from 'node:url'; import chalk from 'chalk'; import { execa } from 'execa'; @@ -30,32 +30,50 @@ function packedPins(packages: PackedPackage[]): Record { ); } -/** Where the repository's own manifest is kept while a run has it pinned. */ -function backupPathOf(outputDir: string): string { - return path.join(outputDir, 'pnpm-workspace.yaml.orig'); +const MANIFEST = 'pnpm-workspace.yaml'; +const LOCKFILE = 'pnpm-lock.yaml'; + +/** Where one of the repository's own files is kept while a run has it pinned. */ +function backupPathOf(outputDir: string, file: string): string { + return path.join(outputDir, `${file}.orig`); +} + +/** + * Sets one of the repository's files aside, unless an earlier ref of the run already did: that copy + * is the original. A file the repository doesn't have is left alone. + */ +async function setAside(repoRoot: string, outputDir: string, file: string): Promise { + try { + await copyFile( + path.join(repoRoot, file), + backupPathOf(outputDir, file), + constants.COPYFILE_EXCL, + ); + } catch (error) { + const { code } = error as NodeJS.ErrnoException; + if (code !== 'EEXIST' && code !== 'ENOENT') { + throw error; + } + } } /** * Points the repository's own install at this ref's packed build, through its overrides. The - * manifest is rewritten from a copy set aside first, so a second ref pins over the original. + * manifest is rewritten from a copy set aside first, so a second ref pins over the original. The + * lockfile is set aside too, so the restore can put back exactly the resolution the repository + * had rather than resolve it again. */ async function pinPackedPackages( repoRoot: string, outputDir: string, packages: PackedPackage[], ): Promise { - const manifestPath = path.join(repoRoot, 'pnpm-workspace.yaml'); - const backupPath = backupPathOf(outputDir); - // The first ref sets the repository's manifest aside; a later one finds that copy and keeps it. - try { - await copyFile(manifestPath, backupPath, constants.COPYFILE_EXCL); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'EEXIST') { - throw error; - } - } + // The lockfile first: the manifest's copy is what marks the repository pinned. + await setAside(repoRoot, outputDir, LOCKFILE); + await setAside(repoRoot, outputDir, MANIFEST); - const config = parse(await readFile(backupPath, 'utf8')) ?? {}; + const manifestPath = path.join(repoRoot, MANIFEST); + const config = parse(await readFile(backupPathOf(outputDir, MANIFEST), 'utf8')) ?? {}; await writeFile( manifestPath, stringify({ ...config, overrides: { ...config.overrides, ...packedPins(packages) } }), @@ -73,12 +91,14 @@ async function installWorkspace(repoRoot: string): Promise { }); } -/** Puts the repository's manifest back, and installs from it. */ +/** + * Puts the repository's manifest and lockfile back as they were, byte for byte, and installs from + * them: the lockfile already satisfies the manifest, so nothing is resolved again. + */ export async function restoreWorkspace(repoRoot: string, outputDir: string): Promise { - const backupPath = backupPathOf(outputDir); - let original: string; + const manifestBackup = backupPathOf(outputDir, MANIFEST); try { - original = await readFile(backupPath, 'utf8'); + await access(manifestBackup); } catch (error) { // No copy set aside: nothing was pinned. if ((error as NodeJS.ErrnoException).code === 'ENOENT') { @@ -87,9 +107,23 @@ export async function restoreWorkspace(repoRoot: string, outputDir: string): Pro throw error; } console.log(chalk.cyan('\nRestoring the repository install…')); - await writeFile(path.join(repoRoot, 'pnpm-workspace.yaml'), original); - await rm(backupPath, { force: true }); + for (const file of [LOCKFILE, MANIFEST]) { + const backup = backupPathOf(outputDir, file); + try { + // eslint-disable-next-line no-await-in-loop + await copyFile(backup, path.join(repoRoot, file)); + } catch (error) { + // A repository without a lockfile had none to set aside. + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { + throw error; + } + } + } await installWorkspace(repoRoot); + // Removed only once the install is back, so an interrupted restore is retried by the next run. + await Promise.all( + [LOCKFILE, MANIFEST].map((file) => rm(backupPathOf(outputDir, file), { force: true })), + ); } /** diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index caac5567fc..8a8b214217 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -6,6 +6,7 @@ import { mkdir, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; import { packRef, packWorkingTree } from '../utils/packWorkspace'; +import { ensurePnpmOnPath } from '../utils/pnpm'; import { getCiMetadata } from '../ciReport'; import type { BenchmarkRunReport } from '../runReport'; import { refLabel, resolveBaselineRef, WORKTREE_REF } from './refs'; @@ -72,6 +73,8 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise { if (ref.variant === 'current') { return packWorkingTree({ repoRoot, outRoot: path.join(packedDir, 'current'), buildCmd }); @@ -123,6 +126,14 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise { + if (result.status === 'rejected') { + throw result.reason; + } + return result.value; + }); // Then build one set of pages per ref, each against its own packed build so both sides of a // comparison resolve the library identically. One at a time: a build pins the repository's own diff --git a/packages/benchmark/src/utils/interrupt.ts b/packages/benchmark/src/utils/interrupt.ts new file mode 100644 index 0000000000..9f12a70478 --- /dev/null +++ b/packages/benchmark/src/utils/interrupt.ts @@ -0,0 +1,39 @@ +// A signal ends the process without running `finally` blocks, so cleanup that must survive a Ctrl-C +// — a temporary checkout, say — registers here instead. + +const SIGNALS: NodeJS.Signals[] = ['SIGINT', 'SIGTERM', 'SIGHUP']; + +const cleanups = new Set<() => Promise>(); + +function handleSignal(signal: NodeJS.Signals): void { + for (const watched of SIGNALS) { + process.off(watched, handleSignal); + } + const pending = [...cleanups]; + cleanups.clear(); + // Ends the process as the signal would have, once every cleanup has had its chance. + void Promise.allSettled(pending.map((cleanup) => cleanup())).then(() => { + process.kill(process.pid, signal); + }); +} + +/** + * Runs `cleanup` if the process is stopped by a signal before the returned function is called, then + * lets the signal end the process. + */ +export function onInterrupt(cleanup: () => Promise): () => void { + if (cleanups.size === 0) { + for (const signal of SIGNALS) { + process.on(signal, handleSignal); + } + } + cleanups.add(cleanup); + return () => { + cleanups.delete(cleanup); + if (cleanups.size === 0) { + for (const signal of SIGNALS) { + process.off(signal, handleSignal); + } + } + }; +} diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index 45721fb798..3e58e9be5e 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -13,6 +13,7 @@ import chalk from 'chalk'; import { mapAsync } from 'es-toolkit/array'; import { execa, parseCommandString } from 'execa'; import { resolveCommit } from './git'; +import { onInterrupt } from './interrupt'; import { listPublishablePackages } from './pnpm'; export interface PackedPackage { @@ -158,6 +159,7 @@ export async function packRef(options: PackRefOptions): Promise await mkdir(outRoot, { recursive: true }); const staging = await mkdtemp(path.join(outRoot, '.staging-')); const checkout = await mkdtemp(path.join(os.tmpdir(), 'pack-workspace-')); + const stopWatching = onInterrupt(() => removeCheckout(repoRoot, checkout)); try { console.log(chalk.cyan(`\nChecking out "${ref}" (${sha.slice(0, 9)}) at ${checkout}`)); await execa('git', ['worktree', 'add', '--detach', checkout, sha], { @@ -183,6 +185,7 @@ export async function packRef(options: PackRefOptions): Promise await rename(staging, dir); return packages.map((pkg) => ({ ...pkg, tarball: path.join(dir, path.basename(pkg.tarball)) })); } finally { + stopWatching(); // On success the staging folder has already become `dir`. await Promise.all([ rm(staging, { recursive: true, force: true }), diff --git a/packages/benchmark/src/utils/pnpm.ts b/packages/benchmark/src/utils/pnpm.ts index e8b377a0dc..aabae19d06 100644 --- a/packages/benchmark/src/utils/pnpm.ts +++ b/packages/benchmark/src/utils/pnpm.ts @@ -1,5 +1,49 @@ +import * as path from 'node:path'; +import { access, constants, mkdir, writeFile } from 'node:fs/promises'; import { $ } from 'execa'; +async function isOnPath(command: string): Promise { + const dirs = (process.env.PATH ?? '').split(path.delimiter).filter(Boolean); + const found = await Promise.allSettled( + dirs.map((dir) => access(path.join(dir, command), constants.X_OK)), + ); + return found.some((result) => result.status === 'fulfilled'); +} + +/** + * What a `pnpm` shim forwards to: corepack when it is there, which runs the version each checkout's + * `packageManager` pins — a baseline may pin another than the working tree — or else the pnpm that + * launched this process, from the `npm_execpath` pnpm sets for what it runs. + */ +async function pnpmTarget(): Promise { + if (await isOnPath('corepack')) { + return 'corepack pnpm'; + } + const execPath = process.env.npm_execpath; + if (!execPath || !path.basename(execPath).startsWith('pnpm')) { + return null; + } + return /\.[cm]?js$/.test(execPath) ? `"${process.execPath}" "${execPath}"` : `"${execPath}"`; +} + +/** + * Puts `pnpm` on `PATH` when it isn't there — when it comes through corepack without its shims, + * say — so the commands a run shells out to, and the scripts those run in turn, all find it. + * POSIX only: Windows resolves commands differently. + */ +export async function ensurePnpmOnPath(binDir: string): Promise { + if (process.platform === 'win32' || (await isOnPath('pnpm'))) { + return; + } + const target = await pnpmTarget(); + if (!target) { + return; + } + await mkdir(binDir, { recursive: true }); + await writeFile(path.join(binDir, 'pnpm'), `#!/bin/sh\nexec ${target} "$@"\n`, { mode: 0o755 }); + process.env.PATH = `${binDir}${path.delimiter}${process.env.PATH ?? ''}`; +} + /** A publishable workspace package, as `pnpm ls` reports it. */ export interface PublishablePackage { name: string; From 2a5145042b0358551e44d28124dcc45df9a6126a Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 17:50:19 +0200 Subject: [PATCH 122/173] [benchmark] Time a paint from when its recording window began A case that starts with React recording paused left the mount out of its renders but not out of `bench:paint`, which was still timed from before the mount. A paint is now timed from the start of the recording window it falls in: the mount, or where the interaction resumed recording. The paused-start Counter click case goes from 31.2 ms to 2.5 ms of paint, the click alone. --- packages/benchmark/src/caseRuntime.tsx | 9 ++++++--- packages/benchmark/src/reactRecording.test.ts | 16 ++++++++++++++-- packages/benchmark/src/reactRecording.ts | 15 ++++++++------- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/packages/benchmark/src/caseRuntime.tsx b/packages/benchmark/src/caseRuntime.tsx index d9a4b0b829..da599ad720 100644 --- a/packages/benchmark/src/caseRuntime.tsx +++ b/packages/benchmark/src/caseRuntime.tsx @@ -318,12 +318,15 @@ export async function measureIteration( for (const entry of timing.elementEntries) { // Skip paints that happened while recording was paused. Attribute by the paint's // `paintTime`, not by when the observer callback fired (which can lag the paint). - if (!recording.activeAt(entry.paintTime)) { + const since = recording.activeSince(entry.paintTime); + if (since === null) { continue; } - // The default sentinel is the base series; named markers become sub-series. + // The default sentinel is the base series; named markers become sub-series. A paint is timed + // from when its recording window began — the mount, or where the interaction resumed + // recording — so a case that starts paused doesn't count the mount it chose to leave out. const id = entry.identifier === 'default' ? undefined : entry.identifier; - paints.push({ id, start: iterationStart, end: entry.paintTime }); + paints.push({ id, start: Math.max(iterationStart, since), end: entry.paintTime }); } if (options?.afterEach) { diff --git a/packages/benchmark/src/reactRecording.test.ts b/packages/benchmark/src/reactRecording.test.ts index 043d14fb81..85a42588b5 100644 --- a/packages/benchmark/src/reactRecording.test.ts +++ b/packages/benchmark/src/reactRecording.test.ts @@ -28,8 +28,20 @@ describe('createReactRecordingControls', () => { it('attributes times before all toggles to the initial state and after to the current', () => { const controls = createReactRecordingControls(false); controls.resumeReactRecording(); - expect(controls.activeAt(-Infinity)).toBe(false); - expect(controls.activeAt(Infinity)).toBe(true); + expect(controls.activeSince(-Infinity)).toBe(null); + expect(controls.activeSince(Infinity)).not.toBe(null); + }); + + it('says when the window around a time began', () => { + const fromStart = createReactRecordingControls(true); + expect(fromStart.activeSince(performance.now())).toBe(-Infinity); + + const resumed = createReactRecordingControls(false); + const beforeResume = performance.now(); + resumed.resumeReactRecording(); + const since = resumed.activeSince(Infinity); + expect(since).toBeGreaterThanOrEqual(beforeResume); + expect(since).toBeLessThanOrEqual(performance.now()); }); }); diff --git a/packages/benchmark/src/reactRecording.ts b/packages/benchmark/src/reactRecording.ts index 915b3b47b0..3887b5f5d1 100644 --- a/packages/benchmark/src/reactRecording.ts +++ b/packages/benchmark/src/reactRecording.ts @@ -4,11 +4,12 @@ export interface ReactRecordingControls { /** Whether any active recording window closed without capturing a render. */ readonly hadEmptyActiveWindow: boolean; /** - * Whether recording was active at `time` (a `performance.now()` timestamp). Paint entries are + * When the recording window around `time` (a `performance.now()` timestamp) began: `-Infinity` + * if recording was active from the start, `null` if it was paused at `time`. Paint entries are * observed asynchronously, so they are attributed by their `paintTime` rather than by the * recording state at the moment the observer callback happens to fire. */ - activeAt(time: number): boolean; + activeSince(time: number): number | null; /** Note that a render was captured in the current window. Called by the harness from `onRender`. */ markRendered(): void; /** Close the final window at the end of the iteration (validates it if recording is still active). */ @@ -33,7 +34,7 @@ export function createReactRecordingControls(initiallyActive: boolean): ReactRec let currentWindowHasRender = false; let emptyActiveWindow = false; // Transitions in chronological order. The implicit state before the first toggle is - // `initiallyActive`; `activeAt` replays this to attribute a paint to its render time. + // `initiallyActive`; `activeSince` replays this to attribute a paint to its window. const transitions: { time: number; active: boolean }[] = []; // Flag the window being closed if it was recording yet captured nothing. @@ -50,15 +51,15 @@ export function createReactRecordingControls(initiallyActive: boolean): ReactRec get hadEmptyActiveWindow() { return emptyActiveWindow; }, - activeAt(time) { - let result = initiallyActive; + activeSince(time) { + let since: number | null = initiallyActive ? -Infinity : null; for (const transition of transitions) { if (transition.time > time) { break; } - result = transition.active; + since = transition.active ? transition.time : null; } - return result; + return since; }, markRendered() { currentWindowHasRender = true; From 125cb1d464f96f1fe7ddb9a90a625506ddb66610 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:13:43 +0200 Subject: [PATCH 123/173] [benchmark] Assume pnpm is on PATH again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repositories standardize on pnpm, so a missing `pnpm` command is the environment's to fix — `corepack enable` provides it — not something a run should shim around. --- .../benchmark/src/runner/runBenchmarks.ts | 3 -- packages/benchmark/src/utils/pnpm.ts | 44 ------------------- 2 files changed, 47 deletions(-) diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index 8a8b214217..2f8cd3a665 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -6,7 +6,6 @@ import { mkdir, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; import { packRef, packWorkingTree } from '../utils/packWorkspace'; -import { ensurePnpmOnPath } from '../utils/pnpm'; import { getCiMetadata } from '../ciReport'; import type { BenchmarkRunReport } from '../runReport'; import { refLabel, resolveBaselineRef, WORKTREE_REF } from './refs'; @@ -73,8 +72,6 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise { - const dirs = (process.env.PATH ?? '').split(path.delimiter).filter(Boolean); - const found = await Promise.allSettled( - dirs.map((dir) => access(path.join(dir, command), constants.X_OK)), - ); - return found.some((result) => result.status === 'fulfilled'); -} - -/** - * What a `pnpm` shim forwards to: corepack when it is there, which runs the version each checkout's - * `packageManager` pins — a baseline may pin another than the working tree — or else the pnpm that - * launched this process, from the `npm_execpath` pnpm sets for what it runs. - */ -async function pnpmTarget(): Promise { - if (await isOnPath('corepack')) { - return 'corepack pnpm'; - } - const execPath = process.env.npm_execpath; - if (!execPath || !path.basename(execPath).startsWith('pnpm')) { - return null; - } - return /\.[cm]?js$/.test(execPath) ? `"${process.execPath}" "${execPath}"` : `"${execPath}"`; -} - -/** - * Puts `pnpm` on `PATH` when it isn't there — when it comes through corepack without its shims, - * say — so the commands a run shells out to, and the scripts those run in turn, all find it. - * POSIX only: Windows resolves commands differently. - */ -export async function ensurePnpmOnPath(binDir: string): Promise { - if (process.platform === 'win32' || (await isOnPath('pnpm'))) { - return; - } - const target = await pnpmTarget(); - if (!target) { - return; - } - await mkdir(binDir, { recursive: true }); - await writeFile(path.join(binDir, 'pnpm'), `#!/bin/sh\nexec ${target} "$@"\n`, { mode: 0o755 }); - process.env.PATH = `${binDir}${path.delimiter}${process.env.PATH ?? ''}`; -} - /** A publishable workspace package, as `pnpm ls` reports it. */ export interface PublishablePackage { name: string; From 701283ab83039807ed3b832f4bfc4c2a740a8ead Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 6 Oct 2026 18:22:19 +0200 Subject: [PATCH 124/173] [benchmark] Simplify interruption recovery, alarm gating and the comment table - A baseline's checkout is recorded next to the packed refs with the process that made it; the next run removes those whose process is gone, however it was stopped, kill -9 included. This replaces the signal handlers, and matches how a pinned manifest is recovered. - compareSamples is told whether a change can alarm, rather than its severity being overwritten afterwards. - A benchmark's duration is stamped once, by the loop that runs it. - The comment's tables take their columns as one list, label benchmarks with formatRounds, and the restore copies files through one helper. --- .../benchmarkRun/buildMarkdownReport.test.ts | 4 +- .../lib/benchmarkRun/buildMarkdownReport.ts | 104 ++++++++---------- .../src/views/BenchmarkRunDetails.tsx | 27 +++-- .../benchmark/src/runReport/analyzeRun.ts | 22 ++-- packages/benchmark/src/runner/buildPages.ts | 58 ++++------ .../benchmark/src/runner/runInterleaved.ts | 16 +-- packages/benchmark/src/utils/interrupt.ts | 39 ------- .../benchmark/src/utils/packWorkspace.test.ts | 35 ++++++ packages/benchmark/src/utils/packWorkspace.ts | 65 ++++++++++- 9 files changed, 202 insertions(+), 168 deletions(-) delete mode 100644 packages/benchmark/src/utils/interrupt.ts diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index c90aa2e781..49fa0f96b7 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -99,10 +99,10 @@ describe('buildBenchmarkRunMarkdownReport', () => { | Benchmark | Metric | baseline | current | Δ vs baseline | |:----------|:----------|----------:|----------:|:----------| - | **Grid scroll** | render | 21.5 ms | 23.5 ms | 🔴 \`worse +9.3% – +9.3%\` | + | **Grid scroll** (3 + 1 rounds · timed out) | render | 21.5 ms | 23.5 ms | 🔴 \`worse +9.3% – +9.3%\` | - **libs / mount** · 4 rounds + **libs / mount** (4 rounds) | Metric | ours | theirs | theirs vs ours | |:----------|----------:|----------:|:----------| diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 416ffb9503..222069ee61 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -4,7 +4,6 @@ import { benchmarkTable, changeTone, findRegressions, - formatDuration, formatRounds, formatComparison, formatRunSummary, @@ -14,8 +13,8 @@ import type { BenchmarkAnalysis, BenchmarkRunReport, BenchmarkTableColumn, - BenchmarkTableRow, ChangeTone, + RunBenchmark, } from '@mui/internal-benchmark/runReport'; interface BuildOptions { @@ -32,36 +31,43 @@ const TONE_MARKERS: Record = { none: '', }; -/** The rows of a benchmark's table that changed — better or worse — with their columns. */ -function changedRowsOf(analysis: BenchmarkAnalysis) { +/** A benchmark's name, with how many rounds it ran and how long it took. */ +function labelOf(benchmark: RunBenchmark): string { + const rounds = formatRounds(benchmark); + return rounds ? `**${benchmark.name}** (${rounds})` : `**${benchmark.name}**`; +} + +/** + * A benchmark's table, keeping the rows that changed — better or worse — as markdown cells: each + * comparison as code, behind a marker of which way it went. + */ +function changedRowsOf(analysis: BenchmarkAnalysis): { + columns: BenchmarkTableColumn[]; + rows: string[][]; +} { const { columns, rows } = benchmarkTable(analysis); + const firstComparison = columns.findIndex((column) => column.kind === 'comparison'); const changed = rows.filter((row) => row.comparisons.some(({ change }) => change === 'better' || change === 'worse'), ); - return { columns, changed }; -} - -/** A row's cells for markdown: each comparison as code, behind a marker of which way it went. */ -function markdownCells(columns: BenchmarkTableColumn[], row: BenchmarkTableRow): string[] { - const firstComparison = columns.findIndex((column) => column.kind === 'comparison'); - return row.cells.map((cell, column) => { - if (columns[column].kind !== 'comparison') { - return cell; - } - return `${TONE_MARKERS[changeTone(row.comparisons[column - firstComparison])]}\`${cell}\``; - }); + return { + columns, + rows: changed.map((row) => + row.cells.map((cell, column) => + columns[column].kind === 'comparison' + ? `${TONE_MARKERS[changeTone(row.comparisons[column - firstComparison])]}\`${cell}\`` + : cell, + ), + ), + }; } -function markdownTable( - headers: string[], - kinds: Array, - rows: string[][], -) { +function markdownTable(columns: BenchmarkTableColumn[], rows: string[][]): string { return formatMarkdownTable( - headers.map((header, column) => ({ + columns.map(({ header, kind }, column) => ({ field: String(column), header, - align: kinds[column] === 'value' ? ('right' as const) : ('left' as const), + align: kind === 'value' ? ('right' as const) : ('left' as const), })), rows.map((cells) => Object.fromEntries(cells.map((cell, column) => [String(column), cell]))), ); @@ -72,49 +78,33 @@ function markdownTable( * columns line up across benchmarks; those where nothing changed are named below it. */ function renderBaselineChanges(analyses: BenchmarkAnalysis[]): string[] { + const tables = analyses.map((analysis) => ({ analysis, ...changedRowsOf(analysis) })); + const changed = tables.filter((table) => table.rows.length > 0); + const unchanged = tables.filter((table) => table.rows.length === 0); const lines: string[] = []; - const unchanged: string[] = []; - let header: { headers: string[]; kinds: Array } | undefined; - const rows: string[][] = []; - for (const analysis of analyses) { - const { name, durationMs } = analysis.benchmark; - const label = - durationMs === undefined ? `**${name}**` : `**${name}** (${formatDuration(durationMs)})`; - const { columns, changed } = changedRowsOf(analysis); - if (changed.length === 0) { - unchanged.push(label); - continue; - } - header ??= { - headers: ['Benchmark', ...columns.map((column) => column.header)], - kinds: ['label', ...columns.map((column) => column.kind)], - }; - changed.forEach((row, index) => { - rows.push([index === 0 ? label : '', ...markdownCells(columns, row)]); - }); - } - if (header) { - lines.push(markdownTable(header.headers, header.kinds, rows), ''); + if (changed.length > 0) { + const columns: BenchmarkTableColumn[] = [ + { header: 'Benchmark', kind: 'label' }, + ...changed[0].columns, + ]; + const rows = changed.flatMap(({ analysis, rows: cells }) => + cells.map((row, index) => [index === 0 ? labelOf(analysis.benchmark) : '', ...row]), + ); + lines.push(markdownTable(columns, rows), ''); } if (unchanged.length > 0) { - lines.push(`No change detected: ${unchanged.join(', ')}`, ''); + const names = unchanged.map(({ analysis }) => labelOf(analysis.benchmark)); + lines.push(`No change detected: ${names.join(', ')}`, ''); } return lines; } /** A `compare()` benchmark: a table of the metrics that changed, or one line when none did. */ function renderComparison(analysis: BenchmarkAnalysis): string { - const heading = `**${analysis.benchmark.name}** · ${formatRounds(analysis.benchmark)}`; - const { columns, changed } = changedRowsOf(analysis); - if (changed.length === 0) { - return `${heading} · no change detected`; - } - const table = markdownTable( - columns.map((column) => column.header), - columns.map((column) => column.kind), - changed.map((row) => markdownCells(columns, row)), - ); - return `${heading}\n\n${table}`; + const { columns, rows } = changedRowsOf(analysis); + return rows.length === 0 + ? `${labelOf(analysis.benchmark)} · no change detected` + : `${labelOf(analysis.benchmark)}\n\n${markdownTable(columns, rows)}`; } /** diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index 29937d93d4..d13ff771b9 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -103,20 +103,19 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { {rows.map((row) => ( - {row.cells.map((cell, column) => - columns[column].kind === 'comparison' ? ( - - {cell} - - ) : ( - - {cell} - - ), - )} + {row.cells.map((cell, column) => ( + + {cell} + + ))} ))} diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index 1296e71f06..a9cecc88f5 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -237,11 +237,21 @@ function differencesOf(subject: number[], against: number[]): number[] { return Array.from({ length: rounds }, (_, round) => subject[round] - against[round]); } +export interface CompareSamplesOptions { + /** The level of the intervals. Defaults to 95%. */ + confidence?: number; + /** + * Whether a change can raise an alarm, so carry a severity. Defaults to true; the variants of a + * `compare()` differ on purpose, so no difference between them can. + */ + canAlarm?: boolean; +} + export function compareSamples( subject: { name: string; values: number[] }, against: { name: string; values: number[] }, definition: RunMetricDefinition, - confidence = CONFIDENCE, + { confidence = CONFIDENCE, canAlarm = true }: CompareSamplesOptions = {}, ): MetricComparison { const differences = differencesOf(subject.values, against.values); const rounds = differences.length; @@ -269,7 +279,7 @@ export function compareSamples( relative, change, }; - return { ...comparison, severity: severityOf(comparison, definition) }; + return { ...comparison, severity: canAlarm ? severityOf(comparison, definition) : 'none' }; } /** The pairs a benchmark's kind compares: the current build against the baseline, or every variant against the reference. */ @@ -350,11 +360,9 @@ export function compareBenchmark( return { metric, definition, - comparisons: pairs.map(({ subject, against }) => { - const comparison = compareSamples(subject, against, definition, confidence); - // Variants of a `compare()` differ on purpose: no difference between them is a regression. - return canAlarm ? comparison : { ...comparison, severity: 'none' as const }; - }), + comparisons: pairs.map(({ subject, against }) => + compareSamples(subject, against, definition, { confidence, canAlarm }), + ), }; }), }; diff --git a/packages/benchmark/src/runner/buildPages.ts b/packages/benchmark/src/runner/buildPages.ts index 0ed7206566..d5f10bfc9f 100644 --- a/packages/benchmark/src/runner/buildPages.ts +++ b/packages/benchmark/src/runner/buildPages.ts @@ -2,7 +2,7 @@ import * as path from 'node:path'; import { createRequire } from 'node:module'; -import { access, constants, copyFile, readFile, rm, writeFile } from 'node:fs/promises'; +import { constants, copyFile, readFile, rm, writeFile } from 'node:fs/promises'; import { pathToFileURL } from 'node:url'; import chalk from 'chalk'; import { execa } from 'execa'; @@ -33,6 +33,23 @@ function packedPins(packages: PackedPackage[]): Record { const MANIFEST = 'pnpm-workspace.yaml'; const LOCKFILE = 'pnpm-lock.yaml'; +/** + * Copies `from` to `to`, and says whether it did: not when `from` doesn't exist, nor when `to` + * already does and `mode` is `COPYFILE_EXCL`. + */ +async function tryCopy(from: string, to: string, mode?: number): Promise { + try { + await copyFile(from, to, mode); + return true; + } catch (error) { + const { code } = error as NodeJS.ErrnoException; + if (code === 'ENOENT' || code === 'EEXIST') { + return false; + } + throw error; + } +} + /** Where one of the repository's own files is kept while a run has it pinned. */ function backupPathOf(outputDir: string, file: string): string { return path.join(outputDir, `${file}.orig`); @@ -43,18 +60,7 @@ function backupPathOf(outputDir: string, file: string): string { * is the original. A file the repository doesn't have is left alone. */ async function setAside(repoRoot: string, outputDir: string, file: string): Promise { - try { - await copyFile( - path.join(repoRoot, file), - backupPathOf(outputDir, file), - constants.COPYFILE_EXCL, - ); - } catch (error) { - const { code } = error as NodeJS.ErrnoException; - if (code !== 'EEXIST' && code !== 'ENOENT') { - throw error; - } - } + await tryCopy(path.join(repoRoot, file), backupPathOf(outputDir, file), constants.COPYFILE_EXCL); } /** @@ -96,29 +102,13 @@ async function installWorkspace(repoRoot: string): Promise { * them: the lockfile already satisfies the manifest, so nothing is resolved again. */ export async function restoreWorkspace(repoRoot: string, outputDir: string): Promise { - const manifestBackup = backupPathOf(outputDir, MANIFEST); - try { - await access(manifestBackup); - } catch (error) { - // No copy set aside: nothing was pinned. - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return; - } - throw error; + // No manifest set aside: nothing was pinned. + if (!(await tryCopy(backupPathOf(outputDir, MANIFEST), path.join(repoRoot, MANIFEST)))) { + return; } console.log(chalk.cyan('\nRestoring the repository install…')); - for (const file of [LOCKFILE, MANIFEST]) { - const backup = backupPathOf(outputDir, file); - try { - // eslint-disable-next-line no-await-in-loop - await copyFile(backup, path.join(repoRoot, file)); - } catch (error) { - // A repository without a lockfile had none to set aside. - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { - throw error; - } - } - } + // A repository without a lockfile had none to set aside. + await tryCopy(backupPathOf(outputDir, LOCKFILE), path.join(repoRoot, LOCKFILE)); await installWorkspace(repoRoot); // Removed only once the install is back, so an interrupted restore is retried by the next run. await Promise.all( diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index ff8273c615..8605afdc22 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -194,7 +194,6 @@ async function runBenchmark( otherBaselineBenchmarks: number, ): Promise { console.log(chalk.cyan(`\nRunning "${entry.name}" (${entry.file})…`)); - const startedAt = Date.now(); const { sampleSize, timeout, autoSampleConditions } = sampling; const horizons = parseHorizons(autoSampleConditions); let opened: Page | undefined; @@ -250,19 +249,12 @@ async function runBenchmark( ); } return { - benchmark: { - ...benchmarkOf(), - sampling: { sampleSize, timedOut: !resolved }, - durationMs: Date.now() - startedAt, - }, + benchmark: { ...benchmarkOf(), sampling: { sampleSize, timedOut: !resolved } }, metrics: definitions, }; } catch (error) { console.error(chalk.red(` ${errorMessage(error)}`)); - return { - benchmark: { ...entry, error: errorMessage(error), durationMs: Date.now() - startedAt }, - metrics: {}, - }; + return { benchmark: { ...entry, error: errorMessage(error) }, metrics: {} }; } finally { await opened?.context().close(); } @@ -362,8 +354,10 @@ export async function runInterleaved(options: RunInterleavedOptions): Promise result.benchmark), diff --git a/packages/benchmark/src/utils/interrupt.ts b/packages/benchmark/src/utils/interrupt.ts deleted file mode 100644 index 9f12a70478..0000000000 --- a/packages/benchmark/src/utils/interrupt.ts +++ /dev/null @@ -1,39 +0,0 @@ -// A signal ends the process without running `finally` blocks, so cleanup that must survive a Ctrl-C -// — a temporary checkout, say — registers here instead. - -const SIGNALS: NodeJS.Signals[] = ['SIGINT', 'SIGTERM', 'SIGHUP']; - -const cleanups = new Set<() => Promise>(); - -function handleSignal(signal: NodeJS.Signals): void { - for (const watched of SIGNALS) { - process.off(watched, handleSignal); - } - const pending = [...cleanups]; - cleanups.clear(); - // Ends the process as the signal would have, once every cleanup has had its chance. - void Promise.allSettled(pending.map((cleanup) => cleanup())).then(() => { - process.kill(process.pid, signal); - }); -} - -/** - * Runs `cleanup` if the process is stopped by a signal before the returned function is called, then - * lets the signal end the process. - */ -export function onInterrupt(cleanup: () => Promise): () => void { - if (cleanups.size === 0) { - for (const signal of SIGNALS) { - process.on(signal, handleSignal); - } - } - cleanups.add(cleanup); - return () => { - cleanups.delete(cleanup); - if (cleanups.size === 0) { - for (const signal of SIGNALS) { - process.off(signal, handleSignal); - } - } - }; -} diff --git a/packages/benchmark/src/utils/packWorkspace.test.ts b/packages/benchmark/src/utils/packWorkspace.test.ts index a4f80182b5..28d31a70cd 100644 --- a/packages/benchmark/src/utils/packWorkspace.test.ts +++ b/packages/benchmark/src/utils/packWorkspace.test.ts @@ -171,6 +171,41 @@ describe('packRef', () => { expect(worktrees.stdout.trim().split('\n')).toHaveLength(1); }); + it( + "removes a checkout an interrupted run left behind, but not a live run's", + { timeout: 120_000 }, + async () => { + const { repoRoot, buildCmd } = await makeFixtureRepo(); + const outRoot = path.join(await makeTempDir(), 'cache'); + await mkdir(outRoot, { recursive: true }); + + // Two checkouts as runs leave them while packing: one by a process that has ended since, + // one by a process that still runs — this one. + const ended = execa('node', ['-e', '']); + const endedPid = ended.pid!; + await ended; + const records = await Promise.all( + [endedPid, process.pid].map(async (pid, index) => { + const checkout = path.join(await makeTempDir(), `pack-workspace-${index}`); + await execa('git', ['worktree', 'add', '--detach', checkout, 'HEAD'], { cwd: repoRoot }); + const record = path.join(outRoot, `.checkout-pack-workspace-${index}.json`); + await writeFile(record, JSON.stringify({ pid, checkout })); + return { checkout, record }; + }), + ); + + await packRef({ repoRoot, ref: 'HEAD', outRoot, installCmd: '', buildCmd }); + + const [abandoned, live] = records; + await expect(stat(abandoned.checkout)).rejects.toThrow(); + await expect(stat(abandoned.record)).rejects.toThrow(); + await expect(stat(live.checkout)).resolves.toBeTruthy(); + const worktrees = await execa('git', ['worktree', 'list'], { cwd: repoRoot }); + expect(worktrees.stdout).not.toContain(abandoned.checkout); + expect(worktrees.stdout).toContain(live.checkout); + }, + ); + it('reports a ref it cannot resolve', async () => { const { repoRoot, buildCmd } = await makeFixtureRepo(); const outRoot = path.join(await makeTempDir(), 'cache'); diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index 3e58e9be5e..01c608057a 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -7,13 +7,12 @@ import * as path from 'node:path'; import * as os from 'node:os'; import { createHash } from 'node:crypto'; import { createReadStream } from 'node:fs'; -import { access, mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { access, mkdir, mkdtemp, readdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; import { pipeline } from 'node:stream/promises'; import chalk from 'chalk'; import { mapAsync } from 'es-toolkit/array'; import { execa, parseCommandString } from 'execa'; import { resolveCommit } from './git'; -import { onInterrupt } from './interrupt'; import { listPublishablePackages } from './pnpm'; export interface PackedPackage { @@ -63,6 +62,59 @@ async function removeCheckout(repoRoot: string, checkout: string): Promise await execa('git', ['worktree', 'prune'], { cwd: repoRoot }); } +const CHECKOUT_RECORD_PREFIX = '.checkout-'; + +/** + * Where a checkout's record is kept while it exists — which process made it, and where it is — next + * to the packed refs. + */ +function checkoutRecordPathOf(outRoot: string, checkout: string): string { + return path.join(outRoot, `${CHECKOUT_RECORD_PREFIX}${path.basename(checkout)}.json`); +} + +interface CheckoutRecord { + pid: number; + checkout: string; +} + +function isRunning(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM: it runs, as someone else. + return (error as NodeJS.ErrnoException).code === 'EPERM'; + } +} + +/** + * Removes the checkouts that interrupted runs left behind — those recorded by a process that is + * no longer running, whatever stopped it — and their records. A live run's checkouts are its own. + */ +async function removeAbandonedCheckouts(repoRoot: string, outRoot: string): Promise { + let names: string[]; + try { + names = await readdir(outRoot); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return; + } + throw error; + } + await Promise.all( + names + .filter((name) => name.startsWith(CHECKOUT_RECORD_PREFIX)) + .map(async (name) => { + const recordPath = path.join(outRoot, name); + const record: CheckoutRecord = JSON.parse(await readFile(recordPath, 'utf8')); + if (!isRunning(record.pid)) { + await removeCheckout(repoRoot, record.checkout); + await rm(recordPath, { force: true }); + } + }), + ); +} + /** Runs `buildCmd` in `cwd`, with the nx daemon off so nothing keeps writing after it returns. */ async function build(cwd: string, buildCmd: string): Promise { const [file, ...args] = parseCommandString(buildCmd); @@ -150,6 +202,7 @@ export async function packRef(options: PackRefOptions): Promise const sha = await resolveCommit(repoRoot, ref); const dir = path.join(outRoot, sha); + await removeAbandonedCheckouts(repoRoot, outRoot); const cached = await readFreshCache(dir, buildCmd); if (cached) { console.log(chalk.green(`\nReusing packed workspace for "${ref}" (${sha.slice(0, 9)}).`)); @@ -159,7 +212,11 @@ export async function packRef(options: PackRefOptions): Promise await mkdir(outRoot, { recursive: true }); const staging = await mkdtemp(path.join(outRoot, '.staging-')); const checkout = await mkdtemp(path.join(os.tmpdir(), 'pack-workspace-')); - const stopWatching = onInterrupt(() => removeCheckout(repoRoot, checkout)); + // Recorded until it is removed: a run stopped before its `finally` leaves the next one to remove + // it. + const recordPath = checkoutRecordPathOf(outRoot, checkout); + const record: CheckoutRecord = { pid: process.pid, checkout }; + await writeFile(recordPath, JSON.stringify(record)); try { console.log(chalk.cyan(`\nChecking out "${ref}" (${sha.slice(0, 9)}) at ${checkout}`)); await execa('git', ['worktree', 'add', '--detach', checkout, sha], { @@ -185,12 +242,12 @@ export async function packRef(options: PackRefOptions): Promise await rename(staging, dir); return packages.map((pkg) => ({ ...pkg, tarball: path.join(dir, path.basename(pkg.tarball)) })); } finally { - stopWatching(); // On success the staging folder has already become `dir`. await Promise.all([ rm(staging, { recursive: true, force: true }), removeCheckout(repoRoot, checkout), ]); + await rm(recordPath, { force: true }); } } From 36b884c9abcb16e703b035a148881934a4dcbb74 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 7 Oct 2026 08:59:35 +0200 Subject: [PATCH 125/173] [benchmark] Default to a 5% horizon; add sampling flags, -t and a JSON reporter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - autoSampleConditions defaults to ['5%']. Against 0%, identical builds never resolve and sample until the timeout, and checking after every round makes a false finding likelier the longer it goes. - --sample-size, --timeout and --auto-sample-conditions override every benchmark's own sampling, to iterate without editing benchmark files. - Named as in Vitest: positional arguments filter files by path, -t / --testNamePattern filters benchmarks by name. - --reporter json prints summarizeRun's analysis to stdout — medians, intervals, verdicts, regressions, durations — with all other output on stderr. NO_COLOR turns colour off. --- packages/benchmark/README.md | 34 ++++++-- packages/benchmark/src/cli/index.ts | 55 +++++++++++- packages/benchmark/src/runReport/index.ts | 1 + .../benchmark/src/runReport/summary.test.ts | 63 ++++++++++++++ packages/benchmark/src/runReport/summary.ts | 85 +++++++++++++++++++ packages/benchmark/src/runner/buildPages.ts | 4 +- .../benchmark/src/runner/runBenchmarks.ts | 27 +++++- .../benchmark/src/runner/runInterleaved.ts | 29 ++++++- packages/benchmark/src/sampling.ts | 6 +- packages/benchmark/src/utils/packWorkspace.ts | 8 +- 10 files changed, 290 insertions(+), 22 deletions(-) create mode 100644 packages/benchmark/src/runReport/summary.test.ts create mode 100644 packages/benchmark/src/runReport/summary.ts diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index bcd3611a17..810e62a05c 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -411,24 +411,26 @@ minutes. A difference is resolved against a horizon once its confidence interval one side of it; `'10%'` stands for both `'-10%'` and `'+10%'`. The metrics that alarm decide, or every metric when none does. -| Option | Default | Meaning | -| :--------------------- | :------- | :-------------------------------------------------------------------- | -| `sampleSize` | `50` | Rounds measured before deciding whether to continue | -| `timeout` | `3` | Minutes to keep sampling while a difference is unresolved | -| `autoSampleConditions` | `['0%']` | Horizons to resolve: by default, until each change is better or worse | +| Option | Default | Meaning | +| :--------------------- | :------- | :------------------------------------------------------------------------------ | +| `sampleSize` | `50` | Rounds measured before deciding whether to continue | +| `timeout` | `3` | Minutes to keep sampling while a difference is unresolved | +| `autoSampleConditions` | `['5%']` | Horizons to resolve: by default, until each change is larger or smaller than 5% | They are set per benchmark: the last argument of `benchmark()` and `compare()`, and among `reactBenchmark()`'s options. A compared case is sampled as its `compare()` asks, so setting its own -is an error. +is an error. `--sample-size`, `--timeout` and `--auto-sample-conditions` override them for every +benchmark of a run, to iterate quickly without editing benchmark files. ```tsx -reactBenchmark('mount', () => , { timeout: 1, autoSampleConditions: ['5%'] }); +reactBenchmark('mount', () => , { timeout: 1, autoSampleConditions: ['10%'] }); compare('scatter', [ours, other], { sampleSize: 100 }); ``` -Two builds that perform the same never resolve against `0%`, so an unchanged benchmark samples until -its timeout; a horizon such as `'5%'` settles once a difference is known to be smaller than that. +A `'0%'` horizon asks whether there is any change at all. Two builds that perform the same never +resolve against it, so an unchanged benchmark samples until its timeout, and checking again after +every round makes a false finding likelier the longer it goes. ### The report @@ -477,6 +479,20 @@ if one was killed outright. Two things follow from it. A tracked file names tarb the workspace for that time, because pnpm scopes an override by parent package name and a harness has none — so nothing else should build against the same checkout meanwhile. +### Choosing what runs, and how it prints + +Named as in Vitest: positional arguments keep the benchmark files whose path contains one of them, +and `-t` / `--testNamePattern` the benchmarks whose name matches a regular expression. + +```bash +benchmark grid -t "scroll$" --sample-size 20 --timeout 0 +``` + +`--reporter json` prints the run's analysis to stdout as JSON — per benchmark and metric, each +variant's median and each comparison's interval, verdict and severity, plus the regressions and how +long it all took — with everything else a run prints sent to stderr. `summarizeRun` from +`@mui/internal-benchmark/runReport` produces the same from a report. `NO_COLOR` turns colour off. + `benchmark --help` lists the remaining options. ## API diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index b75a664853..e40a216d8b 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -11,8 +11,15 @@ import { hideBin } from 'yargs/helpers'; import pkgJson from '@mui/internal-benchmark/package.json' with { type: 'json' }; import type { RunBenchmarksOptions } from '../runner/runBenchmarks'; -/** Everything `runBenchmarks` takes except the directory, which is where the command was run. */ -type Args = Omit; +/** + * Everything `runBenchmarks` takes except the directory, which is where the command was run, with + * its sampling overrides as flags of their own. + */ +type Args = Omit & { + sampleSize?: number; + timeout?: number; + autoSampleConditions?: string[]; +}; const runCommand: CommandModule<{}, Args> = { command: '$0 [filters...]', @@ -44,8 +51,39 @@ const runCommand: CommandModule<{}, Args> = { .option('out', { type: 'string', describe: 'Write the JSON report here. Default: .benchmark/results/report.json', + }) + .option('testNamePattern', { + alias: 't', + type: 'string', + describe: 'Only run benchmarks whose name matches this regular expression', + }) + .option('reporter', { + choices: ['default', 'json'] as const, + default: 'default' as const, + describe: + 'How to print the results: tables, or the analysis as JSON on stdout (medians, intervals, verdicts), with everything else on stderr', + }) + .option('sample-size', { + type: 'number', + describe: "Rounds before deciding whether to continue, overriding every benchmark's own", + }) + .option('timeout', { + type: 'number', + describe: "Minutes to keep sampling while unresolved, overriding every benchmark's own", + }) + .option('auto-sample-conditions', { + type: 'string', + array: true, + describe: "Horizons to resolve, such as 5% or 10%, overriding every benchmark's own", }), handler: async (argv) => { + if (argv.reporter === 'json') { + // Stdout carries the JSON alone; what a run prints along the way is progress. + /* eslint-disable no-console */ + console.log = console.error; + console.info = console.error; + /* eslint-enable no-console */ + } const { runBenchmarks } = await import('../runner/runBenchmarks'); await runBenchmarks({ harnessDir: process.cwd(), @@ -54,10 +92,23 @@ const runCommand: CommandModule<{}, Args> = { buildCmd: argv.buildCmd, out: argv.out, upload: argv.upload, + testNamePattern: argv.testNamePattern, + reporter: argv.reporter, + sampling: { + sampleSize: argv.sampleSize, + timeout: argv.timeout, + autoSampleConditions: argv.autoSampleConditions, + }, }); }, }; +// chalk honours FORCE_COLOR and --no-color, not the NO_COLOR convention (https://no-color.org): +// set to anything but an empty string, it turns colour off. +if (process.env.NO_COLOR) { + chalk.level = 0; +} + let globalArgv: { verbose?: boolean } = {}; await yargs(hideBin(process.argv)) diff --git a/packages/benchmark/src/runReport/index.ts b/packages/benchmark/src/runReport/index.ts index d1ca83e745..a96a0661a8 100644 --- a/packages/benchmark/src/runReport/index.ts +++ b/packages/benchmark/src/runReport/index.ts @@ -1,3 +1,4 @@ export * from './schema'; export * from './analyzeRun'; export * from './format'; +export * from './summary'; diff --git a/packages/benchmark/src/runReport/summary.test.ts b/packages/benchmark/src/runReport/summary.test.ts new file mode 100644 index 0000000000..4f26aba92a --- /dev/null +++ b/packages/benchmark/src/runReport/summary.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest'; +import { summarizeRun } from './summary'; +import type { BenchmarkRunReport } from './schema'; + +const report: BenchmarkRunReport = { + version: 2, + generatedAt: '2026-01-01T00:00:00.000Z', + durationMs: 845_000, + head: { sha: 'abc' }, + environment: { browser: 'Chromium', platform: 'linux', arch: 'x64', launchArgs: [] }, + builds: { current: { label: 'working tree' }, baseline: { sha: 'def', label: 'HEAD~1' } }, + metrics: { render: { kind: 'scalar', alarm: {} } }, + benchmarks: [ + { + name: 'Grid scroll', + file: 'grid.bench.tsx', + kind: 'baseline', + variants: ['current', 'baseline'], + samples: { + current: { render: [22, 24, 23, 25] }, + baseline: { render: [20, 22, 21, 23] }, + }, + sampling: { sampleSize: 4, timedOut: false }, + durationMs: 72_000, + }, + { + name: 'Chart zoom', + file: 'chart.bench.tsx', + kind: 'baseline', + variants: ['current', 'baseline'], + error: 'Render failed', + }, + ], +}; + +describe('summarizeRun', () => { + it('states the verdicts, without the samples behind them', () => { + const summary = summarizeRun(report); + + expect(summary.durationMs).toBe(845_000); + expect(summary.regressions).toEqual([ + expect.objectContaining({ benchmark: 'Grid scroll', metric: 'render', severity: 'error' }), + ]); + const [grid, chart] = summary.benchmarks; + expect(grid).toMatchObject({ + name: 'Grid scroll', + rounds: 4, + timedOut: false, + durationMs: 72_000, + }); + expect(grid.metrics).toEqual([ + { + metric: 'render', + medians: { current: 23.5, baseline: 21.5 }, + comparisons: [ + expect.objectContaining({ subject: 'current', against: 'baseline', change: 'worse' }), + ], + }, + ]); + expect(chart).toMatchObject({ name: 'Chart zoom', error: 'Render failed', metrics: [] }); + expect(JSON.stringify(summary)).not.toContain('"samples"'); + }); +}); diff --git a/packages/benchmark/src/runReport/summary.ts b/packages/benchmark/src/runReport/summary.ts new file mode 100644 index 0000000000..35305c17a4 --- /dev/null +++ b/packages/benchmark/src/runReport/summary.ts @@ -0,0 +1,85 @@ +import { analyzeRun, findRegressions } from './analyzeRun'; +import type { Change, Interval, Severity } from './analyzeRun'; +import type { BenchmarkRunReport, RunBenchmark } from './schema'; + +/** + * A run's conclusions as plain data, for a program to read rather than a person: what the tables, + * the pull request comment and the dashboard show, without the raw samples behind it. + */ +export interface RunSummary { + /** How long the whole run took, when the report says. */ + durationMs?: number; + /** The changes for the worse that fail the check, as the comment lists them. */ + regressions: Array<{ benchmark: string; metric: string; severity: Severity; relative: Interval }>; + benchmarks: BenchmarkSummary[]; +} + +export interface BenchmarkSummary { + name: string; + file: string; + kind: RunBenchmark['kind']; + /** Rounds measured; absent for a benchmark that failed. */ + rounds?: number; + timedOut?: boolean; + durationMs?: number; + error?: string; + metrics: Array<{ + metric: string; + /** Per variant, the median of its samples. */ + medians: Record; + comparisons: Array<{ + subject: string; + against: string; + change: Change; + severity: Severity; + /** The level of both intervals, `0.95` or stricter where a change can raise an alarm. */ + confidence: number; + /** `subject − against`, in the metric's unit. */ + absolute: Interval; + /** The same, as a percentage of `against`. */ + relative: Interval; + }>; + }>; +} + +export function summarizeRun(report: BenchmarkRunReport): RunSummary { + const analyses = analyzeRun(report); + return { + durationMs: report.durationMs, + regressions: findRegressions(analyses).map(({ benchmark, metric, comparison }) => ({ + benchmark, + metric, + severity: comparison.severity, + relative: comparison.relative, + })), + benchmarks: analyses.map(({ benchmark, metrics }) => { + const [reference] = benchmark.variants; + return { + name: benchmark.name, + file: benchmark.file, + kind: benchmark.kind, + rounds: Object.values(benchmark.samples?.[reference] ?? {})[0]?.length, + timedOut: benchmark.sampling?.timedOut, + durationMs: benchmark.durationMs, + error: benchmark.error, + metrics: metrics.map(({ metric, variants, comparisons }) => ({ + metric, + medians: Object.fromEntries( + Object.entries(variants).map(([variant, summary]) => [variant, summary.median]), + ), + comparisons: comparisons.map( + ({ subject, against, change, severity, confidence, absolute, relative }) => ({ + subject, + against, + change, + severity, + confidence, + absolute, + relative, + }), + ), + })), + }; + }), + }; +} diff --git a/packages/benchmark/src/runner/buildPages.ts b/packages/benchmark/src/runner/buildPages.ts index d5f10bfc9f..bfd753fc3e 100644 --- a/packages/benchmark/src/runner/buildPages.ts +++ b/packages/benchmark/src/runner/buildPages.ts @@ -149,7 +149,9 @@ export async function buildRefPages(options: { // it to `src/`, the same as for a plain `vite build`. await vite.build({ root: harnessDir, - logLevel: 'info', + // Warnings and errors only: the run says what it builds, and the bundler's own progress lines go + // to stdout, which stays free for what a caller asks for, such as a JSON report. + logLevel: 'warn', build: { outDir }, }); } diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index 2f8cd3a665..ba08576559 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -7,7 +7,9 @@ import chalk from 'chalk'; import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; import { packRef, packWorkingTree } from '../utils/packWorkspace'; import { getCiMetadata } from '../ciReport'; +import { summarizeRun } from '../runReport'; import type { BenchmarkRunReport } from '../runReport'; +import type { SamplingOptions } from '../sampling'; import { refLabel, resolveBaselineRef, WORKTREE_REF } from './refs'; import type { ResolvedRef } from './refs'; import { discoverBenchFiles } from './benchFiles'; @@ -41,6 +43,15 @@ export interface RunBenchmarksOptions { out?: string; /** Upload the report and refresh the pull request comment. */ upload?: boolean; + /** Sampling options that override every benchmark's own, for the whole run. */ + sampling?: SamplingOptions; + /** Only run benchmarks whose name matches this regular expression, like Vitest's `-t`. */ + testNamePattern?: string; + /** + * How the results are printed: `default`, tables for reading, or `json`, the analysis as JSON on + * stdout — medians, intervals and verdicts — for a program to read. + */ + reporter?: 'default' | 'json'; } /** @@ -59,7 +70,12 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise ({ variant: variants[index], url, caseName })); - return { entry, slots, sampling }; + return { entry, slots, sampling: withOverrides(sampling, options.sampling) }; }); for (const { name, cases, sampling } of reference.comparisons) { const entry: BenchmarkEntry = { name, file: benchFile.file, kind: 'compare', variants: cases }; const slots = cases.map((caseName) => ({ variant: caseName, url: urls[0], caseName })); - planned.push({ entry, slots, sampling }); + planned.push({ entry, slots, sampling: withOverrides(sampling, options.sampling) }); } return planned; } +/** A benchmark's own sampling, with the run's overrides on top: only the options the run set. */ +function withOverrides( + own: Required, + overrides: SamplingOptions = {}, +): Required { + const set = Object.fromEntries( + Object.entries(overrides).filter(([, value]) => value !== undefined), + ); + return resolveSampling({ ...own, ...set }); +} + export async function runInterleaved(options: RunInterleavedOptions): Promise { const { chromium } = await import('@playwright/test'); const [server, browser] = await Promise.all([ @@ -335,13 +350,19 @@ export async function runInterleaved(options: RunInterleavedOptions): Promise planBenchFile(browser, server.origin, benchFile, options), ), ) - ).flat(); + ) + .flat() + .filter((plan) => !testNamePattern || testNamePattern.test(plan.entry.name)); + if (planned.length === 0) { + throw new Error(`No benchmark name matches ${testNamePattern}.`); + } const baselineBenchmarks = planned.filter( (plan) => 'slots' in plan && plan.entry.kind === 'baseline', ).length; diff --git a/packages/benchmark/src/sampling.ts b/packages/benchmark/src/sampling.ts index 483366d613..70af53a7fe 100644 --- a/packages/benchmark/src/sampling.ts +++ b/packages/benchmark/src/sampling.ts @@ -16,7 +16,9 @@ export interface SamplingOptions { /** * Relative differences to resolve before `timeout`. A difference is resolved against a horizon * once its confidence interval lies entirely on one side of it. `'10%'` stands for both `'-10%'` - * and `'+10%'`. Defaults to `['0%']`: sample until each change is known to be better or worse. + * and `'+10%'`. Defaults to `['5%']`: sample until each change is known to be larger or smaller + * than 5%. A `0%` horizon tests whether there is any change at all: identical builds never resolve + * against it, and checking again after every round makes a false finding likelier. * * Judged on the metrics that alarm — the ones a change can be flagged on — or on every metric when * none does. @@ -27,7 +29,7 @@ export interface SamplingOptions { export const DEFAULT_SAMPLING: Required = { sampleSize: 50, timeout: 3, - autoSampleConditions: ['0%'], + autoSampleConditions: ['5%'], }; /** Parses `autoSampleConditions` into signed horizons, in percent. */ diff --git a/packages/benchmark/src/utils/packWorkspace.ts b/packages/benchmark/src/utils/packWorkspace.ts index 01c608057a..ad7ae41079 100644 --- a/packages/benchmark/src/utils/packWorkspace.ts +++ b/packages/benchmark/src/utils/packWorkspace.ts @@ -115,13 +115,17 @@ async function removeAbandonedCheckouts(repoRoot: string, outRoot: string): Prom ); } +// A command's own output is progress, so it goes to stderr: stdout stays free for what a caller +// asks for, such as a JSON report. +const PROGRESS_STDIO = { stdin: 'inherit', stdout: process.stderr, stderr: 'inherit' } as const; + /** Runs `buildCmd` in `cwd`, with the nx daemon off so nothing keeps writing after it returns. */ async function build(cwd: string, buildCmd: string): Promise { const [file, ...args] = parseCommandString(buildCmd); await execa(file, args, { cwd, env: { NX_DAEMON: 'false' }, - stdio: 'inherit', + ...PROGRESS_STDIO, verbose: 'short', }); } @@ -226,7 +230,7 @@ export async function packRef(options: PackRefOptions): Promise if (installCmd) { console.log(chalk.cyan(`\nInstalling dependencies for ${sha.slice(0, 9)}…`)); const [installFile, ...installArgs] = parseCommandString(installCmd); - await execa(installFile, installArgs, { cwd: checkout, stdio: 'inherit', verbose: 'short' }); + await execa(installFile, installArgs, { cwd: checkout, ...PROGRESS_STDIO, verbose: 'short' }); } console.log(chalk.cyan(`\nBuilding packages for ${sha.slice(0, 9)}…`)); await build(checkout, buildCmd); From bc5087648d2b2c10ff82d91932cf83d6d5b35c0d Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:16:12 +0200 Subject: [PATCH 126/173] [benchmark] Record performance traces with --profile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --profile runs every sample as it would be measured — a freshly loaded page in one tab, the variants alternating — with a performance trace around it, and saves the traces under .benchmark/traces/ to open in DevTools' Performance panel. 5 rounds unless --sample-size says otherwise; no report. --- packages/benchmark/README.md | 5 + packages/benchmark/src/cli/index.ts | 10 +- .../benchmark/src/runner/runBenchmarks.ts | 45 ++++++++- .../benchmark/src/runner/runInterleaved.ts | 48 +++++++--- .../benchmark/src/runner/traceBenchmarks.ts | 93 +++++++++++++++++++ 5 files changed, 181 insertions(+), 20 deletions(-) create mode 100644 packages/benchmark/src/runner/traceBenchmarks.ts diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 810e62a05c..8ac2605bf2 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -493,6 +493,11 @@ variant's median and each comparison's interval, verdict and severity, plus the long it all took — with everything else a run prints sent to stderr. `summarizeRun` from `@mui/internal-benchmark/runReport` produces the same from a report. `NO_COLOR` turns colour off. +`--profile` records instead of measuring: each sample runs as it would be measured, with a +performance trace around it, saved under `.benchmark/traces/` to open in DevTools' Performance +panel — 5 rounds unless `--sample-size` says otherwise. It writes no report: tracing slows the code +it records. + `benchmark --help` lists the remaining options. ## API diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index e40a216d8b..5831f87340 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -63,9 +63,16 @@ const runCommand: CommandModule<{}, Args> = { describe: 'How to print the results: tables, or the analysis as JSON on stdout (medians, intervals, verdicts), with everything else on stderr', }) + .option('profile', { + type: 'boolean', + default: false, + describe: + 'Record instead of measuring: a performance trace around every sample, saved to open in DevTools. Writes no report', + }) .option('sample-size', { type: 'number', - describe: "Rounds before deciding whether to continue, overriding every benchmark's own", + describe: + "Rounds before deciding whether to continue, overriding every benchmark's own; with --profile, the rounds to record (default 5)", }) .option('timeout', { type: 'number', @@ -94,6 +101,7 @@ const runCommand: CommandModule<{}, Args> = { upload: argv.upload, testNamePattern: argv.testNamePattern, reporter: argv.reporter, + profile: argv.profile, sampling: { sampleSize: argv.sampleSize, timeout: argv.timeout, diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index ba08576559..cda0830cd1 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -14,6 +14,7 @@ import { refLabel, resolveBaselineRef, WORKTREE_REF } from './refs'; import type { ResolvedRef } from './refs'; import { discoverBenchFiles } from './benchFiles'; import { runInterleaved } from './runInterleaved'; +import { traceBenchmarks } from './traceBenchmarks'; import { buildRefPages, restoreWorkspace } from './buildPages'; import { resolveBrowserBinary } from './browser'; import { printRunReport } from './printReport'; @@ -52,16 +53,27 @@ export interface RunBenchmarksOptions { * stdout — medians, intervals and verdicts — for a program to read. */ reporter?: 'default' | 'json'; + /** + * Record the benchmarks instead of measuring them: a performance trace around every sample, to + * open in DevTools' Performance panel. Writes no report. + */ + profile?: boolean; } +/** Rounds a profile records unless a sample size is given. */ +const PROFILE_ROUNDS = 5; + /** * Benchmarks a harness's `*.bench.tsx` files across two builds of the workspace — the working tree - * and a baseline — and writes, prints and optionally uploads the run report. + * and a baseline — and writes, prints and optionally uploads the run report. With `profile`, it + * traces them instead, and returns no report. * * Both builds are packed from their own commit and installed into the harness the same way, so * neither resolves the library through a workspace link. */ -export async function runBenchmarks(options: RunBenchmarksOptions): Promise { +export async function runBenchmarks( + options: RunBenchmarksOptions, +): Promise { const startedAt = Date.now(); const { harnessDir, @@ -73,6 +85,7 @@ export async function runBenchmarks(options: RunBenchmarksOptions): Promise[0]; /** One measured round: every variant's values by metric name, in slot order. */ type Round = Array>; -function errorMessage(error: unknown): string { +export function errorMessage(error: unknown): string { return error instanceof Error ? (error.stack ?? error.message) : String(error); } /** A random permutation of `0..length-1`, so no variant always runs first in its round. */ -function shuffledIndices(length: number): number[] { +export function shuffledIndices(length: number): number[] { const indices = Array.from({ length }, (_, index) => index); for (let index = indices.length - 1; index > 0; index -= 1) { const swap = Math.floor(Math.random() * (index + 1)); @@ -84,7 +84,7 @@ type PageListing = Pick; * Opens a tab, in a browser context of its own, bridged to a CDP session for trusted input. Closing * its context closes it. */ -async function openBenchTab(browser: Browser): Promise { +export async function openBenchTab(browser: Browser): Promise { const context = await browser.newContext({ viewport: BENCHMARK_VIEWPORT }); try { const page = await context.newPage(); @@ -107,7 +107,7 @@ async function openBenchTab(browser: Browser): Promise { } /** Loads a benchmark page into the tab, and waits until it has registered its cases. */ -async function loadBenchPage( +export async function loadBenchPage( page: Page, url: string, ): Promise { @@ -149,7 +149,7 @@ async function listBenchPages(browser: Browser, urls: string[]): Promise> { +export function sampleBenchCase(page: Page, name: string): Promise> { return page.evaluate((caseName) => window.benchmarkPage!.sample(caseName), name); } @@ -157,7 +157,7 @@ function sampleBenchCase(page: Page, name: string): Promise; } -type BenchmarkEntry = Pick; +export type BenchmarkEntry = Pick; /** A benchmark to measure: its variants' slots, sampled as its options ask. */ -interface MeasuredBenchmark { +export interface MeasuredBenchmark { entry: BenchmarkEntry; slots: PageSlot[]; sampling: Required; } /** A benchmark to measure, or why it can't be. */ -type PlannedBenchmark = MeasuredBenchmark | { entry: BenchmarkEntry; error: string }; +export type PlannedBenchmark = MeasuredBenchmark | { entry: BenchmarkEntry; error: string }; /** * Measures one benchmark in a tab of its own, loading each slot's page before sampling it. Every @@ -336,7 +336,22 @@ function withOverrides( return resolveSampling({ ...own, ...set }); } -export async function runInterleaved(options: RunInterleavedOptions): Promise { +/** What a run drives: the browser, and every benchmark it was asked for, in order. */ +export interface PlannedRun { + browser: Browser; + planned: PlannedBenchmark[]; + /** How many of them measure across the builds. */ + baselineBenchmarks: number; +} + +/** + * Serves the builds, launches the browser and lists every benchmark the run was asked for, then + * hands them to `drive`, which measures or profiles them. + */ +export async function withPlannedRun( + options: RunInterleavedOptions, + drive: (run: PlannedRun) => Promise, +): Promise { const { chromium } = await import('@playwright/test'); const [server, browser] = await Promise.all([ serveDirectory(options.buildsDir), @@ -366,7 +381,15 @@ export async function runInterleaved(options: RunInterleavedOptions): Promise 'slots' in plan && plan.entry.kind === 'baseline', ).length; + return await drive({ browser, planned, baselineBenchmarks }); + } finally { + await browser.close(); + await server.close(); + } +} +export function runInterleaved(options: RunInterleavedOptions): Promise { + return withPlannedRun(options, async ({ browser, planned, baselineBenchmarks }) => { const results: CaseResult[] = []; // Sequential on purpose: concurrent cases would contend for the same machine. for (const plan of planned) { @@ -385,8 +408,5 @@ export async function runInterleaved(options: RunInterleavedOptions): Promise result.metrics)), browserVersion: browser.version(), }; - } finally { - await browser.close(); - await server.close(); - } + }); } diff --git a/packages/benchmark/src/runner/traceBenchmarks.ts b/packages/benchmark/src/runner/traceBenchmarks.ts new file mode 100644 index 0000000000..c6b7160a25 --- /dev/null +++ b/packages/benchmark/src/runner/traceBenchmarks.ts @@ -0,0 +1,93 @@ +/* eslint-disable no-console */ + +import * as path from 'node:path'; +import { mkdir, rm } from 'node:fs/promises'; +import chalk from 'chalk'; +import type { Browser } from '@playwright/test'; +import type { RunBenchmark } from '../runReport'; +import { + errorMessage, + loadBenchPage, + openBenchTab, + sampleBenchCase, + shuffledIndices, + withPlannedRun, +} from './runInterleaved'; +import type { MeasuredBenchmark, RunInterleavedOptions } from './runInterleaved'; + +/** + * Records benchmarks instead of measuring them, to show where their time goes: every sample runs as + * it would be measured — the first iteration of a freshly loaded page, in one tab, the variants + * alternating — with a performance trace recorded around it, to open in DevTools' Performance + * panel. Tracing slows the code it records, so nothing here is a measurement. + */ + +export interface BenchmarkTraces extends Pick { + /** Per variant, its samples' trace files in the order they ran. */ + traces: Record; + error?: string; +} + +async function traceBenchmark( + browser: Browser, + { entry, slots, sampling }: MeasuredBenchmark, + tracesDir: string, +): Promise { + console.log(chalk.cyan(`\nTracing "${entry.name}" (${entry.file})…`)); + const outDir = path.join(tracesDir, entry.name.replace(/[^\w.-]+/g, '-')); + const traces: Record = Object.fromEntries( + slots.map((slot) => [slot.variant, []]), + ); + const page = await openBenchTab(browser); + try { + await mkdir(outDir, { recursive: true }); + for (let round = 1; round <= sampling.sampleSize; round += 1) { + for (const index of shuffledIndices(slots.length)) { + const { variant, url, caseName } = slots[index]; + const file = path.join(outDir, `${variant}-${round}.json`); + // eslint-disable-next-line no-await-in-loop + await loadBenchPage(page, url); + // eslint-disable-next-line no-await-in-loop + await browser.startTracing(page, { path: file }); + // eslint-disable-next-line no-await-in-loop + await sampleBenchCase(page, caseName); + // eslint-disable-next-line no-await-in-loop + await browser.stopTracing(); + traces[variant].push(file); + } + } + return { name: entry.name, file: entry.file, kind: entry.kind, traces }; + } catch (error) { + console.error(chalk.red(` ${errorMessage(error)}`)); + return { + name: entry.name, + file: entry.file, + kind: entry.kind, + traces, + error: errorMessage(error), + }; + } finally { + await page.context().close(); + } +} + +/** Traces every benchmark the run was asked for, one after another, into `tracesDir`. */ +export async function traceBenchmarks( + options: RunInterleavedOptions & { tracesDir: string }, +): Promise { + await rm(options.tracesDir, { recursive: true, force: true }); + return withPlannedRun(options, async ({ browser, planned }) => { + const results: BenchmarkTraces[] = []; + // Sequential on purpose: a browser records one trace at a time. + for (const plan of planned) { + if ('error' in plan) { + const { name, file, kind } = plan.entry; + results.push({ name, file, kind, traces: {}, error: plan.error }); + continue; + } + // eslint-disable-next-line no-await-in-loop + results.push(await traceBenchmark(browser, plan, options.tracesDir)); + } + return results; + }); +} From 73805fbee2cbeddae7de401f688deb028705cf33 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:35:36 +0200 Subject: [PATCH 127/173] [benchmark] Run a nightly A/A on master to track the false-alarm rate The benchmark_aa job measures the commit against itself; any regression is a false alarm and fails the job, so its failure rate over many nights is the false-alarm rate. It runs when a scheduled pipeline sets workflow to benchmark-aa. --- .circleci/config.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d6c61e3fe..81fef815d5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -155,6 +155,34 @@ jobs: key: benchmark-run-cache-v1-{{ checksum ".benchmark-baseline-sha" }} paths: - test/benchmark/.benchmark/packed + # The commit against itself: any regression it reports is a false alarm, so over many nights the + # job's failure rate is the benchmarks' false-alarm rate, about 5% when the statistics are honest. + benchmark_aa: + <<: *default-job + executor: + name: code-infra/mui-node-browser + playwright-img-version: 'v1.62.1-noble' + resource_class: medium+ + steps: + - checkout + - code-infra/install-deps + - run: + name: Build packages + command: pnpm release:build + - run: + name: Benchmark the commit against itself + no_output_timeout: 30m + command: pnpm --silent -F ./test/benchmark benchmark:run --baseline HEAD --reporter json > aa.json + - store_artifacts: + path: aa.json + - run: + name: Fail on a false alarm + command: | + node -e " + const { regressions } = JSON.parse(require('fs').readFileSync('aa.json', 'utf8')); + for (const { benchmark, metric } of regressions) console.error('False alarm: ' + benchmark + ' · ' + metric); + process.exit(regressions.length > 0 ? 1 : 0); + " test_browser: <<: *default-job @@ -201,3 +229,10 @@ workflows: <<: *default-context - test_browser: <<: *default-context + # Started nightly on master by a scheduled pipeline that sets `workflow` to `benchmark-aa`. + benchmark-aa: + when: + equal: [benchmark-aa, << pipeline.parameters.workflow >>] + jobs: + - benchmark_aa: + <<: *default-context From c9bdb60805fdb563f0ba00d6241890f89fc79eec Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 7 Oct 2026 10:43:13 +0200 Subject: [PATCH 128/173] [benchmark] Sampling flags set defaults, as in Vitest, instead of overriding --sample-size, --timeout and --auto-sample-conditions now apply to the benchmarks that don't set their own, the way --testTimeout does for tests: what a benchmark file says always wins. Pages report sampling as written, and the runner fills in the run's defaults and then the built-in ones. --profile takes its own round count, whatever a benchmark samples for. --- packages/benchmark/README.md | 5 ++-- packages/benchmark/src/cli/index.ts | 8 ++++--- packages/benchmark/src/page/page.tsx | 15 ++++++------ .../benchmark/src/runner/runBenchmarks.ts | 5 ++-- .../benchmark/src/runner/runInterleaved.ts | 24 +++++++++---------- .../benchmark/src/runner/traceBenchmarks.ts | 14 +++++++---- packages/benchmark/src/sampling.ts | 14 +++++++---- 7 files changed, 49 insertions(+), 36 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 8ac2605bf2..d4d6b42eae 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -419,8 +419,9 @@ metric when none does. They are set per benchmark: the last argument of `benchmark()` and `compare()`, and among `reactBenchmark()`'s options. A compared case is sampled as its `compare()` asks, so setting its own -is an error. `--sample-size`, `--timeout` and `--auto-sample-conditions` override them for every -benchmark of a run, to iterate quickly without editing benchmark files. +is an error. `--sample-size`, `--timeout` and `--auto-sample-conditions` set them for the benchmarks +that don't set their own, as Vitest's `--testTimeout` does for tests: what a benchmark file says +always wins. ```tsx reactBenchmark('mount', () => , { timeout: 1, autoSampleConditions: ['10%'] }); diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index 5831f87340..0c780226dc 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -72,16 +72,18 @@ const runCommand: CommandModule<{}, Args> = { .option('sample-size', { type: 'number', describe: - "Rounds before deciding whether to continue, overriding every benchmark's own; with --profile, the rounds to record (default 5)", + "Rounds before deciding whether to continue, for benchmarks that don't set their own (default 50); with --profile, the rounds to record (default 5)", }) .option('timeout', { type: 'number', - describe: "Minutes to keep sampling while unresolved, overriding every benchmark's own", + describe: + "Minutes to keep sampling while unresolved, for benchmarks that don't set their own (default 3)", }) .option('auto-sample-conditions', { type: 'string', array: true, - describe: "Horizons to resolve, such as 5% or 10%, overriding every benchmark's own", + describe: + "Horizons to resolve, such as 5% or 10%, for benchmarks that don't set their own (default 5%)", }), handler: async (argv) => { if (argv.reporter === 'json') { diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 47bdaf2728..dd469153b8 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -2,7 +2,7 @@ import { collectGarbage } from '../gc'; import { createInput } from '../input'; import type { BenchmarkInput } from '../input'; import { seriesName, setMetricRecorder } from '../metricCore'; -import { hasSampling, resolveSampling } from '../sampling'; +import { hasSampling, pickSampling, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { MetricDefinition } from '../types'; @@ -54,7 +54,7 @@ export function benchmark( const comparisons: Array<{ name: string; cases: string[]; - sampling: Required; + sampling: SamplingOptions; }> = []; /** @@ -75,7 +75,8 @@ export function compare( if (compared.length < 2) { throw new Error(`Comparison "${name}" needs at least two cases.`); } - const resolved = resolveSampling(sampling); + // Resolved here only to reject invalid options where they were written. + resolveSampling(sampling); for (const benchCase of compared) { const own = cases.get(benchCase.name)?.sampling; if (own && hasSampling(own)) { @@ -92,7 +93,7 @@ export function compare( comparisons.push({ name, cases: compared.map((benchCase) => benchCase.name), - sampling: resolved, + sampling: pickSampling(sampling), }); } @@ -120,9 +121,9 @@ setMetricRecorder((metric, value, options) => { export interface BenchPage { /** The cases measured across the builds: every case no `compare()` took. */ - cases: Array<{ name: string; sampling: Required }>; + cases: Array<{ name: string; sampling: SamplingOptions }>; /** The file's `compare()` calls, with their cases' names in order. */ - comparisons: Array<{ name: string; cases: string[]; sampling: Required }>; + comparisons: Array<{ name: string; cases: string[]; sampling: SamplingOptions }>; /** Every metric recorded so far, by name. */ metricDefinitions: () => Record; /** Runs one iteration of a case, and returns what it recorded, by series. */ @@ -183,7 +184,7 @@ export function markPageReady(): void { window.benchmarkPage = { cases: [...cases] .filter(([name]) => !comparisons.some((comparison) => comparison.cases.includes(name))) - .map(([name, { sampling }]) => ({ name, sampling: resolveSampling(sampling) })), + .map(([name, { sampling }]) => ({ name, sampling: pickSampling(sampling) })), comparisons, metricDefinitions: () => Object.fromEntries(metricDefinitions), sample, diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index cda0830cd1..88c95f0c22 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -44,7 +44,7 @@ export interface RunBenchmarksOptions { out?: string; /** Upload the report and refresh the pull request comment. */ upload?: boolean; - /** Sampling options that override every benchmark's own, for the whole run. */ + /** Defaults, for the whole run, for the sampling options a benchmark doesn't set itself. */ sampling?: SamplingOptions; /** Only run benchmarks whose name matches this regular expression, like Vitest's `-t`. */ testNamePattern?: string; @@ -188,8 +188,7 @@ export async function runBenchmarks( if (profile) { const traced = await traceBenchmarks({ ...runOptions, - // A fixed number of rounds: nothing is being resolved. - sampling: { sampleSize: sampling?.sampleSize ?? PROFILE_ROUNDS, timeout: 0 }, + rounds: sampling?.sampleSize ?? PROFILE_ROUNDS, tracesDir: path.join(outputDir, 'traces'), }); if (reporter === 'json') { diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index e6b061a5d9..60d7e4085e 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -5,7 +5,7 @@ import type { Browser, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; import { compareBenchmark, countAlarmedComparisons } from '../runReport'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; -import { differencesResolved, parseHorizons, resolveSampling } from '../sampling'; +import { differencesResolved, parseHorizons, pickSampling, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { BenchFile } from './benchFiles'; import type { ResolvedRef } from './refs'; @@ -31,7 +31,7 @@ export interface RunInterleavedOptions { benchRefs: ResolvedRef[]; browserBinary: string; launchArgs: string[]; - /** Sampling options that override every benchmark's own. */ + /** Defaults for the sampling options a benchmark doesn't set itself. */ sampling?: SamplingOptions; /** Only run benchmarks whose name matches this, like Vitest's `-t`. */ testNamePattern?: RegExp; @@ -314,26 +314,26 @@ async function planBenchFile( return { entry, error: `"${caseName}" does not exist in [${variants[missing + 1]}].` }; } const slots = urls.map((url, index) => ({ variant: variants[index], url, caseName })); - return { entry, slots, sampling: withOverrides(sampling, options.sampling) }; + return { entry, slots, sampling: samplingOf(sampling, options.sampling) }; }); for (const { name, cases, sampling } of reference.comparisons) { const entry: BenchmarkEntry = { name, file: benchFile.file, kind: 'compare', variants: cases }; const slots = cases.map((caseName) => ({ variant: caseName, url: urls[0], caseName })); - planned.push({ entry, slots, sampling: withOverrides(sampling, options.sampling) }); + planned.push({ entry, slots, sampling: samplingOf(sampling, options.sampling) }); } return planned; } -/** A benchmark's own sampling, with the run's overrides on top: only the options the run set. */ -function withOverrides( - own: Required, - overrides: SamplingOptions = {}, +/** + * A benchmark's sampling, as in Vitest: what it sets itself, then what the run gives as defaults, + * then the built-in defaults. + */ +function samplingOf( + own: SamplingOptions, + runDefaults: SamplingOptions = {}, ): Required { - const set = Object.fromEntries( - Object.entries(overrides).filter(([, value]) => value !== undefined), - ); - return resolveSampling({ ...own, ...set }); + return resolveSampling({ ...pickSampling(runDefaults), ...pickSampling(own) }); } /** What a run drives: the browser, and every benchmark it was asked for, in order. */ diff --git a/packages/benchmark/src/runner/traceBenchmarks.ts b/packages/benchmark/src/runner/traceBenchmarks.ts index c6b7160a25..9dc97f808c 100644 --- a/packages/benchmark/src/runner/traceBenchmarks.ts +++ b/packages/benchmark/src/runner/traceBenchmarks.ts @@ -30,8 +30,8 @@ export interface BenchmarkTraces extends Pick { console.log(chalk.cyan(`\nTracing "${entry.name}" (${entry.file})…`)); const outDir = path.join(tracesDir, entry.name.replace(/[^\w.-]+/g, '-')); @@ -41,7 +41,7 @@ async function traceBenchmark( const page = await openBenchTab(browser); try { await mkdir(outDir, { recursive: true }); - for (let round = 1; round <= sampling.sampleSize; round += 1) { + for (let round = 1; round <= rounds; round += 1) { for (const index of shuffledIndices(slots.length)) { const { variant, url, caseName } = slots[index]; const file = path.join(outDir, `${variant}-${round}.json`); @@ -73,7 +73,11 @@ async function traceBenchmark( /** Traces every benchmark the run was asked for, one after another, into `tracesDir`. */ export async function traceBenchmarks( - options: RunInterleavedOptions & { tracesDir: string }, + options: RunInterleavedOptions & { + tracesDir: string; + /** Rounds to record: every variant once per round, whatever a benchmark samples for. */ + rounds: number; + }, ): Promise { await rm(options.tracesDir, { recursive: true, force: true }); return withPlannedRun(options, async ({ browser, planned }) => { @@ -86,7 +90,7 @@ export async function traceBenchmarks( continue; } // eslint-disable-next-line no-await-in-loop - results.push(await traceBenchmark(browser, plan, options.tracesDir)); + results.push(await traceBenchmark(browser, plan, options)); } return results; }); diff --git a/packages/benchmark/src/sampling.ts b/packages/benchmark/src/sampling.ts index 70af53a7fe..307f17a323 100644 --- a/packages/benchmark/src/sampling.ts +++ b/packages/benchmark/src/sampling.ts @@ -69,10 +69,16 @@ export function resolveSampling(options: SamplingOptions): Required 0; +} + +/** The sampling options `options` sets, leaving out any other key and any left undefined. */ +export function pickSampling(options: SamplingOptions): SamplingOptions { + const { sampleSize, timeout, autoSampleConditions } = options; + return Object.fromEntries( + Object.entries({ sampleSize, timeout, autoSampleConditions }).filter( + ([, value]) => value !== undefined, + ), ); } From e0ff84139a6a4bf59922093e9cfa4d1c176c9bfb Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 7 Oct 2026 11:52:32 +0200 Subject: [PATCH 129/173] [benchmark] Hand cases the DevTools Protocol as cdp benchmark() cases and reactBenchmark() interactions get cdp.send, on the session that drives the page, in benchmark runs and under Vitest alike: to throttle the CPU, or read the engine's counters as custom metrics. It replaces input.send, which hid the same escape hatch under gestures. --- packages/benchmark/README.md | 27 +++++++++++++++++++ packages/benchmark/src/caseRuntime.tsx | 5 ++-- packages/benchmark/src/index.tsx | 12 +++++---- packages/benchmark/src/input.ts | 14 ++++++++-- packages/benchmark/src/page/page.tsx | 12 ++++++--- .../benchmark/src/page/reactBenchmark.tsx | 4 +-- packages/benchmark/src/publicApi.ts | 2 +- packages/benchmark/src/types.ts | 4 ++- 8 files changed, 62 insertions(+), 18 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index d4d6b42eae..59afd9c8c9 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -89,6 +89,33 @@ iteration to the next. Wait for the mount to be painted — `await waitForElemen before starting a gesture on something the case just rendered; until then, the browser may not route the gesture to it. +For anything else the browser can do, the context has `cdp`, the Chrome DevTools Protocol on the +session that drives the page: throttle the CPU, or read the engine's own counters as a custom metric. + +```tsx +const layouts = new DiscreteMetric({ name: 'layout:count' }); + +benchmark( + 'Grid scroll', + () => , + async ({ input, cdp }) => { + await cdp.send('Performance.enable'); + const count = async () => { + const { metrics } = (await cdp.send('Performance.getMetrics')) as { + metrics: Array<{ name: string; value: number }>; + }; + return metrics.find((metric) => metric.name === 'LayoutCount')!.value; + }; + const before = await count(); + await input.scroll({ x: 400, y: 300, deltaY: 2000 }); + layouts.record((await count()) - before); + }, +); +``` + +A setting a case changes this way, such as CPU throttling, stays on the tab for later samples and for +the other cases sampled in it, those of a `compare()` too: a case that changes one sets it every time. + ### Scoping which renders are measured By default a benchmark records every React render and paint, from the mount through the whole interaction. To measure only part of an interaction — or to exclude the mount — pause and resume recording from the interaction callback: diff --git a/packages/benchmark/src/caseRuntime.tsx b/packages/benchmark/src/caseRuntime.tsx index da599ad720..7664fa77fc 100644 --- a/packages/benchmark/src/caseRuntime.tsx +++ b/packages/benchmark/src/caseRuntime.tsx @@ -3,7 +3,6 @@ import * as React from 'react'; import * as ReactDOMClient from 'react-dom/profiling'; import * as ReactDOM from 'react-dom'; import type { RenderEvent, InteractionContext, BenchmarkCaseRuntime } from './types'; -import type { BenchmarkInput } from './input'; import { ElementTiming } from './ElementTiming'; import { createReactRecordingControls } from './reactRecording'; import type { ReactRecordingControls } from './reactRecording'; @@ -261,7 +260,7 @@ export async function measureIteration( renderFn: () => React.ReactElement, interaction: BenchmarkInteraction | undefined, options: CaseOptions | undefined, - input: BenchmarkInput, + driver: Pick, ): Promise { // Per-iteration switch for the harness's React render/paint recording. Starts paused when // `reactRecordingPaused` is set; the interaction callback drives it from there. @@ -285,7 +284,7 @@ export async function measureIteration( waitForElementTiming: timing.waitForElementTiming, pauseReactRecording: recording.pauseReactRecording, resumeReactRecording: recording.resumeReactRecording, - input, + ...driver, }, onUncaughtError: (error) => { renderFailure = { error }; diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index dd8317a49f..2b1f469039 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -7,6 +7,7 @@ import { ScalarMetric } from './ScalarMetric'; import { metricsGate } from './metricsGate'; import { runProfileSession } from './profileSession'; import { createInput } from './input'; +import type { CdpSend } from './input'; import { collectGarbage } from './gc'; import { createCaseRuntime, @@ -33,10 +34,11 @@ const PROFILE_MODE = process.env.BENCHMARK_PROFILE === 'true'; type CdpMethod = Parameters['send']>[0]; -// Gestures go through the CDP session Vitest's Playwright provider holds for the test page. The -// session's `send` only accepts protocol method names it knows; `input.send` takes any string and +// Gestures and `cdp` go through the CDP session Vitest's Playwright provider holds for the test +// page. The session's `send` only accepts protocol method names it knows; ours takes any string and // leaves unknown methods for Chrome to reject. -const input = createInput((method, params) => cdp().send(method as CdpMethod, params)); +const send: CdpSend = (method, params) => cdp().send(method as CdpMethod, params); +const driver = { input: createInput(send), cdp: { send } }; // Paint timings are recorded as one harness-owned `bench:paint` metric: the default sentinel // is the base series (`bench:paint`) and named `elementtiming` markers are sub-series @@ -65,7 +67,7 @@ async function runIteration( await collectGarbage(); metricsGate.setRecordingEnabled(test, !warmup); try { - const { renders, paints } = await measureIteration(renderFn, interaction, options, input); + const { renders, paints } = await measureIteration(renderFn, interaction, options, driver); if (!warmup) { for (const { id, start, end } of paints) { paint.record(end - start, id !== undefined ? { id } : undefined); @@ -101,7 +103,7 @@ export function benchmark( waitForElementTiming: timing.waitForElementTiming, pauseReactRecording: () => {}, resumeReactRecording: () => {}, - input, + ...driver, }, }); await runProfileSession(name, runtime); diff --git a/packages/benchmark/src/input.ts b/packages/benchmark/src/input.ts index 9c3f202d44..e2ff05f740 100644 --- a/packages/benchmark/src/input.ts +++ b/packages/benchmark/src/input.ts @@ -37,7 +37,18 @@ export interface BenchmarkInput { scroll: (options: ScrollGestureOptions) => Promise; pinch: (options: PinchGestureOptions) => Promise; tap: (options: TapGestureOptions) => Promise; - /** Escape hatch for any other CDP command. */ +} + +/** + * The Chrome DevTools Protocol, on the session that drives the page under test: for what a case + * sets up or reads itself — CPU throttling (`Emulation.setCPUThrottlingRate`), the engine's own + * counters (`Performance.getMetrics`) as custom metrics. + * + * A setting changed this way stays on the tab, through later samples and the other cases sampled in + * it — those of a `compare()` too — so a case that changes one sets it every time, and the others + * set it as they need it. + */ +export interface BenchmarkCdp { send: CdpSend; } @@ -66,6 +77,5 @@ export function createInput(send: CdpSend): BenchmarkInput { tap: async ({ x, y }) => { await send('Input.synthesizeTapGesture', { x, y, gestureSourceType: 'mouse' }); }, - send, }; } diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index dd469153b8..09d7e2aa56 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -1,6 +1,6 @@ import { collectGarbage } from '../gc'; import { createInput } from '../input'; -import type { BenchmarkInput } from '../input'; +import type { BenchmarkCdp, BenchmarkInput, CdpSend } from '../input'; import { seriesName, setMetricRecorder } from '../metricCore'; import { hasSampling, pickSampling, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; @@ -15,6 +15,8 @@ import type { MetricDefinition } from '../types'; export interface BenchmarkContext { /** Trusted input — scroll, pinch, tap — dispatched by the browser rather than from script. */ input: BenchmarkInput; + /** The DevTools Protocol, for what the case sets up or reads from the browser itself. */ + cdp: BenchmarkCdp; } /** @@ -139,12 +141,14 @@ declare global { } } -const input = createInput((method, params) => { +const send: CdpSend = (method, params) => { if (!window.benchmarkCdp) { throw new Error('No CDP bridge: the runner must expose `benchmarkCdp` on this page.'); } return window.benchmarkCdp(method, params); -}); +}; +const input = createInput(send); +const cdp: BenchmarkCdp = { send }; async function sample(name: string): Promise> { const run = cases.get(name)?.run; @@ -158,7 +162,7 @@ async function sample(name: string): Promise> { const values = new Map(); sampleValues = values; try { - await run({ input }); + await run({ input, cdp }); } finally { sampleValues = null; } diff --git a/packages/benchmark/src/page/reactBenchmark.tsx b/packages/benchmark/src/page/reactBenchmark.tsx index d1354b7290..15c0f89dd2 100644 --- a/packages/benchmark/src/page/reactBenchmark.tsx +++ b/packages/benchmark/src/page/reactBenchmark.tsx @@ -40,8 +40,8 @@ export function reactBenchmark( ): BenchmarkCase { const { interaction, options } = splitCaseArgs(interactionOrOptions, maybeOptions); - const run = async ({ input }: BenchmarkContext) => { - const result = await measureIteration(renderFn, interaction, options, input); + const run = async ({ input, cdp }: BenchmarkContext) => { + const result = await measureIteration(renderFn, interaction, options, { input, cdp }); // Render time is reported as totals rather than per render: an interaction's render count // follows whatever the browser coalesced, so per-render values would not line up across diff --git a/packages/benchmark/src/publicApi.ts b/packages/benchmark/src/publicApi.ts index 1ded0fd2de..8cc9ae727c 100644 --- a/packages/benchmark/src/publicApi.ts +++ b/packages/benchmark/src/publicApi.ts @@ -9,7 +9,7 @@ export type { MetricConfig, MetricDefinition, } from './types'; -export type { BenchmarkInput } from './input'; +export type { BenchmarkCdp, BenchmarkInput } from './input'; export type { BenchmarkInteraction } from './caseRuntime'; export { ElementTiming } from './ElementTiming'; export { Metric, type MetricRecordOptions } from './metricCore'; diff --git a/packages/benchmark/src/types.ts b/packages/benchmark/src/types.ts index 7aba51eee7..d39f1e62c2 100644 --- a/packages/benchmark/src/types.ts +++ b/packages/benchmark/src/types.ts @@ -1,4 +1,4 @@ -import type { BenchmarkInput } from './input'; +import type { BenchmarkCdp, BenchmarkInput } from './input'; export interface RenderEvent { id: string; @@ -42,6 +42,8 @@ export interface InteractionContext { * that should behave like a real mouse or trackpad rather than synthetic DOM events. */ input: BenchmarkInput; + /** The DevTools Protocol, for what the interaction sets up or reads from the browser itself. */ + cdp: BenchmarkCdp; } // One benchmark case's lifecycle. The measurement loop creates one per iteration and drives it From 776be8f70cbe2c2f438756f39921e81a7f4464ce Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:00:24 +0200 Subject: [PATCH 130/173] [benchmark] Upload into a named timeline with --timeline A run against another baseline, such as a weekly run against the last release, names its dashboard timeline so it neither overwrites the commit's regular report nor lands in the branch's history. Uploads also carry the commit's time, to order them in their timeline, and reports record the filters and name pattern a run was narrowed to. --- packages/benchmark/README.md | 9 +++++++++ packages/benchmark/src/ciReport.ts | 12 ++++++++++-- packages/benchmark/src/cli/index.ts | 7 +++++++ packages/benchmark/src/runReport/schema.ts | 14 ++++++++++++++ packages/benchmark/src/runner/runBenchmarks.ts | 14 +++++++++++++- packages/benchmark/src/runner/upload.ts | 6 +++++- 6 files changed, 58 insertions(+), 4 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 59afd9c8c9..67ec0acfc6 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -466,6 +466,15 @@ The run writes `.benchmark/results/report.json`, prints it as tables, and with ` the dashboard, which renders the pull request comment's Performance section and the repository's history from it. +Uploads from a tracked branch (`master`, `next`, `v7.x`, …) join that branch's timeline, the history +the dashboard draws. A run against another baseline names its own with `--timeline` (or +`BENCHMARK_TIMELINE`), so it neither overwrites the commit's regular report nor lands in the branch's +history, for example a weekly run against the last release: + +```bash +benchmark --baseline "v$(npm view @mui/material version)" --upload --timeline release +``` + - The report holds **raw samples**, round-aligned across variants, plus each metric's kind, format and alarm, how each benchmark's sampling went, the builds, and the environment. - `analyzeRun` from `@mui/internal-benchmark/runReport` draws every conclusion from it: a confidence diff --git a/packages/benchmark/src/ciReport.ts b/packages/benchmark/src/ciReport.ts index 63bc40f30c..5ca1f917ea 100644 --- a/packages/benchmark/src/ciReport.ts +++ b/packages/benchmark/src/ciReport.ts @@ -15,14 +15,21 @@ interface CiInfo { interface GitInfo { commitSha: string; branch: string; + /** When the commit was made, in milliseconds. */ + commitTimestamp: number; } async function getGitInfo(): Promise { - const [commit, branch] = await Promise.all([ + const [commit, branch, commitTime] = await Promise.all([ execa('git', ['rev-parse', 'HEAD']), execa('git', ['branch', '--show-current']), + execa('git', ['log', '-1', '--format=%ct', 'HEAD']), ]); - return { commitSha: commit.stdout, branch: branch.stdout }; + return { + commitSha: commit.stdout, + branch: branch.stdout, + commitTimestamp: Number(commitTime.stdout) * 1000, + }; } /** @@ -117,5 +124,6 @@ export async function getCiMetadata() { (ciInfo.isPr ? ciInfo.prBranch : ciInfo.branch) || '', commitSha: gitInfo.commitSha, + commitTimestamp: gitInfo.commitTimestamp, }; } diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index 0c780226dc..1747d388c0 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -48,6 +48,12 @@ const runCommand: CommandModule<{}, Args> = { describe: 'Upload the report and refresh the PR comment. Defaults to BENCHMARK_UPLOAD=true, so CI can switch it on by environment. Requires CIRCLE_OIDC_TOKEN_V2', }) + .option('timeline', { + type: 'string', + default: process.env.BENCHMARK_TIMELINE, + describe: + "With --upload, the dashboard timeline to upload into in place of the branch's, for a run against another baseline such as the last release. Defaults to BENCHMARK_TIMELINE", + }) .option('out', { type: 'string', describe: 'Write the JSON report here. Default: .benchmark/results/report.json', @@ -101,6 +107,7 @@ const runCommand: CommandModule<{}, Args> = { buildCmd: argv.buildCmd, out: argv.out, upload: argv.upload, + timeline: argv.timeline, testNamePattern: argv.testNamePattern, reporter: argv.reporter, profile: argv.profile, diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts index 249e3c40b4..5a1a9d9a5e 100644 --- a/packages/benchmark/src/runReport/schema.ts +++ b/packages/benchmark/src/runReport/schema.ts @@ -73,6 +73,13 @@ export const benchmarkRunReportSchema = z.object({ /** How long the whole run took, packing and building both refs included. */ durationMs: z.number().optional(), head: z.object({ sha: z.string(), branch: z.string().optional() }), + /** + * What the run was narrowed to: the file filters and the name pattern it was given. Absent when it + * ran every benchmark. + */ + selection: z + .object({ filters: z.array(z.string()), testNamePattern: z.string().optional() }) + .optional(), environment: z.object({ browser: z.string(), platform: z.string(), @@ -99,6 +106,13 @@ export const benchmarkRunUploadSchema = z.object({ reportType: z.literal('benchmark'), prNumber: z.number().int().positive().optional(), branch: z.string(), + /** When the commit was made, in milliseconds: orders it in its timeline. */ + commitTimestamp: z.number().optional(), + /** + * The timeline the upload goes into, in place of its branch's: a run against another baseline, + * such as the last release. The dashboard stores it as `benchmark@.json`. + */ + timeline: z.string().optional(), report: benchmarkRunReportSchema, }); diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index 88c95f0c22..5905870a7c 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -44,6 +44,11 @@ export interface RunBenchmarksOptions { out?: string; /** Upload the report and refresh the pull request comment. */ upload?: boolean; + /** + * The dashboard timeline to upload into, in place of the branch's: for a run against another + * baseline, such as the last release. Needs `upload`. + */ + timeline?: string; /** Defaults, for the whole run, for the sampling options a benchmark doesn't set itself. */ sampling?: SamplingOptions; /** Only run benchmarks whose name matches this regular expression, like Vitest's `-t`. */ @@ -82,11 +87,15 @@ export async function runBenchmarks( buildCmd = 'pnpm release:build', out, upload = false, + timeline, sampling, testNamePattern, reporter = 'default', profile = false, } = options; + if (timeline !== undefined && !upload) { + throw new Error('A timeline only applies to an upload: pass --upload with --timeline.'); + } // Compiled before anything is built, so a bad pattern fails at once. const namePattern = testNamePattern === undefined ? undefined : new RegExp(testNamePattern); @@ -213,6 +222,9 @@ export async function runBenchmarks( generatedAt: new Date().toISOString(), durationMs: Date.now() - startedAt, head: { sha: commitSha, branch }, + ...(filters.length > 0 || testNamePattern !== undefined + ? { selection: { filters, testNamePattern } } + : {}), environment: { browser: `Chromium ${results.browserVersion}`, platform: process.platform, @@ -239,7 +251,7 @@ export async function runBenchmarks( console.log(chalk.green(`\nWrote JSON report to ${outPath}`)); if (upload) { - await publishRunReport(report); + await publishRunReport(report, timeline); } // Written and uploaded first, so the errors are readable and the comment says what happened — diff --git a/packages/benchmark/src/runner/upload.ts b/packages/benchmark/src/runner/upload.ts index ea40965be2..6eb2ad8cc1 100644 --- a/packages/benchmark/src/runner/upload.ts +++ b/packages/benchmark/src/runner/upload.ts @@ -14,7 +14,10 @@ import type { BenchmarkRunReport } from '../runReport'; * refresh the comment does not fail the run: the numbers are already measured, written and * uploaded, and losing them to a comment API hiccup would waste the whole run. */ -export async function publishRunReport(report: BenchmarkRunReport): Promise { +export async function publishRunReport( + report: BenchmarkRunReport, + timeline: string | undefined, +): Promise { const metadata = await getCiMetadata(); if (!metadata.repo) { console.warn( @@ -27,6 +30,7 @@ export async function publishRunReport(report: BenchmarkRunReport): Promise Date: Wed, 7 Oct 2026 16:21:25 +0200 Subject: [PATCH 131/173] [code-infra-dashboard] Give each changed benchmark its own table in the PR comment --- .../benchmarkRun/buildMarkdownReport.test.ts | 8 +++++--- .../lib/benchmarkRun/buildMarkdownReport.ts | 18 ++++++------------ 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index 49fa0f96b7..fd4b17692c 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -97,9 +97,11 @@ describe('buildBenchmarkRunMarkdownReport', () => {
    Changes - | Benchmark | Metric | baseline | current | Δ vs baseline | - |:----------|:----------|----------:|----------:|:----------| - | **Grid scroll** (3 + 1 rounds · timed out) | render | 21.5 ms | 23.5 ms | 🔴 \`worse +9.3% – +9.3%\` | + **Grid scroll** (3 + 1 rounds · timed out) + + | Metric | baseline | current | Δ vs baseline | + |:----------|----------:|----------:|:----------| + | render | 21.5 ms | 23.5 ms | 🔴 \`worse +9.3% – +9.3%\` | **libs / mount** (4 rounds) diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 222069ee61..0ed4649bf4 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -74,23 +74,17 @@ function markdownTable(columns: BenchmarkTableColumn[], rows: string[][]): strin } /** - * The benchmarks measured against the baseline, as one table of the metrics that changed, so the - * columns line up across benchmarks; those where nothing changed are named below it. + * The benchmarks measured against the baseline, each with a table of the metrics that changed; + * those where nothing changed are named together below them. */ function renderBaselineChanges(analyses: BenchmarkAnalysis[]): string[] { const tables = analyses.map((analysis) => ({ analysis, ...changedRowsOf(analysis) })); - const changed = tables.filter((table) => table.rows.length > 0); const unchanged = tables.filter((table) => table.rows.length === 0); const lines: string[] = []; - if (changed.length > 0) { - const columns: BenchmarkTableColumn[] = [ - { header: 'Benchmark', kind: 'label' }, - ...changed[0].columns, - ]; - const rows = changed.flatMap(({ analysis, rows: cells }) => - cells.map((row, index) => [index === 0 ? labelOf(analysis.benchmark) : '', ...row]), - ); - lines.push(markdownTable(columns, rows), ''); + for (const { analysis, columns, rows } of tables) { + if (rows.length > 0) { + lines.push(labelOf(analysis.benchmark), '', markdownTable(columns, rows), ''); + } } if (unchanged.length > 0) { const names = unchanged.map(({ analysis }) => labelOf(analysis.benchmark)); From 65851df59b152f65fd25b4b2b14fb63c2b66475a Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 7 Oct 2026 16:24:27 +0200 Subject: [PATCH 132/173] [code-infra-dashboard] Let the run's tables span the page, the metric column taking the slack --- .../src/views/BenchmarkRunDetails.tsx | 79 ++++++++++--------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index d13ff771b9..97c6fbd31d 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -48,9 +48,10 @@ function comparisonSx(comparison: MetricComparison): SxProps { }); } -// Fixed widths per kind of column, so the columns of every benchmark's table line up. +// Fixed widths per kind of column, so the columns of every benchmark's table line up. The label +// column takes what the page has left, down to its minimum, past which the table scrolls sideways. const COLUMN_WIDTHS: Record = { - label: 220, + label: 160, value: 120, comparison: 280, }; @@ -60,10 +61,10 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { const { columns, rows } = benchmarkTable(analysis); const firstComparison = columns.findIndex((column) => column.kind === 'comparison'); const alignOf = (column: number) => (columns[column].kind === 'value' ? 'right' : undefined); - const tableWidth = columns.reduce((sum, column) => sum + COLUMN_WIDTHS[column.kind], 0); + const minWidth = columns.reduce((sum, column) => sum + COLUMN_WIDTHS[column.kind], 0); return ( - + {benchmark.name}{' '} @@ -85,42 +86,48 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { -
    - - {columns.map((column, index) => ( - - ))} - - - - {columns.map(({ header }, column) => ( - - {header} - + {/* Only the table scrolls sideways on a narrow screen; the heading above it stays put. */} + +
    + + {columns.map((column, index) => ( + ))} - - - - {rows.map((row) => ( - - {row.cells.map((cell, column) => ( - - {cell} + + + + {columns.map(({ header }, column) => ( + + {header} ))} - ))} - -
    -     + + + {rows.map((row) => ( + + {row.cells.map((cell, column) => ( + + {cell} + + ))} + + ))} + + + + ); } From 1bc54457e3e730cb0dfeb866a8f107198a0658b7 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:17:40 +0200 Subject: [PATCH 133/173] [code-infra-dashboard] Keep the PR comment to one line unless something regressed The comment is there to draw attention when something needs it. Without regressions or failures it says so in one line, naming improvements on alarmed metrics; regressions get a table of only the rows that regressed. Informational metrics and comparisons between libraries stay on the dashboard. --- .../benchmarkRun/buildMarkdownReport.test.ts | 64 ++++---- .../lib/benchmarkRun/buildMarkdownReport.ts | 150 +++++++----------- 2 files changed, 83 insertions(+), 131 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index fd4b17692c..e540f87a26 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -58,29 +58,33 @@ const broken: RunBenchmark = { error: 'Render failed', }; -describe('buildBenchmarkRunMarkdownReport', () => { - it('shows a benchmark where nothing changed as one line', () => { - const markdown = buildBenchmarkRunMarkdownReport(reportOf([unchanged]), { - title: 'Performance', - }); +const faster: RunBenchmark = { + name: 'Tooltip mount', + file: 'tooltip.bench.tsx', + kind: 'baseline', + variants: ['current', 'baseline'], + samples: { + current: { render: [5, 6, 5, 6], 'bench:paint': [20, 22, 21, 23] }, + baseline: { render: [10, 11, 10, 11], 'bench:paint': [30, 31, 30, 31] }, + }, +}; - expect(markdown).toContain('No change detected: **Button mount**'); - expect(markdown).not.toContain('| render |'); +describe('buildBenchmarkRunMarkdownReport', () => { + it('is a single line when nothing needs attention', () => { + expect( + buildBenchmarkRunMarkdownReport(reportOf([unchanged, libraries]), { title: 'Performance' }), + ).toBe('## Performance\n\nNo regressions in 2 benchmarks'); }); - it('states nothing but the summary when no benchmark regressed', () => { - const markdown = buildBenchmarkRunMarkdownReport(reportOf([unchanged]), { - title: 'Performance', - }); + it('names improvements on alarmed metrics in that line, and leaves informational ones out', () => { + const markdown = buildBenchmarkRunMarkdownReport(reportOf([faster]), { title: 'Performance' }); - expect(markdown.split('\n').slice(0, 3)).toEqual([ - '## Performance', - '', - '1 benchmark measured · no regressions', - ]); + expect(markdown.split('\n')).toHaveLength(3); + expect(markdown).toContain('**Tooltip mount** render `better'); + expect(markdown).not.toContain('bench:paint'); }); - it('lists regressions above the collapsed results, and marks the heading', () => { + it('tables only the rows that regressed, and marks the heading', () => { expect( buildBenchmarkRunMarkdownReport(reportOf([slower, libraries, broken]), { title: 'Performance', @@ -88,32 +92,18 @@ describe('buildBenchmarkRunMarkdownReport', () => { ).toMatchInlineSnapshot(` "## Performance ⚠️ - 🔴 **Grid scroll** · render · \`worse +9.3% – +9.3%\` - - 2 benchmarks measured · 1 with regressions · 1 timed out · 1 failed + 1 regression in 2 benchmarks ❌ **Chart zoom**: Render failed -
    - Changes - - **Grid scroll** (3 + 1 rounds · timed out) + **Grid scroll** | Metric | baseline | current | Δ vs baseline | |:----------|----------:|----------:|:----------| | render | 21.5 ms | 23.5 ms | 🔴 \`worse +9.3% – +9.3%\` | - **libs / mount** (4 rounds) - - | Metric | ours | theirs | theirs vs ours | - |:----------|----------:|----------:|:----------| - | render | 10 ms | 20.5 ms | 🟠 \`worse +95.8% – +114.2%\` | - - - _Medians; Δ is the confidence interval of the paired per-round difference (95%)._ - -
    " + _Medians; Δ is the confidence interval of the paired per-round difference (95%)._" `); }); @@ -122,8 +112,8 @@ describe('buildBenchmarkRunMarkdownReport', () => { title: 'Performance', }); - expect(markdown).toContain('no regressions'); - expect(markdown).not.toContain('🔴'); + expect(markdown).toContain('No regressions'); + expect(markdown).not.toContain('libs / mount'); }); it('links the full run when given where it is', () => { @@ -132,6 +122,6 @@ describe('buildBenchmarkRunMarkdownReport', () => { detailsUrl: 'https://example.com/run', }); - expect(markdown).toContain('[See the full run](https://example.com/run)'); + expect(markdown).toContain('[details](https://example.com/run)'); }); }); diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 0ed4649bf4..2274abc3ad 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -4,17 +4,14 @@ import { benchmarkTable, changeTone, findRegressions, - formatRounds, formatComparison, - formatRunSummary, + formatDuration, runReportFootnote, } from '@mui/internal-benchmark/runReport'; import type { BenchmarkAnalysis, BenchmarkRunReport, - BenchmarkTableColumn, ChangeTone, - RunBenchmark, } from '@mui/internal-benchmark/runReport'; interface BuildOptions { @@ -31,82 +28,59 @@ const TONE_MARKERS: Record = { none: '', }; -/** A benchmark's name, with how many rounds it ran and how long it took. */ -function labelOf(benchmark: RunBenchmark): string { - const rounds = formatRounds(benchmark); - return rounds ? `**${benchmark.name}** (${rounds})` : `**${benchmark.name}**`; +/** The changes for the better on metrics that alarm, each as a short phrase. */ +function improvementsOf(analyses: BenchmarkAnalysis[]): string[] { + return analyses.flatMap(({ benchmark, metrics }) => + benchmark.kind !== 'baseline' + ? [] + : metrics.flatMap(({ metric, definition, comparisons }) => + definition.alarm === undefined + ? [] + : comparisons + .filter((comparison) => comparison.change === 'better') + .map( + (comparison) => + `**${benchmark.name}** ${metric} \`${formatComparison(comparison)}\``, + ), + ), + ); } -/** - * A benchmark's table, keeping the rows that changed — better or worse — as markdown cells: each - * comparison as code, behind a marker of which way it went. - */ -function changedRowsOf(analysis: BenchmarkAnalysis): { - columns: BenchmarkTableColumn[]; - rows: string[][]; -} { +/** A benchmark's table, keeping only the rows that regressed, each behind its severity's marker. */ +function regressionTable(analysis: BenchmarkAnalysis): string | null { const { columns, rows } = benchmarkTable(analysis); const firstComparison = columns.findIndex((column) => column.kind === 'comparison'); - const changed = rows.filter((row) => - row.comparisons.some(({ change }) => change === 'better' || change === 'worse'), + const regressed = rows.filter((row) => + row.comparisons.some((comparison) => comparison.severity !== 'none'), ); - return { - columns, - rows: changed.map((row) => - row.cells.map((cell, column) => - columns[column].kind === 'comparison' - ? `${TONE_MARKERS[changeTone(row.comparisons[column - firstComparison])]}\`${cell}\`` - : cell, - ), - ), - }; -} - -function markdownTable(columns: BenchmarkTableColumn[], rows: string[][]): string { + if (regressed.length === 0) { + return null; + } return formatMarkdownTable( columns.map(({ header, kind }, column) => ({ field: String(column), header, align: kind === 'value' ? ('right' as const) : ('left' as const), })), - rows.map((cells) => Object.fromEntries(cells.map((cell, column) => [String(column), cell]))), + regressed.map((row) => + Object.fromEntries( + row.cells.map((cell, column) => [ + String(column), + columns[column].kind === 'comparison' + ? `${TONE_MARKERS[changeTone(row.comparisons[column - firstComparison])]}\`${cell}\`` + : cell, + ]), + ), + ), ); } -/** - * The benchmarks measured against the baseline, each with a table of the metrics that changed; - * those where nothing changed are named together below them. - */ -function renderBaselineChanges(analyses: BenchmarkAnalysis[]): string[] { - const tables = analyses.map((analysis) => ({ analysis, ...changedRowsOf(analysis) })); - const unchanged = tables.filter((table) => table.rows.length === 0); - const lines: string[] = []; - for (const { analysis, columns, rows } of tables) { - if (rows.length > 0) { - lines.push(labelOf(analysis.benchmark), '', markdownTable(columns, rows), ''); - } - } - if (unchanged.length > 0) { - const names = unchanged.map(({ analysis }) => labelOf(analysis.benchmark)); - lines.push(`No change detected: ${names.join(', ')}`, ''); - } - return lines; -} - -/** A `compare()` benchmark: a table of the metrics that changed, or one line when none did. */ -function renderComparison(analysis: BenchmarkAnalysis): string { - const { columns, rows } = changedRowsOf(analysis); - return rows.length === 0 - ? `${labelOf(analysis.benchmark)} · no change detected` - : `${labelOf(analysis.benchmark)}\n\n${markdownTable(columns, rows)}`; -} - /** * Renders a version 2 benchmark report as its pull request comment section. * - * Regressions are stated above the collapsed block, and mark the heading, because a reader has to - * see them without expanding anything — GitHub renders `
    ` closed, so anything inside it is - * invisible until someone chooses to look. + * The comment is there to draw attention when something needs it, so a run without regressions or + * failures is a single line. Only regressions get a table, and only of the rows that regressed; + * informational metrics and comparisons between libraries stay on the dashboard. */ export function buildBenchmarkRunMarkdownReport( report: BenchmarkRunReport, @@ -114,44 +88,32 @@ export function buildBenchmarkRunMarkdownReport( ): string { const analyses = analyzeRun(report); const regressions = findRegressions(analyses); - const measured = analyses.filter((analysis) => !analysis.benchmark.error); const failed = analyses.filter((analysis) => analysis.benchmark.error); + const measured = analyses.length - failed.length; + const benchmarks = `${measured} benchmark${measured === 1 ? '' : 's'}`; - const lines = [`## ${options.title}${regressions.length > 0 ? ' ⚠️' : ''}`, '']; - - for (const { benchmark, metric, comparison } of regressions) { - const marker = comparison.severity === 'error' ? '🔴' : '🟡'; - lines.push(`${marker} **${benchmark}** · ${metric} · \`${formatComparison(comparison)}\``); - } - if (regressions.length > 0) { - lines.push(''); - } + const summary = [ + regressions.length > 0 + ? `${regressions.length} regression${regressions.length === 1 ? '' : 's'} in ${benchmarks}` + : `No regressions in ${benchmarks}`, + ...improvementsOf(analyses), + report.durationMs === undefined ? null : `ran ${formatDuration(report.durationMs)}`, + options.detailsUrl ? `[details](${options.detailsUrl})` : null, + ].filter(Boolean); - lines.push(formatRunSummary(analyses, regressions, report.durationMs), ''); + const attention = regressions.length > 0 || failed.length > 0; + const lines = [`## ${options.title}${attention ? ' ⚠️' : ''}`, '', summary.join(' · ')]; for (const { benchmark } of failed) { - lines.push(`❌ **${benchmark.name}**: ${benchmark.error}`); - } - if (failed.length > 0) { - lines.push(''); - } - - if (measured.length > 0) { - lines.push('
    ', 'Changes', ''); - lines.push( - ...renderBaselineChanges( - measured.filter((analysis) => analysis.benchmark.kind === 'baseline'), - ), - ); - const comparisons = measured.filter((analysis) => analysis.benchmark.kind === 'compare'); - if (comparisons.length > 0) { - lines.push(comparisons.map(renderComparison).join('\n\n'), ''); - } - lines.push(`_${runReportFootnote(analyses)}_`, '', '
    '); + lines.push('', `❌ **${benchmark.name}**: ${benchmark.error}`); } - if (options.detailsUrl) { - lines.push('', `[See the full run](${options.detailsUrl})`); + const tables = analyses.flatMap((analysis) => { + const table = regressionTable(analysis); + return table ? [`**${analysis.benchmark.name}**\n\n${table}`] : []; + }); + if (tables.length > 0) { + lines.push('', tables.join('\n\n'), '', `_${runReportFootnote(analyses)}_`); } return lines.join('\n'); From 430cf45fa36c476b69bacf932228471ff350696c Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:33:03 +0200 Subject: [PATCH 134/173] [benchmark] Alarm on paint as on render, with 2% and 5% bands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Paint covers what follows React's render — the commit, injected styles, style recalculation and layout — so a styling or DOM regression can show in paint alone. Both metrics now warn once a change for the worse is confidently 2% and fail at 5%, the size sampling is built to settle, instead of failing on any confirmed change however small. The docs now say how each report version judges a regression: version 1 by a noise band on the means, version 2 by bands on the confidence interval's near end. --- packages/benchmark/README.md | 16 +++++++++++----- packages/benchmark/src/DiscreteMetric.ts | 4 ++-- packages/benchmark/src/ScalarMetric.ts | 3 +-- .../benchmark/src/page/reactBenchmark.tsx | 19 +++++++++++++------ packages/benchmark/src/types.ts | 10 +++++----- 5 files changed, 32 insertions(+), 20 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 67ec0acfc6..8884dc8959 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -186,8 +186,8 @@ This produces a `bench:paint#my-component` sub-series alongside the automatic `b Record your own measurements — a timing, a count, anything measured inside or outside React — from a plain `it()` loop or from inside a `benchmark()`. There are two primitives: -- `ScalarMetric` — a continuous value (timings, sizes). Aggregated as mean ± standard deviation with IQR outlier removal, and compared against a baseline with a relative noise band. -- `DiscreteMetric` — a count of events. Compared as an exact integer (any change is significant) and formatted as a whole number. +- `ScalarMetric` — a continuous value (timings, sizes). +- `DiscreteMetric` — a count of events, formatted as a whole number. Both record values with `record(value)`. `ScalarMetric` additionally offers `time()`/`timeEnd()` — a `console.time`-style shortcut that records the elapsed milliseconds for you. @@ -225,9 +225,14 @@ You can also `record()` or `time()` from inside a `benchmark()` render function - `alarm` — opts the metric into regression flagging. Omit it and the metric is informational (its diff is shown but never flagged). Holds: - `direction` — `'lowerIsBetter'` (default) or `'higherIsBetter'`. - `warn` — softer band; a regression past it is flagged as a warning. - - `error` — harder band; a regression past it is flagged as an error. Defaults to the dashboard's global noise band only when both `warn` and `error` are omitted; with only `warn` set there is no error band (warning-only). + - `error` — harder band; a regression past it is flagged as an error. With only `warn` set there is no error band (warning-only). - Bands are relative fractions for scalar metrics (`0.1` = 10%) and absolute count deltas for discrete metrics (`1`, `2`). Either band is optional. +How a regression is judged depends on which report the repository uploads: + +- **Version 1** (this Vitest reporter) compares the two runs' means. A scalar metric's change must pass a band; with neither band set, `error` is the dashboard's ±20% noise band. A discrete metric flags any change in its mean. +- **Version 2** ([the `benchmark` CLI](#the-benchmark-cli)) first needs the metric's confidence interval to lie wholly on the worse side; the bands then apply to the end of that interval nearest zero, so a change counts only once it is confidently that large. With neither band set, every confirmed change for the worse is an error. + Alarms are evaluated against the baseline when the PR comment is generated, not during the local `vitest run` — a regression never fails the test suite locally. In the PR comment, `error`-band regressions surface as failures and `warn`-band regressions as warnings. #### Sub-series @@ -480,8 +485,9 @@ benchmark --baseline "v$(npm view @mui/material version)" --upload --timeline re - `analyzeRun` from `@mui/internal-benchmark/runReport` draws every conclusion from it: a confidence interval on the paired difference per metric, a change (`better`, `worse`, `no change detected`, or `unchanged` when every round measured the same), and a severity from the metric's alarm. -- `reactBenchmark()`'s `render` alarms on any resolved change for the worse; `render:count`, the - per-phase split and `bench:paint` are informational; every other metric brings its own alarm. +- `reactBenchmark()`'s `render` and `bench:paint` warn once confidently 2% worse and fail at 5%; + `render:count` and the per-phase split are informational; every other metric brings its own + alarm. - Tables list the baseline before the current build, so a row reads old to new. The pull request comment keeps only the metrics that got better or worse, and sums up a benchmark where none did as "no change detected"; the terminal and the dashboard show every metric. diff --git a/packages/benchmark/src/DiscreteMetric.ts b/packages/benchmark/src/DiscreteMetric.ts index b1c27862f8..ed4fcac485 100644 --- a/packages/benchmark/src/DiscreteMetric.ts +++ b/packages/benchmark/src/DiscreteMetric.ts @@ -2,8 +2,8 @@ import { Metric } from './metricCore'; import type { MetricConfig, MetricKind } from './types'; /** - * A discrete count of events or occurrences. Compared against a baseline as an exact integer - * (any change is significant — there is no noise band), and formatted as a whole number by default. + * A discrete count of events or occurrences, whose alarm bands are absolute count deltas. + * Formatted as a whole number by default. * * ```ts * const clicks = new DiscreteMetric({ name: 'button_clicks' }); diff --git a/packages/benchmark/src/ScalarMetric.ts b/packages/benchmark/src/ScalarMetric.ts index 58bbafff3c..fe50c0ddb8 100644 --- a/packages/benchmark/src/ScalarMetric.ts +++ b/packages/benchmark/src/ScalarMetric.ts @@ -2,8 +2,7 @@ import { Metric } from './metricCore'; import type { MetricKind } from './types'; /** - * A continuous measurement (timings, sizes, …). Samples are aggregated with mean ± standard - * deviation and IQR outlier removal, and compared against a baseline with a relative noise band. + * A continuous measurement (timings, sizes, …), whose alarm bands are relative fractions. * * It offers a `console.time`-style timing helper: * diff --git a/packages/benchmark/src/page/reactBenchmark.tsx b/packages/benchmark/src/page/reactBenchmark.tsx index 15c0f89dd2..e36405a21c 100644 --- a/packages/benchmark/src/page/reactBenchmark.tsx +++ b/packages/benchmark/src/page/reactBenchmark.tsx @@ -9,13 +9,20 @@ import type { BenchmarkCase, BenchmarkContext } from './page'; export interface ReactBenchmarkOptions extends CaseOptions, SamplingOptions {} -// Render time alarms on any resolved change for the worse. The render count is informational: an -// extra render matters only through the time it adds, which render time already measures. Paint -// dominates each case's duration and duplicates that signal, so it is informational too, as is the -// per-phase split. -const renderMetric = new ScalarMetric({ name: 'render', format: MILLISECONDS, alarm: {} }); +// Render time and paint both alarm: render is React running the components, while paint adds what +// follows — the commit, injected styles, style recalculation and layout — so a styling or DOM +// regression can show in paint alone. Both warn once a change for the worse is confidently 2% and +// fail at 5%, the size sampling is built to settle; smaller confirmed changes are left to the +// timelines. The render count is informational: an extra render matters only through the time it +// adds, which render time already measures. So is the per-phase split. +const TIME_ALARM = { warn: 0.02, error: 0.05 }; +const renderMetric = new ScalarMetric({ name: 'render', format: MILLISECONDS, alarm: TIME_ALARM }); const renderCountMetric = new DiscreteMetric({ name: 'render:count' }); -const paintMetric = new ScalarMetric({ name: PAINT_METRIC_NAME, format: MILLISECONDS }); +const paintMetric = new ScalarMetric({ + name: PAINT_METRIC_NAME, + format: MILLISECONDS, + alarm: TIME_ALARM, +}); const phaseMetrics = new Map(); function phaseMetricOf(phase: string): ScalarMetric { diff --git a/packages/benchmark/src/types.ts b/packages/benchmark/src/types.ts index d39f1e62c2..8575148a3c 100644 --- a/packages/benchmark/src/types.ts +++ b/packages/benchmark/src/types.ts @@ -61,8 +61,8 @@ export interface BenchmarkCaseRuntime { /** * Whether a custom metric measures a continuous value or a discrete count. - * - `scalar` — continuous measurements (timings, sizes); compared with a relative noise band. - * - `discrete` — counts/events; compared as exact integers. + * - `scalar` — continuous measurements (timings, sizes); alarm bands are relative fractions. + * - `discrete` — counts/events; alarm bands are absolute count deltas. */ export type MetricKind = 'scalar' | 'discrete'; @@ -78,9 +78,9 @@ export interface MetricAlarm { */ warn?: number; /** - * Harder band: a regression past `error` is flagged as an error (the alarm). When **both** - * `warn` and `error` are omitted, `error` defaults to the dashboard's global noise band; with - * only `warn` set there is no error band (warning-only). + * Harder band: a regression past `error` is flagged as an error (the alarm). With only `warn` + * set there is no error band (warning-only). With neither, version 1 reports use the dashboard's + * ±20% noise band and version 2 reports flag every confirmed change for the worse. * Scalar metrics: a relative fraction (`0.25` = 25%). Discrete metrics: an absolute count delta. */ error?: number; From 6cb5034b84fd766d4fce72c93f2122c6c2c38e68 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:02:51 +0200 Subject: [PATCH 135/173] [benchmark] Alarm at a fixed 99% instead of Bonferroni Tightening every alarmed interval with the number of comparisons in a run makes each benchmark sample longer as benchmarks are added: at a few hundred, about 4 times longer than at 95%. Alarmed metrics now use a fixed 99%. The alarm bands, which the interval's near end has to reach, keep a run of unchanged code from alarming; the nightly A/A job tracks how often it still does. --- .../benchmarkRun/buildMarkdownReport.test.ts | 2 +- packages/benchmark/README.md | 11 ++-- .../src/runReport/analyzeRun.test.ts | 42 +++------------ .../benchmark/src/runReport/analyzeRun.ts | 51 ++++--------------- .../benchmark/src/runReport/format.test.ts | 17 ++++--- .../benchmark/src/runner/runInterleaved.ts | 31 +++-------- 6 files changed, 41 insertions(+), 113 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index e540f87a26..080fe1d4bd 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -103,7 +103,7 @@ describe('buildBenchmarkRunMarkdownReport', () => { | render | 21.5 ms | 23.5 ms | 🔴 \`worse +9.3% – +9.3%\` | - _Medians; Δ is the confidence interval of the paired per-round difference (95%)._" + _Medians; Δ is the confidence interval of the paired per-round difference (99% for alarmed metrics, 95% otherwise)._" `); }); diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 8884dc8959..70a7d3a773 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -423,12 +423,11 @@ during a round — thermal throttling, a background process — then affects bot out. Intervals are 95%, except where a change can raise an alarm — an alarmed metric of a benchmark -measured against the baseline. Every such comparison in the run shares one 5% chance of a false alarm -(Bonferroni): with 5 of them, each is a 99% interval, so a run of unchanged code raises a false alarm -at most 5% of the time rather than in about one run in five. A comparison whose per-round differences -never vary — a render count that is the same in every round — can't be flagged by chance, so it takes -no share. Sampling resolves at the same level, counting one alarmed comparison for every other -benchmark in the run; the report counts them exactly. +measured against the baseline — where they are 99%. The level is fixed rather than tightened for +every comparison a run holds, so adding benchmarks doesn't make each one sample longer. What keeps a +run of unchanged code from alarming is mostly the alarm's bands: the interval's near end has to +reach them, so noise has to fake a change bigger than the band, not just one beyond zero. The nightly +A/A job tracks how often a run of unchanged code alarms anyway. Run it with at least 3 logical CPUs (vCPUs in CI); it warns with fewer. On 2, V8's background compilers are starved, so how fast the same code runs depends on how its compilation happened to go, diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index d6ab4e4b94..3a12664b86 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -225,7 +225,7 @@ describe('analyzeRun', () => { }); }); - describe('the run-wide chance of a false alarm', () => { + describe('the confidence level', () => { // Per-round differences of 1, -2, 3, 0, 2, -1, 4, 1: noisy, so the interval has a width. const noisy = { current: [101, 98, 103, 100, 102, 99, 104, 101], @@ -256,44 +256,18 @@ describe('analyzeRun', () => { baseline: { render: noisy.baseline }, }); - it('is shared by the alarmed comparisons of the run', () => { - const [single] = analyzeRun(reportOf([alone], { render: SCALAR })); - const [shared] = analyzeRun(reportOf([alone, other], { render: SCALAR })); - - expect(widthOf(shared) / widthOf(single)).toBeCloseTo(tCritical(7, 0.975) / tCritical(7), 6); - }); + it('is 99% for a metric that can raise an alarm, and 95% otherwise', () => { + const [alarmed] = analyzeRun(reportOf([alone], { render: SCALAR })); + const [plain] = analyzeRun(reportOf([alone], { render: { kind: 'scalar' } })); - it('leaves metrics without an alarm at 95%', () => { - const plain = { render: { kind: 'scalar' as const } }; - const [single] = analyzeRun(reportOf([alone], plain)); - const [shared] = analyzeRun(reportOf([alone, other], plain)); - - expect(widthOf(shared)).toBe(widthOf(single)); + expect(widthOf(alarmed) / widthOf(plain)).toBeCloseTo(tCritical(7, 0.99) / tCritical(7), 6); }); - it('leaves out comparisons that never varied', () => { - const steady = benchmarkOf('c', { - current: { render: [3, 3, 3, 3, 3, 3, 3, 3] }, - baseline: { render: [3, 3, 3, 3, 3, 3, 3, 3] }, - }); + it("doesn't tighten with the number of benchmarks in the run", () => { const [single] = analyzeRun(reportOf([alone], { render: SCALAR })); - const [withSteady] = analyzeRun(reportOf([alone, steady], { render: SCALAR })); - - expect(widthOf(withSteady)).toBe(widthOf(single)); - }); + const [crowded] = analyzeRun(reportOf([alone, other], { render: SCALAR })); - it('can miss a change that 95% would have flagged', () => { - const { base, shifted } = drifting(2); - const flagged = benchmarkOf('d', { - current: { render: shifted.map((value, round) => value + (round % 2 ? 2 : -2)) }, - baseline: { render: base }, - }); - const [single] = analyzeRun(reportOf([flagged], { render: SCALAR })); - const crowd = Array.from({ length: 20 }, (_, index) => ({ ...other, name: `b${index}` })); - const [crowded] = analyzeRun(reportOf([flagged, ...crowd], { render: SCALAR })); - - expect(single.metrics[0].comparisons[0].change).toBe('worse'); - expect(crowded.metrics[0].comparisons[0].change).toBe('undetected'); + expect(widthOf(crowded)).toBe(widthOf(single)); }); }); diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index a9cecc88f5..d7222ed3f6 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -10,9 +10,9 @@ import type { BenchmarkRunReport, RunBenchmark, RunMetricDefinition } from './sc * per-round difference cancels whatever the machine was doing during that round, and its * confidence interval is far tighter than one computed from two independent sets of samples. * - * Intervals are 95%, except where a change can raise an alarm: there, every such comparison in the - * run shares the 5% chance of a false one (Bonferroni), so a run of unchanged code raises one at - * most 5% of the time however many it checks, rather than 5% per comparison. + * Intervals are 95%, except where a change can raise an alarm: there they are 99%, a fixed level so + * a run's cost doesn't grow with how many benchmarks it holds. What keeps a run of unchanged code + * from alarming is mostly the alarm's bands, which the interval's near end has to reach. */ export interface Interval { @@ -66,6 +66,9 @@ export interface BenchmarkAnalysis { /** The confidence level of an interval no alarm depends on. */ export const CONFIDENCE = 0.95; +/** The confidence level of a comparison that can raise an alarm. */ +export const ALARMED_CONFIDENCE = 0.99; + const LANCZOS = [ 0.99999999999980993, 676.5203681218851, -1259.1392167224028, 771.32342877765313, -176.61502916214059, 12.507343278686905, -0.13857109526572012, 9.9843695780195716e-6, @@ -166,14 +169,6 @@ export function meanInterval(values: number[], confidence = CONFIDENCE): Interva return { low: mean - halfWidth, high: mean + halfWidth }; } -/** - * The confidence level of each comparison that can raise an alarm, when `familySize` of them share - * the run's 5% chance of a false alarm. - */ -export function alarmedConfidence(familySize: number): number { - return 1 - (1 - CONFIDENCE) / Math.max(familySize, 1); -} - export function median(values: number[]): number { return quantile( [...values].sort((left, right) => left - right), @@ -324,39 +319,17 @@ function pairedMetricsOf( }); } -/** - * How many comparisons share the run's 5% chance of a false alarm: those that can raise an alarm — - * alarmed metrics of `baseline` benchmarks — leaving out any whose per-round differences never - * varied. Those can't be flagged by chance — a render count that is the same in every round — so - * they would only make the others stricter. - */ -export function countAlarmedComparisons( - metrics: Record, - benchmarks: RunBenchmark[], -): number { - return benchmarks - .flatMap((benchmark) => pairedMetricsOf(metrics, benchmark)) - .filter((paired) => paired.canAlarm) - .flatMap((paired) => paired.pairs) - .filter(({ subject, against }) => { - const differences = differencesOf(subject.values, against.values); - return differences.some((difference) => difference !== differences[0]); - }).length; -} - /** * Compares one benchmark's variants, its metrics described by `metrics` (a report's, or a run's so - * far). The comparisons alone, which is all deciding whether to keep sampling needs. A comparison - * that can raise an alarm is one of `familySize` sharing the run's 5% chance of a false one. + * far). The comparisons alone, which is all deciding whether to keep sampling needs. */ export function compareBenchmark( metrics: Record, benchmark: RunBenchmark, - familySize = 1, ): { metrics: MetricComparisons[] } { return { metrics: pairedMetricsOf(metrics, benchmark).map(({ metric, definition, canAlarm, pairs }) => { - const confidence = canAlarm ? alarmedConfidence(familySize) : CONFIDENCE; + const confidence = canAlarm ? ALARMED_CONFIDENCE : CONFIDENCE; return { metric, definition, @@ -372,12 +345,11 @@ export function compareBenchmark( export function analyzeBenchmark( metrics: Record, benchmark: RunBenchmark, - familySize = 1, ): BenchmarkAnalysis { const { samples } = benchmark; return { benchmark, - metrics: compareBenchmark(metrics, benchmark, familySize).metrics.map((comparison) => ({ + metrics: compareBenchmark(metrics, benchmark).metrics.map((comparison) => ({ ...comparison, variants: Object.fromEntries( benchmark.variants.map((variant) => [ @@ -390,10 +362,7 @@ export function analyzeBenchmark( } export function analyzeRun(report: BenchmarkRunReport): BenchmarkAnalysis[] { - const familySize = countAlarmedComparisons(report.metrics, report.benchmarks); - return report.benchmarks.map((benchmark) => - analyzeBenchmark(report.metrics, benchmark, familySize), - ); + return report.benchmarks.map((benchmark) => analyzeBenchmark(report.metrics, benchmark)); } export interface Regression { diff --git a/packages/benchmark/src/runReport/format.test.ts b/packages/benchmark/src/runReport/format.test.ts index 8310c6441d..29ac00350c 100644 --- a/packages/benchmark/src/runReport/format.test.ts +++ b/packages/benchmark/src/runReport/format.test.ts @@ -104,18 +104,19 @@ describe('formatRunSummary', () => { }); describe('runReportFootnote', () => { - it('states one level when at most one comparison can raise an alarm', () => { - expect(runReportFootnote(analyzeRun(reportOf([benchmarkOf('a')])))).toBe( + it('states one level when nothing can raise an alarm', () => { + const report = { + ...reportOf([benchmarkOf('a')]), + metrics: { render: { kind: 'scalar' as const } }, + }; + expect(runReportFootnote(analyzeRun(report))).toBe( 'Medians; Δ is the confidence interval of the paired per-round difference (95%).', ); }); - it('states the level the alarmed comparisons share', () => { - const footnote = runReportFootnote( - analyzeRun( - reportOf([benchmarkOf('a'), benchmarkOf('b'), benchmarkOf('c'), benchmarkOf('d')]), - ), + it('states the alarmed level beside the other', () => { + expect(runReportFootnote(analyzeRun(reportOf([benchmarkOf('a')])))).toContain( + '(99% for alarmed metrics, 95% otherwise)', ); - expect(footnote).toContain('(98.75% for alarmed metrics, 95% otherwise)'); }); }); diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index 60d7e4085e..4495299fa9 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -3,7 +3,7 @@ import chalk from 'chalk'; import type { Browser, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; -import { compareBenchmark, countAlarmedComparisons } from '../runReport'; +import { compareBenchmark } from '../runReport'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; import { differencesResolved, parseHorizons, pickSampling, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; @@ -187,15 +187,10 @@ export type PlannedBenchmark = MeasuredBenchmark | { entry: BenchmarkEntry; erro * round samples every slot once, in a shuffled order. After `sampleSize` rounds it keeps adding * rounds while a difference is unresolved against the horizons, until the timeout. A failure is * reported as the benchmark's error. - * - * Its alarmed comparisons share the run's 5% chance of a false alarm with those of the other - * `baseline` benchmarks, which haven't been measured yet, or not by this call: each is taken to have - * one, the common case, and the report counts them exactly afterwards. */ async function runBenchmark( browser: Browser, { entry, slots, sampling }: MeasuredBenchmark, - otherBaselineBenchmarks: number, ): Promise { console.log(chalk.cyan(`\nRunning "${entry.name}" (${entry.file})…`)); const { sampleSize, timeout, autoSampleConditions } = sampling; @@ -231,12 +226,8 @@ async function runBenchmark( const definitions = metrics ?? {}; const benchmarkOf = (): RunBenchmark => ({ ...entry, samples: samplesOf(slots, rounds) }); - const isSettled = () => { - const benchmark = benchmarkOf(); - const familySize = - countAlarmedComparisons(definitions, [benchmark]) + otherBaselineBenchmarks; - return differencesResolved(compareBenchmark(definitions, benchmark, familySize), horizons); - }; + const isSettled = () => + differencesResolved(compareBenchmark(definitions, benchmarkOf()), horizons); const deadline = Date.now() + timeout * 60_000; let resolved = isSettled(); @@ -340,8 +331,6 @@ function samplingOf( export interface PlannedRun { browser: Browser; planned: PlannedBenchmark[]; - /** How many of them measure across the builds. */ - baselineBenchmarks: number; } /** @@ -363,8 +352,8 @@ export async function withPlannedRun( ]); try { - // Every file is listed before any is measured, so each benchmark knows how many others share - // the run's chance of a false alarm. Nothing is measured yet, so the files list at once. + // Every file is listed before any is measured, so a name pattern that matches nothing fails + // before the run spends anything. Nothing is measured yet, so the files list at once. const { testNamePattern } = options; const planned = ( await Promise.all( @@ -378,10 +367,7 @@ export async function withPlannedRun( if (planned.length === 0) { throw new Error(`No benchmark name matches ${testNamePattern}.`); } - const baselineBenchmarks = planned.filter( - (plan) => 'slots' in plan && plan.entry.kind === 'baseline', - ).length; - return await drive({ browser, planned, baselineBenchmarks }); + return await drive({ browser, planned }); } finally { await browser.close(); await server.close(); @@ -389,7 +375,7 @@ export async function withPlannedRun( } export function runInterleaved(options: RunInterleavedOptions): Promise { - return withPlannedRun(options, async ({ browser, planned, baselineBenchmarks }) => { + return withPlannedRun(options, async ({ browser, planned }) => { const results: CaseResult[] = []; // Sequential on purpose: concurrent cases would contend for the same machine. for (const plan of planned) { @@ -397,10 +383,9 @@ export function runInterleaved(options: RunInterleavedOptions): Promise Date: Thu, 8 Oct 2026 10:23:34 +0200 Subject: [PATCH 136/173] [code-infra-dashboard] Show each comparison's precision on the run page The precision is the interval's half-width: the smallest change the run could still have missed. On alarmed metrics it is coloured against the alarm's bands, amber when a regression past warn could hide in it and red past error, with the per-round noise and the rounds it takes to resolve the warn band in a tooltip. Comparisons carry that per-round noise. The PR comment is unchanged. --- .../src/views/BenchmarkRunDetails.tsx | 44 ++++++++++++++--- .../benchmark/src/runReport/analyzeRun.ts | 13 +++++ .../benchmark/src/runReport/format.test.ts | 44 +++++++++++++++++ packages/benchmark/src/runReport/format.ts | 49 ++++++++++++++++++- 4 files changed, 142 insertions(+), 8 deletions(-) diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index 97c6fbd31d..fafa2a72e5 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -22,6 +22,7 @@ import { analyzeRun, benchmarkTable, changeTone, + comparisonPrecision, formatDuration, formatRounds, runReportFootnote, @@ -30,15 +31,14 @@ import type { BenchmarkAnalysis, BenchmarkRunReport, BenchmarkTableColumn, - MetricComparison, + ChangeTone, } from '@mui/internal-benchmark/runReport'; import Heading from '../components/Heading'; import ReportHeader from '../components/ReportHeader'; import ErrorDisplay from '../components/ErrorDisplay'; -/** A comparison cell's colours: a tinted background where something moved, so the eye finds it. */ -function comparisonSx(comparison: MetricComparison): SxProps { - const tone = changeTone(comparison); +/** A cell's colours: a tinted background where something needs a look, so the eye finds it. */ +function toneSx(tone: ChangeTone): SxProps { if (tone === 'none') { return { color: 'text.secondary' }; } @@ -56,12 +56,19 @@ const COLUMN_WIDTHS: Record = { comparison: 280, }; +const PRECISION_WIDTH = 110; + function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { const { benchmark } = analysis; const { columns, rows } = benchmarkTable(analysis); const firstComparison = columns.findIndex((column) => column.kind === 'comparison'); const alignOf = (column: number) => (columns[column].kind === 'value' ? 'right' : undefined); - const minWidth = columns.reduce((sum, column) => sum + COLUMN_WIDTHS[column.kind], 0); + const comparisonHeaders = columns + .filter((column) => column.kind === 'comparison') + .map((column) => column.header); + const minWidth = + columns.reduce((sum, column) => sum + COLUMN_WIDTHS[column.kind], 0) + + comparisonHeaders.length * PRECISION_WIDTH; return ( @@ -96,6 +103,9 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { style={column.kind === 'label' ? undefined : { width: COLUMN_WIDTHS[column.kind] }} /> ))} + {comparisonHeaders.map((header) => ( + + ))} @@ -104,10 +114,17 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { {header} ))} + {comparisonHeaders.map((header) => ( + + + {comparisonHeaders.length === 1 ? 'Precision' : `${header} ±`} + + + ))} - {rows.map((row) => ( + {rows.map((row, rowIndex) => ( {row.cells.map((cell, column) => ( {cell} ))} + {row.comparisons.map((comparison, index) => { + const precision = comparisonPrecision( + comparison, + analysis.metrics[rowIndex].definition, + ); + return ( + + + {precision.label} + + + ); + })} ))} diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index d7222ed3f6..a09dab3ac4 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -46,6 +46,12 @@ export interface MetricComparison { absolute: Interval; /** The same, as a percentage of `against`'s mean. */ relative: Interval; + /** + * How much the difference varies from round to round: the standard deviation of the per-round + * differences, as a percentage of `against`'s mean. A property of the benchmark rather than of how + * long it sampled; 0 with fewer than 2 rounds. + */ + noise: number; change: Change; severity: Severity; } @@ -256,6 +262,12 @@ export function compareSamples( low: (absolute.low / reference) * 100, high: (absolute.high / reference) * 100, }; + // The interval's half-width is t × SD / √rounds, so the standard deviation follows from it. + const noise = + rounds < 2 + ? 0 + : ((relative.high - relative.low) / 2 / tCritical(rounds - 1, confidence)) * + Math.sqrt(rounds); let change: Change = 'undetected'; if (rounds > 0 && differences.every((difference) => difference === 0)) { @@ -272,6 +284,7 @@ export function compareSamples( confidence, absolute, relative, + noise, change, }; return { ...comparison, severity: canAlarm ? severityOf(comparison, definition) : 'none' }; diff --git a/packages/benchmark/src/runReport/format.test.ts b/packages/benchmark/src/runReport/format.test.ts index 29ac00350c..aa867b9c8f 100644 --- a/packages/benchmark/src/runReport/format.test.ts +++ b/packages/benchmark/src/runReport/format.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { analyzeRun } from './analyzeRun'; import { benchmarkTable, + comparisonPrecision, formatDuration, formatRounds, formatRunSummary, @@ -120,3 +121,46 @@ describe('runReportFootnote', () => { ); }); }); + +describe('comparisonPrecision', () => { + // Per-round differences of 1, -2, 3, 0, 2, -1, 4, 1 on a baseline of 100: a standard deviation + // of 2, so 2% noise per round, and a 99% half-width of about 2.5% over 8 rounds. + function precisionWith(alarm: { warn?: number; error?: number }) { + const report: BenchmarkRunReport = { + ...reportOf([ + { + ...benchmarkOf('a'), + samples: { + current: { render: [101, 98, 103, 100, 102, 99, 104, 101] }, + baseline: { render: [100, 100, 100, 100, 100, 100, 100, 100] }, + }, + }, + ]), + metrics: { render: { kind: 'scalar', alarm } }, + }; + const [{ metrics }] = analyzeRun(report); + return comparisonPrecision(metrics[0].comparisons[0], metrics[0].definition); + } + + it('states the half-width, uncoloured while it is within the warn band', () => { + expect(precisionWith({ warn: 0.05, error: 0.1 })).toMatchObject({ + label: '±2.5%', + tone: 'none', + }); + }); + + it('warns past the warn band, and errs past the error band', () => { + expect(precisionWith({ warn: 0.02, error: 0.1 }).tone).toBe('warning'); + expect(precisionWith({ warn: 0.01, error: 0.02 }).tone).toBe('error'); + }); + + it('leaves a metric without bands uncoloured', () => { + expect(precisionWith({}).tone).toBe('none'); + }); + + it('gives the per-round noise, and the rounds it takes to resolve the warn band', () => { + expect(precisionWith({ warn: 0.02 }).detail).toBe( + '±2.0% per round · about 7 rounds resolve ±2%', + ); + }); +}); diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index 8a7cfff240..2bd3663ead 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -1,4 +1,4 @@ -import { CONFIDENCE } from './analyzeRun'; +import { CONFIDENCE, tCritical } from './analyzeRun'; import type { BenchmarkAnalysis, Change, @@ -60,6 +60,53 @@ export function changeTone({ change, severity }: MetricComparison): ChangeTone { return change === 'better' ? 'success' : 'none'; } +export interface ComparisonPrecision { + /** The interval's half-width: `±1.4%`, or `±0.5` for a discrete metric, whose bands are counts. */ + label: string; + /** + * Whether a regression the alarm cares about could have hidden in the interval: `warning` past the + * `warn` band, `error` past the `error` band, `none` within them or without bands to judge by. + */ + tone: Exclude; + /** The per-round noise behind it, and the rounds it would take to resolve the `warn` band. */ + detail: string; +} + +/** + * How precisely a run pinned a comparison down: the smallest change it could still have missed, + * judged against the metric's alarm bands. + */ +export function comparisonPrecision( + comparison: MetricComparison, + definition: RunMetricDefinition, +): ComparisonPrecision { + const scalar = definition.kind === 'scalar'; + const interval = scalar ? comparison.relative : comparison.absolute; + const halfWidth = (interval.high - interval.low) / 2; + // Scalar bands are fractions; the relative interval is in percent. + const width = scalar ? halfWidth / 100 : halfWidth; + const { warn, error } = definition.alarm ?? {}; + let tone: ComparisonPrecision['tone'] = 'none'; + if (error !== undefined && width > error) { + tone = 'error'; + } else if (warn !== undefined && width > warn) { + tone = 'warning'; + } + + let detail = `±${comparison.noise.toFixed(1)}% per round`; + if (scalar && warn !== undefined && warn > 0) { + // Rounds for a half-width of `warn`: (z × noise / warn)², z for a run long enough to need them. + const z = tCritical(1000, comparison.confidence); + const rounds = Math.ceil(((z * comparison.noise) / (warn * 100)) ** 2); + detail += ` · about ${rounds} rounds resolve ±${warn * 100}%`; + } + return { + label: `±${scalar ? `${halfWidth.toFixed(1)}%` : halfWidth.toFixed(1)}`, + tone, + detail, + }; +} + /** How a change reads: `better`, `worse`, `no change detected`, `unchanged`. */ export function formatChange(change: Change): string { return change === 'undetected' ? 'no change detected' : change; From 90878e9f02b1444365c940931d409324e4f32ddb Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:36:07 +0200 Subject: [PATCH 137/173] [benchmark] Decide once whether a comparison can alarm Each comparison carries the alarm it answers to (null where none can fire), and its precision with the band that precision reaches. Confidence, severity, regressions, improvements, the footnote and the precision colour all read it, so a compare() benchmark's precision is no longer judged against bands that can't fire. Severity and precision share one band ladder and one conversion to the bands' units. Precision is an optional column of the shared table, which the run page asks for; the PR comment finds improvements with findImprovements and marks regressions by severity. ChangeTone is now Tone. --- .../lib/benchmarkRun/buildMarkdownReport.ts | 53 ++----- .../src/views/BenchmarkRunDetails.tsx | 91 ++++++----- .../benchmark/src/runReport/analyzeRun.ts | 145 +++++++++++------- .../benchmark/src/runReport/format.test.ts | 43 ++++-- packages/benchmark/src/runReport/format.ts | 114 +++++++------- 5 files changed, 230 insertions(+), 216 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 2274abc3ad..8ce4eb299c 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -2,17 +2,13 @@ import { formatMarkdownTable } from '@/utils/formatters'; import { analyzeRun, benchmarkTable, - changeTone, + findImprovements, findRegressions, formatComparison, formatDuration, runReportFootnote, } from '@mui/internal-benchmark/runReport'; -import type { - BenchmarkAnalysis, - BenchmarkRunReport, - ChangeTone, -} from '@mui/internal-benchmark/runReport'; +import type { BenchmarkAnalysis, BenchmarkRunReport } from '@mui/internal-benchmark/runReport'; interface BuildOptions { /** The comment section's heading. */ @@ -21,41 +17,13 @@ interface BuildOptions { detailsUrl?: string; } -const TONE_MARKERS: Record = { - error: '🔴 ', - warning: '🟠 ', - success: '🟢 ', - none: '', -}; - -/** The changes for the better on metrics that alarm, each as a short phrase. */ -function improvementsOf(analyses: BenchmarkAnalysis[]): string[] { - return analyses.flatMap(({ benchmark, metrics }) => - benchmark.kind !== 'baseline' - ? [] - : metrics.flatMap(({ metric, definition, comparisons }) => - definition.alarm === undefined - ? [] - : comparisons - .filter((comparison) => comparison.change === 'better') - .map( - (comparison) => - `**${benchmark.name}** ${metric} \`${formatComparison(comparison)}\``, - ), - ), - ); -} - /** A benchmark's table, keeping only the rows that regressed, each behind its severity's marker. */ -function regressionTable(analysis: BenchmarkAnalysis): string | null { +function regressionTable(analysis: BenchmarkAnalysis): string { const { columns, rows } = benchmarkTable(analysis); const firstComparison = columns.findIndex((column) => column.kind === 'comparison'); const regressed = rows.filter((row) => row.comparisons.some((comparison) => comparison.severity !== 'none'), ); - if (regressed.length === 0) { - return null; - } return formatMarkdownTable( columns.map(({ header, kind }, column) => ({ field: String(column), @@ -67,7 +35,7 @@ function regressionTable(analysis: BenchmarkAnalysis): string | null { row.cells.map((cell, column) => [ String(column), columns[column].kind === 'comparison' - ? `${TONE_MARKERS[changeTone(row.comparisons[column - firstComparison])]}\`${cell}\`` + ? `${row.comparisons[column - firstComparison].severity === 'error' ? '🔴' : '🟠'} \`${cell}\`` : cell, ]), ), @@ -96,7 +64,10 @@ export function buildBenchmarkRunMarkdownReport( regressions.length > 0 ? `${regressions.length} regression${regressions.length === 1 ? '' : 's'} in ${benchmarks}` : `No regressions in ${benchmarks}`, - ...improvementsOf(analyses), + ...findImprovements(analyses).map( + ({ benchmark, metric, comparison }) => + `**${benchmark}** ${metric} \`${formatComparison(comparison)}\``, + ), report.durationMs === undefined ? null : `ran ${formatDuration(report.durationMs)}`, options.detailsUrl ? `[details](${options.detailsUrl})` : null, ].filter(Boolean); @@ -108,10 +79,10 @@ export function buildBenchmarkRunMarkdownReport( lines.push('', `❌ **${benchmark.name}**: ${benchmark.error}`); } - const tables = analyses.flatMap((analysis) => { - const table = regressionTable(analysis); - return table ? [`**${analysis.benchmark.name}**\n\n${table}`] : []; - }); + const regressed = new Set(regressions.map((regression) => regression.benchmark)); + const tables = analyses + .filter((analysis) => regressed.has(analysis.benchmark.name)) + .map((analysis) => `**${analysis.benchmark.name}**\n\n${regressionTable(analysis)}`); if (tables.length > 0) { lines.push('', tables.join('\n\n'), '', `_${runReportFootnote(analyses)}_`); } diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index fafa2a72e5..77a318d2fc 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -22,8 +22,8 @@ import { analyzeRun, benchmarkTable, changeTone, - comparisonPrecision, formatDuration, + formatPrecisionDetail, formatRounds, runReportFootnote, } from '@mui/internal-benchmark/runReport'; @@ -31,14 +31,15 @@ import type { BenchmarkAnalysis, BenchmarkRunReport, BenchmarkTableColumn, - ChangeTone, + BenchmarkTableRow, + Tone, } from '@mui/internal-benchmark/runReport'; import Heading from '../components/Heading'; import ReportHeader from '../components/ReportHeader'; import ErrorDisplay from '../components/ErrorDisplay'; /** A cell's colours: a tinted background where something needs a look, so the eye finds it. */ -function toneSx(tone: ChangeTone): SxProps { +function toneSx(tone: Tone): SxProps { if (tone === 'none') { return { color: 'text.secondary' }; } @@ -54,21 +55,29 @@ const COLUMN_WIDTHS: Record = { label: 160, value: 120, comparison: 280, + precision: 110, }; -const PRECISION_WIDTH = 110; +/** A table cell's colour: a comparison by its change, a precision by whether a regression could hide. */ +function cellSx( + row: BenchmarkTableRow, + columns: BenchmarkTableColumn[], + column: number, +): SxProps | undefined { + const { kind } = columns[column]; + if (kind !== 'comparison' && kind !== 'precision') { + return undefined; + } + const comparison = row.comparisons[column - columns.findIndex((other) => other.kind === kind)]; + return toneSx(kind === 'comparison' ? changeTone(comparison) : comparison.precisionSeverity); +} function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { const { benchmark } = analysis; - const { columns, rows } = benchmarkTable(analysis); - const firstComparison = columns.findIndex((column) => column.kind === 'comparison'); + const { columns, rows } = benchmarkTable(analysis, { precision: true }); + const firstPrecision = columns.findIndex((column) => column.kind === 'precision'); const alignOf = (column: number) => (columns[column].kind === 'value' ? 'right' : undefined); - const comparisonHeaders = columns - .filter((column) => column.kind === 'comparison') - .map((column) => column.header); - const minWidth = - columns.reduce((sum, column) => sum + COLUMN_WIDTHS[column.kind], 0) + - comparisonHeaders.length * PRECISION_WIDTH; + const minWidth = columns.reduce((sum, column) => sum + COLUMN_WIDTHS[column.kind], 0); return ( @@ -103,55 +112,41 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { style={column.kind === 'label' ? undefined : { width: COLUMN_WIDTHS[column.kind] }} /> ))} - {comparisonHeaders.map((header) => ( - - ))} - {columns.map(({ header }, column) => ( + {columns.map(({ header, kind }, column) => ( - {header} - - ))} - {comparisonHeaders.map((header) => ( - - - {comparisonHeaders.length === 1 ? 'Precision' : `${header} ±`} - + {kind === 'precision' ? ( + + {header} + + ) : ( + header + )} ))} - {rows.map((row, rowIndex) => ( + {rows.map((row) => ( {row.cells.map((cell, column) => ( - - {cell} + + {columns[column].kind === 'precision' ? ( + + {cell} + + ) : ( + cell + )} ))} - {row.comparisons.map((comparison, index) => { - const precision = comparisonPrecision( - comparison, - analysis.metrics[rowIndex].definition, - ); - return ( - - - {precision.label} - - - ); - })} ))} diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index a09dab3ac4..cddff20211 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -35,12 +35,20 @@ export type Change = 'better' | 'worse' | 'undetected' | 'unchanged'; /** How much a change for the worse matters, per the metric's alarm. */ export type Severity = 'error' | 'warning' | 'none'; +/** A metric's alarm: its direction and bands. */ +export type MetricAlarm = NonNullable; + export interface MetricComparison { /** The variant the difference is about. */ subject: string; /** The variant it is measured against. */ against: string; - /** The level of the intervals: 95%, or stricter where a change can raise an alarm. */ + /** + * The alarm a change answers to: the metric's, on a `baseline` benchmark; `null` where no change + * can raise one — a metric without an alarm, or a `compare()`, whose variants differ on purpose. + */ + alarm: MetricAlarm | null; + /** The level of the intervals: 99% where a change can raise an alarm, 95% otherwise. */ confidence: number; /** `subject − against`, in the metric's unit: confidence interval of the mean difference. */ absolute: Interval; @@ -52,6 +60,16 @@ export interface MetricComparison { * long it sampled; 0 with fewer than 2 rounds. */ noise: number; + /** + * The smallest change the run could still have missed: the interval's half-width, in the alarm's + * units — a fraction for a scalar metric, a count for a discrete one. + */ + precision: number; + /** + * Which of the alarm's bands a change of `precision` reaches: whether a regression the alarm + * cares about could hide in the interval. `none` without bands. + */ + precisionSeverity: Severity; change: Change; severity: Severity; } @@ -163,15 +181,25 @@ export function tCritical(degreesOfFreedom: number, confidence = CONFIDENCE): nu return high; } +/** The sample standard deviation; 0 for fewer than 2 values. */ +function sampleStandardDeviation(values: number[], mean: number): number { + if (values.length < 2) { + return 0; + } + return Math.sqrt( + values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (values.length - 1), + ); +} + /** The confidence interval of the mean. A single value is its own degenerate interval. */ export function meanInterval(values: number[], confidence = CONFIDENCE): Interval { const mean = calculateMean(values); if (values.length < 2) { return { low: mean, high: mean }; } - const variance = - values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (values.length - 1); - const halfWidth = tCritical(values.length - 1, confidence) * Math.sqrt(variance / values.length); + const halfWidth = + (tCritical(values.length - 1, confidence) * sampleStandardDeviation(values, mean)) / + Math.sqrt(values.length); return { low: mean - halfWidth, high: mean + halfWidth }; } @@ -197,39 +225,42 @@ function definitionOf( } /** - * How far a change went in the bad direction, at the least: the interval bound nearest zero, in - * the unit the alarm thresholds use — a fraction for a scalar metric, a count for a discrete one. + * An interval in the unit the alarm bands use: a fraction for a scalar metric, which bands + * relatively, and a count for a discrete one. */ -function worseningAtLeast( +function inAlarmUnits( comparison: Pick, definition: RunMetricDefinition, -): number { - const lowerIsBetter = definition.alarm?.direction !== 'higherIsBetter'; - if (definition.kind === 'discrete') { - return lowerIsBetter ? comparison.absolute.low : -comparison.absolute.high; +): Interval { + return definition.kind === 'discrete' + ? comparison.absolute + : { low: comparison.relative.low / 100, high: comparison.relative.high / 100 }; +} + +/** The band a change of `amount`, in the alarm's units, reaches. */ +function bandReached(amount: number, { warn, error }: MetricAlarm): Severity { + if (error !== undefined && amount >= error) { + return 'error'; + } + if (warn !== undefined && amount >= warn) { + return 'warning'; } - return (lowerIsBetter ? comparison.relative.low : -comparison.relative.high) / 100; + return 'none'; } -function severityOf( - comparison: Pick, - definition: RunMetricDefinition, -): Severity { - const { alarm } = definition; - if (comparison.change !== 'worse' || !alarm) { +/** + * A change for the worse's severity: judged by how far it went at the least — the interval bound + * nearest zero — against the bands, and an error outright when the alarm has none. + */ +function severityOf(change: Change, interval: Interval, alarm: MetricAlarm): Severity { + if (change !== 'worse') { return 'none'; } if (alarm.error === undefined && alarm.warn === undefined) { return 'error'; } - const worsening = worseningAtLeast(comparison, definition); - if (alarm.error !== undefined && worsening >= alarm.error) { - return 'error'; - } - if (alarm.warn !== undefined && worsening >= alarm.warn) { - return 'warning'; - } - return 'none'; + const worseningAtLeast = alarm.direction === 'higherIsBetter' ? -interval.high : interval.low; + return bandReached(worseningAtLeast, alarm); } /** The per-round differences `subject − against`, over the rounds both have. */ @@ -239,11 +270,9 @@ function differencesOf(subject: number[], against: number[]): number[] { } export interface CompareSamplesOptions { - /** The level of the intervals. Defaults to 95%. */ - confidence?: number; /** - * Whether a change can raise an alarm, so carry a severity. Defaults to true; the variants of a - * `compare()` differ on purpose, so no difference between them can. + * Whether a change can raise the metric's alarm. Defaults to true; the variants of a `compare()` + * differ on purpose, so no difference between them can. */ canAlarm?: boolean; } @@ -252,8 +281,10 @@ export function compareSamples( subject: { name: string; values: number[] }, against: { name: string; values: number[] }, definition: RunMetricDefinition, - { confidence = CONFIDENCE, canAlarm = true }: CompareSamplesOptions = {}, + { canAlarm = true }: CompareSamplesOptions = {}, ): MetricComparison { + const alarm = (canAlarm && definition.alarm) || null; + const confidence = alarm ? ALARMED_CONFIDENCE : CONFIDENCE; const differences = differencesOf(subject.values, against.values); const rounds = differences.length; const absolute = meanInterval(differences, confidence); @@ -262,12 +293,10 @@ export function compareSamples( low: (absolute.low / reference) * 100, high: (absolute.high / reference) * 100, }; - // The interval's half-width is t × SD / √rounds, so the standard deviation follows from it. const noise = - rounds < 2 - ? 0 - : ((relative.high - relative.low) / 2 / tCritical(rounds - 1, confidence)) * - Math.sqrt(rounds); + (sampleStandardDeviation(differences, calculateMean(differences)) / reference) * 100; + const banded = inAlarmUnits({ absolute, relative }, definition); + const precision = (banded.high - banded.low) / 2; let change: Change = 'undetected'; if (rounds > 0 && differences.every((difference) => difference === 0)) { @@ -278,16 +307,19 @@ export function compareSamples( change = increased === lowerIsBetter ? 'worse' : 'better'; } - const comparison = { + return { subject: subject.name, against: against.name, + alarm, confidence, absolute, relative, noise, + precision, + precisionSeverity: alarm ? bandReached(precision, alarm) : 'none', change, + severity: alarm ? severityOf(change, banded, alarm) : 'none', }; - return { ...comparison, severity: canAlarm ? severityOf(comparison, definition) : 'none' }; } /** The pairs a benchmark's kind compares: the current build against the baseline, or every variant against the reference. */ @@ -342,12 +374,11 @@ export function compareBenchmark( ): { metrics: MetricComparisons[] } { return { metrics: pairedMetricsOf(metrics, benchmark).map(({ metric, definition, canAlarm, pairs }) => { - const confidence = canAlarm ? ALARMED_CONFIDENCE : CONFIDENCE; return { metric, definition, comparisons: pairs.map(({ subject, against }) => - compareSamples(subject, against, definition, { confidence, canAlarm }), + compareSamples(subject, against, definition, { canAlarm }), ), }; }), @@ -378,24 +409,34 @@ export function analyzeRun(report: BenchmarkRunReport): BenchmarkAnalysis[] { return report.benchmarks.map((benchmark) => analyzeBenchmark(report.metrics, benchmark)); } -export interface Regression { +/** A comparison that can raise an alarm, with the benchmark and metric it belongs to. */ +export interface AlarmedChange { benchmark: string; metric: string; comparison: MetricComparison; } +export type Regression = AlarmedChange; + +function alarmedChangesOf(analyses: BenchmarkAnalysis[]): AlarmedChange[] { + return analyses.flatMap(({ benchmark, metrics }) => + metrics.flatMap(({ metric, comparisons }) => + comparisons + .filter((comparison) => comparison.alarm !== null) + .map((comparison) => ({ benchmark: benchmark.name, metric, comparison })), + ), + ); +} + /** - * The changes that say something is wrong with the code under test: only a `baseline` benchmark - * measures that, and only an alarmed metric's change for the worse counts. + * The changes that say something is wrong with the code under test: an alarmed metric's change + * for the worse, past its bands. */ export function findRegressions(analyses: BenchmarkAnalysis[]): Regression[] { - return analyses.flatMap(({ benchmark, metrics }) => - benchmark.kind !== 'baseline' - ? [] - : metrics.flatMap(({ metric, comparisons }) => - comparisons - .filter((comparison) => comparison.severity !== 'none') - .map((comparison) => ({ benchmark: benchmark.name, metric, comparison })), - ), - ); + return alarmedChangesOf(analyses).filter(({ comparison }) => comparison.severity !== 'none'); +} + +/** The changes for the better on alarmed metrics: what the code under test sped up. */ +export function findImprovements(analyses: BenchmarkAnalysis[]): AlarmedChange[] { + return alarmedChangesOf(analyses).filter(({ comparison }) => comparison.change === 'better'); } diff --git a/packages/benchmark/src/runReport/format.test.ts b/packages/benchmark/src/runReport/format.test.ts index aa867b9c8f..754703791a 100644 --- a/packages/benchmark/src/runReport/format.test.ts +++ b/packages/benchmark/src/runReport/format.test.ts @@ -2,8 +2,8 @@ import { describe, expect, it } from 'vitest'; import { analyzeRun } from './analyzeRun'; import { benchmarkTable, - comparisonPrecision, formatDuration, + formatPrecisionDetail, formatRounds, formatRunSummary, runReportFootnote, @@ -122,14 +122,18 @@ describe('runReportFootnote', () => { }); }); -describe('comparisonPrecision', () => { +describe('precision', () => { // Per-round differences of 1, -2, 3, 0, 2, -1, 4, 1 on a baseline of 100: a standard deviation // of 2, so 2% noise per round, and a 99% half-width of about 2.5% over 8 rounds. - function precisionWith(alarm: { warn?: number; error?: number }) { + function analysisWith( + alarm: { warn?: number; error?: number }, + kind: 'baseline' | 'compare' = 'baseline', + ) { const report: BenchmarkRunReport = { ...reportOf([ { ...benchmarkOf('a'), + kind, samples: { current: { render: [101, 98, 103, 100, 102, 99, 104, 101] }, baseline: { render: [100, 100, 100, 100, 100, 100, 100, 100] }, @@ -138,28 +142,35 @@ describe('comparisonPrecision', () => { ]), metrics: { render: { kind: 'scalar', alarm } }, }; - const [{ metrics }] = analyzeRun(report); - return comparisonPrecision(metrics[0].comparisons[0], metrics[0].definition); + const [analysis] = analyzeRun(report); + return { analysis, comparison: analysis.metrics[0].comparisons[0] }; } - it('states the half-width, uncoloured while it is within the warn band', () => { - expect(precisionWith({ warn: 0.05, error: 0.1 })).toMatchObject({ - label: '±2.5%', - tone: 'none', - }); + it('adds a column of half-widths to the table when asked', () => { + const { analysis } = analysisWith({ warn: 0.05 }); + const { columns, rows } = benchmarkTable(analysis, { precision: true }); + + expect(columns.at(-1)).toEqual({ header: 'Precision', kind: 'precision' }); + expect(rows[0].cells.at(-1)).toBe('±2.5%'); + expect(benchmarkTable(analysis).columns.map((column) => column.kind)).not.toContain( + 'precision', + ); }); - it('warns past the warn band, and errs past the error band', () => { - expect(precisionWith({ warn: 0.02, error: 0.1 }).tone).toBe('warning'); - expect(precisionWith({ warn: 0.01, error: 0.02 }).tone).toBe('error'); + it('is fine within the warn band, a warning past it, and an error past the error band', () => { + expect(analysisWith({ warn: 0.05, error: 0.1 }).comparison.precisionSeverity).toBe('none'); + expect(analysisWith({ warn: 0.02, error: 0.1 }).comparison.precisionSeverity).toBe('warning'); + expect(analysisWith({ warn: 0.01, error: 0.02 }).comparison.precisionSeverity).toBe('error'); }); - it('leaves a metric without bands uncoloured', () => { - expect(precisionWith({}).tone).toBe('none'); + it('is never judged without bands, or where no change can raise an alarm', () => { + expect(analysisWith({}).comparison.precisionSeverity).toBe('none'); + expect(analysisWith({ warn: 0.01 }, 'compare').comparison.precisionSeverity).toBe('none'); }); it('gives the per-round noise, and the rounds it takes to resolve the warn band', () => { - expect(precisionWith({ warn: 0.02 }).detail).toBe( + const { analysis, comparison } = analysisWith({ warn: 0.02 }); + expect(formatPrecisionDetail(comparison, analysis.metrics[0].definition)).toBe( '±2.0% per round · about 7 rounds resolve ±2%', ); }); diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index 2bd3663ead..b6998df5d8 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -1,4 +1,4 @@ -import { CONFIDENCE, tCritical } from './analyzeRun'; +import { ALARMED_CONFIDENCE, CONFIDENCE, tCritical } from './analyzeRun'; import type { BenchmarkAnalysis, Change, @@ -43,14 +43,15 @@ export function formatPercent(interval: Interval): string { return `${signed(interval.low)} – ${signed(interval.high)}`; } +/** The colour a renderer draws the eye with; a `Severity` is one too. */ +export type Tone = 'error' | 'warning' | 'success' | 'none'; + /** * The colour a comparison draws the eye with: `error` for a regression that fails the check, * `warning` for any other change for the worse, `success` for one for the better, and `none` when * nothing moved. */ -export type ChangeTone = 'error' | 'warning' | 'success' | 'none'; - -export function changeTone({ change, severity }: MetricComparison): ChangeTone { +export function changeTone({ change, severity }: MetricComparison): Tone { if (severity === 'error') { return 'error'; } @@ -60,51 +61,33 @@ export function changeTone({ change, severity }: MetricComparison): ChangeTone { return change === 'better' ? 'success' : 'none'; } -export interface ComparisonPrecision { - /** The interval's half-width: `±1.4%`, or `±0.5` for a discrete metric, whose bands are counts. */ - label: string; - /** - * Whether a regression the alarm cares about could have hidden in the interval: `warning` past the - * `warn` band, `error` past the `error` band, `none` within them or without bands to judge by. - */ - tone: Exclude; - /** The per-round noise behind it, and the rounds it would take to resolve the `warn` band. */ - detail: string; +// Rounds past which the t-distribution is the normal one, for estimating how many a run needs. +const MANY_ROUNDS = 1000; + +/** A comparison's precision as it reads: `±1.4%`, or `±0.5` for a discrete metric. */ +export function formatPrecision(comparison: MetricComparison, definition: RunMetricDefinition) { + return definition.kind === 'discrete' + ? `±${comparison.precision.toFixed(1)}` + : `±${(comparison.precision * 100).toFixed(1)}%`; } /** - * How precisely a run pinned a comparison down: the smallest change it could still have missed, - * judged against the metric's alarm bands. + * What lies behind a precision: the per-round noise, and how many rounds would resolve the alarm's + * `warn` band — `±2.0% per round · about 7 rounds resolve ±2%`. */ -export function comparisonPrecision( +export function formatPrecisionDetail( comparison: MetricComparison, definition: RunMetricDefinition, -): ComparisonPrecision { - const scalar = definition.kind === 'scalar'; - const interval = scalar ? comparison.relative : comparison.absolute; - const halfWidth = (interval.high - interval.low) / 2; - // Scalar bands are fractions; the relative interval is in percent. - const width = scalar ? halfWidth / 100 : halfWidth; - const { warn, error } = definition.alarm ?? {}; - let tone: ComparisonPrecision['tone'] = 'none'; - if (error !== undefined && width > error) { - tone = 'error'; - } else if (warn !== undefined && width > warn) { - tone = 'warning'; - } - - let detail = `±${comparison.noise.toFixed(1)}% per round`; - if (scalar && warn !== undefined && warn > 0) { - // Rounds for a half-width of `warn`: (z × noise / warn)², z for a run long enough to need them. - const z = tCritical(1000, comparison.confidence); - const rounds = Math.ceil(((z * comparison.noise) / (warn * 100)) ** 2); - detail += ` · about ${rounds} rounds resolve ±${warn * 100}%`; +): string { + const noise = `±${comparison.noise.toFixed(1)}% per round`; + const warn = comparison.alarm?.warn; + if (definition.kind === 'discrete' || !warn) { + return noise; } - return { - label: `±${scalar ? `${halfWidth.toFixed(1)}%` : halfWidth.toFixed(1)}`, - tone, - detail, - }; + // Rounds for a half-width of `warn`: (z × noise / warn)². + const z = tCritical(MANY_ROUNDS, comparison.confidence); + const rounds = Math.ceil(((z * comparison.noise) / (warn * 100)) ** 2); + return `${noise} · about ${rounds} rounds resolve ±${warn * 100}%`; } /** How a change reads: `better`, `worse`, `no change detected`, `unchanged`. */ @@ -133,12 +116,16 @@ export function formatComparisonLabel( export interface BenchmarkTableColumn { header: string; - /** `value` columns hold numbers, `comparison` columns a formatted comparison. */ - kind: 'label' | 'value' | 'comparison'; + /** + * `value` columns hold numbers, `comparison` columns a formatted comparison, `precision` columns + * how precisely that comparison was pinned down. + */ + kind: 'label' | 'value' | 'comparison' | 'precision'; } export interface BenchmarkTableRow { metric: string; + definition: RunMetricDefinition; cells: string[]; /** The comparisons behind the row's `comparison` cells, in column order. */ comparisons: MetricComparison[]; @@ -146,9 +133,12 @@ export interface BenchmarkTableRow { /** * A benchmark's results as one table, for every renderer of it: per metric, each variant's median - * and each comparison. + * and each comparison, and with `precision`, how precisely each comparison was pinned down. */ -export function benchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): { +export function benchmarkTable( + { benchmark, metrics }: BenchmarkAnalysis, + { precision = false }: { precision?: boolean } = {}, +): { columns: BenchmarkTableColumn[]; rows: BenchmarkTableRow[]; } { @@ -156,21 +146,31 @@ export function benchmarkTable({ benchmark, metrics }: BenchmarkAnalysis): { // first, as the comparison columns measure against it. const columnOrder = benchmark.kind === 'baseline' ? [...benchmark.variants].reverse() : benchmark.variants; + const comparisonHeaders = (metrics[0]?.comparisons ?? []).map((comparison) => + formatComparisonLabel(benchmark, comparison), + ); const columns: BenchmarkTableColumn[] = [ { header: 'Metric', kind: 'label' }, ...columnOrder.map((variant) => ({ header: variant, kind: 'value' as const })), - ...(metrics[0]?.comparisons ?? []).map((comparison) => ({ - header: formatComparisonLabel(benchmark, comparison), - kind: 'comparison' as const, - })), + ...comparisonHeaders.map((header) => ({ header, kind: 'comparison' as const })), + ...(precision + ? comparisonHeaders.map((header) => ({ + header: comparisonHeaders.length === 1 ? 'Precision' : `${header} ±`, + kind: 'precision' as const, + })) + : []), ]; const rows = metrics.map(({ metric, definition, variants, comparisons }) => ({ metric, + definition, comparisons, cells: [ metric, ...columnOrder.map((variant) => formatValue(variants[variant].median, definition)), ...comparisons.map(formatComparison), + ...(precision + ? comparisons.map((comparison) => formatPrecision(comparison, definition)) + : []), ], })); return { columns, rows }; @@ -239,16 +239,12 @@ export function formatRunSummary( /** How to read a run's numbers, under every rendering of it. */ export function runReportFootnote(analyses: BenchmarkAnalysis[]): string { - const alarmed = Math.max( - CONFIDENCE, - ...analyses.flatMap(({ metrics }) => - metrics.flatMap(({ comparisons }) => comparisons.map((comparison) => comparison.confidence)), - ), + const alarmed = analyses.some(({ metrics }) => + metrics.some(({ comparisons }) => comparisons.some((comparison) => comparison.alarm !== null)), ); - const levels = - alarmed > CONFIDENCE - ? `${formatConfidence(alarmed)} for alarmed metrics, ${formatConfidence(CONFIDENCE)} otherwise` - : formatConfidence(CONFIDENCE); + const levels = alarmed + ? `${formatConfidence(ALARMED_CONFIDENCE)} for alarmed metrics, ${formatConfidence(CONFIDENCE)} otherwise` + : formatConfidence(CONFIDENCE); return `Medians; Δ is the confidence interval of the paired per-round difference (${levels}).`; } From 3e3a4257b4e375b55efe419d8139bb4e8a4fbf34 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:38:53 +0200 Subject: [PATCH 138/173] [benchmark] Drop the warn band from version 2 judgments A change raises its alarm once it is confidently past the error band, 5% for render and paint, and nothing else counts as a regression: the PR comment shows only alarms, while the dashboard already colours every confirmed change for the worse. Precision turns red once a regression past the error band could hide in it. warn still applies to version 1 reports. --- .../lib/benchmarkRun/buildMarkdownReport.ts | 10 +++---- .../src/views/BenchmarkRunDetails.tsx | 2 +- packages/benchmark/README.md | 7 ++--- .../benchmark/src/page/reactBenchmark.tsx | 8 ++--- .../src/runReport/analyzeRun.test.ts | 9 +++--- .../benchmark/src/runReport/analyzeRun.ts | 29 ++++++++----------- .../benchmark/src/runReport/format.test.ts | 20 +++++-------- packages/benchmark/src/runReport/format.ts | 12 ++++---- packages/benchmark/src/runReport/schema.ts | 7 +++-- packages/benchmark/src/types.ts | 3 +- 10 files changed, 50 insertions(+), 57 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 8ce4eb299c..0997e806b5 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -17,10 +17,12 @@ interface BuildOptions { detailsUrl?: string; } -/** A benchmark's table, keeping only the rows that regressed, each behind its severity's marker. */ +/** + * A benchmark's table, keeping only the rows that regressed. A `baseline` benchmark compares one + * pair, so each kept row's comparison is the regression. + */ function regressionTable(analysis: BenchmarkAnalysis): string { const { columns, rows } = benchmarkTable(analysis); - const firstComparison = columns.findIndex((column) => column.kind === 'comparison'); const regressed = rows.filter((row) => row.comparisons.some((comparison) => comparison.severity !== 'none'), ); @@ -34,9 +36,7 @@ function regressionTable(analysis: BenchmarkAnalysis): string { Object.fromEntries( row.cells.map((cell, column) => [ String(column), - columns[column].kind === 'comparison' - ? `${row.comparisons[column - firstComparison].severity === 'error' ? '🔴' : '🟠'} \`${cell}\`` - : cell, + columns[column].kind === 'comparison' ? `🔴 \`${cell}\`` : cell, ]), ), ), diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index 77a318d2fc..570893e383 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -118,7 +118,7 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { {columns.map(({ header, kind }, column) => ( {kind === 'precision' ? ( - + {header} ) : ( diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 70a7d3a773..2a2c621293 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -231,7 +231,7 @@ You can also `record()` or `time()` from inside a `benchmark()` render function How a regression is judged depends on which report the repository uploads: - **Version 1** (this Vitest reporter) compares the two runs' means. A scalar metric's change must pass a band; with neither band set, `error` is the dashboard's ±20% noise band. A discrete metric flags any change in its mean. -- **Version 2** ([the `benchmark` CLI](#the-benchmark-cli)) first needs the metric's confidence interval to lie wholly on the worse side; the bands then apply to the end of that interval nearest zero, so a change counts only once it is confidently that large. With neither band set, every confirmed change for the worse is an error. +- **Version 2** ([the `benchmark` CLI](#the-benchmark-cli)) first needs the metric's confidence interval to lie wholly on the worse side; the `error` band then applies to the end of that interval nearest zero, so a change alarms only once it is confidently that large. Without an `error` band, every confirmed change for the worse alarms. `warn` judges nothing: the dashboard colours every confirmed change for the worse, and the PR comment shows only alarms. Alarms are evaluated against the baseline when the PR comment is generated, not during the local `vitest run` — a regression never fails the test suite locally. In the PR comment, `error`-band regressions surface as failures and `warn`-band regressions as warnings. @@ -484,9 +484,8 @@ benchmark --baseline "v$(npm view @mui/material version)" --upload --timeline re - `analyzeRun` from `@mui/internal-benchmark/runReport` draws every conclusion from it: a confidence interval on the paired difference per metric, a change (`better`, `worse`, `no change detected`, or `unchanged` when every round measured the same), and a severity from the metric's alarm. -- `reactBenchmark()`'s `render` and `bench:paint` warn once confidently 2% worse and fail at 5%; - `render:count` and the per-phase split are informational; every other metric brings its own - alarm. +- `reactBenchmark()`'s `render` and `bench:paint` alarm once confidently 5% worse; `render:count` + and the per-phase split are informational; every other metric brings its own alarm. - Tables list the baseline before the current build, so a row reads old to new. The pull request comment keeps only the metrics that got better or worse, and sums up a benchmark where none did as "no change detected"; the terminal and the dashboard show every metric. diff --git a/packages/benchmark/src/page/reactBenchmark.tsx b/packages/benchmark/src/page/reactBenchmark.tsx index e36405a21c..787f5e6339 100644 --- a/packages/benchmark/src/page/reactBenchmark.tsx +++ b/packages/benchmark/src/page/reactBenchmark.tsx @@ -11,11 +11,11 @@ export interface ReactBenchmarkOptions extends CaseOptions, SamplingOptions {} // Render time and paint both alarm: render is React running the components, while paint adds what // follows — the commit, injected styles, style recalculation and layout — so a styling or DOM -// regression can show in paint alone. Both warn once a change for the worse is confidently 2% and -// fail at 5%, the size sampling is built to settle; smaller confirmed changes are left to the -// timelines. The render count is informational: an extra render matters only through the time it +// regression can show in paint alone. Both alarm once a change for the worse is confidently 5%, the +// size sampling is built to settle; smaller confirmed changes show on the dashboard and are left to +// the timelines. The render count is informational: an extra render matters only through the time it // adds, which render time already measures. So is the per-phase split. -const TIME_ALARM = { warn: 0.02, error: 0.05 }; +const TIME_ALARM = { error: 0.05 }; const renderMetric = new ScalarMetric({ name: 'render', format: MILLISECONDS, alarm: TIME_ALARM }); const renderCountMetric = new DiscreteMetric({ name: 'render:count' }); const paintMetric = new ScalarMetric({ diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index 3a12664b86..961d9fa76f 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -129,7 +129,7 @@ describe('compareSamples', () => { expect(comparison.severity).toBe('none'); }); - it('grades a change against the warn and error thresholds', () => { + it('raises a change only past the error band', () => { const compareWith = (alarm: RunMetricDefinition['alarm']) => compareSamples( { name: 'current', values: shifted }, @@ -138,9 +138,10 @@ describe('compareSamples', () => { ).severity; // The change is +10 on a mean of 105, about +9.5%. - expect(compareWith({ warn: 0.05, error: 0.2 })).toBe('warning'); - expect(compareWith({ warn: 0.05, error: 0.09 })).toBe('error'); - expect(compareWith({ warn: 0.2 })).toBe('none'); + expect(compareWith({ error: 0.09 })).toBe('error'); + expect(compareWith({ error: 0.2 })).toBe('none'); + expect(compareWith({ warn: 0.05, error: 0.2 })).toBe('none'); + expect(compareWith({ warn: 0.2 })).toBe('error'); }); it('compares a discrete metric in absolute counts', () => { diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index cddff20211..845698c374 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -33,7 +33,8 @@ export interface SampleSummary { export type Change = 'better' | 'worse' | 'undetected' | 'unchanged'; /** How much a change for the worse matters, per the metric's alarm. */ -export type Severity = 'error' | 'warning' | 'none'; +/** Whether a change raises its alarm. A metric's `warn` band applies to version 1 reports only. */ +export type Severity = 'error' | 'none'; /** A metric's alarm: its direction and bands. */ export type MetricAlarm = NonNullable; @@ -66,8 +67,8 @@ export interface MetricComparison { */ precision: number; /** - * Which of the alarm's bands a change of `precision` reaches: whether a regression the alarm - * cares about could hide in the interval. `none` without bands. + * Whether a change of `precision` reaches the alarm's `error` band: whether a regression the alarm + * would raise could hide in the interval. `none` without an `error` band. */ precisionSeverity: Severity; change: Change; @@ -237,30 +238,24 @@ function inAlarmUnits( : { low: comparison.relative.low / 100, high: comparison.relative.high / 100 }; } -/** The band a change of `amount`, in the alarm's units, reaches. */ -function bandReached(amount: number, { warn, error }: MetricAlarm): Severity { - if (error !== undefined && amount >= error) { - return 'error'; - } - if (warn !== undefined && amount >= warn) { - return 'warning'; - } - return 'none'; +/** Whether a change of `amount`, in the alarm's units, reaches its `error` band. */ +function reachesErrorBand(amount: number, { error }: MetricAlarm): Severity { + return error !== undefined && amount >= error ? 'error' : 'none'; } /** * A change for the worse's severity: judged by how far it went at the least — the interval bound - * nearest zero — against the bands, and an error outright when the alarm has none. + * nearest zero — against the `error` band, and an error outright when the alarm has none. */ function severityOf(change: Change, interval: Interval, alarm: MetricAlarm): Severity { if (change !== 'worse') { return 'none'; } - if (alarm.error === undefined && alarm.warn === undefined) { + if (alarm.error === undefined) { return 'error'; } const worseningAtLeast = alarm.direction === 'higherIsBetter' ? -interval.high : interval.low; - return bandReached(worseningAtLeast, alarm); + return reachesErrorBand(worseningAtLeast, alarm); } /** The per-round differences `subject − against`, over the rounds both have. */ @@ -316,7 +311,7 @@ export function compareSamples( relative, noise, precision, - precisionSeverity: alarm ? bandReached(precision, alarm) : 'none', + precisionSeverity: alarm ? reachesErrorBand(precision, alarm) : 'none', change, severity: alarm ? severityOf(change, banded, alarm) : 'none', }; @@ -430,7 +425,7 @@ function alarmedChangesOf(analyses: BenchmarkAnalysis[]): AlarmedChange[] { /** * The changes that say something is wrong with the code under test: an alarmed metric's change - * for the worse, past its bands. + * for the worse, past its `error` band. */ export function findRegressions(analyses: BenchmarkAnalysis[]): Regression[] { return alarmedChangesOf(analyses).filter(({ comparison }) => comparison.severity !== 'none'); diff --git a/packages/benchmark/src/runReport/format.test.ts b/packages/benchmark/src/runReport/format.test.ts index 754703791a..660981f4b1 100644 --- a/packages/benchmark/src/runReport/format.test.ts +++ b/packages/benchmark/src/runReport/format.test.ts @@ -125,10 +125,7 @@ describe('runReportFootnote', () => { describe('precision', () => { // Per-round differences of 1, -2, 3, 0, 2, -1, 4, 1 on a baseline of 100: a standard deviation // of 2, so 2% noise per round, and a 99% half-width of about 2.5% over 8 rounds. - function analysisWith( - alarm: { warn?: number; error?: number }, - kind: 'baseline' | 'compare' = 'baseline', - ) { + function analysisWith(alarm: { error?: number }, kind: 'baseline' | 'compare' = 'baseline') { const report: BenchmarkRunReport = { ...reportOf([ { @@ -147,7 +144,7 @@ describe('precision', () => { } it('adds a column of half-widths to the table when asked', () => { - const { analysis } = analysisWith({ warn: 0.05 }); + const { analysis } = analysisWith({ error: 0.05 }); const { columns, rows } = benchmarkTable(analysis, { precision: true }); expect(columns.at(-1)).toEqual({ header: 'Precision', kind: 'precision' }); @@ -157,19 +154,18 @@ describe('precision', () => { ); }); - it('is fine within the warn band, a warning past it, and an error past the error band', () => { - expect(analysisWith({ warn: 0.05, error: 0.1 }).comparison.precisionSeverity).toBe('none'); - expect(analysisWith({ warn: 0.02, error: 0.1 }).comparison.precisionSeverity).toBe('warning'); - expect(analysisWith({ warn: 0.01, error: 0.02 }).comparison.precisionSeverity).toBe('error'); + it('is an error once it is wider than the error band', () => { + expect(analysisWith({ error: 0.05 }).comparison.precisionSeverity).toBe('none'); + expect(analysisWith({ error: 0.02 }).comparison.precisionSeverity).toBe('error'); }); it('is never judged without bands, or where no change can raise an alarm', () => { expect(analysisWith({}).comparison.precisionSeverity).toBe('none'); - expect(analysisWith({ warn: 0.01 }, 'compare').comparison.precisionSeverity).toBe('none'); + expect(analysisWith({ error: 0.01 }, 'compare').comparison.precisionSeverity).toBe('none'); }); - it('gives the per-round noise, and the rounds it takes to resolve the warn band', () => { - const { analysis, comparison } = analysisWith({ warn: 0.02 }); + it('gives the per-round noise, and the rounds it takes to resolve the error band', () => { + const { analysis, comparison } = analysisWith({ error: 0.02 }); expect(formatPrecisionDetail(comparison, analysis.metrics[0].definition)).toBe( '±2.0% per round · about 7 rounds resolve ±2%', ); diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index b6998df5d8..d5b7befd84 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -73,21 +73,21 @@ export function formatPrecision(comparison: MetricComparison, definition: RunMet /** * What lies behind a precision: the per-round noise, and how many rounds would resolve the alarm's - * `warn` band — `±2.0% per round · about 7 rounds resolve ±2%`. + * `error` band — `±2.0% per round · about 2 rounds resolve ±5%`. */ export function formatPrecisionDetail( comparison: MetricComparison, definition: RunMetricDefinition, ): string { const noise = `±${comparison.noise.toFixed(1)}% per round`; - const warn = comparison.alarm?.warn; - if (definition.kind === 'discrete' || !warn) { + const band = comparison.alarm?.error; + if (definition.kind === 'discrete' || !band) { return noise; } - // Rounds for a half-width of `warn`: (z × noise / warn)². + // Rounds for a half-width of the band: (z × noise / band)². const z = tCritical(MANY_ROUNDS, comparison.confidence); - const rounds = Math.ceil(((z * comparison.noise) / (warn * 100)) ** 2); - return `${noise} · about ${rounds} rounds resolve ±${warn * 100}%`; + const rounds = Math.ceil(((z * comparison.noise) / (band * 100)) ** 2); + return `${noise} · about ${rounds} rounds resolve ±${band * 100}%`; } /** How a change reads: `better`, `worse`, `no change detected`, `unchanged`. */ diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts index 5a1a9d9a5e..f0279fcd79 100644 --- a/packages/benchmark/src/runReport/schema.ts +++ b/packages/benchmark/src/runReport/schema.ts @@ -15,9 +15,10 @@ import type { MetricDefinition } from '../types'; */ /** - * The same shape a metric's `alarm` config has. `warn` and `error` are the smallest changes that - * count at each level — a relative fraction (`0.1` = 10%) for a scalar metric, an absolute count - * for a discrete one. With both omitted, any resolved change for the worse is an error. + * The same shape a metric's `alarm` config has. `error` is the smallest change that raises the alarm + * — a relative fraction (`0.1` = 10%) for a scalar metric, an absolute count for a discrete one; + * without it, any resolved change for the worse does. `warn` is carried for version 1 reports and + * judges nothing here. */ const alarmSchema = z.object({ direction: z.enum(['lowerIsBetter', 'higherIsBetter']).optional(), diff --git a/packages/benchmark/src/types.ts b/packages/benchmark/src/types.ts index 8575148a3c..5a4b2c62be 100644 --- a/packages/benchmark/src/types.ts +++ b/packages/benchmark/src/types.ts @@ -73,7 +73,8 @@ export interface MetricAlarm { /** Defaults to `lowerIsBetter`. */ direction?: MetricDirection; /** - * Softer band: a regression past `warn` (but within `error`) is flagged as a warning. + * Softer band, for version 1 reports only: a regression past `warn` (but within `error`) is + * flagged as a warning. Version 2 colours every confirmed change for the worse on the dashboard. * Scalar metrics: a relative fraction (`0.1` = 10%). Discrete metrics: an absolute count delta. */ warn?: number; From c12e1b42dabbdf3be8b9fe057070b54e7019009f Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:46:14 +0200 Subject: [PATCH 139/173] [benchmark] Tidy version 2 alarms after dropping the warn band - A metric with only warn never alarms, as in version 1, rather than alarming on every confirmed change for the worse. - MetricAlarm comes from the package's types; the sample standard deviation lives in stats and is computed once per comparison; Regression is AlarmedChange. - The precision colour is derived (precisionTone) rather than stored, and table columns name the comparison they show, so renderers do no index maths. - The summary's regressions and the terminal drop the severity that is now always error; the PR comment no longer joins tables to regressions by name. - Docs mark warn as version 1 only; band labels are rounded. --- .../lib/benchmarkRun/buildMarkdownReport.ts | 15 +++-- .../src/views/BenchmarkRunDetails.tsx | 51 ++++++++-------- packages/benchmark/README.md | 13 ++-- .../src/runReport/analyzeRun.test.ts | 3 +- .../benchmark/src/runReport/analyzeRun.ts | 59 ++++++------------- .../benchmark/src/runReport/format.test.ts | 11 ++-- packages/benchmark/src/runReport/format.ts | 25 ++++++-- .../benchmark/src/runReport/summary.test.ts | 2 +- packages/benchmark/src/runReport/summary.ts | 3 +- packages/benchmark/src/runner/printReport.ts | 5 +- packages/benchmark/src/sampling.ts | 6 +- packages/benchmark/src/stats.ts | 9 +++ packages/benchmark/src/types.ts | 5 +- 13 files changed, 107 insertions(+), 100 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 0997e806b5..f239c77dd8 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -21,11 +21,14 @@ interface BuildOptions { * A benchmark's table, keeping only the rows that regressed. A `baseline` benchmark compares one * pair, so each kept row's comparison is the regression. */ -function regressionTable(analysis: BenchmarkAnalysis): string { +function regressionTable(analysis: BenchmarkAnalysis): string | null { const { columns, rows } = benchmarkTable(analysis); const regressed = rows.filter((row) => - row.comparisons.some((comparison) => comparison.severity !== 'none'), + row.comparisons.some((comparison) => comparison.severity === 'error'), ); + if (regressed.length === 0) { + return null; + } return formatMarkdownTable( columns.map(({ header, kind }, column) => ({ field: String(column), @@ -79,10 +82,10 @@ export function buildBenchmarkRunMarkdownReport( lines.push('', `❌ **${benchmark.name}**: ${benchmark.error}`); } - const regressed = new Set(regressions.map((regression) => regression.benchmark)); - const tables = analyses - .filter((analysis) => regressed.has(analysis.benchmark.name)) - .map((analysis) => `**${analysis.benchmark.name}**\n\n${regressionTable(analysis)}`); + const tables = analyses.flatMap((analysis) => { + const table = regressionTable(analysis); + return table ? [`**${analysis.benchmark.name}**\n\n${table}`] : []; + }); if (tables.length > 0) { lines.push('', tables.join('\n\n'), '', `_${runReportFootnote(analyses)}_`); } diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index 570893e383..9b08150b70 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -24,6 +24,7 @@ import { changeTone, formatDuration, formatPrecisionDetail, + precisionTone, formatRounds, runReportFootnote, } from '@mui/internal-benchmark/runReport'; @@ -59,23 +60,17 @@ const COLUMN_WIDTHS: Record = { }; /** A table cell's colour: a comparison by its change, a precision by whether a regression could hide. */ -function cellSx( - row: BenchmarkTableRow, - columns: BenchmarkTableColumn[], - column: number, -): SxProps | undefined { - const { kind } = columns[column]; - if (kind !== 'comparison' && kind !== 'precision') { +function cellSx(row: BenchmarkTableRow, column: BenchmarkTableColumn): SxProps | undefined { + if (column.comparison === undefined) { return undefined; } - const comparison = row.comparisons[column - columns.findIndex((other) => other.kind === kind)]; - return toneSx(kind === 'comparison' ? changeTone(comparison) : comparison.precisionSeverity); + const comparison = row.comparisons[column.comparison]; + return toneSx(column.kind === 'comparison' ? changeTone(comparison) : precisionTone(comparison)); } function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { const { benchmark } = analysis; const { columns, rows } = benchmarkTable(analysis, { precision: true }); - const firstPrecision = columns.findIndex((column) => column.kind === 'precision'); const alignOf = (column: number) => (columns[column].kind === 'value' ? 'right' : undefined); const minWidth = columns.reduce((sum, column) => sum + COLUMN_WIDTHS[column.kind], 0); @@ -131,22 +126,26 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { {rows.map((row) => ( - {row.cells.map((cell, column) => ( - - {columns[column].kind === 'precision' ? ( - - {cell} - - ) : ( - cell - )} - - ))} + {row.cells.map((cell, column) => { + const { kind, comparison } = columns[column]; + return ( + + {kind === 'precision' && comparison !== undefined ? ( + + {cell} + + ) : ( + cell + )} + + ); + })} ))} diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 2a2c621293..17f0e30fdf 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -224,16 +224,16 @@ You can also `record()` or `time()` from inside a `benchmark()` render function - `format` — an [`Intl.NumberFormatOptions`](https://developer.mozilla.org/en-US/docs/Web/API/Intl/NumberFormat/NumberFormat) object used to display the value. - `alarm` — opts the metric into regression flagging. Omit it and the metric is informational (its diff is shown but never flagged). Holds: - `direction` — `'lowerIsBetter'` (default) or `'higherIsBetter'`. - - `warn` — softer band; a regression past it is flagged as a warning. - - `error` — harder band; a regression past it is flagged as an error. With only `warn` set there is no error band (warning-only). + - `warn` — softer band, for version 1 reports; a regression past it is flagged as a warning. + - `error` — harder band; a regression past it is flagged as an error. With only `warn` set there is no error band. - Bands are relative fractions for scalar metrics (`0.1` = 10%) and absolute count deltas for discrete metrics (`1`, `2`). Either band is optional. How a regression is judged depends on which report the repository uploads: - **Version 1** (this Vitest reporter) compares the two runs' means. A scalar metric's change must pass a band; with neither band set, `error` is the dashboard's ±20% noise band. A discrete metric flags any change in its mean. -- **Version 2** ([the `benchmark` CLI](#the-benchmark-cli)) first needs the metric's confidence interval to lie wholly on the worse side; the `error` band then applies to the end of that interval nearest zero, so a change alarms only once it is confidently that large. Without an `error` band, every confirmed change for the worse alarms. `warn` judges nothing: the dashboard colours every confirmed change for the worse, and the PR comment shows only alarms. +- **Version 2** ([the `benchmark` CLI](#the-benchmark-cli)) first needs the metric's confidence interval to lie wholly on the worse side; the `error` band then applies to the end of that interval nearest zero, so a change alarms only once it is confidently that large. With neither band, every confirmed change for the worse alarms; with only `warn`, none does. `warn` judges nothing: the dashboard colours every confirmed change for the worse, and the PR comment shows only alarms. -Alarms are evaluated against the baseline when the PR comment is generated, not during the local `vitest run` — a regression never fails the test suite locally. In the PR comment, `error`-band regressions surface as failures and `warn`-band regressions as warnings. +Alarms are evaluated against the baseline when the PR comment is generated, not during the local `vitest run` — a regression never fails the test suite locally. In a version 1 PR comment, `error`-band regressions surface as failures and `warn`-band regressions as warnings. #### Sub-series @@ -487,8 +487,9 @@ benchmark --baseline "v$(npm view @mui/material version)" --upload --timeline re - `reactBenchmark()`'s `render` and `bench:paint` alarm once confidently 5% worse; `render:count` and the per-phase split are informational; every other metric brings its own alarm. - Tables list the baseline before the current build, so a row reads old to new. The pull request - comment keeps only the metrics that got better or worse, and sums up a benchmark where none did - as "no change detected"; the terminal and the dashboard show every metric. + comment is one line unless something alarmed — the count, improvements on alarmed metrics and a + link — and then tables only the rows that alarmed; the terminal and the dashboard show every + metric. ### Choosing the baseline diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index 961d9fa76f..f3d560b9be 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -141,7 +141,8 @@ describe('compareSamples', () => { expect(compareWith({ error: 0.09 })).toBe('error'); expect(compareWith({ error: 0.2 })).toBe('none'); expect(compareWith({ warn: 0.05, error: 0.2 })).toBe('none'); - expect(compareWith({ warn: 0.2 })).toBe('error'); + expect(compareWith({ warn: 0.2 })).toBe('none'); + expect(compareWith({})).toBe('error'); }); it('compares a discrete metric in absolute counts', () => { diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index 845698c374..ce48d4f60d 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -1,4 +1,5 @@ -import { calculateMean, quantile } from '../stats'; +import { calculateMean, calculateSampleStdDev, quantile } from '../stats'; +import type { MetricAlarm } from '../types'; import { baseMetricName } from '../metricCore'; import type { BenchmarkRunReport, RunBenchmark, RunMetricDefinition } from './schema'; @@ -32,13 +33,9 @@ export interface SampleSummary { */ export type Change = 'better' | 'worse' | 'undetected' | 'unchanged'; -/** How much a change for the worse matters, per the metric's alarm. */ /** Whether a change raises its alarm. A metric's `warn` band applies to version 1 reports only. */ export type Severity = 'error' | 'none'; -/** A metric's alarm: its direction and bands. */ -export type MetricAlarm = NonNullable; - export interface MetricComparison { /** The variant the difference is about. */ subject: string; @@ -66,11 +63,6 @@ export interface MetricComparison { * units — a fraction for a scalar metric, a count for a discrete one. */ precision: number; - /** - * Whether a change of `precision` reaches the alarm's `error` band: whether a regression the alarm - * would raise could hide in the interval. `none` without an `error` band. - */ - precisionSeverity: Severity; change: Change; severity: Severity; } @@ -182,26 +174,19 @@ export function tCritical(degreesOfFreedom: number, confidence = CONFIDENCE): nu return high; } -/** The sample standard deviation; 0 for fewer than 2 values. */ -function sampleStandardDeviation(values: number[], mean: number): number { - if (values.length < 2) { - return 0; +/** The confidence interval of a mean from `count` values with standard deviation `stdDev`. */ +function intervalAround(mean: number, stdDev: number, count: number, confidence: number): Interval { + if (count < 2) { + return { low: mean, high: mean }; } - return Math.sqrt( - values.reduce((sum, value) => sum + (value - mean) ** 2, 0) / (values.length - 1), - ); + const halfWidth = (tCritical(count - 1, confidence) * stdDev) / Math.sqrt(count); + return { low: mean - halfWidth, high: mean + halfWidth }; } /** The confidence interval of the mean. A single value is its own degenerate interval. */ export function meanInterval(values: number[], confidence = CONFIDENCE): Interval { const mean = calculateMean(values); - if (values.length < 2) { - return { low: mean, high: mean }; - } - const halfWidth = - (tCritical(values.length - 1, confidence) * sampleStandardDeviation(values, mean)) / - Math.sqrt(values.length); - return { low: mean - halfWidth, high: mean + halfWidth }; + return intervalAround(mean, calculateSampleStdDev(values, mean), values.length, confidence); } export function median(values: number[]): number { @@ -238,24 +223,20 @@ function inAlarmUnits( : { low: comparison.relative.low / 100, high: comparison.relative.high / 100 }; } -/** Whether a change of `amount`, in the alarm's units, reaches its `error` band. */ -function reachesErrorBand(amount: number, { error }: MetricAlarm): Severity { - return error !== undefined && amount >= error ? 'error' : 'none'; -} - /** * A change for the worse's severity: judged by how far it went at the least — the interval bound - * nearest zero — against the `error` band, and an error outright when the alarm has none. + * nearest zero — against the `error` band. Without bands, every confirmed change for the worse + * alarms; with only `warn`, none does, as `warn` judges nothing here. */ function severityOf(change: Change, interval: Interval, alarm: MetricAlarm): Severity { if (change !== 'worse') { return 'none'; } if (alarm.error === undefined) { - return 'error'; + return alarm.warn === undefined ? 'error' : 'none'; } const worseningAtLeast = alarm.direction === 'higherIsBetter' ? -interval.high : interval.low; - return reachesErrorBand(worseningAtLeast, alarm); + return worseningAtLeast >= alarm.error ? 'error' : 'none'; } /** The per-round differences `subject − against`, over the rounds both have. */ @@ -282,14 +263,15 @@ export function compareSamples( const confidence = alarm ? ALARMED_CONFIDENCE : CONFIDENCE; const differences = differencesOf(subject.values, against.values); const rounds = differences.length; - const absolute = meanInterval(differences, confidence); + const mean = calculateMean(differences); + const stdDev = calculateSampleStdDev(differences, mean); + const absolute = intervalAround(mean, stdDev, rounds, confidence); const reference = calculateMean(against.values.slice(0, rounds)); const relative = { low: (absolute.low / reference) * 100, high: (absolute.high / reference) * 100, }; - const noise = - (sampleStandardDeviation(differences, calculateMean(differences)) / reference) * 100; + const noise = (stdDev / reference) * 100; const banded = inAlarmUnits({ absolute, relative }, definition); const precision = (banded.high - banded.low) / 2; @@ -311,7 +293,6 @@ export function compareSamples( relative, noise, precision, - precisionSeverity: alarm ? reachesErrorBand(precision, alarm) : 'none', change, severity: alarm ? severityOf(change, banded, alarm) : 'none', }; @@ -350,7 +331,7 @@ function pairedMetricsOf( return { metric, definition, - canAlarm: benchmark.kind === 'baseline' && definition.alarm !== undefined, + canAlarm: benchmark.kind === 'baseline', pairs: pairsOf(benchmark).map(({ subject, against }) => ({ subject: sideOf(subject), against: sideOf(against), @@ -411,8 +392,6 @@ export interface AlarmedChange { comparison: MetricComparison; } -export type Regression = AlarmedChange; - function alarmedChangesOf(analyses: BenchmarkAnalysis[]): AlarmedChange[] { return analyses.flatMap(({ benchmark, metrics }) => metrics.flatMap(({ metric, comparisons }) => @@ -427,7 +406,7 @@ function alarmedChangesOf(analyses: BenchmarkAnalysis[]): AlarmedChange[] { * The changes that say something is wrong with the code under test: an alarmed metric's change * for the worse, past its `error` band. */ -export function findRegressions(analyses: BenchmarkAnalysis[]): Regression[] { +export function findRegressions(analyses: BenchmarkAnalysis[]): AlarmedChange[] { return alarmedChangesOf(analyses).filter(({ comparison }) => comparison.severity !== 'none'); } diff --git a/packages/benchmark/src/runReport/format.test.ts b/packages/benchmark/src/runReport/format.test.ts index 660981f4b1..00b97e665c 100644 --- a/packages/benchmark/src/runReport/format.test.ts +++ b/packages/benchmark/src/runReport/format.test.ts @@ -4,6 +4,7 @@ import { benchmarkTable, formatDuration, formatPrecisionDetail, + precisionTone, formatRounds, formatRunSummary, runReportFootnote, @@ -147,7 +148,7 @@ describe('precision', () => { const { analysis } = analysisWith({ error: 0.05 }); const { columns, rows } = benchmarkTable(analysis, { precision: true }); - expect(columns.at(-1)).toEqual({ header: 'Precision', kind: 'precision' }); + expect(columns.at(-1)).toEqual({ header: 'Precision', kind: 'precision', comparison: 0 }); expect(rows[0].cells.at(-1)).toBe('±2.5%'); expect(benchmarkTable(analysis).columns.map((column) => column.kind)).not.toContain( 'precision', @@ -155,13 +156,13 @@ describe('precision', () => { }); it('is an error once it is wider than the error band', () => { - expect(analysisWith({ error: 0.05 }).comparison.precisionSeverity).toBe('none'); - expect(analysisWith({ error: 0.02 }).comparison.precisionSeverity).toBe('error'); + expect(precisionTone(analysisWith({ error: 0.05 }).comparison)).toBe('none'); + expect(precisionTone(analysisWith({ error: 0.02 }).comparison)).toBe('error'); }); it('is never judged without bands, or where no change can raise an alarm', () => { - expect(analysisWith({}).comparison.precisionSeverity).toBe('none'); - expect(analysisWith({ error: 0.01 }, 'compare').comparison.precisionSeverity).toBe('none'); + expect(precisionTone(analysisWith({}).comparison)).toBe('none'); + expect(precisionTone(analysisWith({ error: 0.01 }, 'compare').comparison)).toBe('none'); }); it('gives the per-round noise, and the rounds it takes to resolve the error band', () => { diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index d5b7befd84..2f9988d9f8 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -3,8 +3,8 @@ import type { BenchmarkAnalysis, Change, Interval, + AlarmedChange, MetricComparison, - Regression, } from './analyzeRun'; import type { RunBenchmark, RunMetricDefinition } from './schema'; @@ -64,6 +64,14 @@ export function changeTone({ change, severity }: MetricComparison): Tone { // Rounds past which the t-distribution is the normal one, for estimating how many a run needs. const MANY_ROUNDS = 1000; +/** + * The colour a comparison's precision draws the eye with: `error` when a regression the alarm would + * raise — one past its `error` band — could hide in the interval, `none` otherwise. + */ +export function precisionTone({ alarm, precision }: MetricComparison): Tone { + return alarm?.error !== undefined && precision >= alarm.error ? 'error' : 'none'; +} + /** A comparison's precision as it reads: `±1.4%`, or `±0.5` for a discrete metric. */ export function formatPrecision(comparison: MetricComparison, definition: RunMetricDefinition) { return definition.kind === 'discrete' @@ -87,7 +95,7 @@ export function formatPrecisionDetail( // Rounds for a half-width of the band: (z × noise / band)². const z = tCritical(MANY_ROUNDS, comparison.confidence); const rounds = Math.ceil(((z * comparison.noise) / (band * 100)) ** 2); - return `${noise} · about ${rounds} rounds resolve ±${band * 100}%`; + return `${noise} · about ${rounds} rounds resolve ±${formatterFor(DEFAULT_SCALAR_FORMAT).format(band * 100)}%`; } /** How a change reads: `better`, `worse`, `no change detected`, `unchanged`. */ @@ -121,6 +129,8 @@ export interface BenchmarkTableColumn { * how precisely that comparison was pinned down. */ kind: 'label' | 'value' | 'comparison' | 'precision'; + /** For a `comparison` or `precision` column: which of a row's `comparisons` it shows. */ + comparison?: number; } export interface BenchmarkTableRow { @@ -152,11 +162,16 @@ export function benchmarkTable( const columns: BenchmarkTableColumn[] = [ { header: 'Metric', kind: 'label' }, ...columnOrder.map((variant) => ({ header: variant, kind: 'value' as const })), - ...comparisonHeaders.map((header) => ({ header, kind: 'comparison' as const })), + ...comparisonHeaders.map((header, comparison) => ({ + header, + kind: 'comparison' as const, + comparison, + })), ...(precision - ? comparisonHeaders.map((header) => ({ + ? comparisonHeaders.map((header, comparison) => ({ header: comparisonHeaders.length === 1 ? 'Precision' : `${header} ±`, kind: 'precision' as const, + comparison, })) : []), ]; @@ -219,7 +234,7 @@ export function formatRounds(benchmark: RunBenchmark): string | null { */ export function formatRunSummary( analyses: BenchmarkAnalysis[], - regressions: Regression[], + regressions: AlarmedChange[], durationMs?: number, ): string { const measured = analyses.filter((analysis) => !analysis.benchmark.error).length; diff --git a/packages/benchmark/src/runReport/summary.test.ts b/packages/benchmark/src/runReport/summary.test.ts index 4f26aba92a..d64870b92e 100644 --- a/packages/benchmark/src/runReport/summary.test.ts +++ b/packages/benchmark/src/runReport/summary.test.ts @@ -39,7 +39,7 @@ describe('summarizeRun', () => { expect(summary.durationMs).toBe(845_000); expect(summary.regressions).toEqual([ - expect.objectContaining({ benchmark: 'Grid scroll', metric: 'render', severity: 'error' }), + expect.objectContaining({ benchmark: 'Grid scroll', metric: 'render' }), ]); const [grid, chart] = summary.benchmarks; expect(grid).toMatchObject({ diff --git a/packages/benchmark/src/runReport/summary.ts b/packages/benchmark/src/runReport/summary.ts index 35305c17a4..f1472fed3e 100644 --- a/packages/benchmark/src/runReport/summary.ts +++ b/packages/benchmark/src/runReport/summary.ts @@ -10,7 +10,7 @@ export interface RunSummary { /** How long the whole run took, when the report says. */ durationMs?: number; /** The changes for the worse that fail the check, as the comment lists them. */ - regressions: Array<{ benchmark: string; metric: string; severity: Severity; relative: Interval }>; + regressions: Array<{ benchmark: string; metric: string; relative: Interval }>; benchmarks: BenchmarkSummary[]; } @@ -49,7 +49,6 @@ export function summarizeRun(report: BenchmarkRunReport): RunSummary { regressions: findRegressions(analyses).map(({ benchmark, metric, comparison }) => ({ benchmark, metric, - severity: comparison.severity, relative: comparison.relative, })), benchmarks: analyses.map(({ benchmark, metrics }) => { diff --git a/packages/benchmark/src/runner/printReport.ts b/packages/benchmark/src/runner/printReport.ts index b88ccb940d..d1d6ad2a38 100644 --- a/packages/benchmark/src/runner/printReport.ts +++ b/packages/benchmark/src/runner/printReport.ts @@ -1,6 +1,6 @@ /* eslint-disable no-console */ -import { dim, printTable, red, yellow } from '../format'; +import { dim, printTable, red } from '../format'; import { analyzeRun, benchmarkTable, @@ -44,8 +44,7 @@ export function printRunReport(report: BenchmarkRunReport): void { const regressions = findRegressions(analyses); console.log(''); for (const { benchmark, metric, comparison } of regressions) { - const color = comparison.severity === 'error' ? red : yellow; - console.log(color(`⚠ ${benchmark} · ${metric} · ${formatComparison(comparison)}`)); + console.log(red(`⚠ ${benchmark} · ${metric} · ${formatComparison(comparison)}`)); } console.log(formatRunSummary(analyses, regressions, report.durationMs)); console.log(dim(runReportFootnote(analyses))); diff --git a/packages/benchmark/src/sampling.ts b/packages/benchmark/src/sampling.ts index 307f17a323..8988a7155f 100644 --- a/packages/benchmark/src/sampling.ts +++ b/packages/benchmark/src/sampling.ts @@ -20,8 +20,8 @@ export interface SamplingOptions { * than 5%. A `0%` horizon tests whether there is any change at all: identical builds never resolve * against it, and checking again after every round makes a false finding likelier. * - * Judged on the metrics that alarm — the ones a change can be flagged on — or on every metric when - * none does. + * Judged on the metrics that carry an alarm, the ones a benchmark cares about most, whether or + * not its kind lets them raise one; on every metric when none carries one. */ autoSampleConditions?: string[]; } @@ -89,7 +89,7 @@ export function isResolved(interval: { low: number; high: number }, horizons: nu /** * Whether sampling a benchmark can stop: every comparison it reports has resolved against the - * horizons, on the metrics that alarm, or on every metric when none does. + * horizons, on the metrics that carry an alarm, or on every metric when none does. */ export function differencesResolved( analysis: { metrics: MetricComparisons[] }, diff --git a/packages/benchmark/src/stats.ts b/packages/benchmark/src/stats.ts index 70756a8d68..3d42eb3374 100644 --- a/packages/benchmark/src/stats.ts +++ b/packages/benchmark/src/stats.ts @@ -7,6 +7,15 @@ export function calculateStdDev(values: number[], mean: number): number { return Math.sqrt(squaredDiffs.reduce((sum, v) => sum + v, 0) / values.length); } +/** The sample standard deviation, which an interval estimated from the values needs; 0 for fewer than 2. */ +export function calculateSampleStdDev(values: number[], mean: number): number { + if (values.length < 2) { + return 0; + } + const squaredDiffs = values.map((v) => (v - mean) ** 2); + return Math.sqrt(squaredDiffs.reduce((sum, v) => sum + v, 0) / (values.length - 1)); +} + export function quantile(sorted: number[], q: number): number { const pos = (sorted.length - 1) * q; const base = Math.floor(pos); diff --git a/packages/benchmark/src/types.ts b/packages/benchmark/src/types.ts index 5a4b2c62be..e05a30674b 100644 --- a/packages/benchmark/src/types.ts +++ b/packages/benchmark/src/types.ts @@ -80,8 +80,9 @@ export interface MetricAlarm { warn?: number; /** * Harder band: a regression past `error` is flagged as an error (the alarm). With only `warn` - * set there is no error band (warning-only). With neither, version 1 reports use the dashboard's - * ±20% noise band and version 2 reports flag every confirmed change for the worse. + * set there is no error band: version 1 reports warn, version 2 reports never alarm. With + * neither, version 1 reports use the dashboard's ±20% noise band and version 2 reports flag every + * confirmed change for the worse. * Scalar metrics: a relative fraction (`0.25` = 25%). Discrete metrics: an absolute count delta. */ error?: number; From c1f2f2d818018ae88b520d1c2c6f1527f6b6671d Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:25:51 +0200 Subject: [PATCH 140/173] [benchmark] One 99% level, and only alarmable differences extend sampling Every interval is now 99%, so the dashboard colours fewer chance changes on informational metrics and a report reads at one level. Sampling past sampleSize waits only on differences that can raise an alarm: a compare(), or a benchmark without alarmed metrics, stops at sampleSize and shows how precise its numbers got. Comparisons and the JSON summary no longer carry a confidence, which is now the same everywhere. --- .../src/components/BenchmarkRunHistory.tsx | 2 +- .../benchmarkRun/buildMarkdownReport.test.ts | 2 +- packages/benchmark/README.md | 16 ++++++++------ .../src/runReport/analyzeRun.test.ts | 15 +++++++------ .../benchmark/src/runReport/analyzeRun.ts | 21 +++++++------------ .../benchmark/src/runReport/format.test.ts | 16 +++----------- packages/benchmark/src/runReport/format.ts | 12 +++-------- packages/benchmark/src/runReport/summary.ts | 5 +---- packages/benchmark/src/sampling.test.ts | 17 +++++++++------ packages/benchmark/src/sampling.ts | 20 +++++++++--------- 10 files changed, 54 insertions(+), 72 deletions(-) diff --git a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx index c02f3334e2..c987befff1 100644 --- a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx +++ b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx @@ -153,7 +153,7 @@ export default function BenchmarkRunHistory({ repo }: BenchmarkRunHistoryProps) ) : ( - Each master commit against its parent, measured in the same run: the midpoint of the 95% + Each master commit against its parent, measured in the same run: the midpoint of the 99% confidence interval on the paired change. Click a bar to open that run. diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index 080fe1d4bd..3787403551 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -103,7 +103,7 @@ describe('buildBenchmarkRunMarkdownReport', () => { | render | 21.5 ms | 23.5 ms | 🔴 \`worse +9.3% – +9.3%\` | - _Medians; Δ is the confidence interval of the paired per-round difference (99% for alarmed metrics, 95% otherwise)._" + _Medians; Δ is the confidence interval of the paired per-round difference (99%)._" `); }); diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 17f0e30fdf..d1ff2d4def 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -422,12 +422,16 @@ per-round differences rather than on two independent sets of samples. Whatever t during a round — thermal throttling, a background process — then affects both sides of it and cancels out. -Intervals are 95%, except where a change can raise an alarm — an alarmed metric of a benchmark -measured against the baseline — where they are 99%. The level is fixed rather than tightened for -every comparison a run holds, so adding benchmarks doesn't make each one sample longer. What keeps a -run of unchanged code from alarming is mostly the alarm's bands: the interval's near end has to -reach them, so noise has to fake a change bigger than the band, not just one beyond zero. The nightly -A/A job tracks how often a run of unchanged code alarms anyway. +Every interval is 99%. The level is fixed rather than tightened for every comparison a run holds, +so adding benchmarks doesn't make each one sample longer. What keeps a run of unchanged code from +alarming is mostly the alarm's bands: the interval's near end has to reach them, so noise has to fake +a change bigger than the band, not just one beyond zero. The nightly A/A job tracks how often a run +of unchanged code alarms anyway. + +Only differences that can raise an alarm — an alarmed metric of a benchmark measured against the +baseline — keep a run sampling past `sampleSize`. The rest are there to read: a `compare()`, or a +benchmark without alarmed metrics, stops at `sampleSize`, and the run page shows how precise its +numbers got. Run it with at least 3 logical CPUs (vCPUs in CI); it warns with fewer. On 2, V8's background compilers are starved, so how fast the same code runs depends on how its compilation happened to go, diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index f3d560b9be..252055b193 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -35,21 +35,20 @@ function reportOf(benchmarks: RunBenchmark[], metrics: BenchmarkRunReport['metri } describe('tCritical', () => { - it('matches the published table at 95%', () => { - expect(tCritical(1)).toBeCloseTo(12.706, 3); - expect(tCritical(9)).toBeCloseTo(2.262, 3); - expect(tCritical(29)).toBeCloseTo(2.045, 3); + it('matches the published table at 99%, the default', () => { + expect(tCritical(1)).toBeCloseTo(63.657, 2); + expect(tCritical(9)).toBeCloseTo(3.25, 3); }); it('matches the published table at other levels', () => { - expect(tCritical(1, 0.99)).toBeCloseTo(63.657, 2); - expect(tCritical(9, 0.99)).toBeCloseTo(3.25, 3); + expect(tCritical(1, 0.95)).toBeCloseTo(12.706, 3); + expect(tCritical(29, 0.95)).toBeCloseTo(2.045, 3); expect(tCritical(29, 0.999)).toBeCloseTo(3.659, 3); }); it('approaches the normal quantile for large samples', () => { - expect(tCritical(100_000)).toBeCloseTo(1.96, 2); - expect(tCritical(100_000, 0.99)).toBeCloseTo(2.576, 2); + expect(tCritical(100_000)).toBeCloseTo(2.576, 2); + expect(tCritical(100_000, 0.95)).toBeCloseTo(1.96, 2); }); }); diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index ce48d4f60d..434a48e69a 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -11,9 +11,9 @@ import type { BenchmarkRunReport, RunBenchmark, RunMetricDefinition } from './sc * per-round difference cancels whatever the machine was doing during that round, and its * confidence interval is far tighter than one computed from two independent sets of samples. * - * Intervals are 95%, except where a change can raise an alarm: there they are 99%, a fixed level so - * a run's cost doesn't grow with how many benchmarks it holds. What keeps a run of unchanged code - * from alarming is mostly the alarm's bands, which the interval's near end has to reach. + * Every interval is 99%, a fixed level, so a run's cost doesn't grow with how many benchmarks it + * holds. What keeps a run of unchanged code from alarming is mostly the alarm's bands, which the + * interval's near end has to reach. */ export interface Interval { @@ -46,8 +46,6 @@ export interface MetricComparison { * can raise one — a metric without an alarm, or a `compare()`, whose variants differ on purpose. */ alarm: MetricAlarm | null; - /** The level of the intervals: 99% where a change can raise an alarm, 95% otherwise. */ - confidence: number; /** `subject − against`, in the metric's unit: confidence interval of the mean difference. */ absolute: Interval; /** The same, as a percentage of `against`'s mean. */ @@ -80,11 +78,8 @@ export interface BenchmarkAnalysis { metrics: MetricAnalysis[]; } -/** The confidence level of an interval no alarm depends on. */ -export const CONFIDENCE = 0.95; - -/** The confidence level of a comparison that can raise an alarm. */ -export const ALARMED_CONFIDENCE = 0.99; +/** The confidence level of every interval. */ +export const CONFIDENCE = 0.99; const LANCZOS = [ 0.99999999999980993, 676.5203681218851, -1259.1392167224028, 771.32342877765313, @@ -146,7 +141,7 @@ function incompleteBeta(x: number, a: number, b: number): number { const criticalValues = new Map(); -/** The two-sided critical value of Student's t at a confidence level, 95% by default. */ +/** The two-sided critical value of Student's t at a confidence level, 99% by default. */ export function tCritical(degreesOfFreedom: number, confidence = CONFIDENCE): number { const key = `${degreesOfFreedom}:${confidence}`; const cached = criticalValues.get(key); @@ -260,12 +255,11 @@ export function compareSamples( { canAlarm = true }: CompareSamplesOptions = {}, ): MetricComparison { const alarm = (canAlarm && definition.alarm) || null; - const confidence = alarm ? ALARMED_CONFIDENCE : CONFIDENCE; const differences = differencesOf(subject.values, against.values); const rounds = differences.length; const mean = calculateMean(differences); const stdDev = calculateSampleStdDev(differences, mean); - const absolute = intervalAround(mean, stdDev, rounds, confidence); + const absolute = intervalAround(mean, stdDev, rounds, CONFIDENCE); const reference = calculateMean(against.values.slice(0, rounds)); const relative = { low: (absolute.low / reference) * 100, @@ -288,7 +282,6 @@ export function compareSamples( subject: subject.name, against: against.name, alarm, - confidence, absolute, relative, noise, diff --git a/packages/benchmark/src/runReport/format.test.ts b/packages/benchmark/src/runReport/format.test.ts index 00b97e665c..1789baf046 100644 --- a/packages/benchmark/src/runReport/format.test.ts +++ b/packages/benchmark/src/runReport/format.test.ts @@ -106,19 +106,9 @@ describe('formatRunSummary', () => { }); describe('runReportFootnote', () => { - it('states one level when nothing can raise an alarm', () => { - const report = { - ...reportOf([benchmarkOf('a')]), - metrics: { render: { kind: 'scalar' as const } }, - }; - expect(runReportFootnote(analyzeRun(report))).toBe( - 'Medians; Δ is the confidence interval of the paired per-round difference (95%).', - ); - }); - - it('states the alarmed level beside the other', () => { - expect(runReportFootnote(analyzeRun(reportOf([benchmarkOf('a')])))).toContain( - '(99% for alarmed metrics, 95% otherwise)', + it('states the level every interval uses', () => { + expect(runReportFootnote(analyzeRun(reportOf([benchmarkOf('a')])))).toBe( + 'Medians; Δ is the confidence interval of the paired per-round difference (99%).', ); }); }); diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index 2f9988d9f8..74b7b8e5df 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -1,4 +1,4 @@ -import { ALARMED_CONFIDENCE, CONFIDENCE, tCritical } from './analyzeRun'; +import { CONFIDENCE, tCritical } from './analyzeRun'; import type { BenchmarkAnalysis, Change, @@ -93,7 +93,7 @@ export function formatPrecisionDetail( return noise; } // Rounds for a half-width of the band: (z × noise / band)². - const z = tCritical(MANY_ROUNDS, comparison.confidence); + const z = tCritical(MANY_ROUNDS, CONFIDENCE); const rounds = Math.ceil(((z * comparison.noise) / (band * 100)) ** 2); return `${noise} · about ${rounds} rounds resolve ±${formatterFor(DEFAULT_SCALAR_FORMAT).format(band * 100)}%`; } @@ -254,13 +254,7 @@ export function formatRunSummary( /** How to read a run's numbers, under every rendering of it. */ export function runReportFootnote(analyses: BenchmarkAnalysis[]): string { - const alarmed = analyses.some(({ metrics }) => - metrics.some(({ comparisons }) => comparisons.some((comparison) => comparison.alarm !== null)), - ); - const levels = alarmed - ? `${formatConfidence(ALARMED_CONFIDENCE)} for alarmed metrics, ${formatConfidence(CONFIDENCE)} otherwise` - : formatConfidence(CONFIDENCE); - return `Medians; Δ is the confidence interval of the paired per-round difference (${levels}).`; + return `Medians; Δ is the confidence interval of the paired per-round difference (${formatConfidence(CONFIDENCE)}).`; } function formatConfidence(fraction: number): string { diff --git a/packages/benchmark/src/runReport/summary.ts b/packages/benchmark/src/runReport/summary.ts index f1472fed3e..5c7655c204 100644 --- a/packages/benchmark/src/runReport/summary.ts +++ b/packages/benchmark/src/runReport/summary.ts @@ -32,8 +32,6 @@ export interface BenchmarkSummary { against: string; change: Change; severity: Severity; - /** The level of both intervals, `0.95` or stricter where a change can raise an alarm. */ - confidence: number; /** `subject − against`, in the metric's unit. */ absolute: Interval; /** The same, as a percentage of `against`. */ @@ -67,12 +65,11 @@ export function summarizeRun(report: BenchmarkRunReport): RunSummary { Object.entries(variants).map(([variant, summary]) => [variant, summary.median]), ), comparisons: comparisons.map( - ({ subject, against, change, severity, confidence, absolute, relative }) => ({ + ({ subject, against, change, severity, absolute, relative }) => ({ subject, against, change, severity, - confidence, absolute, relative, }), diff --git a/packages/benchmark/src/sampling.test.ts b/packages/benchmark/src/sampling.test.ts index e5d8932f27..a62ffeb4d7 100644 --- a/packages/benchmark/src/sampling.test.ts +++ b/packages/benchmark/src/sampling.test.ts @@ -130,15 +130,20 @@ describe('differencesResolved', () => { expect(differencesResolved(analysis, [0])).toBe(true); }); - it('judges every metric when none alarms', () => { + it('stops at once when nothing can alarm', () => { const analysis = analyzeBenchmark( { render: { kind: 'scalar' }, paint: { kind: 'scalar' } }, - benchmarkOf( - { current: clearlySlower, baseline: noisy }, - { current: sameish, baseline: noisy }, - ), + benchmarkOf({ current: sameish, baseline: noisy }), ); - expect(differencesResolved(analysis, [0])).toBe(false); + expect(differencesResolved(analysis, [0])).toBe(true); + }); + + it('stops at once for a compare(), whose differences never alarm', () => { + const analysis = analyzeBenchmark(alarmedRender, { + ...benchmarkOf({ current: sameish, baseline: noisy }), + kind: 'compare', + }); + expect(differencesResolved(analysis, [0])).toBe(true); }); it('resolves a small difference against a wider horizon', () => { diff --git a/packages/benchmark/src/sampling.ts b/packages/benchmark/src/sampling.ts index 8988a7155f..65166e8701 100644 --- a/packages/benchmark/src/sampling.ts +++ b/packages/benchmark/src/sampling.ts @@ -9,8 +9,8 @@ export interface SamplingOptions { /** Rounds measured before deciding whether to continue. Defaults to 50. */ sampleSize?: number; /** - * Minutes to keep sampling after `sampleSize` rounds while a difference is unresolved. `0` - * measures exactly `sampleSize` rounds. Defaults to 3. + * Minutes to keep sampling after `sampleSize` rounds while a difference that can raise an alarm is + * unresolved. `0` measures exactly `sampleSize` rounds. Defaults to 3. */ timeout?: number; /** @@ -20,8 +20,8 @@ export interface SamplingOptions { * than 5%. A `0%` horizon tests whether there is any change at all: identical builds never resolve * against it, and checking again after every round makes a false finding likelier. * - * Judged on the metrics that carry an alarm, the ones a benchmark cares about most, whether or - * not its kind lets them raise one; on every metric when none carries one. + * Judged only on differences that can raise an alarm: an alarmed metric of a `baseline` benchmark. + * A benchmark with none, a `compare()` among them, stops at `sampleSize`. */ autoSampleConditions?: string[]; } @@ -88,16 +88,16 @@ export function isResolved(interval: { low: number; high: number }, horizons: nu } /** - * Whether sampling a benchmark can stop: every comparison it reports has resolved against the - * horizons, on the metrics that carry an alarm, or on every metric when none does. + * Whether sampling a benchmark can stop: every difference that can raise an alarm has resolved + * against the horizons. The others are only read, so they never keep a run going. */ export function differencesResolved( analysis: { metrics: MetricComparisons[] }, horizons: number[], ): boolean { - const alarmed = analysis.metrics.filter((metric) => metric.definition.alarm !== undefined); - const judged = alarmed.length > 0 ? alarmed : analysis.metrics; - return judged.every((metric) => - metric.comparisons.every((comparison) => isResolved(comparison.relative, horizons)), + return analysis.metrics.every((metric) => + metric.comparisons.every( + (comparison) => comparison.alarm === null || isResolved(comparison.relative, horizons), + ), ); } From 5b3253dcbbbccf0f826187a0e7ccb22da071e797 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:42:36 +0200 Subject: [PATCH 141/173] [benchmark] Let each benchmark set its own alarm; sample until it settles A benchmark can override a metric's alarm for itself: reactBenchmark's alarm sets render and bench:paint's band (or false never to alarm), and benchmark() takes the same per metric as alarms. The report carries these per benchmark and the analysis merges them over the metric's, so the PR comment, the dashboard and the A/A check all follow. Sampling now settles each alarmed difference against its own error band rather than a shared horizon, so autoSampleConditions and --auto-sample-conditions are gone. Version 2 alarms no longer carry warn. --- packages/benchmark/README.md | 58 ++++++++------ packages/benchmark/src/cli/index.ts | 10 +-- packages/benchmark/src/page/index.ts | 8 +- packages/benchmark/src/page/page.tsx | 51 ++++++++++--- .../benchmark/src/page/reactBenchmark.tsx | 22 ++++-- .../src/runReport/analyzeRun.test.ts | 44 ++++++++++- .../benchmark/src/runReport/analyzeRun.ts | 42 +++++++--- packages/benchmark/src/runReport/schema.ts | 16 ++-- .../benchmark/src/runner/runInterleaved.ts | 49 ++++++------ packages/benchmark/src/sampling.test.ts | 75 ++++++------------ packages/benchmark/src/sampling.ts | 76 ++++++------------- packages/benchmark/src/types.ts | 9 +-- test/benchmark/src/_shared/sampling.ts | 8 +- 13 files changed, 266 insertions(+), 202 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index d1ff2d4def..a544fb5b04 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -224,14 +224,14 @@ You can also `record()` or `time()` from inside a `benchmark()` render function - `format` — an [`Intl.NumberFormatOptions`](https://developer.mozilla.org/en-US/docs/Web/API/Intl/NumberFormat/NumberFormat) object used to display the value. - `alarm` — opts the metric into regression flagging. Omit it and the metric is informational (its diff is shown but never flagged). Holds: - `direction` — `'lowerIsBetter'` (default) or `'higherIsBetter'`. - - `warn` — softer band, for version 1 reports; a regression past it is flagged as a warning. - - `error` — harder band; a regression past it is flagged as an error. With only `warn` set there is no error band. + - `warn` — softer band, for version 1 reports only; a regression past it is flagged as a warning. + - `error` — harder band; a regression past it is flagged as an error. - Bands are relative fractions for scalar metrics (`0.1` = 10%) and absolute count deltas for discrete metrics (`1`, `2`). Either band is optional. How a regression is judged depends on which report the repository uploads: - **Version 1** (this Vitest reporter) compares the two runs' means. A scalar metric's change must pass a band; with neither band set, `error` is the dashboard's ±20% noise band. A discrete metric flags any change in its mean. -- **Version 2** ([the `benchmark` CLI](#the-benchmark-cli)) first needs the metric's confidence interval to lie wholly on the worse side; the `error` band then applies to the end of that interval nearest zero, so a change alarms only once it is confidently that large. With neither band, every confirmed change for the worse alarms; with only `warn`, none does. `warn` judges nothing: the dashboard colours every confirmed change for the worse, and the PR comment shows only alarms. +- **Version 2** ([the `benchmark` CLI](#the-benchmark-cli)) first needs the metric's confidence interval to lie wholly on the worse side; the `error` band then applies to the end of that interval nearest zero, so a change alarms only once it is confidently that large. Without an `error` band, every confirmed change for the worse alarms. Version 2 has no `warn`: the dashboard colours every confirmed change for the worse, and the PR comment shows only alarms. A benchmark can override a metric's alarm for itself; see [Choosing what alarms](#choosing-what-alarms). Alarms are evaluated against the baseline when the PR comment is generated, not during the local `vitest run` — a regression never fails the test suite locally. In a version 1 PR comment, `error`-band regressions surface as failures and `warn`-band regressions as warnings. @@ -440,33 +440,44 @@ which costs the results precision. ### Sampling How many rounds a benchmark is measured for adapts to its results, the way tachometer's -auto-sampling does and under tachometer's names. After `sampleSize` rounds, rounds keep being added -while any difference is unresolved against an `autoSampleConditions` horizon, for up to `timeout` -minutes. A difference is resolved against a horizon once its confidence interval lies entirely on -one side of it; `'10%'` stands for both `'-10%'` and `'+10%'`. The metrics that alarm decide, or every -metric when none does. - -| Option | Default | Meaning | -| :--------------------- | :------- | :------------------------------------------------------------------------------ | -| `sampleSize` | `50` | Rounds measured before deciding whether to continue | -| `timeout` | `3` | Minutes to keep sampling while a difference is unresolved | -| `autoSampleConditions` | `['5%']` | Horizons to resolve: by default, until each change is larger or smaller than 5% | +auto-sampling does. After `sampleSize` rounds, rounds keep being added while a difference that can +raise an alarm is unsettled, for up to `timeout` minutes. A difference is settled once its interval +lies entirely past its alarm's `error` band or entirely within it: known to alarm, or known not to. +An alarm without a band settles against zero, so an unchanged benchmark measured with one samples +until its timeout. Differences that can't alarm, in a `compare()` or a metric without an alarm, never +keep a run going. + +| Option | Default | Meaning | +| :----------- | :------ | :---------------------------------------------------------------------- | +| `sampleSize` | `50` | Rounds measured before deciding whether to continue | +| `timeout` | `3` | Minutes to keep sampling while an alarmed difference is still unsettled | They are set per benchmark: the last argument of `benchmark()` and `compare()`, and among `reactBenchmark()`'s options. A compared case is sampled as its `compare()` asks, so setting its own -is an error. `--sample-size`, `--timeout` and `--auto-sample-conditions` set them for the benchmarks -that don't set their own, as Vitest's `--testTimeout` does for tests: what a benchmark file says -always wins. +is an error. `--sample-size` and `--timeout` set them for the benchmarks that don't set their own, as +Vitest's `--testTimeout` does for tests: what a benchmark file says always wins. ```tsx -reactBenchmark('mount', () => , { timeout: 1, autoSampleConditions: ['10%'] }); +reactBenchmark('mount', () => , { timeout: 1 }); compare('scatter', [ours, other], { sampleSize: 100 }); ``` -A `'0%'` horizon asks whether there is any change at all. Two builds that perform the same never -resolve against it, so an unchanged benchmark samples until its timeout, and checking again after -every round makes a false finding likelier the longer it goes. +### Choosing what alarms + +`render` and `bench:paint` alarm once a change is confidently 5% worse. A `reactBenchmark()` can set +its own band for both with `alarm`: a lower one for a benchmark that matters more, a higher one for +one that matters less, or `false` for one that never alarms. Sampling follows the band, so a tighter +band also samples longer. + +```tsx +reactBenchmark('Data grid scroll', () => , scroll, { alarm: { error: 0.03 } }); +reactBenchmark('Docs demo mount', () => , { alarm: { error: 0.15 } }); +reactBenchmark('Experimental', () => , { alarm: false }); +``` + +`benchmark()` takes the same per metric, as `alarms`: `{ alarms: { 'json:parse': { error: 0.1 } } }`. +A benchmark's alarm is merged over the metric's, and a `name#id` sub-series follows its base metric. ### The report @@ -488,8 +499,9 @@ benchmark --baseline "v$(npm view @mui/material version)" --upload --timeline re - `analyzeRun` from `@mui/internal-benchmark/runReport` draws every conclusion from it: a confidence interval on the paired difference per metric, a change (`better`, `worse`, `no change detected`, or `unchanged` when every round measured the same), and a severity from the metric's alarm. -- `reactBenchmark()`'s `render` and `bench:paint` alarm once confidently 5% worse; `render:count` - and the per-phase split are informational; every other metric brings its own alarm. +- `reactBenchmark()`'s `render` and `bench:paint` alarm once confidently 5% worse, unless the + benchmark sets its own `alarm`; `render:count` and the per-phase split are informational; every + other metric brings its own alarm. - Tables list the baseline before the current build, so a row reads old to new. The pull request comment is one line unless something alarmed — the count, improvements on alarmed metrics and a link — and then tables only the rows that alarmed; the terminal and the dashboard show every diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index 1747d388c0..ad82cd53ce 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -18,7 +18,6 @@ import type { RunBenchmarksOptions } from '../runner/runBenchmarks'; type Args = Omit & { sampleSize?: number; timeout?: number; - autoSampleConditions?: string[]; }; const runCommand: CommandModule<{}, Args> = { @@ -83,13 +82,7 @@ const runCommand: CommandModule<{}, Args> = { .option('timeout', { type: 'number', describe: - "Minutes to keep sampling while unresolved, for benchmarks that don't set their own (default 3)", - }) - .option('auto-sample-conditions', { - type: 'string', - array: true, - describe: - "Horizons to resolve, such as 5% or 10%, for benchmarks that don't set their own (default 5%)", + "Minutes to keep sampling while an alarmed difference is unsettled against its band, for benchmarks that don't set their own (default 3)", }), handler: async (argv) => { if (argv.reporter === 'json') { @@ -114,7 +107,6 @@ const runCommand: CommandModule<{}, Args> = { sampling: { sampleSize: argv.sampleSize, timeout: argv.timeout, - autoSampleConditions: argv.autoSampleConditions, }, }); }, diff --git a/packages/benchmark/src/page/index.ts b/packages/benchmark/src/page/index.ts index 8567725861..009565591d 100644 --- a/packages/benchmark/src/page/index.ts +++ b/packages/benchmark/src/page/index.ts @@ -2,7 +2,13 @@ export * from '../publicApi'; export { benchmark, compare, markPageReady } from './page'; -export type { BenchmarkCase, BenchmarkContext, BenchmarkRun, BenchPage } from './page'; +export type { + BenchmarkCase, + BenchmarkCaseOptions, + BenchmarkContext, + BenchmarkRun, + BenchPage, +} from './page'; export { reactBenchmark } from './reactBenchmark'; export type { ReactBenchmarkOptions } from './reactBenchmark'; export type { SamplingOptions } from '../sampling'; diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 09d7e2aa56..d90a8dfd48 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -5,6 +5,7 @@ import { seriesName, setMetricRecorder } from '../metricCore'; import { hasSampling, pickSampling, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { MetricDefinition } from '../types'; +import type { RunMetricAlarm } from '../runReport/schema'; // The page runtime the `benchmark` CLI measures. `benchmark()` registers a case, and the runner calls // `window.benchmarkPage.sample()` to run one iteration of it at a time. What a case measures is @@ -30,25 +31,36 @@ export interface BenchmarkCase { readonly name: string; } -/** Every case the file registered, by name, with its sampling options as given. */ -const cases = new Map(); +export interface BenchmarkCaseOptions extends SamplingOptions { + /** + * This benchmark's own alarms, by metric name, in place of the metric's: its fields override the + * metric's alarm — a lower `error` band for a benchmark that matters more, a higher one for one + * that matters less — and `false` means the metric never alarms here. A `name#id` sub-series + * follows its base metric. + */ + alarms?: Record; +} + +/** Every case the file registered, by name, with its options as given. */ +const cases = new Map(); /** * Registers a benchmark case. The runner calls `run` once per sample, each time in a freshly loaded - * page, for as many rounds as `sampling` asks. A case on its own is measured across the builds; one passed to - * `compare()` is measured against the other cases there instead, sampled as the `compare()` asks. + * page, for as many rounds as `options` asks. A case on its own is measured across the builds; one + * passed to `compare()` is measured against the other cases there instead, sampled as the + * `compare()` asks, and never alarms. */ export function benchmark( name: string, run: BenchmarkRun, - sampling: SamplingOptions = {}, + options: BenchmarkCaseOptions = {}, ): BenchmarkCase { if (cases.has(name)) { throw new Error(`Two benchmarks share the name "${name}". Benchmark names must be unique.`); } // Resolved here only to reject invalid options where they were written. - resolveSampling(sampling); - cases.set(name, { run, sampling }); + resolveSampling(options); + cases.set(name, { run, options }); return { name }; } @@ -80,7 +92,7 @@ export function compare( // Resolved here only to reject invalid options where they were written. resolveSampling(sampling); for (const benchCase of compared) { - const own = cases.get(benchCase.name)?.sampling; + const own = cases.get(benchCase.name)?.options; if (own && hasSampling(own)) { throw new Error( `"${benchCase.name}" sets its own sampling, but a compared case is sampled as its ` + @@ -122,8 +134,15 @@ setMetricRecorder((metric, value, options) => { }); export interface BenchPage { - /** The cases measured across the builds: every case no `compare()` took. */ - cases: Array<{ name: string; sampling: SamplingOptions }>; + /** + * The cases measured across the builds: every case no `compare()` took. `alarms` is the case's own, + * with `null` for a metric that never alarms there. + */ + cases: Array<{ + name: string; + sampling: SamplingOptions; + alarms?: Record; + }>; /** The file's `compare()` calls, with their cases' names in order. */ comparisons: Array<{ name: string; cases: string[]; sampling: SamplingOptions }>; /** Every metric recorded so far, by name. */ @@ -188,7 +207,17 @@ export function markPageReady(): void { window.benchmarkPage = { cases: [...cases] .filter(([name]) => !comparisons.some((comparison) => comparison.cases.includes(name))) - .map(([name, { sampling }]) => ({ name, sampling: pickSampling(sampling) })), + .map(([name, { options }]) => ({ + name, + sampling: pickSampling(options), + ...(options.alarms + ? { + alarms: Object.fromEntries( + Object.entries(options.alarms).map(([metric, alarm]) => [metric, alarm || null]), + ), + } + : {}), + })), comparisons, metricDefinitions: () => Object.fromEntries(metricDefinitions), sample, diff --git a/packages/benchmark/src/page/reactBenchmark.tsx b/packages/benchmark/src/page/reactBenchmark.tsx index 787f5e6339..429964a824 100644 --- a/packages/benchmark/src/page/reactBenchmark.tsx +++ b/packages/benchmark/src/page/reactBenchmark.tsx @@ -4,16 +4,24 @@ import type { BenchmarkInteraction, CaseOptions } from '../caseRuntime'; import { DiscreteMetric } from '../DiscreteMetric'; import { ScalarMetric } from '../ScalarMetric'; import type { SamplingOptions } from '../sampling'; +import type { RunMetricAlarm } from '../runReport/schema'; import { benchmark } from './page'; import type { BenchmarkCase, BenchmarkContext } from './page'; -export interface ReactBenchmarkOptions extends CaseOptions, SamplingOptions {} +export interface ReactBenchmarkOptions extends CaseOptions, SamplingOptions { + /** + * This benchmark's own alarm for `render` and `bench:paint`, in place of the default 5% `error` + * band: a lower band for a benchmark that matters more, a higher one for one that matters less, + * or `false` for a benchmark that never alarms. + */ + alarm?: RunMetricAlarm | false; +} // Render time and paint both alarm: render is React running the components, while paint adds what // follows — the commit, injected styles, style recalculation and layout — so a styling or DOM -// regression can show in paint alone. Both alarm once a change for the worse is confidently 5%, the -// size sampling is built to settle; smaller confirmed changes show on the dashboard and are left to -// the timelines. The render count is informational: an extra render matters only through the time it +// regression can show in paint alone. Both alarm once a change for the worse is confidently 5%, +// unless the benchmark sets its own `alarm`; smaller confirmed changes show on the dashboard and are +// left to the timelines. The render count is informational: an extra render matters only through the time it // adds, which render time already measures. So is the per-phase split. const TIME_ALARM = { error: 0.05 }; const renderMetric = new ScalarMetric({ name: 'render', format: MILLISECONDS, alarm: TIME_ALARM }); @@ -72,5 +80,9 @@ export function reactBenchmark( paintMetric.record(end - start, id === undefined ? undefined : { id }); } }; - return benchmark(name, run, options); + const alarm = options?.alarm; + return benchmark(name, run, { + ...options, + alarms: alarm === undefined ? undefined : { render: alarm, [PAINT_METRIC_NAME]: alarm }, + }); } diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index 252055b193..a619267914 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -139,8 +139,6 @@ describe('compareSamples', () => { // The change is +10 on a mean of 105, about +9.5%. expect(compareWith({ error: 0.09 })).toBe('error'); expect(compareWith({ error: 0.2 })).toBe('none'); - expect(compareWith({ warn: 0.05, error: 0.2 })).toBe('none'); - expect(compareWith({ warn: 0.2 })).toBe('none'); expect(compareWith({})).toBe('error'); }); @@ -289,6 +287,48 @@ describe('analyzeRun', () => { }); }); +describe("a benchmark's own alarms", () => { + // Every round +10 on a mean of 105: about +9.5%, with no doubt about it. + const { base, shifted } = drifting(10); + function analyze(alarms: RunBenchmark['alarms'], metric = 'render') { + const [analysis] = analyzeRun( + reportOf( + [ + { + name: 'grid', + file: 'grid.bench.tsx', + kind: 'baseline', + variants: ['current', 'baseline'], + samples: { current: { [metric]: shifted }, baseline: { [metric]: base } }, + alarms, + }, + ], + { render: { kind: 'scalar', alarm: { error: 0.2 } } }, + ), + ); + return analysis.metrics[0].comparisons[0]; + } + + it("uses the metric's alarm when the benchmark sets none", () => { + expect(analyze(undefined).severity).toBe('none'); + }); + + it("overrides the metric's band", () => { + expect(analyze({ render: { error: 0.05 } })).toMatchObject({ + alarm: { error: 0.05 }, + severity: 'error', + }); + }); + + it('turns the alarm off with null', () => { + expect(analyze({ render: null })).toMatchObject({ alarm: null, severity: 'none' }); + }); + + it("applies a base metric's entry to its sub-series", () => { + expect(analyze({ render: { error: 0.05 } }, 'render#rows').severity).toBe('error'); + }); +}); + describe('findRegressions', () => { it('reports alarmed changes for the worse, on baseline benchmarks only', () => { const { base, shifted } = drifting(2); diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index 434a48e69a..df30d9c531 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -1,7 +1,11 @@ import { calculateMean, calculateSampleStdDev, quantile } from '../stats'; -import type { MetricAlarm } from '../types'; import { baseMetricName } from '../metricCore'; -import type { BenchmarkRunReport, RunBenchmark, RunMetricDefinition } from './schema'; +import type { + BenchmarkRunReport, + RunBenchmark, + RunMetricAlarm, + RunMetricDefinition, +} from './schema'; /** * The one analysis of a run report, shared by the runner's table, the pull request comment and the @@ -33,7 +37,7 @@ export interface SampleSummary { */ export type Change = 'better' | 'worse' | 'undetected' | 'unchanged'; -/** Whether a change raises its alarm. A metric's `warn` band applies to version 1 reports only. */ +/** Whether a change raises its alarm. */ export type Severity = 'error' | 'none'; export interface MetricComparison { @@ -45,7 +49,7 @@ export interface MetricComparison { * The alarm a change answers to: the metric's, on a `baseline` benchmark; `null` where no change * can raise one — a metric without an alarm, or a `compare()`, whose variants differ on purpose. */ - alarm: MetricAlarm | null; + alarm: RunMetricAlarm | null; /** `subject − against`, in the metric's unit: confidence interval of the mean difference. */ absolute: Interval; /** The same, as a percentage of `against`'s mean. */ @@ -205,6 +209,22 @@ function definitionOf( return metrics[metric] ?? metrics[baseMetricName(metric)] ?? DEFAULT_DEFINITION; } +/** + * The benchmark's own alarm for a metric, if it sets one: an alarm to merge over the metric's, or + * `null` when the metric never alarms in this benchmark. A sub-series follows its base metric. + */ +function alarmOverrideOf( + benchmark: RunBenchmark, + metric: string, +): RunMetricAlarm | null | undefined { + const { alarms } = benchmark; + if (!alarms) { + return undefined; + } + const key = metric in alarms ? metric : baseMetricName(metric); + return key in alarms ? alarms[key] : undefined; +} + /** * An interval in the unit the alarm bands use: a fraction for a scalar metric, which bands * relatively, and a count for a discrete one. @@ -220,15 +240,15 @@ function inAlarmUnits( /** * A change for the worse's severity: judged by how far it went at the least — the interval bound - * nearest zero — against the `error` band. Without bands, every confirmed change for the worse - * alarms; with only `warn`, none does, as `warn` judges nothing here. + * nearest zero — against the `error` band. Without a band, every confirmed change for the worse + * alarms. */ -function severityOf(change: Change, interval: Interval, alarm: MetricAlarm): Severity { +function severityOf(change: Change, interval: Interval, alarm: RunMetricAlarm): Severity { if (change !== 'worse') { return 'none'; } if (alarm.error === undefined) { - return alarm.warn === undefined ? 'error' : 'none'; + return 'error'; } const worseningAtLeast = alarm.direction === 'higherIsBetter' ? -interval.high : interval.low; return worseningAtLeast >= alarm.error ? 'error' : 'none'; @@ -319,12 +339,14 @@ function pairedMetricsOf( const { samples } = benchmark; const [reference] = benchmark.variants; return Object.keys(samples?.[reference] ?? {}).map((metric) => { - const definition = definitionOf(metrics, metric); + const override = alarmOverrideOf(benchmark, metric); + const shared = definitionOf(metrics, metric); + const definition = override ? { ...shared, alarm: { ...shared.alarm, ...override } } : shared; const sideOf = (name: string) => ({ name, values: samples?.[name]?.[metric] ?? [] }); return { metric, definition, - canAlarm: benchmark.kind === 'baseline', + canAlarm: benchmark.kind === 'baseline' && override !== null, pairs: pairsOf(benchmark).map(({ subject, against }) => ({ subject: sideOf(subject), against: sideOf(against), diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts index f0279fcd79..80dd2cf947 100644 --- a/packages/benchmark/src/runReport/schema.ts +++ b/packages/benchmark/src/runReport/schema.ts @@ -15,14 +15,13 @@ import type { MetricDefinition } from '../types'; */ /** - * The same shape a metric's `alarm` config has. `error` is the smallest change that raises the alarm - * — a relative fraction (`0.1` = 10%) for a scalar metric, an absolute count for a discrete one; - * without it, any resolved change for the worse does. `warn` is carried for version 1 reports and - * judges nothing here. + * A metric's alarm, as version 2 judges it. `error` is the smallest change that raises the alarm — a + * relative fraction (`0.1` = 10%) for a scalar metric, an absolute count for a discrete one; without + * it, any confirmed change for the worse does. A metric config's `warn` is for version 1 reports and + * isn't carried here. */ const alarmSchema = z.object({ direction: z.enum(['lowerIsBetter', 'higherIsBetter']).optional(), - warn: z.number().min(0).optional(), error: z.number().min(0).optional(), }); @@ -57,6 +56,12 @@ const benchmarkSchema = z.object({ variants: z.array(z.string()), /** Per variant, per metric: one value per round, round-aligned across variants and metrics. */ samples: z.record(z.string(), z.record(z.string(), z.array(z.number()))).optional(), + /** + * This benchmark's own alarms, by metric name, in place of the metric's: its fields override the + * metric's alarm, and `null` means the metric never alarms here. A `name#id` sub-series follows + * its base metric's entry. + */ + alarms: z.record(z.string(), alarmSchema.nullable()).optional(), /** * How sampling went: rounds past `sampleSize` were added while a difference was unresolved, and * `timedOut` says it was still unresolved when the timeout stopped it. @@ -120,4 +125,5 @@ export const benchmarkRunUploadSchema = z.object({ export type BenchmarkRunReport = z.infer; export type BenchmarkRunUpload = z.infer; export type RunMetricDefinition = z.infer; +export type RunMetricAlarm = z.infer; export type RunBenchmark = z.infer; diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index 4495299fa9..e155a3788f 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -5,7 +5,7 @@ import type { Browser, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; import { compareBenchmark } from '../runReport'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; -import { differencesResolved, parseHorizons, pickSampling, resolveSampling } from '../sampling'; +import { differencesResolved, pickSampling, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { BenchFile } from './benchFiles'; import type { ResolvedRef } from './refs'; @@ -170,7 +170,7 @@ interface CaseResult { metrics: Record; } -export type BenchmarkEntry = Pick; +export type BenchmarkEntry = Pick; /** A benchmark to measure: its variants' slots, sampled as its options ask. */ export interface MeasuredBenchmark { @@ -185,7 +185,7 @@ export type PlannedBenchmark = MeasuredBenchmark | { entry: BenchmarkEntry; erro /** * Measures one benchmark in a tab of its own, loading each slot's page before sampling it. Every * round samples every slot once, in a shuffled order. After `sampleSize` rounds it keeps adding - * rounds while a difference is unresolved against the horizons, until the timeout. A failure is + * rounds while a difference that can alarm is unsettled against its band, until the timeout. A failure is * reported as the benchmark's error. */ async function runBenchmark( @@ -193,8 +193,7 @@ async function runBenchmark( { entry, slots, sampling }: MeasuredBenchmark, ): Promise { console.log(chalk.cyan(`\nRunning "${entry.name}" (${entry.file})…`)); - const { sampleSize, timeout, autoSampleConditions } = sampling; - const horizons = parseHorizons(autoSampleConditions); + const { sampleSize, timeout } = sampling; let opened: Page | undefined; try { const page = await openBenchTab(browser); @@ -226,8 +225,7 @@ async function runBenchmark( const definitions = metrics ?? {}; const benchmarkOf = (): RunBenchmark => ({ ...entry, samples: samplesOf(slots, rounds) }); - const isSettled = () => - differencesResolved(compareBenchmark(definitions, benchmarkOf()), horizons); + const isSettled = () => differencesResolved(compareBenchmark(definitions, benchmarkOf())); const deadline = Date.now() + timeout * 60_000; let resolved = isSettled(); @@ -290,23 +288,26 @@ async function planBenchFile( const [reference] = await listBenchPages(browser, urls.slice(0, 1)); const others = reference.cases.length > 0 ? await listBenchPages(browser, urls.slice(1)) : []; - const planned: PlannedBenchmark[] = reference.cases.map(({ name: caseName, sampling }) => { - const entry: BenchmarkEntry = { - name: caseName, - file: benchFile.file, - kind: 'baseline', - variants, - }; - const missing = others.findIndex( - (listing) => !listing.cases.some((benchCase) => benchCase.name === caseName), - ); - if (missing !== -1) { - // A case added by the change under test has nothing to compare with. - return { entry, error: `"${caseName}" does not exist in [${variants[missing + 1]}].` }; - } - const slots = urls.map((url, index) => ({ variant: variants[index], url, caseName })); - return { entry, slots, sampling: samplingOf(sampling, options.sampling) }; - }); + const planned: PlannedBenchmark[] = reference.cases.map( + ({ name: caseName, sampling, alarms }) => { + const entry: BenchmarkEntry = { + name: caseName, + file: benchFile.file, + kind: 'baseline', + variants, + ...(alarms ? { alarms } : {}), + }; + const missing = others.findIndex( + (listing) => !listing.cases.some((benchCase) => benchCase.name === caseName), + ); + if (missing !== -1) { + // A case added by the change under test has nothing to compare with. + return { entry, error: `"${caseName}" does not exist in [${variants[missing + 1]}].` }; + } + const slots = urls.map((url, index) => ({ variant: variants[index], url, caseName })); + return { entry, slots, sampling: samplingOf(sampling, options.sampling) }; + }, + ); for (const { name, cases, sampling } of reference.comparisons) { const entry: BenchmarkEntry = { name, file: benchFile.file, kind: 'compare', variants: cases }; diff --git a/packages/benchmark/src/sampling.test.ts b/packages/benchmark/src/sampling.test.ts index a62ffeb4d7..0d86bcc36b 100644 --- a/packages/benchmark/src/sampling.test.ts +++ b/packages/benchmark/src/sampling.test.ts @@ -1,33 +1,7 @@ import { describe, expect, it } from 'vitest'; import { analyzeBenchmark } from './runReport/analyzeRun'; import type { RunBenchmark, RunMetricDefinition } from './runReport'; -import { - DEFAULT_SAMPLING, - differencesResolved, - isResolved, - parseHorizons, - resolveSampling, -} from './sampling'; - -describe('parseHorizons', () => { - it('expands an unsigned percentage to both signs', () => { - expect(parseHorizons(['10%'])).toEqual([-10, 10]); - }); - - it('keeps a signed percentage to its own side', () => { - expect(parseHorizons(['+5%', '-2.5%'])).toEqual([5, -2.5]); - }); - - it('reads zero as a single horizon', () => { - expect(parseHorizons(['0%'])).toEqual([0]); - }); - - it('rejects anything but a percentage', () => { - expect(() => parseHorizons(['5ms'])).toThrow(/Invalid auto-sample condition "5ms"/); - expect(() => parseHorizons(['%'])).toThrow(/Invalid auto-sample condition "%"/); - expect(() => parseHorizons(['ten%'])).toThrow(/Invalid auto-sample condition "ten%"/); - }); -}); +import { DEFAULT_SAMPLING, differencesResolved, isResolved, resolveSampling } from './sampling'; describe('isResolved', () => { it('is resolved when the interval lies on one side of every horizon', () => { @@ -48,16 +22,14 @@ describe('isResolved', () => { describe('resolveSampling', () => { it('fills in a default for every option left out or undefined', () => { - expect(resolveSampling({ timeout: 1, autoSampleConditions: undefined })).toEqual({ + expect(resolveSampling({ timeout: 1, sampleSize: undefined })).toEqual({ ...DEFAULT_SAMPLING, timeout: 1, }); }); it('accepts the documented ranges', () => { - expect(() => - resolveSampling({ sampleSize: 2, timeout: 0, autoSampleConditions: ['1%'] }), - ).not.toThrow(); + expect(() => resolveSampling({ sampleSize: 2, timeout: 0 })).not.toThrow(); }); it('rejects a sample size that cannot form an interval', () => { @@ -68,12 +40,6 @@ describe('resolveSampling', () => { it('rejects a negative timeout', () => { expect(() => resolveSampling({ timeout: -1 })).toThrow(/Invalid timeout -1/); }); - - it('rejects an invalid condition', () => { - expect(() => resolveSampling({ autoSampleConditions: ['5ms'] })).toThrow( - /Invalid auto-sample condition "5ms"/, - ); - }); }); describe('differencesResolved', () => { @@ -103,20 +69,20 @@ describe('differencesResolved', () => { paint: { kind: 'scalar' }, }; - it('stops once every alarmed difference is clear of the horizon', () => { + it('stops once an alarmed difference without a band is known to be a change', () => { const analysis = analyzeBenchmark( alarmedRender, benchmarkOf({ current: clearlySlower, baseline: noisy }), ); - expect(differencesResolved(analysis, [0])).toBe(true); + expect(differencesResolved(analysis)).toBe(true); }); - it('keeps sampling while an alarmed difference straddles the horizon', () => { + it('keeps sampling while an alarmed difference without a band could still be none', () => { const analysis = analyzeBenchmark( alarmedRender, benchmarkOf({ current: sameish, baseline: noisy }), ); - expect(differencesResolved(analysis, [0])).toBe(false); + expect(differencesResolved(analysis)).toBe(false); }); it('ignores metrics that do not alarm when some do', () => { @@ -127,7 +93,7 @@ describe('differencesResolved', () => { { current: sameish, baseline: noisy }, ), ); - expect(differencesResolved(analysis, [0])).toBe(true); + expect(differencesResolved(analysis)).toBe(true); }); it('stops at once when nothing can alarm', () => { @@ -135,7 +101,7 @@ describe('differencesResolved', () => { { render: { kind: 'scalar' }, paint: { kind: 'scalar' } }, benchmarkOf({ current: sameish, baseline: noisy }), ); - expect(differencesResolved(analysis, [0])).toBe(true); + expect(differencesResolved(analysis)).toBe(true); }); it('stops at once for a compare(), whose differences never alarm', () => { @@ -143,14 +109,23 @@ describe('differencesResolved', () => { ...benchmarkOf({ current: sameish, baseline: noisy }), kind: 'compare', }); - expect(differencesResolved(analysis, [0])).toBe(true); + expect(differencesResolved(analysis)).toBe(true); }); - it('resolves a small difference against a wider horizon', () => { - const analysis = analyzeBenchmark( - alarmedRender, - benchmarkOf({ current: sameish, baseline: noisy }), - ); - expect(differencesResolved(analysis, [-50, 50])).toBe(true); + it("settles against the alarm's error band", () => { + const banded = (error: number) => ({ render: { kind: 'scalar' as const, alarm: { error } } }); + // About +100%: clearly within a 200% band, still straddling a 100% one. + const slower = benchmarkOf({ current: clearlySlower, baseline: noisy }); + expect(differencesResolved(analyzeBenchmark(banded(2), slower))).toBe(true); + expect(differencesResolved(analyzeBenchmark(banded(1), slower))).toBe(false); + }); + + it("follows the benchmark's own alarm over the metric's", () => { + const unsure = benchmarkOf({ current: sameish, baseline: noisy }); + const withAlarms = (alarms: RunBenchmark['alarms']) => + differencesResolved(analyzeBenchmark(alarmedRender, { ...unsure, alarms })); + expect(withAlarms(undefined)).toBe(false); + expect(withAlarms({ render: { error: 0.5 } })).toBe(true); + expect(withAlarms({ render: null })).toBe(true); }); }); diff --git a/packages/benchmark/src/sampling.ts b/packages/benchmark/src/sampling.ts index 65166e8701..334a235e7b 100644 --- a/packages/benchmark/src/sampling.ts +++ b/packages/benchmark/src/sampling.ts @@ -1,70 +1,39 @@ import type { MetricComparisons } from './runReport/analyzeRun'; /** - * How many rounds a benchmark is sampled for: tachometer's knobs, under tachometer's names. After - * `sampleSize` rounds, sampling continues while any difference is still unresolved against an - * `autoSampleConditions` horizon, for up to `timeout` minutes. + * How many rounds a benchmark is sampled for, under tachometer's names. After `sampleSize` rounds, + * sampling continues while a difference that can raise an alarm is still unsettled against its + * alarm's `error` band, for up to `timeout` minutes. */ export interface SamplingOptions { /** Rounds measured before deciding whether to continue. Defaults to 50. */ sampleSize?: number; /** * Minutes to keep sampling after `sampleSize` rounds while a difference that can raise an alarm is - * unresolved. `0` measures exactly `sampleSize` rounds. Defaults to 3. + * unsettled: its interval still straddles the alarm's `error` band, or zero for an alarm without + * one. `0` measures exactly `sampleSize` rounds. Defaults to 3. */ timeout?: number; - /** - * Relative differences to resolve before `timeout`. A difference is resolved against a horizon - * once its confidence interval lies entirely on one side of it. `'10%'` stands for both `'-10%'` - * and `'+10%'`. Defaults to `['5%']`: sample until each change is known to be larger or smaller - * than 5%. A `0%` horizon tests whether there is any change at all: identical builds never resolve - * against it, and checking again after every round makes a false finding likelier. - * - * Judged only on differences that can raise an alarm: an alarmed metric of a `baseline` benchmark. - * A benchmark with none, a `compare()` among them, stops at `sampleSize`. - */ - autoSampleConditions?: string[]; } export const DEFAULT_SAMPLING: Required = { sampleSize: 50, timeout: 3, - autoSampleConditions: ['5%'], }; -/** Parses `autoSampleConditions` into signed horizons, in percent. */ -export function parseHorizons(conditions: string[]): number[] { - return conditions.flatMap((condition) => { - const trimmed = condition.trim(); - const value = Number(trimmed.slice(0, -1)); - if (!trimmed.endsWith('%') || trimmed === '%' || Number.isNaN(value)) { - throw new Error( - `Invalid auto-sample condition "${condition}": expected a percentage, like "5%", "+5%" or "-5%".`, - ); - } - const signed = trimmed.startsWith('+') || trimmed.startsWith('-'); - return signed || value === 0 ? [value] : [-value, value]; - }); -} - /** * Checks sampling options and fills in the defaults. Called when a case registers, so a mistake fails * the page rather than a round, and the runner gets complete options. */ export function resolveSampling(options: SamplingOptions): Required { - const { - sampleSize = DEFAULT_SAMPLING.sampleSize, - timeout = DEFAULT_SAMPLING.timeout, - autoSampleConditions = DEFAULT_SAMPLING.autoSampleConditions, - } = options; + const { sampleSize = DEFAULT_SAMPLING.sampleSize, timeout = DEFAULT_SAMPLING.timeout } = options; if (!(Number.isInteger(sampleSize) && sampleSize >= 2)) { throw new Error(`Invalid sampleSize ${sampleSize}: expected an integer of at least 2.`); } if (!(timeout >= 0)) { throw new Error(`Invalid timeout ${timeout}: expected a number of minutes, 0 or more.`); } - parseHorizons(autoSampleConditions); - return { sampleSize, timeout, autoSampleConditions }; + return { sampleSize, timeout }; } /** Whether any sampling option is set. */ @@ -74,11 +43,9 @@ export function hasSampling(options: SamplingOptions): boolean { /** The sampling options `options` sets, leaving out any other key and any left undefined. */ export function pickSampling(options: SamplingOptions): SamplingOptions { - const { sampleSize, timeout, autoSampleConditions } = options; + const { sampleSize, timeout } = options; return Object.fromEntries( - Object.entries({ sampleSize, timeout, autoSampleConditions }).filter( - ([, value]) => value !== undefined, - ), + Object.entries({ sampleSize, timeout }).filter(([, value]) => value !== undefined), ); } @@ -88,16 +55,21 @@ export function isResolved(interval: { low: number; high: number }, horizons: nu } /** - * Whether sampling a benchmark can stop: every difference that can raise an alarm has resolved - * against the horizons. The others are only read, so they never keep a run going. + * Whether sampling a benchmark can stop: every difference that can raise an alarm is known to be past + * its `error` band or within it, either way — or, for an alarm without a band, known to be a change + * or not. The others are only read, so they never keep a run going. */ -export function differencesResolved( - analysis: { metrics: MetricComparisons[] }, - horizons: number[], -): boolean { - return analysis.metrics.every((metric) => - metric.comparisons.every( - (comparison) => comparison.alarm === null || isResolved(comparison.relative, horizons), - ), +export function differencesResolved(analysis: { metrics: MetricComparisons[] }): boolean { + return analysis.metrics.every(({ definition, comparisons }) => + comparisons.every(({ alarm, relative, absolute }) => { + if (alarm === null) { + return true; + } + // Bands are fractions on a scalar metric, whose relative interval is in percent, and counts on + // a discrete one. + const discrete = definition.kind === 'discrete'; + const band = (alarm.error ?? 0) * (discrete ? 1 : 100); + return isResolved(discrete ? absolute : relative, band === 0 ? [0] : [-band, band]); + }), ); } diff --git a/packages/benchmark/src/types.ts b/packages/benchmark/src/types.ts index e05a30674b..c6e5e0cf8a 100644 --- a/packages/benchmark/src/types.ts +++ b/packages/benchmark/src/types.ts @@ -74,15 +74,14 @@ export interface MetricAlarm { direction?: MetricDirection; /** * Softer band, for version 1 reports only: a regression past `warn` (but within `error`) is - * flagged as a warning. Version 2 colours every confirmed change for the worse on the dashboard. + * flagged as a warning. Version 2 reports don't carry it. * Scalar metrics: a relative fraction (`0.1` = 10%). Discrete metrics: an absolute count delta. */ warn?: number; /** - * Harder band: a regression past `error` is flagged as an error (the alarm). With only `warn` - * set there is no error band: version 1 reports warn, version 2 reports never alarm. With - * neither, version 1 reports use the dashboard's ±20% noise band and version 2 reports flag every - * confirmed change for the worse. + * Harder band: a regression past `error` is flagged as an error (the alarm). Without it, version 1 + * reports use the dashboard's ±20% noise band (or warn only, with `warn` set) and version 2 reports + * flag every confirmed change for the worse. * Scalar metrics: a relative fraction (`0.25` = 25%). Discrete metrics: an absolute count delta. */ error?: number; diff --git a/test/benchmark/src/_shared/sampling.ts b/test/benchmark/src/_shared/sampling.ts index 1dfd88ce8f..e21c0491e9 100644 --- a/test/benchmark/src/_shared/sampling.ts +++ b/test/benchmark/src/_shared/sampling.ts @@ -1,9 +1,7 @@ import type { SamplingOptions } from '@mui/internal-benchmark/page'; /** - * Sampling for this smoke-test harness, which should stay quick. Its baseline comparisons run - * identical code on both sides, so they never resolve against the default `0%` horizon and would - * sample until the timeout; a close `compare()` can take as long. A `10%` horizon settles as soon as - * a difference is known to be larger or smaller than that. + * Sampling for this smoke-test harness, which should stay quick: a benchmark whose alarmed + * difference doesn't settle against its band stops after a minute instead of three. */ -export const SMOKE_SAMPLING: SamplingOptions = { timeout: 1, autoSampleConditions: ['10%'] }; +export const SMOKE_SAMPLING: SamplingOptions = { timeout: 1 }; From 153dabd05876fd497663b2e03104afb0824e88ae Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:53:03 +0200 Subject: [PATCH 142/173] [benchmark] Default scalar alarms to a 5% band, and settle only the worse side A scalar metric's alarm without an error band settled against zero, so an unchanged benchmark measured with one sampled until the timeout; it now uses 5%, as render and paint do. A discrete count without a band still alarms on any confirmed change. Sampling settles only against the band's worse edge, since an improvement can't alarm. Also tidied after the per-benchmark alarms: each comparison carries whether it is settled, computed beside its severity, so the sampler reads it instead of re-deriving band units; false is the one 'never alarms' value from the page to the report; the effective alarm is resolved once; reactBenchmark keys its override off its metrics; compare() rejects cases that set their own alarms; and the leftovers of two confidence levels (meanInterval, confidence parameters, the footnote's argument) are gone. --- .../lib/benchmarkRun/buildMarkdownReport.ts | 2 +- .../src/views/BenchmarkRunDetails.tsx | 2 +- packages/benchmark/README.md | 13 +-- packages/benchmark/src/page/page.tsx | 40 ++++----- .../benchmark/src/page/reactBenchmark.tsx | 6 +- .../src/runReport/analyzeRun.test.ts | 61 +++++++------ .../benchmark/src/runReport/analyzeRun.ts | 86 +++++++++++-------- .../benchmark/src/runReport/format.test.ts | 2 +- packages/benchmark/src/runReport/format.ts | 4 +- packages/benchmark/src/runReport/schema.ts | 6 +- packages/benchmark/src/runner/printReport.ts | 2 +- .../benchmark/src/runner/runInterleaved.ts | 18 ++-- packages/benchmark/src/sampling.test.ts | 54 ++++++------ packages/benchmark/src/sampling.ts | 35 ++------ packages/benchmark/src/types.ts | 4 +- 15 files changed, 163 insertions(+), 172 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index f239c77dd8..7207b92d61 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -87,7 +87,7 @@ export function buildBenchmarkRunMarkdownReport( return table ? [`**${analysis.benchmark.name}**\n\n${table}`] : []; }); if (tables.length > 0) { - lines.push('', tables.join('\n\n'), '', `_${runReportFootnote(analyses)}_`); + lines.push('', tables.join('\n\n'), '', `_${runReportFootnote()}_`); } return lines.join('\n'); diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index 9b08150b70..c819772527 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -241,7 +241,7 @@ export default function BenchmarkRunDetails() { {report && ( - {runReportFootnote(analyses)} + {runReportFootnote()} {report.durationMs !== undefined && `Ran ${formatDuration(report.durationMs)} · `} diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index a544fb5b04..375e4f1c75 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -231,7 +231,7 @@ You can also `record()` or `time()` from inside a `benchmark()` render function How a regression is judged depends on which report the repository uploads: - **Version 1** (this Vitest reporter) compares the two runs' means. A scalar metric's change must pass a band; with neither band set, `error` is the dashboard's ±20% noise band. A discrete metric flags any change in its mean. -- **Version 2** ([the `benchmark` CLI](#the-benchmark-cli)) first needs the metric's confidence interval to lie wholly on the worse side; the `error` band then applies to the end of that interval nearest zero, so a change alarms only once it is confidently that large. Without an `error` band, every confirmed change for the worse alarms. Version 2 has no `warn`: the dashboard colours every confirmed change for the worse, and the PR comment shows only alarms. A benchmark can override a metric's alarm for itself; see [Choosing what alarms](#choosing-what-alarms). +- **Version 2** ([the `benchmark` CLI](#the-benchmark-cli)) first needs the metric's confidence interval to lie wholly on the worse side; the `error` band then applies to the end of that interval nearest zero, so a change alarms only once it is confidently that large. Without an `error` band, a scalar metric uses 5% and a discrete one alarms on any confirmed change in its count. Version 2 has no `warn`: the dashboard colours every confirmed change for the worse, and the PR comment shows only alarms. A benchmark can override a metric's alarm for itself; see [Choosing what alarms](#choosing-what-alarms). Alarms are evaluated against the baseline when the PR comment is generated, not during the local `vitest run` — a regression never fails the test suite locally. In a version 1 PR comment, `error`-band regressions surface as failures and `warn`-band regressions as warnings. @@ -442,9 +442,10 @@ which costs the results precision. How many rounds a benchmark is measured for adapts to its results, the way tachometer's auto-sampling does. After `sampleSize` rounds, rounds keep being added while a difference that can raise an alarm is unsettled, for up to `timeout` minutes. A difference is settled once its interval -lies entirely past its alarm's `error` band or entirely within it: known to alarm, or known not to. -An alarm without a band settles against zero, so an unchanged benchmark measured with one samples -until its timeout. Differences that can't alarm, in a `compare()` or a metric without an alarm, never +lies entirely past its alarm's `error` band on the worse side, or entirely short of it: known to +alarm, or known not to. Improvements never keep a run going. +A scalar metric's alarm without a band uses 5%; a discrete one's settles against zero, which an +unchanged count does at once. Differences that can't alarm, in a `compare()` or a metric without an alarm, never keep a run going. | Option | Default | Meaning | @@ -467,8 +468,8 @@ compare('scatter', [ours, other], { sampleSize: 100 }); `render` and `bench:paint` alarm once a change is confidently 5% worse. A `reactBenchmark()` can set its own band for both with `alarm`: a lower one for a benchmark that matters more, a higher one for -one that matters less, or `false` for one that never alarms. Sampling follows the band, so a tighter -band also samples longer. +one that matters less, or `false` for one that never alarms. `{}` keeps the band it would have had. +Sampling follows the band, so a tighter band also samples longer. ```tsx reactBenchmark('Data grid scroll', () => , scroll, { alarm: { error: 0.03 } }); diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index d90a8dfd48..8628d75e9b 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -2,7 +2,7 @@ import { collectGarbage } from '../gc'; import { createInput } from '../input'; import type { BenchmarkCdp, BenchmarkInput, CdpSend } from '../input'; import { seriesName, setMetricRecorder } from '../metricCore'; -import { hasSampling, pickSampling, resolveSampling } from '../sampling'; +import { pickSampling, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { MetricDefinition } from '../types'; import type { RunMetricAlarm } from '../runReport/schema'; @@ -41,8 +41,11 @@ export interface BenchmarkCaseOptions extends SamplingOptions { alarms?: Record; } -/** Every case the file registered, by name, with its options as given. */ -const cases = new Map(); +/** Every case the file registered, by name, with the sampling and alarms it set. */ +const cases = new Map< + string, + { run: BenchmarkRun; sampling: SamplingOptions; alarms?: BenchmarkCaseOptions['alarms'] } +>(); /** * Registers a benchmark case. The runner calls `run` once per sample, each time in a freshly loaded @@ -60,7 +63,7 @@ export function benchmark( } // Resolved here only to reject invalid options where they were written. resolveSampling(options); - cases.set(name, { run, options }); + cases.set(name, { run, sampling: pickSampling(options), alarms: options.alarms }); return { name }; } @@ -92,13 +95,19 @@ export function compare( // Resolved here only to reject invalid options where they were written. resolveSampling(sampling); for (const benchCase of compared) { - const own = cases.get(benchCase.name)?.options; - if (own && hasSampling(own)) { + const own = cases.get(benchCase.name); + if (own && Object.keys(own.sampling).length > 0) { throw new Error( `"${benchCase.name}" sets its own sampling, but a compared case is sampled as its ` + `compare() asks: pass the options to compare("${name}", …) instead.`, ); } + if (own?.alarms) { + throw new Error( + `"${benchCase.name}" sets its own alarms, but a compared case never alarms: its ` + + `variants differ on purpose.`, + ); + } const owner = comparisons.find((comparison) => comparison.cases.includes(benchCase.name)); if (owner) { throw new Error(`"${benchCase.name}" is already compared in "${owner.name}".`); @@ -134,14 +143,11 @@ setMetricRecorder((metric, value, options) => { }); export interface BenchPage { - /** - * The cases measured across the builds: every case no `compare()` took. `alarms` is the case's own, - * with `null` for a metric that never alarms there. - */ + /** The cases measured across the builds: every case no `compare()` took. */ cases: Array<{ name: string; sampling: SamplingOptions; - alarms?: Record; + alarms?: BenchmarkCaseOptions['alarms']; }>; /** The file's `compare()` calls, with their cases' names in order. */ comparisons: Array<{ name: string; cases: string[]; sampling: SamplingOptions }>; @@ -207,17 +213,7 @@ export function markPageReady(): void { window.benchmarkPage = { cases: [...cases] .filter(([name]) => !comparisons.some((comparison) => comparison.cases.includes(name))) - .map(([name, { options }]) => ({ - name, - sampling: pickSampling(options), - ...(options.alarms - ? { - alarms: Object.fromEntries( - Object.entries(options.alarms).map(([metric, alarm]) => [metric, alarm || null]), - ), - } - : {}), - })), + .map(([name, { sampling, alarms }]) => ({ name, sampling, ...(alarms ? { alarms } : {}) })), comparisons, metricDefinitions: () => Object.fromEntries(metricDefinitions), sample, diff --git a/packages/benchmark/src/page/reactBenchmark.tsx b/packages/benchmark/src/page/reactBenchmark.tsx index 429964a824..ebd0c8ba2a 100644 --- a/packages/benchmark/src/page/reactBenchmark.tsx +++ b/packages/benchmark/src/page/reactBenchmark.tsx @@ -5,6 +5,7 @@ import { DiscreteMetric } from '../DiscreteMetric'; import { ScalarMetric } from '../ScalarMetric'; import type { SamplingOptions } from '../sampling'; import type { RunMetricAlarm } from '../runReport/schema'; +import { DEFAULT_ERROR_BAND } from '../runReport/analyzeRun'; import { benchmark } from './page'; import type { BenchmarkCase, BenchmarkContext } from './page'; @@ -23,7 +24,7 @@ export interface ReactBenchmarkOptions extends CaseOptions, SamplingOptions { // unless the benchmark sets its own `alarm`; smaller confirmed changes show on the dashboard and are // left to the timelines. The render count is informational: an extra render matters only through the time it // adds, which render time already measures. So is the per-phase split. -const TIME_ALARM = { error: 0.05 }; +const TIME_ALARM = { error: DEFAULT_ERROR_BAND }; const renderMetric = new ScalarMetric({ name: 'render', format: MILLISECONDS, alarm: TIME_ALARM }); const renderCountMetric = new DiscreteMetric({ name: 'render:count' }); const paintMetric = new ScalarMetric({ @@ -83,6 +84,7 @@ export function reactBenchmark( const alarm = options?.alarm; return benchmark(name, run, { ...options, - alarms: alarm === undefined ? undefined : { render: alarm, [PAINT_METRIC_NAME]: alarm }, + alarms: + alarm === undefined ? undefined : { [renderMetric.name]: alarm, [paintMetric.name]: alarm }, }); } diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index a619267914..183321958e 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -1,12 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { - analyzeRun, - compareSamples, - findRegressions, - meanInterval, - median, - tCritical, -} from './analyzeRun'; +import { analyzeRun, compareSamples, findRegressions, median, tCritical } from './analyzeRun'; import type { BenchmarkAnalysis } from './analyzeRun'; import type { BenchmarkRunReport, RunBenchmark, RunMetricDefinition } from './schema'; @@ -52,18 +45,6 @@ describe('tCritical', () => { }); }); -describe('meanInterval', () => { - it('centers on the mean', () => { - const interval = meanInterval([1, 2, 3, 4, 5]); - expect((interval.low + interval.high) / 2).toBeCloseTo(3); - expect(interval.low).toBeLessThan(3); - }); - - it('is a point for a single value', () => { - expect(meanInterval([7])).toEqual({ low: 7, high: 7 }); - }); -}); - describe('median', () => { it('takes the middle value, or the mean of the middle two', () => { expect(median([3, 1, 2])).toBe(2); @@ -155,7 +136,7 @@ describe('compareSamples', () => { describe('analyzeRun', () => { it('compares the current build against the baseline', () => { - const { base, shifted } = drifting(2); + const { base, shifted } = drifting(10); const [analysis] = analyzeRun( reportOf( [ @@ -255,13 +236,6 @@ describe('analyzeRun', () => { baseline: { render: noisy.baseline }, }); - it('is 99% for a metric that can raise an alarm, and 95% otherwise', () => { - const [alarmed] = analyzeRun(reportOf([alone], { render: SCALAR })); - const [plain] = analyzeRun(reportOf([alone], { render: { kind: 'scalar' } })); - - expect(widthOf(alarmed) / widthOf(plain)).toBeCloseTo(tCritical(7, 0.99) / tCritical(7), 6); - }); - it("doesn't tighten with the number of benchmarks in the run", () => { const [single] = analyzeRun(reportOf([alone], { render: SCALAR })); const [crowded] = analyzeRun(reportOf([alone, other], { render: SCALAR })); @@ -320,8 +294,8 @@ describe("a benchmark's own alarms", () => { }); }); - it('turns the alarm off with null', () => { - expect(analyze({ render: null })).toMatchObject({ alarm: null, severity: 'none' }); + it('turns the alarm off with false', () => { + expect(analyze({ render: false })).toMatchObject({ alarm: null, severity: 'none' }); }); it("applies a base metric's entry to its sub-series", () => { @@ -329,9 +303,34 @@ describe("a benchmark's own alarms", () => { }); }); +describe('the default band', () => { + it('lets a scalar alarm without a band through under 5%, and alarms past it', () => { + const severityAt = (shift: number) => { + const { base, shifted } = drifting(shift); + return compareSamples( + { name: 'current', values: shifted }, + { name: 'baseline', values: base }, + { kind: 'scalar', alarm: {} }, + ).severity; + }; + // +3 and +10 on a mean of 105: about +2.9% and +9.5%, with no doubt about either. + expect(severityAt(3)).toBe('none'); + expect(severityAt(10)).toBe('error'); + }); + + it('alarms on any confirmed change in a discrete count without a band', () => { + const comparison = compareSamples( + { name: 'current', values: [6, 6, 6] }, + { name: 'baseline', values: [5, 5, 5] }, + { kind: 'discrete', alarm: {} }, + ); + expect(comparison.severity).toBe('error'); + }); +}); + describe('findRegressions', () => { it('reports alarmed changes for the worse, on baseline benchmarks only', () => { - const { base, shifted } = drifting(2); + const { base, shifted } = drifting(10); const samples = { render: shifted }; const analyses = analyzeRun( reportOf( diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index df30d9c531..2a961c8f60 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -46,8 +46,9 @@ export interface MetricComparison { /** The variant it is measured against. */ against: string; /** - * The alarm a change answers to: the metric's, on a `baseline` benchmark; `null` where no change - * can raise one — a metric without an alarm, or a `compare()`, whose variants differ on purpose. + * The alarm a change answers to: the metric's, with the benchmark's own merged over it, on a + * `baseline` benchmark; `null` where no change can raise one — a metric without an alarm, one the + * benchmark turned off, or a `compare()`, whose variants differ on purpose. */ alarm: RunMetricAlarm | null; /** `subject − against`, in the metric's unit: confidence interval of the mean difference. */ @@ -65,6 +66,13 @@ export interface MetricComparison { * units — a fraction for a scalar metric, a count for a discrete one. */ precision: number; + /** + * Whether more rounds could still change what the alarm says: settled once the interval lies + * wholly on one side of the alarm's `error` band on the worse side (of zero, without a band). An + * improvement can't alarm, so the better side never keeps a run going, and nothing does where + * nothing can alarm. + */ + settled: boolean; change: Change; severity: Severity; } @@ -85,6 +93,13 @@ export interface BenchmarkAnalysis { /** The confidence level of every interval. */ export const CONFIDENCE = 0.99; +/** + * The `error` band of a scalar metric's alarm that sets none: a change has to be confidently 5% + * worse to alarm, and sampling settles against 5%. A discrete metric's alarm without a band alarms + * on any confirmed change in its count. + */ +export const DEFAULT_ERROR_BAND = 0.05; + const LANCZOS = [ 0.99999999999980993, 676.5203681218851, -1259.1392167224028, 771.32342877765313, -176.61502916214059, 12.507343278686905, -0.13857109526572012, 9.9843695780195716e-6, @@ -174,20 +189,14 @@ export function tCritical(degreesOfFreedom: number, confidence = CONFIDENCE): nu } /** The confidence interval of a mean from `count` values with standard deviation `stdDev`. */ -function intervalAround(mean: number, stdDev: number, count: number, confidence: number): Interval { +function intervalAround(mean: number, stdDev: number, count: number): Interval { if (count < 2) { return { low: mean, high: mean }; } - const halfWidth = (tCritical(count - 1, confidence) * stdDev) / Math.sqrt(count); + const halfWidth = (tCritical(count - 1) * stdDev) / Math.sqrt(count); return { low: mean - halfWidth, high: mean + halfWidth }; } -/** The confidence interval of the mean. A single value is its own degenerate interval. */ -export function meanInterval(values: number[], confidence = CONFIDENCE): Interval { - const mean = calculateMean(values); - return intervalAround(mean, calculateSampleStdDev(values, mean), values.length, confidence); -} - export function median(values: number[]): number { return quantile( [...values].sort((left, right) => left - right), @@ -211,12 +220,12 @@ function definitionOf( /** * The benchmark's own alarm for a metric, if it sets one: an alarm to merge over the metric's, or - * `null` when the metric never alarms in this benchmark. A sub-series follows its base metric. + * `false` when the metric never alarms in this benchmark. A sub-series follows its base metric. */ function alarmOverrideOf( benchmark: RunBenchmark, metric: string, -): RunMetricAlarm | null | undefined { +): RunMetricAlarm | false | undefined { const { alarms } = benchmark; if (!alarms) { return undefined; @@ -240,8 +249,8 @@ function inAlarmUnits( /** * A change for the worse's severity: judged by how far it went at the least — the interval bound - * nearest zero — against the `error` band. Without a band, every confirmed change for the worse - * alarms. + * nearest zero — against the `error` band. Without a band, which only a discrete metric's alarm + * keeps, every confirmed change for the worse alarms. */ function severityOf(change: Change, interval: Interval, alarm: RunMetricAlarm): Severity { if (change !== 'worse') { @@ -262,24 +271,27 @@ function differencesOf(subject: number[], against: number[]): number[] { export interface CompareSamplesOptions { /** - * Whether a change can raise the metric's alarm. Defaults to true; the variants of a `compare()` - * differ on purpose, so no difference between them can. + * The alarm a change answers to. Defaults to the metric's; `null` where no change can raise one, + * such as between the variants of a `compare()`, which differ on purpose. */ - canAlarm?: boolean; + alarm?: RunMetricAlarm | null; } export function compareSamples( subject: { name: string; values: number[] }, against: { name: string; values: number[] }, definition: RunMetricDefinition, - { canAlarm = true }: CompareSamplesOptions = {}, + { alarm: given = definition.alarm ?? null }: CompareSamplesOptions = {}, ): MetricComparison { - const alarm = (canAlarm && definition.alarm) || null; + const alarm = + given && definition.kind === 'scalar' && given.error === undefined + ? { ...given, error: DEFAULT_ERROR_BAND } + : given; const differences = differencesOf(subject.values, against.values); const rounds = differences.length; const mean = calculateMean(differences); const stdDev = calculateSampleStdDev(differences, mean); - const absolute = intervalAround(mean, stdDev, rounds, CONFIDENCE); + const absolute = intervalAround(mean, stdDev, rounds); const reference = calculateMean(against.values.slice(0, rounds)); const relative = { low: (absolute.low / reference) * 100, @@ -288,13 +300,16 @@ export function compareSamples( const noise = (stdDev / reference) * 100; const banded = inAlarmUnits({ absolute, relative }, definition); const precision = (banded.high - banded.low) / 2; + // Only the worse side's edge matters: an improvement never raises an alarm, however large. + const worseEdge = + alarm?.direction === 'higherIsBetter' ? -(alarm.error ?? 0) : (alarm?.error ?? 0); let change: Change = 'undetected'; if (rounds > 0 && differences.every((difference) => difference === 0)) { change = 'unchanged'; } else if (absolute.low > 0 || absolute.high < 0) { const increased = absolute.low > 0; - const lowerIsBetter = definition.alarm?.direction !== 'higherIsBetter'; + const lowerIsBetter = (alarm ?? definition.alarm)?.direction !== 'higherIsBetter'; change = increased === lowerIsBetter ? 'worse' : 'better'; } @@ -306,6 +321,7 @@ export function compareSamples( relative, noise, precision, + settled: alarm === null || !(banded.low < worseEdge && worseEdge < banded.high), change, severity: alarm ? severityOf(change, banded, alarm) : 'none', }; @@ -324,7 +340,8 @@ export type MetricComparisons = Omit; interface PairedMetric { metric: string; definition: RunMetricDefinition; - canAlarm: boolean; + /** The alarm its changes answer to here, or `null` where none can raise one. */ + alarm: RunMetricAlarm | null; pairs: Array<{ subject: { name: string; values: number[] }; against: { name: string; values: number[] }; @@ -339,14 +356,17 @@ function pairedMetricsOf( const { samples } = benchmark; const [reference] = benchmark.variants; return Object.keys(samples?.[reference] ?? {}).map((metric) => { + const definition = definitionOf(metrics, metric); const override = alarmOverrideOf(benchmark, metric); - const shared = definitionOf(metrics, metric); - const definition = override ? { ...shared, alarm: { ...shared.alarm, ...override } } : shared; + let alarm: RunMetricAlarm | null = null; + if (benchmark.kind === 'baseline' && override !== false) { + alarm = override ? { ...definition.alarm, ...override } : (definition.alarm ?? null); + } const sideOf = (name: string) => ({ name, values: samples?.[name]?.[metric] ?? [] }); return { metric, definition, - canAlarm: benchmark.kind === 'baseline' && override !== null, + alarm, pairs: pairsOf(benchmark).map(({ subject, against }) => ({ subject: sideOf(subject), against: sideOf(against), @@ -364,15 +384,13 @@ export function compareBenchmark( benchmark: RunBenchmark, ): { metrics: MetricComparisons[] } { return { - metrics: pairedMetricsOf(metrics, benchmark).map(({ metric, definition, canAlarm, pairs }) => { - return { - metric, - definition, - comparisons: pairs.map(({ subject, against }) => - compareSamples(subject, against, definition, { canAlarm }), - ), - }; - }), + metrics: pairedMetricsOf(metrics, benchmark).map(({ metric, definition, alarm, pairs }) => ({ + metric, + definition, + comparisons: pairs.map(({ subject, against }) => + compareSamples(subject, against, definition, { alarm }), + ), + })), }; } diff --git a/packages/benchmark/src/runReport/format.test.ts b/packages/benchmark/src/runReport/format.test.ts index 1789baf046..39da9d4ff1 100644 --- a/packages/benchmark/src/runReport/format.test.ts +++ b/packages/benchmark/src/runReport/format.test.ts @@ -107,7 +107,7 @@ describe('formatRunSummary', () => { describe('runReportFootnote', () => { it('states the level every interval uses', () => { - expect(runReportFootnote(analyzeRun(reportOf([benchmarkOf('a')])))).toBe( + expect(runReportFootnote()).toBe( 'Medians; Δ is the confidence interval of the paired per-round difference (99%).', ); }); diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index 74b7b8e5df..315b9966a0 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -93,7 +93,7 @@ export function formatPrecisionDetail( return noise; } // Rounds for a half-width of the band: (z × noise / band)². - const z = tCritical(MANY_ROUNDS, CONFIDENCE); + const z = tCritical(MANY_ROUNDS); const rounds = Math.ceil(((z * comparison.noise) / (band * 100)) ** 2); return `${noise} · about ${rounds} rounds resolve ±${formatterFor(DEFAULT_SCALAR_FORMAT).format(band * 100)}%`; } @@ -253,7 +253,7 @@ export function formatRunSummary( } /** How to read a run's numbers, under every rendering of it. */ -export function runReportFootnote(analyses: BenchmarkAnalysis[]): string { +export function runReportFootnote(): string { return `Medians; Δ is the confidence interval of the paired per-round difference (${formatConfidence(CONFIDENCE)}).`; } diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts index 80dd2cf947..a897b7de89 100644 --- a/packages/benchmark/src/runReport/schema.ts +++ b/packages/benchmark/src/runReport/schema.ts @@ -17,7 +17,7 @@ import type { MetricDefinition } from '../types'; /** * A metric's alarm, as version 2 judges it. `error` is the smallest change that raises the alarm — a * relative fraction (`0.1` = 10%) for a scalar metric, an absolute count for a discrete one; without - * it, any confirmed change for the worse does. A metric config's `warn` is for version 1 reports and + * it, 5% for a scalar metric and any confirmed change for a discrete one. A metric config's `warn` is for version 1 reports and * isn't carried here. */ const alarmSchema = z.object({ @@ -58,10 +58,10 @@ const benchmarkSchema = z.object({ samples: z.record(z.string(), z.record(z.string(), z.array(z.number()))).optional(), /** * This benchmark's own alarms, by metric name, in place of the metric's: its fields override the - * metric's alarm, and `null` means the metric never alarms here. A `name#id` sub-series follows + * metric's alarm, and `false` means the metric never alarms here. A `name#id` sub-series follows * its base metric's entry. */ - alarms: z.record(z.string(), alarmSchema.nullable()).optional(), + alarms: z.record(z.string(), z.union([alarmSchema, z.literal(false)])).optional(), /** * How sampling went: rounds past `sampleSize` were added while a difference was unresolved, and * `timedOut` says it was still unresolved when the timeout stopped it. diff --git a/packages/benchmark/src/runner/printReport.ts b/packages/benchmark/src/runner/printReport.ts index d1d6ad2a38..ce655f32d1 100644 --- a/packages/benchmark/src/runner/printReport.ts +++ b/packages/benchmark/src/runner/printReport.ts @@ -47,5 +47,5 @@ export function printRunReport(report: BenchmarkRunReport): void { console.log(red(`⚠ ${benchmark} · ${metric} · ${formatComparison(comparison)}`)); } console.log(formatRunSummary(analyses, regressions, report.durationMs)); - console.log(dim(runReportFootnote(analyses))); + console.log(dim(runReportFootnote())); } diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index e155a3788f..72d6a72d31 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -5,7 +5,7 @@ import type { Browser, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; import { compareBenchmark } from '../runReport'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; -import { differencesResolved, pickSampling, resolveSampling } from '../sampling'; +import { differencesSettled, pickSampling, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { BenchFile } from './benchFiles'; import type { ResolvedRef } from './refs'; @@ -225,24 +225,24 @@ async function runBenchmark( const definitions = metrics ?? {}; const benchmarkOf = (): RunBenchmark => ({ ...entry, samples: samplesOf(slots, rounds) }); - const isSettled = () => differencesResolved(compareBenchmark(definitions, benchmarkOf())); + const isSettled = () => differencesSettled(compareBenchmark(definitions, benchmarkOf())); const deadline = Date.now() + timeout * 60_000; - let resolved = isSettled(); - while (!resolved && Date.now() < deadline) { + let settled = isSettled(); + while (!settled && Date.now() < deadline) { // eslint-disable-next-line no-await-in-loop rounds.push(await runRound()); - resolved = isSettled(); + settled = isSettled(); } - if (rounds.length > sampleSize || !resolved) { + if (rounds.length > sampleSize || !settled) { console.log( chalk.dim( - ` ${rounds.length} rounds, ${resolved ? 'resolved' : `unresolved after ${timeout} min`}`, + ` ${rounds.length} rounds, ${settled ? 'settled' : `unsettled after ${timeout} min`}`, ), ); } return { - benchmark: { ...benchmarkOf(), sampling: { sampleSize, timedOut: !resolved } }, + benchmark: { ...benchmarkOf(), sampling: { sampleSize, timedOut: !settled } }, metrics: definitions, }; } catch (error) { @@ -325,7 +325,7 @@ function samplingOf( own: SamplingOptions, runDefaults: SamplingOptions = {}, ): Required { - return resolveSampling({ ...pickSampling(runDefaults), ...pickSampling(own) }); + return resolveSampling({ ...pickSampling(runDefaults), ...own }); } /** What a run drives: the browser, and every benchmark it was asked for, in order. */ diff --git a/packages/benchmark/src/sampling.test.ts b/packages/benchmark/src/sampling.test.ts index 0d86bcc36b..eb7823c985 100644 --- a/packages/benchmark/src/sampling.test.ts +++ b/packages/benchmark/src/sampling.test.ts @@ -1,24 +1,7 @@ import { describe, expect, it } from 'vitest'; import { analyzeBenchmark } from './runReport/analyzeRun'; import type { RunBenchmark, RunMetricDefinition } from './runReport'; -import { DEFAULT_SAMPLING, differencesResolved, isResolved, resolveSampling } from './sampling'; - -describe('isResolved', () => { - it('is resolved when the interval lies on one side of every horizon', () => { - expect(isResolved({ low: 2, high: 8 }, [0])).toBe(true); - expect(isResolved({ low: -8, high: -2 }, [0])).toBe(true); - expect(isResolved({ low: -4, high: 4 }, [-10, 10])).toBe(true); - }); - - it('is unresolved while the interval straddles a horizon', () => { - expect(isResolved({ low: -1, high: 3 }, [0])).toBe(false); - expect(isResolved({ low: 5, high: 15 }, [-10, 10])).toBe(false); - }); - - it('treats an exact, zero-width difference as resolved', () => { - expect(isResolved({ low: 0, high: 0 }, [0])).toBe(true); - }); -}); +import { DEFAULT_SAMPLING, differencesSettled, resolveSampling } from './sampling'; describe('resolveSampling', () => { it('fills in a default for every option left out or undefined', () => { @@ -42,7 +25,7 @@ describe('resolveSampling', () => { }); }); -describe('differencesResolved', () => { +describe('differencesSettled', () => { const noisy = [10, 12, 9, 11, 10, 13, 8, 11]; const clearlySlower = noisy.map((value) => value * 2); const sameish = [11, 9, 10, 12, 11, 10, 9, 12]; @@ -69,20 +52,20 @@ describe('differencesResolved', () => { paint: { kind: 'scalar' }, }; - it('stops once an alarmed difference without a band is known to be a change', () => { + it('stops once an alarmed difference is known to be past the default band', () => { const analysis = analyzeBenchmark( alarmedRender, benchmarkOf({ current: clearlySlower, baseline: noisy }), ); - expect(differencesResolved(analysis)).toBe(true); + expect(differencesSettled(analysis)).toBe(true); }); - it('keeps sampling while an alarmed difference without a band could still be none', () => { + it('keeps sampling while an alarmed difference could still be past the default band', () => { const analysis = analyzeBenchmark( alarmedRender, benchmarkOf({ current: sameish, baseline: noisy }), ); - expect(differencesResolved(analysis)).toBe(false); + expect(differencesSettled(analysis)).toBe(false); }); it('ignores metrics that do not alarm when some do', () => { @@ -93,7 +76,7 @@ describe('differencesResolved', () => { { current: sameish, baseline: noisy }, ), ); - expect(differencesResolved(analysis)).toBe(true); + expect(differencesSettled(analysis)).toBe(true); }); it('stops at once when nothing can alarm', () => { @@ -101,7 +84,7 @@ describe('differencesResolved', () => { { render: { kind: 'scalar' }, paint: { kind: 'scalar' } }, benchmarkOf({ current: sameish, baseline: noisy }), ); - expect(differencesResolved(analysis)).toBe(true); + expect(differencesSettled(analysis)).toBe(true); }); it('stops at once for a compare(), whose differences never alarm', () => { @@ -109,23 +92,34 @@ describe('differencesResolved', () => { ...benchmarkOf({ current: sameish, baseline: noisy }), kind: 'compare', }); - expect(differencesResolved(analysis)).toBe(true); + expect(differencesSettled(analysis)).toBe(true); }); it("settles against the alarm's error band", () => { const banded = (error: number) => ({ render: { kind: 'scalar' as const, alarm: { error } } }); // About +100%: clearly within a 200% band, still straddling a 100% one. const slower = benchmarkOf({ current: clearlySlower, baseline: noisy }); - expect(differencesResolved(analyzeBenchmark(banded(2), slower))).toBe(true); - expect(differencesResolved(analyzeBenchmark(banded(1), slower))).toBe(false); + expect(differencesSettled(analyzeBenchmark(banded(2), slower))).toBe(true); + expect(differencesSettled(analyzeBenchmark(banded(1), slower))).toBe(false); + }); + + it('never waits on an improvement, however close to the band', () => { + // About 50% faster: straddling the band's better edge, which no alarm asks about. + const faster = noisy.map((value) => value / 2); + const banded = { render: { kind: 'scalar' as const, alarm: { error: 0.5 } } }; + expect( + differencesSettled( + analyzeBenchmark(banded, benchmarkOf({ current: faster, baseline: noisy })), + ), + ).toBe(true); }); it("follows the benchmark's own alarm over the metric's", () => { const unsure = benchmarkOf({ current: sameish, baseline: noisy }); const withAlarms = (alarms: RunBenchmark['alarms']) => - differencesResolved(analyzeBenchmark(alarmedRender, { ...unsure, alarms })); + differencesSettled(analyzeBenchmark(alarmedRender, { ...unsure, alarms })); expect(withAlarms(undefined)).toBe(false); expect(withAlarms({ render: { error: 0.5 } })).toBe(true); - expect(withAlarms({ render: null })).toBe(true); + expect(withAlarms({ render: false })).toBe(true); }); }); diff --git a/packages/benchmark/src/sampling.ts b/packages/benchmark/src/sampling.ts index 334a235e7b..65be470115 100644 --- a/packages/benchmark/src/sampling.ts +++ b/packages/benchmark/src/sampling.ts @@ -10,8 +10,9 @@ export interface SamplingOptions { sampleSize?: number; /** * Minutes to keep sampling after `sampleSize` rounds while a difference that can raise an alarm is - * unsettled: its interval still straddles the alarm's `error` band, or zero for an alarm without - * one. `0` measures exactly `sampleSize` rounds. Defaults to 3. + * unsettled: its interval still straddles the alarm's `error` band (5% for a scalar metric that sets + * none), or zero for a discrete one without a band. `0` measures exactly `sampleSize` rounds. + * Defaults to 3. */ timeout?: number; } @@ -36,11 +37,6 @@ export function resolveSampling(options: SamplingOptions): Required 0; -} - /** The sampling options `options` sets, leaving out any other key and any left undefined. */ export function pickSampling(options: SamplingOptions): SamplingOptions { const { sampleSize, timeout } = options; @@ -49,27 +45,12 @@ export function pickSampling(options: SamplingOptions): SamplingOptions { ); } -/** Whether an interval lies entirely on one side of every horizon. */ -export function isResolved(interval: { low: number; high: number }, horizons: number[]): boolean { - return horizons.every((horizon) => !(interval.low < horizon && horizon < interval.high)); -} - /** - * Whether sampling a benchmark can stop: every difference that can raise an alarm is known to be past - * its `error` band or within it, either way — or, for an alarm without a band, known to be a change - * or not. The others are only read, so they never keep a run going. + * Whether sampling a benchmark can stop: every difference that can raise an alarm is settled. The + * others are only read, so they never keep a run going. */ -export function differencesResolved(analysis: { metrics: MetricComparisons[] }): boolean { - return analysis.metrics.every(({ definition, comparisons }) => - comparisons.every(({ alarm, relative, absolute }) => { - if (alarm === null) { - return true; - } - // Bands are fractions on a scalar metric, whose relative interval is in percent, and counts on - // a discrete one. - const discrete = definition.kind === 'discrete'; - const band = (alarm.error ?? 0) * (discrete ? 1 : 100); - return isResolved(discrete ? absolute : relative, band === 0 ? [0] : [-band, band]); - }), +export function differencesSettled(analysis: { metrics: MetricComparisons[] }): boolean { + return analysis.metrics.every((metric) => + metric.comparisons.every((comparison) => comparison.settled), ); } diff --git a/packages/benchmark/src/types.ts b/packages/benchmark/src/types.ts index c6e5e0cf8a..333f93cd8a 100644 --- a/packages/benchmark/src/types.ts +++ b/packages/benchmark/src/types.ts @@ -80,8 +80,8 @@ export interface MetricAlarm { warn?: number; /** * Harder band: a regression past `error` is flagged as an error (the alarm). Without it, version 1 - * reports use the dashboard's ±20% noise band (or warn only, with `warn` set) and version 2 reports - * flag every confirmed change for the worse. + * reports use the dashboard's ±20% noise band (or warn only, with `warn` set); version 2 reports + * use 5% for a scalar metric and flag any confirmed change for the worse in a discrete one. * Scalar metrics: a relative fraction (`0.25` = 25%). Discrete metrics: an absolute count delta. */ error?: number; From c231ea907c5193c03e3e8074bd7b357b3c022682 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:57:58 +0200 Subject: [PATCH 143/173] [benchmark] Name each improved benchmark once; expect a near-0% A/A rate The PR comment's summary line lists a benchmark's improved alarmed metrics together under its name. The nightly A/A job's comment expects close to 0% false alarms, now that an alarm needs a change confidently past its band, and the README notes { error: 0 } for alarming on any confirmed change. --- .circleci/config.yml | 3 ++- .../lib/benchmarkRun/buildMarkdownReport.test.ts | 11 +++++++++++ .../src/lib/benchmarkRun/buildMarkdownReport.ts | 16 ++++++++++++---- packages/benchmark/README.md | 3 ++- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 81fef815d5..829995cef5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -156,7 +156,8 @@ jobs: paths: - test/benchmark/.benchmark/packed # The commit against itself: any regression it reports is a false alarm, so over many nights the - # job's failure rate is the benchmarks' false-alarm rate, about 5% when the statistics are honest. + # job's failure rate is the benchmarks' false-alarm rate. An alarm needs a change confidently past + # its error band, so expect close to 0%; a rate that climbs means noise or bias in a benchmark. benchmark_aa: <<: *default-job executor: diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index 3787403551..f20a2411fa 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -84,6 +84,17 @@ describe('buildBenchmarkRunMarkdownReport', () => { expect(markdown).not.toContain('bench:paint'); }); + it('names a benchmark once, however many of its alarmed metrics improved', () => { + const report = reportOf([faster]); + const markdown = buildBenchmarkRunMarkdownReport( + { ...report, metrics: { ...report.metrics, 'bench:paint': { kind: 'scalar', alarm: {} } } }, + { title: 'Performance' }, + ); + + expect(markdown.match(/Tooltip mount/g)).toHaveLength(1); + expect(markdown).toMatch(/\*\*Tooltip mount\*\* render `better[^`]*`, bench:paint `better/); + }); + it('tables only the rows that regressed, and marks the heading', () => { expect( buildBenchmarkRunMarkdownReport(reportOf([slower, libraries, broken]), { diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 7207b92d61..7299622b1f 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -17,6 +17,17 @@ interface BuildOptions { detailsUrl?: string; } +/** The improvements on alarmed metrics, one phrase per benchmark: `**Grid** render \`…\`, bench:paint \`…\``. */ +function improvementsByBenchmark(analyses: BenchmarkAnalysis[]): string[] { + const byBenchmark = new Map(); + for (const { benchmark, metric, comparison } of findImprovements(analyses)) { + const phrases = byBenchmark.get(benchmark) ?? []; + phrases.push(`${metric} \`${formatComparison(comparison)}\``); + byBenchmark.set(benchmark, phrases); + } + return [...byBenchmark].map(([benchmark, phrases]) => `**${benchmark}** ${phrases.join(', ')}`); +} + /** * A benchmark's table, keeping only the rows that regressed. A `baseline` benchmark compares one * pair, so each kept row's comparison is the regression. @@ -67,10 +78,7 @@ export function buildBenchmarkRunMarkdownReport( regressions.length > 0 ? `${regressions.length} regression${regressions.length === 1 ? '' : 's'} in ${benchmarks}` : `No regressions in ${benchmarks}`, - ...findImprovements(analyses).map( - ({ benchmark, metric, comparison }) => - `**${benchmark}** ${metric} \`${formatComparison(comparison)}\``, - ), + ...improvementsByBenchmark(analyses), report.durationMs === undefined ? null : `ran ${formatDuration(report.durationMs)}`, options.detailsUrl ? `[details](${options.detailsUrl})` : null, ].filter(Boolean); diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 375e4f1c75..b1cb0c6595 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -468,7 +468,8 @@ compare('scatter', [ours, other], { sampleSize: 100 }); `render` and `bench:paint` alarm once a change is confidently 5% worse. A `reactBenchmark()` can set its own band for both with `alarm`: a lower one for a benchmark that matters more, a higher one for -one that matters less, or `false` for one that never alarms. `{}` keeps the band it would have had. +one that matters less, or `false` for one that never alarms. `{}` keeps the band it would have had, +and `{ error: 0 }` alarms on any confirmed change for the worse, however small. Sampling follows the band, so a tighter band also samples longer. ```tsx From b9c34f7212f1439eee3937217b8a3d095c164ef5 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 13 Oct 2026 10:48:03 +0200 Subject: [PATCH 144/173] [code-infra-dashboard] Table improvements in the PR comment like regressions Each benchmark that changed gets one table of the rows that regressed or improved, marked red or green, benchmarks with a regression first. The summary line is back to just the status. --- .../benchmarkRun/buildMarkdownReport.test.ts | 53 ++++++++++--- .../lib/benchmarkRun/buildMarkdownReport.ts | 75 +++++++++++-------- 2 files changed, 85 insertions(+), 43 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts index f20a2411fa..41922f87df 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.test.ts @@ -69,6 +69,17 @@ const faster: RunBenchmark = { }, }; +const mixed: RunBenchmark = { + name: 'Toolbar mount', + file: 'toolbar.bench.tsx', + kind: 'baseline', + variants: ['current', 'baseline'], + samples: { + current: { render: [5, 6, 5, 6], 'bench:paint': [30, 31, 30, 31] }, + baseline: { render: [10, 11, 10, 11], 'bench:paint': [20, 21, 20, 21] }, + }, +}; + describe('buildBenchmarkRunMarkdownReport', () => { it('is a single line when nothing needs attention', () => { expect( @@ -76,23 +87,45 @@ describe('buildBenchmarkRunMarkdownReport', () => { ).toBe('## Performance\n\nNo regressions in 2 benchmarks'); }); - it('names improvements on alarmed metrics in that line, and leaves informational ones out', () => { - const markdown = buildBenchmarkRunMarkdownReport(reportOf([faster]), { title: 'Performance' }); + it('tables the rows that improved on alarmed metrics, and leaves informational ones out', () => { + expect(buildBenchmarkRunMarkdownReport(reportOf([faster]), { title: 'Performance' })) + .toMatchInlineSnapshot(` + "## Performance + + No regressions in 1 benchmark + + **Tooltip mount** - expect(markdown.split('\n')).toHaveLength(3); - expect(markdown).toContain('**Tooltip mount** render `better'); - expect(markdown).not.toContain('bench:paint'); + | Metric | baseline | current | Δ vs baseline | + |:----------|----------:|----------:|:----------| + | render | 10.5 ms | 5.5 ms | 🟢 \`better -47.6% – -47.6%\` | + + + _Medians; Δ is the confidence interval of the paired per-round difference (99%)._" + `); }); - it('names a benchmark once, however many of its alarmed metrics improved', () => { - const report = reportOf([faster]); + it('puts the regressed and improved rows of a benchmark in one table', () => { + const report = reportOf([mixed]); const markdown = buildBenchmarkRunMarkdownReport( - { ...report, metrics: { ...report.metrics, 'bench:paint': { kind: 'scalar', alarm: {} } } }, + { + ...report, + metrics: { ...report.metrics, 'bench:paint': { kind: 'scalar', format: MS, alarm: {} } }, + }, { title: 'Performance' }, ); - expect(markdown.match(/Tooltip mount/g)).toHaveLength(1); - expect(markdown).toMatch(/\*\*Tooltip mount\*\* render `better[^`]*`, bench:paint `better/); + expect(markdown.match(/Toolbar mount/g)).toHaveLength(1); + expect(markdown).toContain('| render | 10.5 ms | 5.5 ms | 🟢 `better'); + expect(markdown).toContain('| bench:paint | 20.5 ms | 30.5 ms | 🔴 `worse'); + }); + + it('lists the benchmarks with a regression before the ones that only improved', () => { + const markdown = buildBenchmarkRunMarkdownReport(reportOf([faster, slower]), { + title: 'Performance', + }); + + expect(markdown.indexOf('**Grid scroll**')).toBeLessThan(markdown.indexOf('**Tooltip mount**')); }); it('tables only the rows that regressed, and marks the heading', () => { diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 7299622b1f..169ae5e346 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -2,13 +2,15 @@ import { formatMarkdownTable } from '@/utils/formatters'; import { analyzeRun, benchmarkTable, - findImprovements, findRegressions, - formatComparison, formatDuration, runReportFootnote, } from '@mui/internal-benchmark/runReport'; -import type { BenchmarkAnalysis, BenchmarkRunReport } from '@mui/internal-benchmark/runReport'; +import type { + BenchmarkAnalysis, + BenchmarkRunReport, + MetricComparison, +} from '@mui/internal-benchmark/runReport'; interface BuildOptions { /** The comment section's heading. */ @@ -17,27 +19,25 @@ interface BuildOptions { detailsUrl?: string; } -/** The improvements on alarmed metrics, one phrase per benchmark: `**Grid** render \`…\`, bench:paint \`…\``. */ -function improvementsByBenchmark(analyses: BenchmarkAnalysis[]): string[] { - const byBenchmark = new Map(); - for (const { benchmark, metric, comparison } of findImprovements(analyses)) { - const phrases = byBenchmark.get(benchmark) ?? []; - phrases.push(`${metric} \`${formatComparison(comparison)}\``); - byBenchmark.set(benchmark, phrases); +/** + * How a comparison is flagged: 🔴 for a regression, 🟢 for a change for the better on an alarmed + * metric (what `findImprovements` counts), nothing for no change or an informational metric. + */ +function markerOf(comparison: MetricComparison): string | null { + if (comparison.severity === 'error') { + return '🔴'; + } + if (comparison.alarm !== null && comparison.change === 'better') { + return '🟢'; } - return [...byBenchmark].map(([benchmark, phrases]) => `**${benchmark}** ${phrases.join(', ')}`); + return null; } -/** - * A benchmark's table, keeping only the rows that regressed. A `baseline` benchmark compares one - * pair, so each kept row's comparison is the regression. - */ -function regressionTable(analysis: BenchmarkAnalysis): string | null { +/** A benchmark's table, keeping only the rows that regressed or improved. */ +function changesTable(analysis: BenchmarkAnalysis): string | null { const { columns, rows } = benchmarkTable(analysis); - const regressed = rows.filter((row) => - row.comparisons.some((comparison) => comparison.severity === 'error'), - ); - if (regressed.length === 0) { + const changed = rows.filter((row) => row.comparisons.some(markerOf)); + if (changed.length === 0) { return null; } return formatMarkdownTable( @@ -46,12 +46,16 @@ function regressionTable(analysis: BenchmarkAnalysis): string | null { header, align: kind === 'value' ? ('right' as const) : ('left' as const), })), - regressed.map((row) => + changed.map((row) => Object.fromEntries( - row.cells.map((cell, column) => [ - String(column), - columns[column].kind === 'comparison' ? `🔴 \`${cell}\`` : cell, - ]), + row.cells.map((cell, column) => { + const { kind, comparison } = columns[column]; + if (kind !== 'comparison' || comparison === undefined) { + return [String(column), cell]; + } + const marker = markerOf(row.comparisons[comparison]); + return [String(column), marker ? `${marker} \`${cell}\`` : `\`${cell}\``]; + }), ), ), ); @@ -60,9 +64,10 @@ function regressionTable(analysis: BenchmarkAnalysis): string | null { /** * Renders a version 2 benchmark report as its pull request comment section. * - * The comment is there to draw attention when something needs it, so a run without regressions or - * failures is a single line. Only regressions get a table, and only of the rows that regressed; - * informational metrics and comparisons between libraries stay on the dashboard. + * The comment is there to draw attention to what changed, so a run where nothing did is a single + * line. Each benchmark that regressed or improved gets a table of just the rows that did, the ones + * with a regression first; informational metrics and comparisons between libraries stay on the + * dashboard. */ export function buildBenchmarkRunMarkdownReport( report: BenchmarkRunReport, @@ -78,7 +83,6 @@ export function buildBenchmarkRunMarkdownReport( regressions.length > 0 ? `${regressions.length} regression${regressions.length === 1 ? '' : 's'} in ${benchmarks}` : `No regressions in ${benchmarks}`, - ...improvementsByBenchmark(analyses), report.durationMs === undefined ? null : `ran ${formatDuration(report.durationMs)}`, options.detailsUrl ? `[details](${options.detailsUrl})` : null, ].filter(Boolean); @@ -90,10 +94,15 @@ export function buildBenchmarkRunMarkdownReport( lines.push('', `❌ **${benchmark.name}**: ${benchmark.error}`); } - const tables = analyses.flatMap((analysis) => { - const table = regressionTable(analysis); - return table ? [`**${analysis.benchmark.name}**\n\n${table}`] : []; - }); + const regressed = new Set(regressions.map(({ benchmark }) => benchmark)); + const tables = [...analyses] + .sort( + (a, b) => Number(regressed.has(b.benchmark.name)) - Number(regressed.has(a.benchmark.name)), + ) + .flatMap((analysis) => { + const table = changesTable(analysis); + return table ? [`**${analysis.benchmark.name}**\n\n${table}`] : []; + }); if (tables.length > 0) { lines.push('', tables.join('\n\n'), '', `_${runReportFootnote()}_`); } From a16175c008054d5fbf6f33c9337228d762f2f01f Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:35:30 +0200 Subject: [PATCH 145/173] [benchmark] Share what counts as a regression or an improvement isRegression and isImprovement classify one comparison; findRegressions and the PR comment's markers both use them. findImprovements had no callers left. --- .../lib/benchmarkRun/buildMarkdownReport.ts | 45 ++++++++++--------- .../benchmark/src/runReport/analyzeRun.ts | 31 ++++++------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts index 169ae5e346..7b560d4e9f 100644 --- a/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts +++ b/apps/code-infra-dashboard/src/lib/benchmarkRun/buildMarkdownReport.ts @@ -4,6 +4,8 @@ import { benchmarkTable, findRegressions, formatDuration, + isImprovement, + isRegression, runReportFootnote, } from '@mui/internal-benchmark/runReport'; import type { @@ -19,24 +21,24 @@ interface BuildOptions { detailsUrl?: string; } -/** - * How a comparison is flagged: 🔴 for a regression, 🟢 for a change for the better on an alarmed - * metric (what `findImprovements` counts), nothing for no change or an informational metric. - */ +/** How a comparison is flagged: 🔴 for a regression, 🟢 for an improvement, nothing otherwise. */ function markerOf(comparison: MetricComparison): string | null { - if (comparison.severity === 'error') { + if (isRegression(comparison)) { return '🔴'; } - if (comparison.alarm !== null && comparison.change === 'better') { - return '🟢'; - } - return null; + return isImprovement(comparison) ? '🟢' : null; +} + +function hasRegression({ metrics }: BenchmarkAnalysis): boolean { + return metrics.some(({ comparisons }) => comparisons.some(isRegression)); } /** A benchmark's table, keeping only the rows that regressed or improved. */ function changesTable(analysis: BenchmarkAnalysis): string | null { const { columns, rows } = benchmarkTable(analysis); - const changed = rows.filter((row) => row.comparisons.some(markerOf)); + const changed = rows + .map((row) => ({ row, markers: row.comparisons.map(markerOf) })) + .filter(({ markers }) => markers.some(Boolean)); if (changed.length === 0) { return null; } @@ -46,15 +48,15 @@ function changesTable(analysis: BenchmarkAnalysis): string | null { header, align: kind === 'value' ? ('right' as const) : ('left' as const), })), - changed.map((row) => + changed.map(({ row, markers }) => Object.fromEntries( row.cells.map((cell, column) => { const { kind, comparison } = columns[column]; if (kind !== 'comparison' || comparison === undefined) { return [String(column), cell]; } - const marker = markerOf(row.comparisons[comparison]); - return [String(column), marker ? `${marker} \`${cell}\`` : `\`${cell}\``]; + const marker = markers[comparison]; + return [String(column), `${marker ? `${marker} ` : ''}\`${cell}\``]; }), ), ), @@ -94,15 +96,14 @@ export function buildBenchmarkRunMarkdownReport( lines.push('', `❌ **${benchmark.name}**: ${benchmark.error}`); } - const regressed = new Set(regressions.map(({ benchmark }) => benchmark)); - const tables = [...analyses] - .sort( - (a, b) => Number(regressed.has(b.benchmark.name)) - Number(regressed.has(a.benchmark.name)), - ) - .flatMap((analysis) => { - const table = changesTable(analysis); - return table ? [`**${analysis.benchmark.name}**\n\n${table}`] : []; - }); + const regressedFirst = [ + ...analyses.filter(hasRegression), + ...analyses.filter((analysis) => !hasRegression(analysis)), + ]; + const tables = regressedFirst.flatMap((analysis) => { + const table = changesTable(analysis); + return table ? [`**${analysis.benchmark.name}**\n\n${table}`] : []; + }); if (tables.length > 0) { lines.push('', tables.join('\n\n'), '', `_${runReportFootnote()}_`); } diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index 2a961c8f60..a35dc42196 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -425,25 +425,26 @@ export interface AlarmedChange { comparison: MetricComparison; } -function alarmedChangesOf(analyses: BenchmarkAnalysis[]): AlarmedChange[] { +/** + * Whether a change says something is wrong with the code under test: an alarmed metric's change + * for the worse, past its `error` band. + */ +export function isRegression(comparison: MetricComparison): boolean { + return comparison.alarm !== null && comparison.severity !== 'none'; +} + +/** Whether a change is one for the better on an alarmed metric: what the code under test sped up. */ +export function isImprovement(comparison: MetricComparison): boolean { + return comparison.alarm !== null && comparison.change === 'better'; +} + +/** Every regression in a run, with the benchmark and metric it belongs to. */ +export function findRegressions(analyses: BenchmarkAnalysis[]): AlarmedChange[] { return analyses.flatMap(({ benchmark, metrics }) => metrics.flatMap(({ metric, comparisons }) => comparisons - .filter((comparison) => comparison.alarm !== null) + .filter(isRegression) .map((comparison) => ({ benchmark: benchmark.name, metric, comparison })), ), ); } - -/** - * The changes that say something is wrong with the code under test: an alarmed metric's change - * for the worse, past its `error` band. - */ -export function findRegressions(analyses: BenchmarkAnalysis[]): AlarmedChange[] { - return alarmedChangesOf(analyses).filter(({ comparison }) => comparison.severity !== 'none'); -} - -/** The changes for the better on alarmed metrics: what the code under test sped up. */ -export function findImprovements(analyses: BenchmarkAnalysis[]): AlarmedChange[] { - return alarmedChangesOf(analyses).filter(({ comparison }) => comparison.change === 'better'); -} From c10b35298bd62d6a2496a18f1dd2b978221dc462 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:44:03 +0200 Subject: [PATCH 146/173] [code-infra-dashboard] Show only "timed out" in the warning colour The rounds and duration stay secondary; formatRounds can leave the timeout out for a caller that shows it itself. --- .../src/views/BenchmarkRunDetails.tsx | 18 +++++++++++------- .../benchmark/src/runReport/format.test.ts | 6 ++++++ packages/benchmark/src/runReport/format.ts | 10 +++++++--- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index c819772527..57ddaabb8c 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -87,13 +87,17 @@ function BenchmarkTable({ analysis }: { analysis: BenchmarkAnalysis }) { : 'Rounds past the sample size were added while a difference was unresolved' } > - - {formatRounds(benchmark)} - + + {formatRounds(benchmark, { timedOut: false })} + {benchmark.sampling?.timedOut ? ( + + {' · '} + + timed out + + + ) : null} + diff --git a/packages/benchmark/src/runReport/format.test.ts b/packages/benchmark/src/runReport/format.test.ts index 39da9d4ff1..be049c4177 100644 --- a/packages/benchmark/src/runReport/format.test.ts +++ b/packages/benchmark/src/runReport/format.test.ts @@ -95,6 +95,12 @@ describe('formatRounds', () => { it('says how long a benchmark took next to its rounds', () => { expect(formatRounds({ ...benchmarkOf('a'), durationMs: 72_000 })).toBe('4 rounds · 1m 12s'); }); + + it('says when sampling timed out, unless the caller shows that itself', () => { + const benchmark = { ...benchmarkOf('a'), sampling: { sampleSize: 4, timedOut: true } }; + expect(formatRounds(benchmark)).toBe('4 rounds · timed out'); + expect(formatRounds(benchmark, { timedOut: false })).toBe('4 rounds'); + }); }); describe('formatRunSummary', () => { diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index 315b9966a0..90863aa142 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -210,9 +210,13 @@ export function formatDuration(durationMs: number): string { /** * How many rounds a benchmark ran, how long it took, and whether sampling resolved: `50 rounds`, * `50 + 23 rounds · 1m 12s` past the sample size, `50 + 312 rounds · 3m 04s · timed out`. `null` - * for a benchmark that measured nothing. + * for a benchmark that measured nothing. Without `timedOut`, the timeout is left for the caller to + * show on its own. */ -export function formatRounds(benchmark: RunBenchmark): string | null { +export function formatRounds( + benchmark: RunBenchmark, + { timedOut = true }: { timedOut?: boolean } = {}, +): string | null { const [reference] = benchmark.variants; const rounds = Object.values(benchmark.samples?.[reference] ?? {})[0]?.length; if (rounds === undefined) { @@ -222,7 +226,7 @@ export function formatRounds(benchmark: RunBenchmark): string | null { return [ rounds > sampleSize ? `${sampleSize} + ${rounds - sampleSize} rounds` : `${rounds} rounds`, benchmark.durationMs === undefined ? null : formatDuration(benchmark.durationMs), - benchmark.sampling?.timedOut ? 'timed out' : null, + timedOut && benchmark.sampling?.timedOut ? 'timed out' : null, ] .filter(Boolean) .join(' · '); From a8889d26a73ee1afc267842bfb335076b361fa02 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:59:11 +0200 Subject: [PATCH 147/173] [benchmark] Share one page load per round between a file's benchmarks Each round loads every build's page once and samples all of the file's benchmarks still sampling there, in declaration order, instead of loading a page per sample. Each benchmark keeps its own rounds, sample size and timeout; a compare()'s variants still run in a shuffled order. Profiling keeps a fresh page per sample. --- packages/benchmark/README.md | 25 +- packages/benchmark/src/page/page.tsx | 8 +- .../benchmark/src/runner/runInterleaved.ts | 225 +++++++++++++----- .../benchmark/src/runner/traceBenchmarks.ts | 9 +- 4 files changed, 184 insertions(+), 83 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index b1cb0c6595..845bdcfb25 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -402,20 +402,23 @@ compare('scatter', [ ]); ``` -Every sample loads the whole benchmark file in a fresh page, so a static import reaches every case. -Import what only one case needs inside that case, before the part it times: it then loads in that -case's samples only. +Every page load runs the whole benchmark file, so a static import reaches every case. Import what +only one case needs inside that case, before the part it times: it then loads where that case runs +only. ### How it measures The plugin generates a page per benchmark file under `src/__bench__/`. A benchmark file that imports `vitest`, or the Vitest entry `@mui/internal-benchmark`, fails the build. The runner drives Chromium -through Playwright and runs each benchmark in one tab, which loads the page of the variant to be -measured — the current or the baseline build, or a case of a `compare()` — before every sample. -Every sample is the first iteration of a freshly loaded page, with garbage collected right before -it, and every variant runs in the same renderer process. Whatever a page or a process holds on to — -the JIT tier its code settled into, the heap it grew — then never favours one variant for the whole -benchmark, which the paired rounds below could not tell from a real difference. +through Playwright and runs a file's benchmarks in one tab. Every round, it loads each build's page +once and samples every benchmark of the file that is still sampling there, with garbage collected +right before each sample; the cases of a `compare()` share the working tree's page. Benchmarks run in +the order the file declares them. A page is loaded afresh every round, and every variant runs in the same renderer process. +Whatever a page or a process holds on to — the JIT tier its code settled into, the heap it grew — +then never favours one variant for the whole benchmark, which the paired rounds below could not tell +from a real difference. A file's benchmarks do share a page within a round, so code one of them +warmed can speed up the next: a file is a unit of related benchmarks, and an unrelated one belongs in +a file of its own. Every variant is sampled once per round, in a shuffled order, and a difference is judged on the per-round differences rather than on two independent sets of samples. Whatever the machine was doing @@ -553,8 +556,8 @@ variant's median and each comparison's interval, verdict and severity, plus the long it all took — with everything else a run prints sent to stderr. `summarizeRun` from `@mui/internal-benchmark/runReport` produces the same from a report. `NO_COLOR` turns colour off. -`--profile` records instead of measuring: each sample runs as it would be measured, with a -performance trace around it, saved under `.benchmark/traces/` to open in DevTools' Performance +`--profile` records instead of measuring: each sample runs in a freshly loaded page of its own, with +a performance trace around it, saved under `.benchmark/traces/` to open in DevTools' Performance panel — 5 rounds unless `--sample-size` says otherwise. It writes no report: tracing slows the code it records. diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 8628d75e9b..d0d5f9b040 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -48,8 +48,8 @@ const cases = new Map< >(); /** - * Registers a benchmark case. The runner calls `run` once per sample, each time in a freshly loaded - * page, for as many rounds as `options` asks. A case on its own is measured across the builds; one + * Registers a benchmark case. The runner calls `run` once per sample, in a page loaded afresh every + * round and shared with the file's other cases, for as many rounds as `options` asks. A case on its own is measured across the builds; one * passed to `compare()` is measured against the other cases there instead, sampled as the * `compare()` asks, and never alarms. */ @@ -78,8 +78,8 @@ const comparisons: Array<{ * Compares cases with each other — one library's implementation against another's, say — on the * working tree's build, rather than each against its baseline. The first case is the reference. * - * Every sample loads the whole file in a fresh page. Import what only one case needs inside that - * case, before the part it times, to keep it out of the others' samples. + * Every page load runs the whole file. Import what only one case needs inside that case, before the + * part it times, to keep it out of the others' samples. */ export function compare( name: string, diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index 72d6a72d31..fcc146b1c9 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -13,14 +13,15 @@ import { serveDirectory } from './serveDirectory'; import type { BenchPage } from '../page/page'; /** - * Runs the cases of `*.bench.tsx` files in Playwright. A benchmark runs in one tab, which loads the - * variant to be measured before every sample: every sample is the first iteration of a freshly - * loaded page, and every variant runs in the same renderer process. Whatever a page or a process - * holds on to — the JIT tier its code settled into, the heap it grew — then never favours one - * variant for the whole benchmark, which paired rounds could not tell from a real difference. - * Interactions get trusted input through a CDP session the tab is bridged to. Variants are sampled - * once per round in a shuffled order, so the samples are round-aligned and can be compared on paired - * differences. + * Runs the cases of `*.bench.tsx` files in Playwright. A file's benchmarks run in one tab, which + * loads each build's page once per round and samples every benchmark of the file still sampling + * there, with garbage collected before each sample. A page is reloaded every round, and every + * variant runs in the same renderer process: whatever a page or a process holds on to — the JIT tier + * its code settled into, the heap it grew — then never favours one variant for the whole benchmark, + * which paired rounds could not tell from a real difference. Interactions get trusted input through + * a CDP session the tab is bridged to. Pages load in a shuffled order every round, and a benchmark's + * variants on the same page are sampled in a shuffled order too, so the samples are round-aligned and + * can be compared on paired differences. */ export interface RunInterleavedOptions { @@ -65,6 +66,10 @@ export function shuffledIndices(length: number): number[] { return indices; } +function shuffled(items: T[]): T[] { + return shuffledIndices(items.length).map((index) => items[index]); +} + /** * The metrics every variant reported in every round. One some iteration did not produce — a * per-phase render split that only sometimes has a second phase, say — cannot be paired round by @@ -182,75 +187,161 @@ export interface MeasuredBenchmark { /** A benchmark to measure, or why it can't be. */ export type PlannedBenchmark = MeasuredBenchmark | { entry: BenchmarkEntry; error: string }; +/** A benchmark being measured, and how far it got. */ +interface Progress { + plan: MeasuredBenchmark; + rounds: Round[]; + /** Read from its reference page after its first sample there. */ + metrics?: Record; + /** When it has to stop sampling past its sample size; set once it reaches it. */ + deadline?: number; + settled: boolean; + error?: string; + /** Its samples, plus its share of the page loads they ran in. */ + durationMs: number; +} + +function benchmarkOf({ plan, rounds }: Progress): RunBenchmark { + return { ...plan.entry, samples: samplesOf(plan.slots, rounds) }; +} + +/** Whether a benchmark still samples: below its sample size, or unsettled before its timeout. */ +function isSampling(progress: Progress): boolean { + if (progress.error !== undefined) { + return false; + } + if (progress.rounds.length < progress.plan.sampling.sampleSize) { + return true; + } + return !progress.settled && Date.now() < (progress.deadline ?? 0); +} + /** - * Measures one benchmark in a tab of its own, loading each slot's page before sampling it. Every - * round samples every slot once, in a shuffled order. After `sampleSize` rounds it keeps adding - * rounds while a difference that can alarm is unsettled against its band, until the timeout. A failure is - * reported as the benchmark's error. + * Runs one round of a file's benchmarks still sampling: each page they use loads once, in a shuffled + * order, and samples them there. A benchmark gets the round only if every one of its variants was + * sampled in it; one whose sample fails stops, with its error, and the page reloads before the next. */ -async function runBenchmark( - browser: Browser, - { entry, slots, sampling }: MeasuredBenchmark, -): Promise { - console.log(chalk.cyan(`\nRunning "${entry.name}" (${entry.file})…`)); - const { sampleSize, timeout } = sampling; - let opened: Page | undefined; - try { - const page = await openBenchTab(browser); - opened = page; - let metrics: Record | undefined; - const runRound = async (): Promise => { - const round: Round = []; - for (const index of shuffledIndices(slots.length)) { - // eslint-disable-next-line no-await-in-loop - await loadBenchPage(page, slots[index].url); +async function runFileRound(page: Page, active: Progress[]): Promise { + const partial = new Map(active.map((progress) => [progress, [] as Round])); + const urls = [...new Set(active.flatMap(({ plan }) => plan.slots.map((slot) => slot.url)))]; + for (const url of shuffled(urls)) { + // A benchmark's variants on this page run back to back, in a shuffled order, so none of them + // always runs on the code the others warmed. + const benchmarks = active.filter(({ plan }) => plan.slots.some((slot) => slot.url === url)); + const samples = benchmarks.flatMap((progress) => + shuffled( + progress.plan.slots.flatMap((slot, index) => + slot.url === url ? [{ progress, slot, index }] : [], + ), + ), + ); + const loadStartedAt = Date.now(); + // eslint-disable-next-line no-await-in-loop + await loadBenchPage(page, url); + const loadShare = (Date.now() - loadStartedAt) / benchmarks.length; + for (const progress of benchmarks) { + progress.durationMs += loadShare; + } + let reload = false; + for (const { progress, slot, index } of samples) { + if (progress.error !== undefined) { + continue; + } + const startedAt = Date.now(); + try { + if (reload) { + // eslint-disable-next-line no-await-in-loop + await loadBenchPage(page, url); + reload = false; + } // eslint-disable-next-line no-await-in-loop - round[index] = await sampleBenchCase(page, slots[index].caseName); - if (index === 0 && !metrics) { + partial.get(progress)![index] = await sampleBenchCase(page, slot.caseName); + if (index === 0 && !progress.metrics) { // Read from the reference page: a metric whose config the change under test altered is // described the way the change describes it. Its first sample recorded every metric the // report keeps, as those are the ones recorded in every round. // eslint-disable-next-line no-await-in-loop - metrics = await page.evaluate(() => window.benchmarkPage!.metricDefinitions()); + progress.metrics = await page.evaluate(() => window.benchmarkPage!.metricDefinitions()); } + } catch (error) { + console.error(chalk.red(` ${progress.plan.entry.name}: ${errorMessage(error)}`)); + progress.error = errorMessage(error); + // What the failed case left behind on the page must not reach the next one. + reload = true; } - return round; - }; + progress.durationMs += Date.now() - startedAt; + } + } + for (const progress of active) { + if (progress.error !== undefined) { + continue; + } + progress.rounds.push(partial.get(progress)!); + const { sampleSize, timeout } = progress.plan.sampling; + if (progress.rounds.length >= sampleSize) { + progress.deadline ??= Date.now() + timeout * 60_000; + progress.settled = differencesSettled( + compareBenchmark(progress.metrics ?? {}, benchmarkOf(progress)), + ); + } + } +} - const rounds: Round[] = []; - for (let roundIndex = 0; roundIndex < sampleSize; roundIndex += 1) { +/** + * Measures a file's benchmarks in one tab, a round at a time, until none is still sampling. Each + * keeps its own rounds: after `sampleSize` of them it goes on while a difference that can alarm is + * unsettled against its band, until its timeout. A failure is reported as the benchmark's error. + */ +async function runBenchFile(browser: Browser, plans: MeasuredBenchmark[]): Promise { + const count = `${plans.length} benchmark${plans.length === 1 ? '' : 's'}`; + console.log(chalk.cyan(`\nRunning ${plans[0].entry.file} (${count})…`)); + const progresses: Progress[] = plans.map((plan) => ({ + plan, + rounds: [], + settled: false, + durationMs: 0, + })); + let opened: Page | undefined; + try { + const page = await openBenchTab(browser); + opened = page; + let active = progresses.filter(isSampling); + while (active.length > 0) { // eslint-disable-next-line no-await-in-loop - rounds.push(await runRound()); + await runFileRound(page, active); + active = progresses.filter(isSampling); } - const definitions = metrics ?? {}; - const benchmarkOf = (): RunBenchmark => ({ ...entry, samples: samplesOf(slots, rounds) }); - - const isSettled = () => differencesSettled(compareBenchmark(definitions, benchmarkOf())); + } catch (error) { + console.error(chalk.red(` ${errorMessage(error)}`)); + for (const progress of progresses) { + progress.error ??= errorMessage(error); + } + } finally { + await opened?.context().close(); + } - const deadline = Date.now() + timeout * 60_000; - let settled = isSettled(); - while (!settled && Date.now() < deadline) { - // eslint-disable-next-line no-await-in-loop - rounds.push(await runRound()); - settled = isSettled(); + return progresses.map((progress) => { + const { plan, rounds, settled, error, durationMs } = progress; + if (error !== undefined) { + return { benchmark: { ...plan.entry, error, durationMs }, metrics: {} }; } + const { sampleSize, timeout } = plan.sampling; if (rounds.length > sampleSize || !settled) { console.log( chalk.dim( - ` ${rounds.length} rounds, ${settled ? 'settled' : `unsettled after ${timeout} min`}`, + ` ${plan.entry.name}: ${rounds.length} rounds, ${settled ? 'settled' : `unsettled after ${timeout} min`}`, ), ); } return { - benchmark: { ...benchmarkOf(), sampling: { sampleSize, timedOut: !settled } }, - metrics: definitions, + benchmark: { + ...benchmarkOf(progress), + sampling: { sampleSize, timedOut: !settled }, + durationMs: Math.round(durationMs), + }, + metrics: progress.metrics ?? {}, }; - } catch (error) { - console.error(chalk.red(` ${errorMessage(error)}`)); - return { benchmark: { ...entry, error: errorMessage(error) }, metrics: {} }; - } finally { - await opened?.context().close(); - } + }); } /** The round-aligned samples of a benchmark, per variant and metric. */ @@ -377,21 +468,27 @@ export async function withPlannedRun( export function runInterleaved(options: RunInterleavedOptions): Promise { return withPlannedRun(options, async ({ browser, planned }) => { - const results: CaseResult[] = []; - // Sequential on purpose: concurrent cases would contend for the same machine. + const results = new Map(); + const measuredByFile = new Map(); for (const plan of planned) { if ('error' in plan) { - results.push({ benchmark: { ...plan.entry, error: plan.error }, metrics: {} }); - continue; + results.set(plan, { benchmark: { ...plan.entry, error: plan.error }, metrics: {} }); + } else { + const plans = measuredByFile.get(plan.entry.file) ?? []; + plans.push(plan); + measuredByFile.set(plan.entry.file, plans); } - const startedAt = Date.now(); + } + // Sequential on purpose: concurrent files would contend for the same machine. + for (const plans of measuredByFile.values()) { // eslint-disable-next-line no-await-in-loop - const { benchmark, metrics } = await runBenchmark(browser, plan); - results.push({ benchmark: { ...benchmark, durationMs: Date.now() - startedAt }, metrics }); + const fileResults = await runBenchFile(browser, plans); + plans.forEach((plan, index) => results.set(plan, fileResults[index])); } + const ordered = planned.map((plan) => results.get(plan)!); return { - benchmarks: results.map((result) => result.benchmark), - metrics: Object.assign({}, ...results.map((result) => result.metrics)), + benchmarks: ordered.map((result) => result.benchmark), + metrics: Object.assign({}, ...ordered.map((result) => result.metrics)), browserVersion: browser.version(), }; }); diff --git a/packages/benchmark/src/runner/traceBenchmarks.ts b/packages/benchmark/src/runner/traceBenchmarks.ts index 9dc97f808c..e274ba864f 100644 --- a/packages/benchmark/src/runner/traceBenchmarks.ts +++ b/packages/benchmark/src/runner/traceBenchmarks.ts @@ -16,10 +16,11 @@ import { import type { MeasuredBenchmark, RunInterleavedOptions } from './runInterleaved'; /** - * Records benchmarks instead of measuring them, to show where their time goes: every sample runs as - * it would be measured — the first iteration of a freshly loaded page, in one tab, the variants - * alternating — with a performance trace recorded around it, to open in DevTools' Performance - * panel. Tracing slows the code it records, so nothing here is a measurement. + * Records benchmarks instead of measuring them, to show where their time goes: every sample runs in a + * freshly loaded page of its own, in one tab, the variants alternating, with a performance trace + * recorded around it to open in DevTools' Performance panel. A measured run shares a page between a + * file's benchmarks; a trace leaves the others out, to show one benchmark alone. Tracing slows the + * code it records, so nothing here is a measurement. */ export interface BenchmarkTraces extends Pick { From feb43284d130d66127b4a11dcd21231483464bba Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:29:04 +0200 Subject: [PATCH 148/173] [benchmark] Don't warn about 'use client' directives when building the pages Component libraries mark their modules 'use client'; bundling them into a page's chunks drops the directive, which means nothing on a client-only page, and Rolldown warned once per module. --- packages/benchmark/src/runner/plugin.ts | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/benchmark/src/runner/plugin.ts b/packages/benchmark/src/runner/plugin.ts index ee2b536263..693d218635 100644 --- a/packages/benchmark/src/runner/plugin.ts +++ b/packages/benchmark/src/runner/plugin.ts @@ -175,7 +175,18 @@ export function benchmarkPlugin(options: BenchmarkPluginOptions = {}): Plugin { // page nothing once it is parsed, and parsing happens on load, before a sample. // Compression stays on, so the code is the production code otherwise. sourcemap: true, - rolldownOptions: { input, output: { minify: { compress: true, mangle: false } } }, + rolldownOptions: { + input, + output: { minify: { compress: true, mangle: false } }, + // Component libraries mark their modules 'use client' for React Server Components. A + // page bundles them into shared chunks, where a per-module directive can't hold, and on + // a client-only page it means nothing: one warning per module would bury the build log. + onLog(level, log, handler) { + if (log.code !== 'MODULE_LEVEL_DIRECTIVE') { + handler(level, log); + } + }, + }, }, }; }, From 115961dac5925aaa335437ed762ed077d0de91ff Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:49:47 +0200 Subject: [PATCH 149/173] [benchmark] Print each file's runtime, and its benchmarks' rounds as base + adaptive A file's progress ends with how long it ran, and a benchmark that sampled past its sample size reads `30 + 15 rounds`, the way the report and dashboard write it. --- packages/benchmark/src/runner/runInterleaved.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index fcc146b1c9..eca9e538eb 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -3,7 +3,7 @@ import chalk from 'chalk'; import type { Browser, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; -import { compareBenchmark } from '../runReport'; +import { compareBenchmark, formatDuration, formatRounds } from '../runReport'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; import { differencesSettled, pickSampling, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; @@ -295,6 +295,7 @@ async function runFileRound(page: Page, active: Progress[]): Promise { async function runBenchFile(browser: Browser, plans: MeasuredBenchmark[]): Promise { const count = `${plans.length} benchmark${plans.length === 1 ? '' : 's'}`; console.log(chalk.cyan(`\nRunning ${plans[0].entry.file} (${count})…`)); + const startedAt = Date.now(); const progresses: Progress[] = plans.map((plan) => ({ plan, rounds: [], @@ -320,16 +321,21 @@ async function runBenchFile(browser: Browser, plans: MeasuredBenchmark[]): Promi await opened?.context().close(); } - return progresses.map((progress) => { + const results = progresses.map((progress) => { const { plan, rounds, settled, error, durationMs } = progress; if (error !== undefined) { return { benchmark: { ...plan.entry, error, durationMs }, metrics: {} }; } const { sampleSize, timeout } = plan.sampling; if (rounds.length > sampleSize || !settled) { + // The rounds past the sample size are the adaptive ones: `30 + 15 rounds`. + const roundsLabel = formatRounds( + { ...benchmarkOf(progress), sampling: { sampleSize, timedOut: !settled } }, + { timedOut: false }, + ); console.log( chalk.dim( - ` ${plan.entry.name}: ${rounds.length} rounds, ${settled ? 'settled' : `unsettled after ${timeout} min`}`, + ` ${plan.entry.name}: ${roundsLabel}, ${settled ? 'settled' : `unsettled after ${timeout} min`}`, ), ); } @@ -342,6 +348,8 @@ async function runBenchFile(browser: Browser, plans: MeasuredBenchmark[]): Promi metrics: progress.metrics ?? {}, }; }); + console.log(chalk.dim(` ${plans[0].entry.file} ran ${formatDuration(Date.now() - startedAt)}`)); + return results; } /** The round-aligned samples of a benchmark, per variant and metric. */ From f172e988c1c36e34da6c0b1fef46b1a60a9213f5 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 13 Oct 2026 15:58:41 +0200 Subject: [PATCH 150/173] [benchmark] Simplify the per-file runner Plans group by file with Map.groupBy, and a file returns its results in order, planning errors included. Each round keeps its samples next to their benchmark, one 'loaded URL' path covers both the first load and the reload after a failure, and the progress line counts rounds with formatRoundCount rather than building a report entry. The trace loop uses shuffled(). --- packages/benchmark/src/runReport/format.ts | 8 +- packages/benchmark/src/runner/plugin.ts | 7 +- .../benchmark/src/runner/runInterleaved.ts | 118 ++++++++---------- .../benchmark/src/runner/traceBenchmarks.ts | 5 +- 4 files changed, 66 insertions(+), 72 deletions(-) diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index 90863aa142..f3254b4390 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -207,6 +207,11 @@ export function formatDuration(durationMs: number): string { : `${Math.floor(minutes / 60)}h ${twoDigits(minutes % 60)}m`; } +/** Rounds, with those past the sample size apart: `50 rounds`, `50 + 23 rounds`. */ +export function formatRoundCount(rounds: number, sampleSize: number): string { + return rounds > sampleSize ? `${sampleSize} + ${rounds - sampleSize} rounds` : `${rounds} rounds`; +} + /** * How many rounds a benchmark ran, how long it took, and whether sampling resolved: `50 rounds`, * `50 + 23 rounds · 1m 12s` past the sample size, `50 + 312 rounds · 3m 04s · timed out`. `null` @@ -222,9 +227,8 @@ export function formatRounds( if (rounds === undefined) { return null; } - const sampleSize = benchmark.sampling?.sampleSize ?? rounds; return [ - rounds > sampleSize ? `${sampleSize} + ${rounds - sampleSize} rounds` : `${rounds} rounds`, + formatRoundCount(rounds, benchmark.sampling?.sampleSize ?? rounds), benchmark.durationMs === undefined ? null : formatDuration(benchmark.durationMs), timedOut && benchmark.sampling?.timedOut ? 'timed out' : null, ] diff --git a/packages/benchmark/src/runner/plugin.ts b/packages/benchmark/src/runner/plugin.ts index 693d218635..9fef3ea27f 100644 --- a/packages/benchmark/src/runner/plugin.ts +++ b/packages/benchmark/src/runner/plugin.ts @@ -178,9 +178,10 @@ export function benchmarkPlugin(options: BenchmarkPluginOptions = {}): Plugin { rolldownOptions: { input, output: { minify: { compress: true, mangle: false } }, - // Component libraries mark their modules 'use client' for React Server Components. A - // page bundles them into shared chunks, where a per-module directive can't hold, and on - // a client-only page it means nothing: one warning per module would bury the build log. + // Component libraries mark their modules with directives such as 'use client', for React + // Server Components. A page bundles them into shared chunks, where a per-module directive + // can't hold, and on a client-only page it means nothing: one warning per module would + // bury the build log. onLog(level, log, handler) { if (log.code !== 'MODULE_LEVEL_DIRECTIVE') { handler(level, log); diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index eca9e538eb..1bfcc2ff0b 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -3,7 +3,7 @@ import chalk from 'chalk'; import type { Browser, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; -import { compareBenchmark, formatDuration, formatRounds } from '../runReport'; +import { compareBenchmark, formatDuration, formatRoundCount } from '../runReport'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; import { differencesSettled, pickSampling, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; @@ -57,7 +57,7 @@ export function errorMessage(error: unknown): string { } /** A random permutation of `0..length-1`, so no variant always runs first in its round. */ -export function shuffledIndices(length: number): number[] { +function shuffledIndices(length: number): number[] { const indices = Array.from({ length }, (_, index) => index); for (let index = indices.length - 1; index > 0; index -= 1) { const swap = Math.floor(Math.random() * (index + 1)); @@ -66,7 +66,7 @@ export function shuffledIndices(length: number): number[] { return indices; } -function shuffled(items: T[]): T[] { +export function shuffled(items: T[]): T[] { return shuffledIndices(items.length).map((index) => items[index]); } @@ -222,40 +222,41 @@ function isSampling(progress: Progress): boolean { * sampled in it; one whose sample fails stops, with its error, and the page reloads before the next. */ async function runFileRound(page: Page, active: Progress[]): Promise { - const partial = new Map(active.map((progress) => [progress, [] as Round])); + const entries = active.map((progress) => ({ progress, round: [] as Round })); const urls = [...new Set(active.flatMap(({ plan }) => plan.slots.map((slot) => slot.url)))]; + // Unset after a failed sample, so what the failed case left behind doesn't reach the next one. + let loadedUrl: string | undefined; for (const url of shuffled(urls)) { // A benchmark's variants on this page run back to back, in a shuffled order, so none of them // always runs on the code the others warmed. - const benchmarks = active.filter(({ plan }) => plan.slots.some((slot) => slot.url === url)); - const samples = benchmarks.flatMap((progress) => + const onPage = entries.filter(({ progress }) => + progress.plan.slots.some((slot) => slot.url === url), + ); + const samples = onPage.flatMap(({ progress, round }) => shuffled( progress.plan.slots.flatMap((slot, index) => - slot.url === url ? [{ progress, slot, index }] : [], + slot.url === url ? [{ progress, round, slot, index }] : [], ), ), ); - const loadStartedAt = Date.now(); - // eslint-disable-next-line no-await-in-loop - await loadBenchPage(page, url); - const loadShare = (Date.now() - loadStartedAt) / benchmarks.length; - for (const progress of benchmarks) { - progress.durationMs += loadShare; - } - let reload = false; - for (const { progress, slot, index } of samples) { + for (const { progress, round, slot, index } of samples) { if (progress.error !== undefined) { continue; } - const startedAt = Date.now(); try { - if (reload) { + if (loadedUrl !== url) { + const loadStartedAt = Date.now(); // eslint-disable-next-line no-await-in-loop await loadBenchPage(page, url); - reload = false; + loadedUrl = url; + const loadShare = (Date.now() - loadStartedAt) / onPage.length; + for (const entry of onPage) { + entry.progress.durationMs += loadShare; + } } + const startedAt = Date.now(); // eslint-disable-next-line no-await-in-loop - partial.get(progress)![index] = await sampleBenchCase(page, slot.caseName); + round[index] = await sampleBenchCase(page, slot.caseName); if (index === 0 && !progress.metrics) { // Read from the reference page: a metric whose config the change under test altered is // described the way the change describes it. Its first sample recorded every metric the @@ -263,20 +264,19 @@ async function runFileRound(page: Page, active: Progress[]): Promise { // eslint-disable-next-line no-await-in-loop progress.metrics = await page.evaluate(() => window.benchmarkPage!.metricDefinitions()); } + progress.durationMs += Date.now() - startedAt; } catch (error) { console.error(chalk.red(` ${progress.plan.entry.name}: ${errorMessage(error)}`)); progress.error = errorMessage(error); - // What the failed case left behind on the page must not reach the next one. - reload = true; + loadedUrl = undefined; } - progress.durationMs += Date.now() - startedAt; } } - for (const progress of active) { + for (const { progress, round } of entries) { if (progress.error !== undefined) { continue; } - progress.rounds.push(partial.get(progress)!); + progress.rounds.push(round); const { sampleSize, timeout } = progress.plan.sampling; if (progress.rounds.length >= sampleSize) { progress.deadline ??= Date.now() + timeout * 60_000; @@ -288,26 +288,26 @@ async function runFileRound(page: Page, active: Progress[]): Promise { } /** - * Measures a file's benchmarks in one tab, a round at a time, until none is still sampling. Each - * keeps its own rounds: after `sampleSize` of them it goes on while a difference that can alarm is - * unsettled against its band, until its timeout. A failure is reported as the benchmark's error. + * Measures a file's benchmarks in one tab, a round at a time, until none is still sampling, and + * returns their results in the plans' order. Each keeps its own rounds: after `sampleSize` of them + * it goes on while a difference that can alarm is unsettled against its band, until its timeout. A + * failure is reported as the benchmark's error, as is a plan that couldn't be measured at all. */ -async function runBenchFile(browser: Browser, plans: MeasuredBenchmark[]): Promise { +async function runBenchFile(browser: Browser, plans: PlannedBenchmark[]): Promise { const count = `${plans.length} benchmark${plans.length === 1 ? '' : 's'}`; console.log(chalk.cyan(`\nRunning ${plans[0].entry.file} (${count})…`)); const startedAt = Date.now(); - const progresses: Progress[] = plans.map((plan) => ({ - plan, - rounds: [], - settled: false, - durationMs: 0, - })); + // A plan that couldn't be measured keeps its error; the others get a progress to sample into. + const tracked = plans.map((plan): Progress | Extract => + 'error' in plan ? plan : { plan, rounds: [], settled: false, durationMs: 0 }, + ); + const progresses = tracked.filter((item): item is Progress => 'rounds' in item); let opened: Page | undefined; try { - const page = await openBenchTab(browser); - opened = page; let active = progresses.filter(isSampling); - while (active.length > 0) { + const page = active.length > 0 ? await openBenchTab(browser) : undefined; + opened = page; + while (page && active.length > 0) { // eslint-disable-next-line no-await-in-loop await runFileRound(page, active); active = progresses.filter(isSampling); @@ -321,21 +321,17 @@ async function runBenchFile(browser: Browser, plans: MeasuredBenchmark[]): Promi await opened?.context().close(); } - const results = progresses.map((progress) => { + const resultOf = (progress: Progress): CaseResult => { const { plan, rounds, settled, error, durationMs } = progress; if (error !== undefined) { return { benchmark: { ...plan.entry, error, durationMs }, metrics: {} }; } const { sampleSize, timeout } = plan.sampling; if (rounds.length > sampleSize || !settled) { - // The rounds past the sample size are the adaptive ones: `30 + 15 rounds`. - const roundsLabel = formatRounds( - { ...benchmarkOf(progress), sampling: { sampleSize, timedOut: !settled } }, - { timedOut: false }, - ); + const outcome = settled ? 'settled' : `unsettled after ${timeout} min`; console.log( chalk.dim( - ` ${plan.entry.name}: ${roundsLabel}, ${settled ? 'settled' : `unsettled after ${timeout} min`}`, + ` ${plan.entry.name}: ${formatRoundCount(rounds.length, sampleSize)}, ${outcome}`, ), ); } @@ -347,7 +343,12 @@ async function runBenchFile(browser: Browser, plans: MeasuredBenchmark[]): Promi }, metrics: progress.metrics ?? {}, }; - }); + }; + const results = tracked.map((item) => + 'rounds' in item + ? resultOf(item) + : { benchmark: { ...item.entry, error: item.error }, metrics: {} }, + ); console.log(chalk.dim(` ${plans[0].entry.file} ran ${formatDuration(Date.now() - startedAt)}`)); return results; } @@ -476,27 +477,16 @@ export async function withPlannedRun( export function runInterleaved(options: RunInterleavedOptions): Promise { return withPlannedRun(options, async ({ browser, planned }) => { - const results = new Map(); - const measuredByFile = new Map(); - for (const plan of planned) { - if ('error' in plan) { - results.set(plan, { benchmark: { ...plan.entry, error: plan.error }, metrics: {} }); - } else { - const plans = measuredByFile.get(plan.entry.file) ?? []; - plans.push(plan); - measuredByFile.set(plan.entry.file, plans); - } - } - // Sequential on purpose: concurrent files would contend for the same machine. - for (const plans of measuredByFile.values()) { + const results: CaseResult[] = []; + // A file's plans are listed together, so grouping keeps the run's order. Sequential on purpose: + // concurrent files would contend for the same machine. + for (const plans of Map.groupBy(planned, (plan) => plan.entry.file).values()) { // eslint-disable-next-line no-await-in-loop - const fileResults = await runBenchFile(browser, plans); - plans.forEach((plan, index) => results.set(plan, fileResults[index])); + results.push(...(await runBenchFile(browser, plans))); } - const ordered = planned.map((plan) => results.get(plan)!); return { - benchmarks: ordered.map((result) => result.benchmark), - metrics: Object.assign({}, ...ordered.map((result) => result.metrics)), + benchmarks: results.map((result) => result.benchmark), + metrics: Object.assign({}, ...results.map((result) => result.metrics)), browserVersion: browser.version(), }; }); diff --git a/packages/benchmark/src/runner/traceBenchmarks.ts b/packages/benchmark/src/runner/traceBenchmarks.ts index e274ba864f..1b62f79223 100644 --- a/packages/benchmark/src/runner/traceBenchmarks.ts +++ b/packages/benchmark/src/runner/traceBenchmarks.ts @@ -10,7 +10,7 @@ import { loadBenchPage, openBenchTab, sampleBenchCase, - shuffledIndices, + shuffled, withPlannedRun, } from './runInterleaved'; import type { MeasuredBenchmark, RunInterleavedOptions } from './runInterleaved'; @@ -43,8 +43,7 @@ async function traceBenchmark( try { await mkdir(outDir, { recursive: true }); for (let round = 1; round <= rounds; round += 1) { - for (const index of shuffledIndices(slots.length)) { - const { variant, url, caseName } = slots[index]; + for (const { variant, url, caseName } of shuffled(slots)) { const file = path.join(outDir, `${variant}-${round}.json`); // eslint-disable-next-line no-await-in-loop await loadBenchPage(page, url); From a898de2ddb114cf42f45d77e07751f8e697236cd Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:18:33 +0200 Subject: [PATCH 151/173] [benchmark] Say "timed out" in the progress line, as the summary and report do --- packages/benchmark/src/runner/runInterleaved.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index 1bfcc2ff0b..c42bf9af57 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -328,7 +328,7 @@ async function runBenchFile(browser: Browser, plans: PlannedBenchmark[]): Promis } const { sampleSize, timeout } = plan.sampling; if (rounds.length > sampleSize || !settled) { - const outcome = settled ? 'settled' : `unsettled after ${timeout} min`; + const outcome = settled ? 'settled' : `timed out after ${timeout} min`; console.log( chalk.dim( ` ${plan.entry.name}: ${formatRoundCount(rounds.length, sampleSize)}, ${outcome}`, From 565949e87762ede4ba58c4892ec6cd91e1e8c50d Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:04:02 +0200 Subject: [PATCH 152/173] [benchmark] Count the sample size in the time limit, and stop gracefully on a signal timeout is now the minutes a benchmark may sample for in all, its sampleSize rounds included (which always run), rather than minutes past them. SIGINT and SIGTERM stop the run: the browser closes, the workspace is put back, and the CLI exits with the signal's code, without a stack trace per benchmark. A second signal exits at once. --- packages/benchmark/README.md | 11 ++-- packages/benchmark/src/cli/index.ts | 59 +++++++++++++------ .../benchmark/src/runner/runBenchmarks.ts | 9 +++ .../benchmark/src/runner/runInterleaved.ts | 51 ++++++++++++---- .../benchmark/src/runner/traceBenchmarks.ts | 3 +- packages/benchmark/src/sampling.ts | 10 ++-- 6 files changed, 103 insertions(+), 40 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 845bdcfb25..e97129552d 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -444,17 +444,18 @@ which costs the results precision. How many rounds a benchmark is measured for adapts to its results, the way tachometer's auto-sampling does. After `sampleSize` rounds, rounds keep being added while a difference that can -raise an alarm is unsettled, for up to `timeout` minutes. A difference is settled once its interval +raise an alarm is unsettled, until `timeout` minutes after the benchmark began sampling. The +`sampleSize` rounds count toward that time but always run, even past it. A difference is settled once its interval lies entirely past its alarm's `error` band on the worse side, or entirely short of it: known to alarm, or known not to. Improvements never keep a run going. A scalar metric's alarm without a band uses 5%; a discrete one's settles against zero, which an unchanged count does at once. Differences that can't alarm, in a `compare()` or a metric without an alarm, never keep a run going. -| Option | Default | Meaning | -| :----------- | :------ | :---------------------------------------------------------------------- | -| `sampleSize` | `50` | Rounds measured before deciding whether to continue | -| `timeout` | `3` | Minutes to keep sampling while an alarmed difference is still unsettled | +| Option | Default | Meaning | +| :----------- | :------ | :-------------------------------------------------------------------------- | +| `sampleSize` | `50` | Rounds measured before deciding whether to continue | +| `timeout` | `3` | Minutes a benchmark may sample for while an alarmed difference is unsettled | They are set per benchmark: the last argument of `benchmark()` and `compare()`, and among `reactBenchmark()`'s options. A compared case is sampled as its `compare()` asks, so setting its own diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index ad82cd53ce..d8d486c1e8 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node +import * as os from 'node:os'; import chalk from 'chalk'; import yargs from 'yargs'; import type { CommandModule } from 'yargs'; @@ -82,7 +83,7 @@ const runCommand: CommandModule<{}, Args> = { .option('timeout', { type: 'number', describe: - "Minutes to keep sampling while an alarmed difference is unsettled against its band, for benchmarks that don't set their own (default 3)", + "Minutes a benchmark may sample for, its sample size included, while an alarmed difference is unsettled against its band, for benchmarks that don't set their own (default 3)", }), handler: async (argv) => { if (argv.reporter === 'json') { @@ -93,22 +94,46 @@ const runCommand: CommandModule<{}, Args> = { /* eslint-enable no-console */ } const { runBenchmarks } = await import('../runner/runBenchmarks'); - await runBenchmarks({ - harnessDir: process.cwd(), - filters: argv.filters ?? [], - baseline: argv.baseline, - buildCmd: argv.buildCmd, - out: argv.out, - upload: argv.upload, - timeline: argv.timeline, - testNamePattern: argv.testNamePattern, - reporter: argv.reporter, - profile: argv.profile, - sampling: { - sampleSize: argv.sampleSize, - timeout: argv.timeout, - }, - }); + // A canceled CI job or a Ctrl+C stops the run: the browser closes and the workspace is put back, + // and the run exits as the signal would have made it, without a stack trace per benchmark. A + // second signal exits at once. + const stop = new AbortController(); + const onSignal = (signal: NodeJS.Signals) => { + process.exitCode = 128 + os.constants.signals[signal]; + if (stop.signal.aborted) { + process.exit(); + } + console.error(chalk.yellow(`\n${signal}: stopping, and putting the repository back…`)); + stop.abort(new Error(`Stopped by ${signal}.`)); + }; + process.on('SIGINT', onSignal); + process.on('SIGTERM', onSignal); + try { + await runBenchmarks({ + harnessDir: process.cwd(), + filters: argv.filters ?? [], + baseline: argv.baseline, + buildCmd: argv.buildCmd, + out: argv.out, + upload: argv.upload, + timeline: argv.timeline, + testNamePattern: argv.testNamePattern, + reporter: argv.reporter, + profile: argv.profile, + sampling: { + sampleSize: argv.sampleSize, + timeout: argv.timeout, + }, + signal: stop.signal, + }); + } catch (error) { + if (!stop.signal.aborted) { + throw error; + } + } finally { + process.off('SIGINT', onSignal); + process.off('SIGTERM', onSignal); + } }, }; diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index 5905870a7c..79578dff14 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -58,6 +58,11 @@ export interface RunBenchmarksOptions { * stdout — medians, intervals and verdicts — for a program to read. */ reporter?: 'default' | 'json'; + /** + * Stops the run: it rejects with the signal's reason once the workspace is put back, without a + * report. + */ + signal?: AbortSignal; /** * Record the benchmarks instead of measuring them: a performance trace around every sample, to * open in DevTools' Performance panel. Writes no report. @@ -92,6 +97,7 @@ export async function runBenchmarks( testNamePattern, reporter = 'default', profile = false, + signal, } = options; if (timeline !== undefined && !upload) { throw new Error('A timeline only applies to an upload: pass --upload with --timeline.'); @@ -170,6 +176,7 @@ export async function runBenchmarks( return result.value; }); + signal?.throwIfAborted(); // Then build one set of pages per ref, each against its own packed build so both sides of a // comparison resolve the library identically. One at a time: a build pins the repository's own // workspace file. @@ -185,7 +192,9 @@ export async function runBenchmarks( }); } + signal?.throwIfAborted(); const runOptions = { + signal, buildsDir, benchFiles, benchRefs: refs, diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index c42bf9af57..e110a143f6 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -36,6 +36,8 @@ export interface RunInterleavedOptions { sampling?: SamplingOptions; /** Only run benchmarks whose name matches this, like Vitest's `-t`. */ testNamePattern?: RegExp; + /** Stops the run: the browser closes, and the run rejects with the signal's reason. */ + signal?: AbortSignal; } export interface InterleavedResults { @@ -193,8 +195,8 @@ interface Progress { rounds: Round[]; /** Read from its reference page after its first sample there. */ metrics?: Record; - /** When it has to stop sampling past its sample size; set once it reaches it. */ - deadline?: number; + /** When its time is up: past it, only rounds short of its sample size still run. */ + deadline: number; settled: boolean; error?: string; /** Its samples, plus its share of the page loads they ran in. */ @@ -213,7 +215,7 @@ function isSampling(progress: Progress): boolean { if (progress.rounds.length < progress.plan.sampling.sampleSize) { return true; } - return !progress.settled && Date.now() < (progress.deadline ?? 0); + return !progress.settled && Date.now() < progress.deadline; } /** @@ -221,7 +223,7 @@ function isSampling(progress: Progress): boolean { * order, and samples them there. A benchmark gets the round only if every one of its variants was * sampled in it; one whose sample fails stops, with its error, and the page reloads before the next. */ -async function runFileRound(page: Page, active: Progress[]): Promise { +async function runFileRound(page: Page, active: Progress[], signal?: AbortSignal): Promise { const entries = active.map((progress) => ({ progress, round: [] as Round })); const urls = [...new Set(active.flatMap(({ plan }) => plan.slots.map((slot) => slot.url)))]; // Unset after a failed sample, so what the failed case left behind doesn't reach the next one. @@ -243,6 +245,7 @@ async function runFileRound(page: Page, active: Progress[]): Promise { if (progress.error !== undefined) { continue; } + signal?.throwIfAborted(); try { if (loadedUrl !== url) { const loadStartedAt = Date.now(); @@ -266,6 +269,8 @@ async function runFileRound(page: Page, active: Progress[]): Promise { } progress.durationMs += Date.now() - startedAt; } catch (error) { + // A stopped run closes the browser under the sample; that is no failure of the benchmark. + signal?.throwIfAborted(); console.error(chalk.red(` ${progress.plan.entry.name}: ${errorMessage(error)}`)); progress.error = errorMessage(error); loadedUrl = undefined; @@ -277,9 +282,8 @@ async function runFileRound(page: Page, active: Progress[]): Promise { continue; } progress.rounds.push(round); - const { sampleSize, timeout } = progress.plan.sampling; + const { sampleSize } = progress.plan.sampling; if (progress.rounds.length >= sampleSize) { - progress.deadline ??= Date.now() + timeout * 60_000; progress.settled = differencesSettled( compareBenchmark(progress.metrics ?? {}, benchmarkOf(progress)), ); @@ -289,17 +293,31 @@ async function runFileRound(page: Page, active: Progress[]): Promise { /** * Measures a file's benchmarks in one tab, a round at a time, until none is still sampling, and - * returns their results in the plans' order. Each keeps its own rounds: after `sampleSize` of them - * it goes on while a difference that can alarm is unsettled against its band, until its timeout. A + * returns their results in the plans' order. Each keeps its own rounds: it measures `sampleSize` of + * them, then goes on while a difference that can alarm is unsettled against its band, until + * `timeout` minutes after its sampling began. A * failure is reported as the benchmark's error, as is a plan that couldn't be measured at all. */ -async function runBenchFile(browser: Browser, plans: PlannedBenchmark[]): Promise { +async function runBenchFile( + browser: Browser, + plans: PlannedBenchmark[], + signal?: AbortSignal, +): Promise { const count = `${plans.length} benchmark${plans.length === 1 ? '' : 's'}`; console.log(chalk.cyan(`\nRunning ${plans[0].entry.file} (${count})…`)); const startedAt = Date.now(); // A plan that couldn't be measured keeps its error; the others get a progress to sample into. const tracked = plans.map((plan): Progress | Extract => - 'error' in plan ? plan : { plan, rounds: [], settled: false, durationMs: 0 }, + 'error' in plan + ? plan + : { + plan, + rounds: [], + settled: false, + // The time limit covers all of its sampling, the rounds of its sample size included. + deadline: startedAt + plan.sampling.timeout * 60_000, + durationMs: 0, + }, ); const progresses = tracked.filter((item): item is Progress => 'rounds' in item); let opened: Page | undefined; @@ -309,10 +327,11 @@ async function runBenchFile(browser: Browser, plans: PlannedBenchmark[]): Promis opened = page; while (page && active.length > 0) { // eslint-disable-next-line no-await-in-loop - await runFileRound(page, active); + await runFileRound(page, active, signal); active = progresses.filter(isSampling); } } catch (error) { + signal?.throwIfAborted(); console.error(chalk.red(` ${errorMessage(error)}`)); for (const progress of progresses) { progress.error ??= errorMessage(error); @@ -452,7 +471,11 @@ export async function withPlannedRun( }), ]); + // Closing the browser ends whatever a page is doing at once, rather than when its sample would. + const closeOnAbort = () => browser.close(); + options.signal?.addEventListener('abort', closeOnAbort, { once: true }); try { + options.signal?.throwIfAborted(); // Every file is listed before any is measured, so a name pattern that matches nothing fails // before the run spends anything. Nothing is measured yet, so the files list at once. const { testNamePattern } = options; @@ -469,7 +492,11 @@ export async function withPlannedRun( throw new Error(`No benchmark name matches ${testNamePattern}.`); } return await drive({ browser, planned }); + } catch (error) { + options.signal?.throwIfAborted(); + throw error; } finally { + options.signal?.removeEventListener('abort', closeOnAbort); await browser.close(); await server.close(); } @@ -482,7 +509,7 @@ export function runInterleaved(options: RunInterleavedOptions): Promise plan.entry.file).values()) { // eslint-disable-next-line no-await-in-loop - results.push(...(await runBenchFile(browser, plans))); + results.push(...(await runBenchFile(browser, plans, options.signal))); } return { benchmarks: results.map((result) => result.benchmark), diff --git a/packages/benchmark/src/runner/traceBenchmarks.ts b/packages/benchmark/src/runner/traceBenchmarks.ts index 1b62f79223..737f13d58b 100644 --- a/packages/benchmark/src/runner/traceBenchmarks.ts +++ b/packages/benchmark/src/runner/traceBenchmarks.ts @@ -32,7 +32,7 @@ export interface BenchmarkTraces extends Pick { console.log(chalk.cyan(`\nTracing "${entry.name}" (${entry.file})…`)); const outDir = path.join(tracesDir, entry.name.replace(/[^\w.-]+/g, '-')); @@ -58,6 +58,7 @@ async function traceBenchmark( } return { name: entry.name, file: entry.file, kind: entry.kind, traces }; } catch (error) { + signal?.throwIfAborted(); console.error(chalk.red(` ${errorMessage(error)}`)); return { name: entry.name, diff --git a/packages/benchmark/src/sampling.ts b/packages/benchmark/src/sampling.ts index 65be470115..eee3020970 100644 --- a/packages/benchmark/src/sampling.ts +++ b/packages/benchmark/src/sampling.ts @@ -3,16 +3,16 @@ import type { MetricComparisons } from './runReport/analyzeRun'; /** * How many rounds a benchmark is sampled for, under tachometer's names. After `sampleSize` rounds, * sampling continues while a difference that can raise an alarm is still unsettled against its - * alarm's `error` band, for up to `timeout` minutes. + * alarm's `error` band, until `timeout` minutes after sampling began. */ export interface SamplingOptions { /** Rounds measured before deciding whether to continue. Defaults to 50. */ sampleSize?: number; /** - * Minutes to keep sampling after `sampleSize` rounds while a difference that can raise an alarm is - * unsettled: its interval still straddles the alarm's `error` band (5% for a scalar metric that sets - * none), or zero for a discrete one without a band. `0` measures exactly `sampleSize` rounds. - * Defaults to 3. + * Minutes a benchmark may sample for, its first `sampleSize` rounds included, which always run. + * Past them it keeps sampling while a difference that can raise an alarm is unsettled: its interval + * still straddles the alarm's `error` band (5% for a scalar metric that sets none), or zero for a + * discrete one without a band. `0` measures exactly `sampleSize` rounds. Defaults to 3. */ timeout?: number; } From 201c1e81ffeb6b86e350bb32d63d47fd374cecf6 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:10:18 +0200 Subject: [PATCH 153/173] [benchmark] Trim the abort checks to the ones that matter Closing the browser already fails the next sample, so the per-sample and catch-and- rethrow checks go; the remaining ones keep a stop from reading as benchmark errors. A stop is checked before each ref's page build, the abort listener uses events.addAbortListener, and the CLI no longer removes its signal handlers on the way out. --- packages/benchmark/README.md | 6 +++--- packages/benchmark/src/cli/index.ts | 3 --- .../benchmark/src/runner/runBenchmarks.ts | 6 ++---- .../benchmark/src/runner/runInterleaved.ts | 20 ++++++++----------- 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index e97129552d..ff552c85dc 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -445,9 +445,9 @@ which costs the results precision. How many rounds a benchmark is measured for adapts to its results, the way tachometer's auto-sampling does. After `sampleSize` rounds, rounds keep being added while a difference that can raise an alarm is unsettled, until `timeout` minutes after the benchmark began sampling. The -`sampleSize` rounds count toward that time but always run, even past it. A difference is settled once its interval -lies entirely past its alarm's `error` band on the worse side, or entirely short of it: known to -alarm, or known not to. Improvements never keep a run going. +`sampleSize` rounds count toward that time but always run, even past it. A difference is settled +once its interval lies entirely past its alarm's `error` band on the worse side, or entirely short +of it: known to alarm, or known not to. Improvements never keep a run going. A scalar metric's alarm without a band uses 5%; a discrete one's settles against zero, which an unchanged count does at once. Differences that can't alarm, in a `compare()` or a metric without an alarm, never keep a run going. diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index d8d486c1e8..c4ccee3331 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -130,9 +130,6 @@ const runCommand: CommandModule<{}, Args> = { if (!stop.signal.aborted) { throw error; } - } finally { - process.off('SIGINT', onSignal); - process.off('SIGTERM', onSignal); } }, }; diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index 79578dff14..7671f8dc0c 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -59,8 +59,7 @@ export interface RunBenchmarksOptions { */ reporter?: 'default' | 'json'; /** - * Stops the run: it rejects with the signal's reason once the workspace is put back, without a - * report. + * Stops the run: it rejects once the workspace is put back, without a report. */ signal?: AbortSignal; /** @@ -176,11 +175,11 @@ export async function runBenchmarks( return result.value; }); - signal?.throwIfAborted(); // Then build one set of pages per ref, each against its own packed build so both sides of a // comparison resolve the library identically. One at a time: a build pins the repository's own // workspace file. for (const [index, ref] of refs.entries()) { + signal?.throwIfAborted(); // eslint-disable-next-line no-await-in-loop await buildRefPages({ harnessDir, @@ -192,7 +191,6 @@ export async function runBenchmarks( }); } - signal?.throwIfAborted(); const runOptions = { signal, buildsDir, diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index e110a143f6..de749e5b91 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -1,5 +1,6 @@ /* eslint-disable no-console */ +import { addAbortListener } from 'node:events'; import chalk from 'chalk'; import type { Browser, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; @@ -36,7 +37,7 @@ export interface RunInterleavedOptions { sampling?: SamplingOptions; /** Only run benchmarks whose name matches this, like Vitest's `-t`. */ testNamePattern?: RegExp; - /** Stops the run: the browser closes, and the run rejects with the signal's reason. */ + /** Stops the run: the browser closes, and the run rejects. */ signal?: AbortSignal; } @@ -245,7 +246,6 @@ async function runFileRound(page: Page, active: Progress[], signal?: AbortSignal if (progress.error !== undefined) { continue; } - signal?.throwIfAborted(); try { if (loadedUrl !== url) { const loadStartedAt = Date.now(); @@ -282,8 +282,7 @@ async function runFileRound(page: Page, active: Progress[], signal?: AbortSignal continue; } progress.rounds.push(round); - const { sampleSize } = progress.plan.sampling; - if (progress.rounds.length >= sampleSize) { + if (progress.rounds.length >= progress.plan.sampling.sampleSize) { progress.settled = differencesSettled( compareBenchmark(progress.metrics ?? {}, benchmarkOf(progress)), ); @@ -295,8 +294,8 @@ async function runFileRound(page: Page, active: Progress[], signal?: AbortSignal * Measures a file's benchmarks in one tab, a round at a time, until none is still sampling, and * returns their results in the plans' order. Each keeps its own rounds: it measures `sampleSize` of * them, then goes on while a difference that can alarm is unsettled against its band, until - * `timeout` minutes after its sampling began. A - * failure is reported as the benchmark's error, as is a plan that couldn't be measured at all. + * `timeout` minutes after its sampling began. A failure is reported as the benchmark's error, as is + * a plan that couldn't be measured at all. */ async function runBenchFile( browser: Browser, @@ -472,8 +471,8 @@ export async function withPlannedRun( ]); // Closing the browser ends whatever a page is doing at once, rather than when its sample would. - const closeOnAbort = () => browser.close(); - options.signal?.addEventListener('abort', closeOnAbort, { once: true }); + const closeOnAbort = + options.signal && addAbortListener(options.signal, () => void browser.close()); try { options.signal?.throwIfAborted(); // Every file is listed before any is measured, so a name pattern that matches nothing fails @@ -492,11 +491,8 @@ export async function withPlannedRun( throw new Error(`No benchmark name matches ${testNamePattern}.`); } return await drive({ browser, planned }); - } catch (error) { - options.signal?.throwIfAborted(); - throw error; } finally { - options.signal?.removeEventListener('abort', closeOnAbort); + closeOnAbort?.[Symbol.dispose](); await browser.close(); await server.close(); } From edbf53ce73a4cdd6102c163e2727386a625bc10b Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:09:22 +0200 Subject: [PATCH 154/173] [code-infra-dashboard] Comment benchmark results on base-ui-plus pull requests --- apps/code-infra-dashboard/src/constants.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/code-infra-dashboard/src/constants.ts b/apps/code-infra-dashboard/src/constants.ts index 2eb9c161dd..15b978e1fa 100644 --- a/apps/code-infra-dashboard/src/constants.ts +++ b/apps/code-infra-dashboard/src/constants.ts @@ -177,6 +177,7 @@ export const repositories = new Map( packages: [], prComment: { bundleSize: true, + benchmark: true, }, }, { From 1c1cb8c32e867bae09e56306cc0399a0af395e35 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:35:20 +0200 Subject: [PATCH 155/173] [benchmark] Upload a report with its own command `benchmark upload [file]` sends a report a run wrote, so an upload can be retried without measuring again, and a local run never touches the upload. It replaces the run's --upload; --timeline moves with it. --- .circleci/config.yml | 10 +++-- packages/benchmark/README.md | 23 ++++++++-- packages/benchmark/src/cli/index.ts | 43 ++++++++++++------- .../benchmark/src/runner/runBenchmarks.ts | 26 +++-------- packages/benchmark/src/runner/upload.ts | 20 ++++++++- 5 files changed, 79 insertions(+), 43 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d33ae9bbd5..e03bc993c4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -143,13 +143,17 @@ jobs: - run: name: Run the benchmark smoke test no_output_timeout: 30m + command: pnpm test:benchmark-run --baseline "$(cat .benchmark-baseline-sha)" + - run: + name: Upload the benchmark report + # Also after a failed run, so the comment says what happened. + when: always environment: # TEMPORARY: uploads in place of `test_benchmark`, since both write the commit's one # `benchmark.json`, and to this pull request's dashboard preview, since production does - # not read version 2 reports yet. Remove both lines before merging. - BENCHMARK_UPLOAD: 'true' + # not read version 2 reports yet. Remove this step's environment before merging. CI_REPORT_API_URL: https://code-infra-dashboard-pr-1898.onrender.com - command: pnpm test:benchmark-run --baseline "$(cat .benchmark-baseline-sha)" + command: pnpm -F ./test/benchmark benchmark:run upload - save_cache: name: Save the packed-baseline cache key: benchmark-run-cache-v1-{{ checksum ".benchmark-baseline-sha" }} diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index ff552c85dc..7f4a541cba 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -487,9 +487,23 @@ A benchmark's alarm is merged over the metric's, and a `name#id` sub-series foll ### The report -The run writes `.benchmark/results/report.json`, prints it as tables, and with `--upload` sends it to -the dashboard, which renders the pull request comment's Performance section and the repository's -history from it. +The run writes `.benchmark/results/report.json` (or `--out`) and prints it as tables. In CI, +`benchmark upload` sends that file to the dashboard, which renders the pull request comment's +Performance section and the repository's history from it. A separate step, so a failed upload can be +retried without measuring again; run it even when the run step fails (CircleCI's `when: always`), so +the comment reports a run whose benchmarks all errored: + +```yaml +- run: + name: Run benchmarks + command: pnpm -F ./test/benchmark benchmark --baseline "$(cat .benchmark-baseline-sha)" +- run: + name: Upload the benchmark report + when: always + command: pnpm -F ./test/benchmark benchmark upload +``` + +`benchmark upload [file]` refuses a report measured on another commit than the one CI is building. Uploads from a tracked branch (`master`, `next`, `v7.x`, …) join that branch's timeline, the history the dashboard draws. A run against another baseline names its own with `--timeline` (or @@ -497,7 +511,8 @@ the dashboard draws. A run against another baseline names its own with `--timeli history, for example a weekly run against the last release: ```bash -benchmark --baseline "v$(npm view @mui/material version)" --upload --timeline release +benchmark --baseline "v$(npm view @mui/material version)" +benchmark upload --timeline release ``` - The report holds **raw samples**, round-aligned across variants, plus each metric's kind, format diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index c4ccee3331..0bc1e5b46d 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -42,21 +42,10 @@ const runCommand: CommandModule<{}, Args> = { describe: "Command that builds the publishable workspace packages, in each ref's checkout and in the working tree. Default: pnpm release:build", }) - .option('upload', { - type: 'boolean', - default: process.env.BENCHMARK_UPLOAD === 'true', - describe: - 'Upload the report and refresh the PR comment. Defaults to BENCHMARK_UPLOAD=true, so CI can switch it on by environment. Requires CIRCLE_OIDC_TOKEN_V2', - }) - .option('timeline', { - type: 'string', - default: process.env.BENCHMARK_TIMELINE, - describe: - "With --upload, the dashboard timeline to upload into in place of the branch's, for a run against another baseline such as the last release. Defaults to BENCHMARK_TIMELINE", - }) .option('out', { type: 'string', - describe: 'Write the JSON report here. Default: .benchmark/results/report.json', + describe: + 'Write the JSON report here, for `benchmark upload`. Default: .benchmark/results/report.json', }) .option('testNamePattern', { alias: 't', @@ -115,8 +104,6 @@ const runCommand: CommandModule<{}, Args> = { baseline: argv.baseline, buildCmd: argv.buildCmd, out: argv.out, - upload: argv.upload, - timeline: argv.timeline, testNamePattern: argv.testNamePattern, reporter: argv.reporter, profile: argv.profile, @@ -134,6 +121,31 @@ const runCommand: CommandModule<{}, Args> = { }, }; +const uploadCommand: CommandModule<{}, { file?: string; timeline?: string }> = { + command: 'upload [file]', + describe: + 'Upload a report that a run wrote to the dashboard, and refresh the PR comment. Run from the harness package, in CI. Requires CIRCLE_OIDC_TOKEN_V2', + builder: (args) => + args + .positional('file', { + type: 'string', + describe: 'The report to upload. Default: .benchmark/results/report.json', + }) + .option('timeline', { + type: 'string', + default: process.env.BENCHMARK_TIMELINE, + describe: + "The dashboard timeline to upload into in place of the branch's, for a run against another baseline such as the last release. Defaults to BENCHMARK_TIMELINE", + }), + handler: async (argv) => { + const [{ uploadReportFile }, { resultsPathOf }] = await Promise.all([ + import('../runner/upload'), + import('../runner/outputDir'), + ]); + await uploadReportFile(argv.file ?? resultsPathOf(process.cwd()), argv.timeline); + }, +}; + // chalk honours FORCE_COLOR and --no-color, not the NO_COLOR convention (https://no-color.org): // set to anything but an empty string, it turns colour off. if (process.env.NO_COLOR) { @@ -154,6 +166,7 @@ await yargs(hideBin(process.argv)) .middleware((argv) => { globalArgv = argv; }, true) + .command(uploadCommand) .command(runCommand) .fail((msg, err, yargsInstance) => { if (msg) { diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index 7671f8dc0c..a36e46cd7b 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -18,7 +18,6 @@ import { traceBenchmarks } from './traceBenchmarks'; import { buildRefPages, restoreWorkspace } from './buildPages'; import { resolveBrowserBinary } from './browser'; import { printRunReport } from './printReport'; -import { publishRunReport } from './upload'; import { buildsDirOf, prepareOutputDir, resultsPathOf } from './outputDir'; /** Fewer logical CPUs than this starve V8's background compilers. */ @@ -40,15 +39,11 @@ export interface RunBenchmarksOptions { * Defaults to `pnpm release:build`. */ buildCmd?: string; - /** Where to write the JSON report. Defaults to `.benchmark/results/report.json`. */ - out?: string; - /** Upload the report and refresh the pull request comment. */ - upload?: boolean; /** - * The dashboard timeline to upload into, in place of the branch's: for a run against another - * baseline, such as the last release. Needs `upload`. + * Where to write the JSON report, which `benchmark upload` sends to the dashboard. Defaults to + * `.benchmark/results/report.json`. */ - timeline?: string; + out?: string; /** Defaults, for the whole run, for the sampling options a benchmark doesn't set itself. */ sampling?: SamplingOptions; /** Only run benchmarks whose name matches this regular expression, like Vitest's `-t`. */ @@ -74,7 +69,7 @@ const PROFILE_ROUNDS = 5; /** * Benchmarks a harness's `*.bench.tsx` files across two builds of the workspace — the working tree - * and a baseline — and writes, prints and optionally uploads the run report. With `profile`, it + * and a baseline — and writes and prints the run report. With `profile`, it * traces them instead, and returns no report. * * Both builds are packed from their own commit and installed into the harness the same way, so @@ -90,17 +85,12 @@ export async function runBenchmarks( baseline, buildCmd = 'pnpm release:build', out, - upload = false, - timeline, sampling, testNamePattern, reporter = 'default', profile = false, signal, } = options; - if (timeline !== undefined && !upload) { - throw new Error('A timeline only applies to an upload: pass --upload with --timeline.'); - } // Compiled before anything is built, so a bad pattern fails at once. const namePattern = testNamePattern === undefined ? undefined : new RegExp(testNamePattern); @@ -257,12 +247,8 @@ export async function runBenchmarks( } console.log(chalk.green(`\nWrote JSON report to ${outPath}`)); - if (upload) { - await publishRunReport(report, timeline); - } - - // Written and uploaded first, so the errors are readable and the comment says what happened — - // then fail, because a harness that measured nothing at all otherwise reports a passing job. + // Written first, so the errors are readable and an upload can still say what happened — then + // fail, because a harness that measured nothing at all otherwise reports a passing job. if (report.benchmarks.every((benchmark) => benchmark.error !== undefined)) { throw new Error('No benchmark produced any measurement.'); } diff --git a/packages/benchmark/src/runner/upload.ts b/packages/benchmark/src/runner/upload.ts index 6eb2ad8cc1..ab9e8505c9 100644 --- a/packages/benchmark/src/runner/upload.ts +++ b/packages/benchmark/src/runner/upload.ts @@ -1,10 +1,11 @@ /* eslint-disable no-console -- progress belongs in the CI log. */ +import { readFile } from 'node:fs/promises'; import chalk from 'chalk'; import { postToDashboard } from '../ciApi'; import { getCiMetadata } from '../ciReport'; import { syncPrComment } from '../syncPrComment'; -import { benchmarkRunUploadSchema } from '../runReport'; +import { benchmarkRunReportSchema, benchmarkRunUploadSchema } from '../runReport'; import type { BenchmarkRunReport } from '../runReport'; /** @@ -54,3 +55,20 @@ export async function publishRunReport( ); } } + +/** + * Uploads a report file that `benchmark` wrote, and refreshes the pull request comment. + * + * The dashboard files the report under the commit CI is building, so a report measured on another + * commit is refused rather than filed under the wrong one. + */ +export async function uploadReportFile(file: string, timeline: string | undefined): Promise { + const report = benchmarkRunReportSchema.parse(JSON.parse(await readFile(file, 'utf8'))); + const { commitSha } = await getCiMetadata(); + if (commitSha && report.head.sha && report.head.sha !== commitSha) { + throw new Error( + `${file} was measured on ${report.head.sha}, but CI is building ${commitSha}. Upload a report from this commit's run.`, + ); + } + await publishRunReport(report, timeline); +} From 70c0524f22ccf168a9a66240c0ee0d92db9ce021 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:47:53 +0200 Subject: [PATCH 156/173] [benchmark] Run the benchmark CLI as pnpm test:benchmark The Vitest suite moves to pnpm test:performance, after its folder. --- .circleci/config.yml | 6 +++--- package.json | 4 ++-- test/benchmark/README.md | 8 ++++---- test/benchmark/package.json | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e03bc993c4..2a24e86b2f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -143,7 +143,7 @@ jobs: - run: name: Run the benchmark smoke test no_output_timeout: 30m - command: pnpm test:benchmark-run --baseline "$(cat .benchmark-baseline-sha)" + command: pnpm test:benchmark --baseline "$(cat .benchmark-baseline-sha)" - run: name: Upload the benchmark report # Also after a failed run, so the comment says what happened. @@ -153,7 +153,7 @@ jobs: # `benchmark.json`, and to this pull request's dashboard preview, since production does # not read version 2 reports yet. Remove this step's environment before merging. CI_REPORT_API_URL: https://code-infra-dashboard-pr-1898.onrender.com - command: pnpm -F ./test/benchmark benchmark:run upload + command: pnpm test:benchmark upload - save_cache: name: Save the packed-baseline cache key: benchmark-run-cache-v1-{{ checksum ".benchmark-baseline-sha" }} @@ -177,7 +177,7 @@ jobs: - run: name: Benchmark the commit against itself no_output_timeout: 30m - command: pnpm --silent -F ./test/benchmark benchmark:run --baseline HEAD --reporter json > aa.json + command: pnpm --silent -F ./test/benchmark benchmark --baseline HEAD --reporter json > aa.json - store_artifacts: path: aa.json - run: diff --git a/package.json b/package.json index 8c6fe97105..95f727dc29 100644 --- a/package.json +++ b/package.json @@ -30,8 +30,8 @@ "docs:validate": "pnpm -F \"./docs\" run docs-infra validate --command 'pnpm docs:validate'", "docs:lint": "pnpm -F \"./docs\" run lint", "docs:lib": "pnpm -F \"./packages/docs-infra\" run build", - "test:benchmark": "pnpm -F ./test/performance benchmark", - "test:benchmark-run": "pnpm -F ./test/benchmark benchmark:run", + "test:benchmark": "pnpm -F ./test/benchmark benchmark", + "test:performance": "pnpm -F ./test/performance benchmark", "org-versions": "tsx scripts/orgVersions.mts" }, "dependencies": { diff --git a/test/benchmark/README.md b/test/benchmark/README.md index 812b65fc18..0c6dbcfb14 100644 --- a/test/benchmark/README.md +++ b/test/benchmark/README.md @@ -27,14 +27,14 @@ build. ```bash pnpm release:build # the harness resolves workspace packages through their build output -pnpm test:benchmark-run # from the repo root +pnpm test:benchmark # from the repo root ``` Or from here, with the usual filters and flags: ```bash -pnpm -F ./test/benchmark benchmark:run workload -pnpm -F ./test/benchmark benchmark:run --baseline HEAD +pnpm -F ./test/benchmark benchmark workload +pnpm -F ./test/benchmark benchmark --baseline HEAD ``` A filter is a case-insensitive substring of a benchmark file's path under `src/`, the way vitest @@ -59,7 +59,7 @@ every job that compares a branch to its base, so bundle size and the benchmarks instead of each deriving it. Pass it in: ```bash -pnpm test:benchmark-run --baseline "$(pnpm code-infra baseline)" +pnpm test:benchmark --baseline "$(pnpm code-infra baseline)" ``` Without `--baseline` the previous commit is used. diff --git a/test/benchmark/package.json b/test/benchmark/package.json index 0f419f37d4..dffb959b24 100644 --- a/test/benchmark/package.json +++ b/test/benchmark/package.json @@ -6,7 +6,7 @@ "build": "vite build", "preview": "vite preview", "typescript": "tsc -noEmit", - "benchmark:run": "node ./scripts/runBenchmark.mjs" + "benchmark": "node ./scripts/runBenchmark.mjs" }, "dependencies": { "@mui/internal-test-utils": "workspace:*", From 86723885e1d041b991929588703cbef75fa508ee Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Wed, 14 Oct 2026 16:58:56 +0200 Subject: [PATCH 157/173] [benchmark] Drop the pinch and tap gestures No benchmark uses them; scroll stays. --- packages/benchmark/README.md | 1 - packages/benchmark/src/input.ts | 28 ---------------------------- packages/benchmark/src/page/page.tsx | 2 +- packages/benchmark/src/types.ts | 2 +- 4 files changed, 2 insertions(+), 31 deletions(-) diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 7f4a541cba..c6eb329bcf 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -79,7 +79,6 @@ benchmark( () => , async ({ input }) => { await input.scroll({ x: 400, y: 300, deltaY: 600 }); - await input.pinch({ x: 400, y: 300, scaleFactor: 2 }); }, ); ``` diff --git a/packages/benchmark/src/input.ts b/packages/benchmark/src/input.ts index e2ff05f740..e770068e10 100644 --- a/packages/benchmark/src/input.ts +++ b/packages/benchmark/src/input.ts @@ -14,20 +14,6 @@ export interface ScrollGestureOptions { speed?: number; } -export interface PinchGestureOptions { - x: number; - y: number; - /** Relative scale at the end of the pinch: `> 1` zooms in, `< 1` zooms out. */ - scaleFactor: number; - /** Relative pointer speed in pixels per second. Defaults to the browser's 800. */ - relativeSpeed?: number; -} - -export interface TapGestureOptions { - x: number; - y: number; -} - /** * Trusted input for interactions. Each gesture is one command: the browser generates the whole * event sequence itself at frame cadence, so the round trip to the driver is paid once per gesture, @@ -35,8 +21,6 @@ export interface TapGestureOptions { */ export interface BenchmarkInput { scroll: (options: ScrollGestureOptions) => Promise; - pinch: (options: PinchGestureOptions) => Promise; - tap: (options: TapGestureOptions) => Promise; } /** @@ -65,17 +49,5 @@ export function createInput(send: CdpSend): BenchmarkInput { gestureSourceType: 'mouse', }); }, - pinch: async ({ x, y, scaleFactor, relativeSpeed }) => { - await send('Input.synthesizePinchGesture', { - x, - y, - scaleFactor, - relativeSpeed, - gestureSourceType: 'mouse', - }); - }, - tap: async ({ x, y }) => { - await send('Input.synthesizeTapGesture', { x, y, gestureSourceType: 'mouse' }); - }, }; } diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index d0d5f9b040..5365c27e71 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -14,7 +14,7 @@ import type { RunMetricAlarm } from '../runReport/schema'; /** What a case's `run` is handed on every iteration. */ export interface BenchmarkContext { - /** Trusted input — scroll, pinch, tap — dispatched by the browser rather than from script. */ + /** Trusted input — scroll gestures — dispatched by the browser rather than from script. */ input: BenchmarkInput; /** The DevTools Protocol, for what the case sets up or reads from the browser itself. */ cdp: BenchmarkCdp; diff --git a/packages/benchmark/src/types.ts b/packages/benchmark/src/types.ts index 333f93cd8a..e5c90d159c 100644 --- a/packages/benchmark/src/types.ts +++ b/packages/benchmark/src/types.ts @@ -38,7 +38,7 @@ export interface InteractionContext { */ resumeReactRecording: () => void; /** - * Trusted input gestures (scroll, pinch, tap) dispatched by the browser itself, for interactions + * Trusted input gestures (scroll) dispatched by the browser itself, for interactions * that should behave like a real mouse or trackpad rather than synthetic DOM events. */ input: BenchmarkInput; From f55753bb9e10aaa50b7a07d9159b37cfb8ae6fdc Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:39:50 +0200 Subject: [PATCH 158/173] [benchmark] Drop input; gestures go through cdp A case sends Input.synthesizeScrollGesture itself. A gesture API can come back once more than scrolling needs one. --- packages/benchmark/README.md | 36 +++++++++---- packages/benchmark/src/caseRuntime.tsx | 2 +- packages/benchmark/src/cdp.ts | 16 ++++++ packages/benchmark/src/index.tsx | 7 ++- packages/benchmark/src/input.ts | 53 ------------------- packages/benchmark/src/page/page.tsx | 8 +-- .../benchmark/src/page/reactBenchmark.tsx | 4 +- packages/benchmark/src/publicApi.ts | 2 +- packages/benchmark/src/types.ts | 7 +-- test/benchmark/src/list/list.bench.tsx | 10 +++- test/benchmark/src/lists/lists.bench.tsx | 10 +++- 11 files changed, 68 insertions(+), 87 deletions(-) create mode 100644 packages/benchmark/src/cdp.ts delete mode 100644 packages/benchmark/src/input.ts diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index c6eb329bcf..43400b5837 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -70,15 +70,22 @@ benchmark( ); ``` -For input that should behave like a real mouse or trackpad, the interaction context has `input`: -trusted gestures the browser generates itself at frame cadence, one command per gesture. +The interaction context has `cdp`, the Chrome DevTools Protocol on the session that drives the page. +For input that should behave like a real mouse or trackpad, send a gesture: the browser generates the +whole event sequence itself at frame cadence, one command per gesture. Its distances point the other +way from a wheel's: a negative `yDistance` scrolls down. ```tsx benchmark( 'Chart zoom', () => , - async ({ input }) => { - await input.scroll({ x: 400, y: 300, deltaY: 600 }); + async ({ cdp }) => { + await cdp.send('Input.synthesizeScrollGesture', { + x: 400, + y: 300, + yDistance: -600, + gestureSourceType: 'mouse', + }); }, ); ``` @@ -88,8 +95,7 @@ iteration to the next. Wait for the mount to be painted — `await waitForElemen before starting a gesture on something the case just rendered; until then, the browser may not route the gesture to it. -For anything else the browser can do, the context has `cdp`, the Chrome DevTools Protocol on the -session that drives the page: throttle the CPU, or read the engine's own counters as a custom metric. +The same session can throttle the CPU, or read the engine's own counters as a custom metric. ```tsx const layouts = new DiscreteMetric({ name: 'layout:count' }); @@ -97,7 +103,7 @@ const layouts = new DiscreteMetric({ name: 'layout:count' }); benchmark( 'Grid scroll', () => , - async ({ input, cdp }) => { + async ({ cdp }) => { await cdp.send('Performance.enable'); const count = async () => { const { metrics } = (await cdp.send('Performance.getMetrics')) as { @@ -106,7 +112,12 @@ benchmark( return metrics.find((metric) => metric.name === 'LayoutCount')!.value; }; const before = await count(); - await input.scroll({ x: 400, y: 300, deltaY: 2000 }); + await cdp.send('Input.synthesizeScrollGesture', { + x: 400, + y: 300, + yDistance: -2000, + gestureSourceType: 'mouse', + }); layouts.record((await count()) - before); }, ); @@ -382,8 +393,13 @@ reactBenchmark('mount', () => ); reactBenchmark( 'scroll', () => , - async ({ input }) => { - await input.scroll({ x: 100, y: 100, deltaY: 2000 }); + async ({ cdp }) => { + await cdp.send('Input.synthesizeScrollGesture', { + x: 100, + y: 100, + yDistance: -2000, + gestureSourceType: 'mouse', + }); }, ); ``` diff --git a/packages/benchmark/src/caseRuntime.tsx b/packages/benchmark/src/caseRuntime.tsx index 7664fa77fc..509fe64791 100644 --- a/packages/benchmark/src/caseRuntime.tsx +++ b/packages/benchmark/src/caseRuntime.tsx @@ -260,7 +260,7 @@ export async function measureIteration( renderFn: () => React.ReactElement, interaction: BenchmarkInteraction | undefined, options: CaseOptions | undefined, - driver: Pick, + driver: Pick, ): Promise { // Per-iteration switch for the harness's React render/paint recording. Starts paused when // `reactRecordingPaused` is set; the interaction callback drives it from there. diff --git a/packages/benchmark/src/cdp.ts b/packages/benchmark/src/cdp.ts new file mode 100644 index 0000000000..4d4d8aa755 --- /dev/null +++ b/packages/benchmark/src/cdp.ts @@ -0,0 +1,16 @@ +/** Sends a raw Chrome DevTools Protocol command to the page under test. */ +export type CdpSend = (method: string, params?: Record) => Promise; + +/** + * The Chrome DevTools Protocol, on the session that drives the page under test: for what a case + * sets up or reads itself — trusted input gestures (`Input.synthesizeScrollGesture`), CPU + * throttling (`Emulation.setCPUThrottlingRate`), the engine's own counters + * (`Performance.getMetrics`) as custom metrics. + * + * A setting changed this way stays on the tab, through later samples and the other cases sampled in + * it — those of a `compare()` too — so a case that changes one sets it every time, and the others + * set it as they need it. + */ +export interface BenchmarkCdp { + send: CdpSend; +} diff --git a/packages/benchmark/src/index.tsx b/packages/benchmark/src/index.tsx index 2b1f469039..8493001b4c 100644 --- a/packages/benchmark/src/index.tsx +++ b/packages/benchmark/src/index.tsx @@ -6,8 +6,7 @@ import type { RenderEvent, IterationData } from './types'; import { ScalarMetric } from './ScalarMetric'; import { metricsGate } from './metricsGate'; import { runProfileSession } from './profileSession'; -import { createInput } from './input'; -import type { CdpSend } from './input'; +import type { CdpSend } from './cdp'; import { collectGarbage } from './gc'; import { createCaseRuntime, @@ -34,11 +33,11 @@ const PROFILE_MODE = process.env.BENCHMARK_PROFILE === 'true'; type CdpMethod = Parameters['send']>[0]; -// Gestures and `cdp` go through the CDP session Vitest's Playwright provider holds for the test +// `cdp` goes through the CDP session Vitest's Playwright provider holds for the test // page. The session's `send` only accepts protocol method names it knows; ours takes any string and // leaves unknown methods for Chrome to reject. const send: CdpSend = (method, params) => cdp().send(method as CdpMethod, params); -const driver = { input: createInput(send), cdp: { send } }; +const driver = { cdp: { send } }; // Paint timings are recorded as one harness-owned `bench:paint` metric: the default sentinel // is the base series (`bench:paint`) and named `elementtiming` markers are sub-series diff --git a/packages/benchmark/src/input.ts b/packages/benchmark/src/input.ts deleted file mode 100644 index e770068e10..0000000000 --- a/packages/benchmark/src/input.ts +++ /dev/null @@ -1,53 +0,0 @@ -/** Sends a raw Chrome DevTools Protocol command to the page under test. */ -export type CdpSend = (method: string, params?: Record) => Promise; - -export interface ScrollGestureOptions { - /** Viewport x coordinate (CSS pixels) where the gesture starts. */ - x: number; - /** Viewport y coordinate (CSS pixels) where the gesture starts. */ - y: number; - /** Horizontal distance to scroll, positive to scroll right. */ - deltaX?: number; - /** Vertical distance to scroll, positive to scroll down (like a wheel's `deltaY`). */ - deltaY?: number; - /** Gesture speed in pixels per second. Defaults to the browser's 800. */ - speed?: number; -} - -/** - * Trusted input for interactions. Each gesture is one command: the browser generates the whole - * event sequence itself at frame cadence, so the round trip to the driver is paid once per gesture, - * not once per event, and the page sees the event timing a real mouse or trackpad would produce. - */ -export interface BenchmarkInput { - scroll: (options: ScrollGestureOptions) => Promise; -} - -/** - * The Chrome DevTools Protocol, on the session that drives the page under test: for what a case - * sets up or reads itself — CPU throttling (`Emulation.setCPUThrottlingRate`), the engine's own - * counters (`Performance.getMetrics`) as custom metrics. - * - * A setting changed this way stays on the tab, through later samples and the other cases sampled in - * it — those of a `compare()` too — so a case that changes one sets it every time, and the others - * set it as they need it. - */ -export interface BenchmarkCdp { - send: CdpSend; -} - -export function createInput(send: CdpSend): BenchmarkInput { - return { - scroll: async ({ x, y, deltaX = 0, deltaY = 0, speed }) => { - // CDP distances point the other way: positive `yDistance` scrolls up. - await send('Input.synthesizeScrollGesture', { - x, - y, - xDistance: -deltaX, - yDistance: -deltaY, - speed, - gestureSourceType: 'mouse', - }); - }, - }; -} diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 5365c27e71..3df821279f 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -1,6 +1,5 @@ import { collectGarbage } from '../gc'; -import { createInput } from '../input'; -import type { BenchmarkCdp, BenchmarkInput, CdpSend } from '../input'; +import type { BenchmarkCdp, CdpSend } from '../cdp'; import { seriesName, setMetricRecorder } from '../metricCore'; import { pickSampling, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; @@ -14,8 +13,6 @@ import type { RunMetricAlarm } from '../runReport/schema'; /** What a case's `run` is handed on every iteration. */ export interface BenchmarkContext { - /** Trusted input — scroll gestures — dispatched by the browser rather than from script. */ - input: BenchmarkInput; /** The DevTools Protocol, for what the case sets up or reads from the browser itself. */ cdp: BenchmarkCdp; } @@ -172,7 +169,6 @@ const send: CdpSend = (method, params) => { } return window.benchmarkCdp(method, params); }; -const input = createInput(send); const cdp: BenchmarkCdp = { send }; async function sample(name: string): Promise> { @@ -187,7 +183,7 @@ async function sample(name: string): Promise> { const values = new Map(); sampleValues = values; try { - await run({ input, cdp }); + await run({ cdp }); } finally { sampleValues = null; } diff --git a/packages/benchmark/src/page/reactBenchmark.tsx b/packages/benchmark/src/page/reactBenchmark.tsx index ebd0c8ba2a..51dad476c8 100644 --- a/packages/benchmark/src/page/reactBenchmark.tsx +++ b/packages/benchmark/src/page/reactBenchmark.tsx @@ -56,8 +56,8 @@ export function reactBenchmark( ): BenchmarkCase { const { interaction, options } = splitCaseArgs(interactionOrOptions, maybeOptions); - const run = async ({ input, cdp }: BenchmarkContext) => { - const result = await measureIteration(renderFn, interaction, options, { input, cdp }); + const run = async ({ cdp }: BenchmarkContext) => { + const result = await measureIteration(renderFn, interaction, options, { cdp }); // Render time is reported as totals rather than per render: an interaction's render count // follows whatever the browser coalesced, so per-render values would not line up across diff --git a/packages/benchmark/src/publicApi.ts b/packages/benchmark/src/publicApi.ts index 8cc9ae727c..e27c7ef6c3 100644 --- a/packages/benchmark/src/publicApi.ts +++ b/packages/benchmark/src/publicApi.ts @@ -9,7 +9,7 @@ export type { MetricConfig, MetricDefinition, } from './types'; -export type { BenchmarkCdp, BenchmarkInput } from './input'; +export type { BenchmarkCdp } from './cdp'; export type { BenchmarkInteraction } from './caseRuntime'; export { ElementTiming } from './ElementTiming'; export { Metric, type MetricRecordOptions } from './metricCore'; diff --git a/packages/benchmark/src/types.ts b/packages/benchmark/src/types.ts index e5c90d159c..e67470eb1f 100644 --- a/packages/benchmark/src/types.ts +++ b/packages/benchmark/src/types.ts @@ -1,4 +1,4 @@ -import type { BenchmarkCdp, BenchmarkInput } from './input'; +import type { BenchmarkCdp } from './cdp'; export interface RenderEvent { id: string; @@ -37,11 +37,6 @@ export interface InteractionContext { * already active. */ resumeReactRecording: () => void; - /** - * Trusted input gestures (scroll) dispatched by the browser itself, for interactions - * that should behave like a real mouse or trackpad rather than synthetic DOM events. - */ - input: BenchmarkInput; /** The DevTools Protocol, for what the interaction sets up or reads from the browser itself. */ cdp: BenchmarkCdp; } diff --git a/test/benchmark/src/list/list.bench.tsx b/test/benchmark/src/list/list.bench.tsx index 092de49f8c..918c0ba6d2 100644 --- a/test/benchmark/src/list/list.bench.tsx +++ b/test/benchmark/src/list/list.bench.tsx @@ -38,12 +38,18 @@ reactBenchmark( ), - async ({ input, resumeReactRecording, waitForElementTiming }) => { + async ({ cdp, resumeReactRecording, waitForElementTiming }) => { // The scroller has to be painted before a gesture can hit it. await waitForElementTiming('default'); resumeReactRecording(); scrollDuration.time(); - await input.scroll({ x: 200, y: 150, deltaY: 1200, speed: 2400 }); + await cdp.send('Input.synthesizeScrollGesture', { + x: 200, + y: 150, + yDistance: -1200, + speed: 2400, + gestureSourceType: 'mouse', + }); scrollDuration.timeEnd(); }, { reactRecordingPaused: true, ...SMOKE_SAMPLING }, diff --git a/test/benchmark/src/lists/lists.bench.tsx b/test/benchmark/src/lists/lists.bench.tsx index 62c87ecea5..77625c3a72 100644 --- a/test/benchmark/src/lists/lists.bench.tsx +++ b/test/benchmark/src/lists/lists.bench.tsx @@ -13,14 +13,20 @@ import { UlList } from './lists.ul'; */ const scroll: BenchmarkInteraction = async ({ - input, + cdp, resumeReactRecording, waitForElementTiming, }) => { // The scroller has to be painted before a gesture can hit it. await waitForElementTiming('default'); resumeReactRecording(); - await input.scroll({ x: 200, y: 150, deltaY: 1200, speed: 2400 }); + await cdp.send('Input.synthesizeScrollGesture', { + x: 200, + y: 150, + yDistance: -1200, + speed: 2400, + gestureSourceType: 'mouse', + }); }; compare( From 56af0f3684eacfbe3938f62380e0bd75029f3f14 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:21:59 +0200 Subject: [PATCH 159/173] [benchmark] Validate sampling options without resolving them A case's registration only checks its options; the runner's merge drops a pick that changed nothing, since own options are already picked. --- packages/benchmark/src/page/page.tsx | 8 +++---- .../benchmark/src/runner/runInterleaved.ts | 4 ++-- packages/benchmark/src/sampling.test.ts | 24 ++++++++++++++----- packages/benchmark/src/sampling.ts | 18 +++++++++----- 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 3df821279f..bcc244fd32 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -1,7 +1,7 @@ import { collectGarbage } from '../gc'; import type { BenchmarkCdp, CdpSend } from '../cdp'; import { seriesName, setMetricRecorder } from '../metricCore'; -import { pickSampling, resolveSampling } from '../sampling'; +import { pickSampling, validateSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { MetricDefinition } from '../types'; import type { RunMetricAlarm } from '../runReport/schema'; @@ -58,8 +58,7 @@ export function benchmark( if (cases.has(name)) { throw new Error(`Two benchmarks share the name "${name}". Benchmark names must be unique.`); } - // Resolved here only to reject invalid options where they were written. - resolveSampling(options); + validateSampling(options); cases.set(name, { run, sampling: pickSampling(options), alarms: options.alarms }); return { name }; } @@ -89,8 +88,7 @@ export function compare( if (compared.length < 2) { throw new Error(`Comparison "${name}" needs at least two cases.`); } - // Resolved here only to reject invalid options where they were written. - resolveSampling(sampling); + validateSampling(sampling); for (const benchCase of compared) { const own = cases.get(benchCase.name); if (own && Object.keys(own.sampling).length > 0) { diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index de749e5b91..626ddf33e3 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -6,7 +6,7 @@ import type { Browser, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; import { compareBenchmark, formatDuration, formatRoundCount } from '../runReport'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; -import { differencesSettled, pickSampling, resolveSampling } from '../sampling'; +import { differencesSettled, resolveSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { BenchFile } from './benchFiles'; import type { ResolvedRef } from './refs'; @@ -443,7 +443,7 @@ function samplingOf( own: SamplingOptions, runDefaults: SamplingOptions = {}, ): Required { - return resolveSampling({ ...pickSampling(runDefaults), ...own }); + return resolveSampling({ ...runDefaults, ...own }); } /** What a run drives: the browser, and every benchmark it was asked for, in order. */ diff --git a/packages/benchmark/src/sampling.test.ts b/packages/benchmark/src/sampling.test.ts index eb7823c985..31249ab738 100644 --- a/packages/benchmark/src/sampling.test.ts +++ b/packages/benchmark/src/sampling.test.ts @@ -1,7 +1,12 @@ import { describe, expect, it } from 'vitest'; import { analyzeBenchmark } from './runReport/analyzeRun'; import type { RunBenchmark, RunMetricDefinition } from './runReport'; -import { DEFAULT_SAMPLING, differencesSettled, resolveSampling } from './sampling'; +import { + DEFAULT_SAMPLING, + differencesSettled, + resolveSampling, + validateSampling, +} from './sampling'; describe('resolveSampling', () => { it('fills in a default for every option left out or undefined', () => { @@ -11,17 +16,24 @@ describe('resolveSampling', () => { }); }); - it('accepts the documented ranges', () => { - expect(() => resolveSampling({ sampleSize: 2, timeout: 0 })).not.toThrow(); + it('rejects options out of range', () => { + expect(() => resolveSampling({ sampleSize: 1 })).toThrow(/Invalid sampleSize 1/); + }); +}); + +describe('validateSampling', () => { + it('accepts the documented ranges, and options left out', () => { + expect(() => validateSampling({ sampleSize: 2, timeout: 0 })).not.toThrow(); + expect(() => validateSampling({})).not.toThrow(); }); it('rejects a sample size that cannot form an interval', () => { - expect(() => resolveSampling({ sampleSize: 1 })).toThrow(/Invalid sampleSize 1/); - expect(() => resolveSampling({ sampleSize: 2.5 })).toThrow(/Invalid sampleSize 2.5/); + expect(() => validateSampling({ sampleSize: 1 })).toThrow(/Invalid sampleSize 1/); + expect(() => validateSampling({ sampleSize: 2.5 })).toThrow(/Invalid sampleSize 2.5/); }); it('rejects a negative timeout', () => { - expect(() => resolveSampling({ timeout: -1 })).toThrow(/Invalid timeout -1/); + expect(() => validateSampling({ timeout: -1 })).toThrow(/Invalid timeout -1/); }); }); diff --git a/packages/benchmark/src/sampling.ts b/packages/benchmark/src/sampling.ts index eee3020970..10a8e24fe3 100644 --- a/packages/benchmark/src/sampling.ts +++ b/packages/benchmark/src/sampling.ts @@ -23,17 +23,23 @@ export const DEFAULT_SAMPLING: Required = { }; /** - * Checks sampling options and fills in the defaults. Called when a case registers, so a mistake fails - * the page rather than a round, and the runner gets complete options. + * Throws on a sampling option out of range. Called when a case registers, so a mistake fails the + * page where it was written rather than a round. */ -export function resolveSampling(options: SamplingOptions): Required { - const { sampleSize = DEFAULT_SAMPLING.sampleSize, timeout = DEFAULT_SAMPLING.timeout } = options; - if (!(Number.isInteger(sampleSize) && sampleSize >= 2)) { +export function validateSampling(options: SamplingOptions): void { + const { sampleSize, timeout } = options; + if (sampleSize !== undefined && !(Number.isInteger(sampleSize) && sampleSize >= 2)) { throw new Error(`Invalid sampleSize ${sampleSize}: expected an integer of at least 2.`); } - if (!(timeout >= 0)) { + if (timeout !== undefined && !(timeout >= 0)) { throw new Error(`Invalid timeout ${timeout}: expected a number of minutes, 0 or more.`); } +} + +/** Validates sampling options and fills in the defaults for those left out or undefined. */ +export function resolveSampling(options: SamplingOptions): Required { + validateSampling(options); + const { sampleSize = DEFAULT_SAMPLING.sampleSize, timeout = DEFAULT_SAMPLING.timeout } = options; return { sampleSize, timeout }; } From 0405b7620fa82640ea6cab74a88c0494f0fbb471 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:25:42 +0200 Subject: [PATCH 160/173] [benchmark] Check sampling options once, where they come in getSampling validates and keeps the options set, when a case registers and when a run starts; from there a plain spread over the defaults merges them. A bad --sample-size now fails before anything is built. --- packages/benchmark/src/page/page.tsx | 9 +++-- .../benchmark/src/runner/runBenchmarks.ts | 5 +-- .../benchmark/src/runner/runInterleaved.ts | 9 ++--- packages/benchmark/src/sampling.test.ts | 34 ++++++------------- packages/benchmark/src/sampling.ts | 27 +++++---------- 5 files changed, 32 insertions(+), 52 deletions(-) diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index bcc244fd32..47bd075f0f 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -1,7 +1,7 @@ import { collectGarbage } from '../gc'; import type { BenchmarkCdp, CdpSend } from '../cdp'; import { seriesName, setMetricRecorder } from '../metricCore'; -import { pickSampling, validateSampling } from '../sampling'; +import { getSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { MetricDefinition } from '../types'; import type { RunMetricAlarm } from '../runReport/schema'; @@ -58,8 +58,7 @@ export function benchmark( if (cases.has(name)) { throw new Error(`Two benchmarks share the name "${name}". Benchmark names must be unique.`); } - validateSampling(options); - cases.set(name, { run, sampling: pickSampling(options), alarms: options.alarms }); + cases.set(name, { run, sampling: getSampling(options), alarms: options.alarms }); return { name }; } @@ -88,7 +87,7 @@ export function compare( if (compared.length < 2) { throw new Error(`Comparison "${name}" needs at least two cases.`); } - validateSampling(sampling); + const ownSampling = getSampling(sampling); for (const benchCase of compared) { const own = cases.get(benchCase.name); if (own && Object.keys(own.sampling).length > 0) { @@ -111,7 +110,7 @@ export function compare( comparisons.push({ name, cases: compared.map((benchCase) => benchCase.name), - sampling: pickSampling(sampling), + sampling: ownSampling, }); } diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index a36e46cd7b..1257952ca9 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -9,6 +9,7 @@ import { packRef, packWorkingTree } from '../utils/packWorkspace'; import { getCiMetadata } from '../ciReport'; import { summarizeRun } from '../runReport'; import type { BenchmarkRunReport } from '../runReport'; +import { getSampling } from '../sampling'; import type { SamplingOptions } from '../sampling'; import { refLabel, resolveBaselineRef, WORKTREE_REF } from './refs'; import type { ResolvedRef } from './refs'; @@ -85,13 +86,13 @@ export async function runBenchmarks( baseline, buildCmd = 'pnpm release:build', out, - sampling, testNamePattern, reporter = 'default', profile = false, signal, } = options; - // Compiled before anything is built, so a bad pattern fails at once. + // Checked before anything is built, so a bad option fails at once. + const sampling = getSampling(options.sampling ?? {}); const namePattern = testNamePattern === undefined ? undefined : new RegExp(testNamePattern); const repoRoot = await findWorkspaceDir(harnessDir); diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index 626ddf33e3..ea16cdfffa 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -6,7 +6,7 @@ import type { Browser, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; import { compareBenchmark, formatDuration, formatRoundCount } from '../runReport'; import type { RunBenchmark, RunMetricDefinition } from '../runReport'; -import { differencesSettled, resolveSampling } from '../sampling'; +import { DEFAULT_SAMPLING, differencesSettled } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { BenchFile } from './benchFiles'; import type { ResolvedRef } from './refs'; @@ -33,7 +33,7 @@ export interface RunInterleavedOptions { benchRefs: ResolvedRef[]; browserBinary: string; launchArgs: string[]; - /** Defaults for the sampling options a benchmark doesn't set itself. */ + /** Defaults for the sampling options a benchmark doesn't set itself, from `getSampling`. */ sampling?: SamplingOptions; /** Only run benchmarks whose name matches this, like Vitest's `-t`. */ testNamePattern?: RegExp; @@ -437,13 +437,14 @@ async function planBenchFile( /** * A benchmark's sampling, as in Vitest: what it sets itself, then what the run gives as defaults, - * then the built-in defaults. + * then the built-in defaults. Both come from `getSampling`, so they are valid and set no key to + * undefined. */ function samplingOf( own: SamplingOptions, runDefaults: SamplingOptions = {}, ): Required { - return resolveSampling({ ...runDefaults, ...own }); + return { ...DEFAULT_SAMPLING, ...runDefaults, ...own }; } /** What a run drives: the browser, and every benchmark it was asked for, in order. */ diff --git a/packages/benchmark/src/sampling.test.ts b/packages/benchmark/src/sampling.test.ts index 31249ab738..0495ae3de7 100644 --- a/packages/benchmark/src/sampling.test.ts +++ b/packages/benchmark/src/sampling.test.ts @@ -1,39 +1,27 @@ import { describe, expect, it } from 'vitest'; import { analyzeBenchmark } from './runReport/analyzeRun'; import type { RunBenchmark, RunMetricDefinition } from './runReport'; -import { - DEFAULT_SAMPLING, - differencesSettled, - resolveSampling, - validateSampling, -} from './sampling'; +import { differencesSettled, getSampling } from './sampling'; -describe('resolveSampling', () => { - it('fills in a default for every option left out or undefined', () => { - expect(resolveSampling({ timeout: 1, sampleSize: undefined })).toEqual({ - ...DEFAULT_SAMPLING, - timeout: 1, - }); - }); - - it('rejects options out of range', () => { - expect(() => resolveSampling({ sampleSize: 1 })).toThrow(/Invalid sampleSize 1/); +describe('getSampling', () => { + it('keeps only the sampling options that are set', () => { + const options = { sampleSize: undefined, timeout: 1, alarms: {} }; + expect(getSampling(options)).toEqual({ timeout: 1 }); + expect(Object.keys(getSampling(options))).toEqual(['timeout']); }); -}); -describe('validateSampling', () => { it('accepts the documented ranges, and options left out', () => { - expect(() => validateSampling({ sampleSize: 2, timeout: 0 })).not.toThrow(); - expect(() => validateSampling({})).not.toThrow(); + expect(getSampling({ sampleSize: 2, timeout: 0 })).toEqual({ sampleSize: 2, timeout: 0 }); + expect(getSampling({})).toEqual({}); }); it('rejects a sample size that cannot form an interval', () => { - expect(() => validateSampling({ sampleSize: 1 })).toThrow(/Invalid sampleSize 1/); - expect(() => validateSampling({ sampleSize: 2.5 })).toThrow(/Invalid sampleSize 2.5/); + expect(() => getSampling({ sampleSize: 1 })).toThrow(/Invalid sampleSize 1/); + expect(() => getSampling({ sampleSize: 2.5 })).toThrow(/Invalid sampleSize 2.5/); }); it('rejects a negative timeout', () => { - expect(() => validateSampling({ timeout: -1 })).toThrow(/Invalid timeout -1/); + expect(() => getSampling({ timeout: -1 })).toThrow(/Invalid timeout -1/); }); }); diff --git a/packages/benchmark/src/sampling.ts b/packages/benchmark/src/sampling.ts index 10a8e24fe3..a2d17cc25f 100644 --- a/packages/benchmark/src/sampling.ts +++ b/packages/benchmark/src/sampling.ts @@ -23,10 +23,12 @@ export const DEFAULT_SAMPLING: Required = { }; /** - * Throws on a sampling option out of range. Called when a case registers, so a mistake fails the - * page where it was written rather than a round. + * The sampling options `options` sets, checked: only `sampleSize` and `timeout`, and only those not + * undefined, so spreading them over defaults keeps the defaults for the rest. Throws on one out of + * range. Called where options come in, when a case registers and when a run starts, so a mistake + * fails there rather than in a round, and what it returns is valid from then on. */ -export function validateSampling(options: SamplingOptions): void { +export function getSampling(options: SamplingOptions): SamplingOptions { const { sampleSize, timeout } = options; if (sampleSize !== undefined && !(Number.isInteger(sampleSize) && sampleSize >= 2)) { throw new Error(`Invalid sampleSize ${sampleSize}: expected an integer of at least 2.`); @@ -34,21 +36,10 @@ export function validateSampling(options: SamplingOptions): void { if (timeout !== undefined && !(timeout >= 0)) { throw new Error(`Invalid timeout ${timeout}: expected a number of minutes, 0 or more.`); } -} - -/** Validates sampling options and fills in the defaults for those left out or undefined. */ -export function resolveSampling(options: SamplingOptions): Required { - validateSampling(options); - const { sampleSize = DEFAULT_SAMPLING.sampleSize, timeout = DEFAULT_SAMPLING.timeout } = options; - return { sampleSize, timeout }; -} - -/** The sampling options `options` sets, leaving out any other key and any left undefined. */ -export function pickSampling(options: SamplingOptions): SamplingOptions { - const { sampleSize, timeout } = options; - return Object.fromEntries( - Object.entries({ sampleSize, timeout }).filter(([, value]) => value !== undefined), - ); + return { + ...(sampleSize !== undefined ? { sampleSize } : {}), + ...(timeout !== undefined ? { timeout } : {}), + }; } /** From b2ea5df646617f4733ed512c6df1b362c2c9024c Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:35:10 +0200 Subject: [PATCH 161/173] [benchmark] Require @playwright/test as a peer The benchmark CLI launches Chromium through it, so it is no longer optional. --- packages/benchmark/package.json | 3 --- packages/benchmark/src/runner/browser.ts | 17 ++++------------- packages/benchmark/src/runner/runBenchmarks.ts | 2 +- packages/benchmark/src/runner/runInterleaved.ts | 2 +- 4 files changed, 6 insertions(+), 18 deletions(-) diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index 48309adcef..5dced1a71c 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -61,9 +61,6 @@ "benchmark": "./src/cli/index.ts" }, "peerDependenciesMeta": { - "@playwright/test": { - "optional": true - }, "vite": { "optional": true } diff --git a/packages/benchmark/src/runner/browser.ts b/packages/benchmark/src/runner/browser.ts index 8de1402dc9..4ba3961087 100644 --- a/packages/benchmark/src/runner/browser.ts +++ b/packages/benchmark/src/runner/browser.ts @@ -1,22 +1,13 @@ import { access } from 'node:fs/promises'; -import type * as PlaywrightTest from '@playwright/test'; +import { chromium } from '@playwright/test'; /** * Resolves the browser every benchmark runs on: Playwright's pinned Chrome for Testing, rather than * whatever Chrome the machine has auto-updated to. */ -export async function resolveBrowserBinary(harnessDir: string): Promise { - let playwright: typeof PlaywrightTest; - try { - playwright = await import('@playwright/test'); - } catch { - // An optional peer, so it installs cleanly when absent and only fails here. - throw new Error( - `"@playwright/test" is not installed. Add it as a devDependency of ${harnessDir}.`, - ); - } - - const binary = playwright.chromium.executablePath(); +export async function resolveBrowserBinary(): Promise { + // `pnpm install` brings the package, not its browser. + const binary = chromium.executablePath(); try { await access(binary); } catch { diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index 1257952ca9..f3e290ac9a 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -122,7 +122,7 @@ export async function runBenchmarks( const refs: ResolvedRef[] = [WORKTREE_REF, await resolveBaselineRef(baseline, repoRoot)]; // Before any build, so a missing browser fails in seconds rather than after the packing. - const browserBinary = await resolveBrowserBinary(harnessDir); + const browserBinary = await resolveBrowserBinary(); // `getuid` is POSIX-only; on Windows nobody is root. Chrome cannot enter its sandbox as root. const launchArgs = process.getuid?.() === 0 ? ['--no-sandbox'] : []; diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index ea16cdfffa..766df1f2c5 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -2,6 +2,7 @@ import { addAbortListener } from 'node:events'; import chalk from 'chalk'; +import { chromium } from '@playwright/test'; import type { Browser, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; import { compareBenchmark, formatDuration, formatRoundCount } from '../runReport'; @@ -461,7 +462,6 @@ export async function withPlannedRun( options: RunInterleavedOptions, drive: (run: PlannedRun) => Promise, ): Promise { - const { chromium } = await import('@playwright/test'); const [server, browser] = await Promise.all([ serveDirectory(options.buildsDir), chromium.launch({ From d84a6af44e5884c6710da4f48e66b97b1db96047 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:38:26 +0200 Subject: [PATCH 162/173] [benchmark] Let Playwright find and report its own browser Its launch error says how to install a missing Chromium; checking ahead of the builds only saved time on a fresh machine. --- packages/benchmark/src/runner/browser.ts | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 packages/benchmark/src/runner/browser.ts diff --git a/packages/benchmark/src/runner/browser.ts b/packages/benchmark/src/runner/browser.ts deleted file mode 100644 index 4ba3961087..0000000000 --- a/packages/benchmark/src/runner/browser.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { access } from 'node:fs/promises'; -import { chromium } from '@playwright/test'; - -/** - * Resolves the browser every benchmark runs on: Playwright's pinned Chrome for Testing, rather than - * whatever Chrome the machine has auto-updated to. - */ -export async function resolveBrowserBinary(): Promise { - // `pnpm install` brings the package, not its browser. - const binary = chromium.executablePath(); - try { - await access(binary); - } catch { - throw new Error( - `Playwright's Chromium is not installed at "${binary}". Run \`pnpm exec playwright install chromium\`.`, - ); - } - return binary; -} From aca68d706045c25da2275cedb61cc43e4c39ecbb Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:39:01 +0200 Subject: [PATCH 163/173] [benchmark] Launch Playwright's own Chromium without browser.ts Completes the previous commit, which only deleted the file. --- packages/benchmark/src/runner/runBenchmarks.ts | 7 ++----- packages/benchmark/src/runner/runInterleaved.ts | 2 -- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index f3e290ac9a..8c2d8163bf 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -4,6 +4,7 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { mkdir, writeFile } from 'node:fs/promises'; import chalk from 'chalk'; +import { chromium } from '@playwright/test'; import { findWorkspaceDir } from '@pnpm/find-workspace-dir'; import { packRef, packWorkingTree } from '../utils/packWorkspace'; import { getCiMetadata } from '../ciReport'; @@ -17,7 +18,6 @@ import { discoverBenchFiles } from './benchFiles'; import { runInterleaved } from './runInterleaved'; import { traceBenchmarks } from './traceBenchmarks'; import { buildRefPages, restoreWorkspace } from './buildPages'; -import { resolveBrowserBinary } from './browser'; import { printRunReport } from './printReport'; import { buildsDirOf, prepareOutputDir, resultsPathOf } from './outputDir'; @@ -121,8 +121,6 @@ export async function runBenchmarks( // The working tree first: it is the reference every comparison is judged from. const refs: ResolvedRef[] = [WORKTREE_REF, await resolveBaselineRef(baseline, repoRoot)]; - // Before any build, so a missing browser fails in seconds rather than after the packing. - const browserBinary = await resolveBrowserBinary(); // `getuid` is POSIX-only; on Windows nobody is root. Chrome cannot enter its sandbox as root. const launchArgs = process.getuid?.() === 0 ? ['--no-sandbox'] : []; @@ -131,7 +129,7 @@ export async function runBenchmarks( ref.sha ? `${refLabel(ref)} (${ref.sha.slice(0, 9)})` : refLabel(ref); console.log(chalk.cyan(`Files: ${benchFiles.map((benchFile) => benchFile.file).join(', ')}`)); console.log(chalk.cyan(`Refs: ${refs.map(describeRef).join(', ')}`)); - console.log(chalk.cyan(`Browser: ${browserBinary} ${launchArgs.join(' ')}`)); + console.log(chalk.cyan(`Browser: ${chromium.executablePath()} ${launchArgs.join(' ')}`)); // Logical CPUs (hardware threads, vCPUs in CI), not physical cores; it honours CPU affinity, and // in practice a container's CPU quota. const cpus = os.availableParallelism(); @@ -187,7 +185,6 @@ export async function runBenchmarks( buildsDir, benchFiles, benchRefs: refs, - browserBinary, launchArgs, testNamePattern: namePattern, }; diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index 766df1f2c5..fe9bd3b549 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -32,7 +32,6 @@ export interface RunInterleavedOptions { benchFiles: BenchFile[]; /** The builds a benchmark file compares: the working tree first, then the baseline. */ benchRefs: ResolvedRef[]; - browserBinary: string; launchArgs: string[]; /** Defaults for the sampling options a benchmark doesn't set itself, from `getSampling`. */ sampling?: SamplingOptions; @@ -465,7 +464,6 @@ export async function withPlannedRun( const [server, browser] = await Promise.all([ serveDirectory(options.buildsDir), chromium.launch({ - executablePath: options.browserBinary, headless: true, args: [...BENCHMARK_LAUNCH_ARGS, ...options.launchArgs], }), From e6d1e1ee3d4b2eea00afbc42c2cc207f766ed9d6 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:41:49 +0200 Subject: [PATCH 164/173] [benchmark] Upload in one function uploadReportFile resolves the report the way a run writes it, reads the CI metadata once, and says when there is no report rather than failing with ENOENT. compare() checks a case's own sampling by value. --- .circleci/config.yml | 3 +- packages/benchmark/README.md | 2 +- packages/benchmark/src/cli/index.ts | 7 +-- packages/benchmark/src/page/page.tsx | 2 +- .../benchmark/src/runner/runBenchmarks.ts | 2 +- packages/benchmark/src/runner/upload.ts | 62 +++++++++++-------- 6 files changed, 42 insertions(+), 36 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2a24e86b2f..15d94f512e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -146,7 +146,8 @@ jobs: command: pnpm test:benchmark --baseline "$(cat .benchmark-baseline-sha)" - run: name: Upload the benchmark report - # Also after a failed run, so the comment says what happened. + # Also after a failed run that wrote its report, such as one where every benchmark errored, + # so the comment says what happened. when: always environment: # TEMPORARY: uploads in place of `test_benchmark`, since both write the commit's one diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 43400b5837..1f0d12eaa7 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -521,7 +521,7 @@ the comment reports a run whose benchmarks all errored: `benchmark upload [file]` refuses a report measured on another commit than the one CI is building. Uploads from a tracked branch (`master`, `next`, `v7.x`, …) join that branch's timeline, the history -the dashboard draws. A run against another baseline names its own with `--timeline` (or +the dashboard draws. A run against another baseline uploads into its own with `benchmark upload --timeline` (or `BENCHMARK_TIMELINE`), so it neither overwrites the commit's regular report nor lands in the branch's history, for example a weekly run against the last release: diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index 0bc1e5b46d..85a67568a6 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -138,11 +138,8 @@ const uploadCommand: CommandModule<{}, { file?: string; timeline?: string }> = { "The dashboard timeline to upload into in place of the branch's, for a run against another baseline such as the last release. Defaults to BENCHMARK_TIMELINE", }), handler: async (argv) => { - const [{ uploadReportFile }, { resultsPathOf }] = await Promise.all([ - import('../runner/upload'), - import('../runner/outputDir'), - ]); - await uploadReportFile(argv.file ?? resultsPathOf(process.cwd()), argv.timeline); + const { uploadReportFile } = await import('../runner/upload'); + await uploadReportFile({ harnessDir: process.cwd(), file: argv.file, timeline: argv.timeline }); }, }; diff --git a/packages/benchmark/src/page/page.tsx b/packages/benchmark/src/page/page.tsx index 47bd075f0f..bf52761608 100644 --- a/packages/benchmark/src/page/page.tsx +++ b/packages/benchmark/src/page/page.tsx @@ -90,7 +90,7 @@ export function compare( const ownSampling = getSampling(sampling); for (const benchCase of compared) { const own = cases.get(benchCase.name); - if (own && Object.keys(own.sampling).length > 0) { + if (own && (own.sampling.sampleSize !== undefined || own.sampling.timeout !== undefined)) { throw new Error( `"${benchCase.name}" sets its own sampling, but a compared case is sampled as its ` + `compare() asks: pass the options to compare("${name}", …) instead.`, diff --git a/packages/benchmark/src/runner/runBenchmarks.ts b/packages/benchmark/src/runner/runBenchmarks.ts index 8c2d8163bf..dadde5bc6f 100644 --- a/packages/benchmark/src/runner/runBenchmarks.ts +++ b/packages/benchmark/src/runner/runBenchmarks.ts @@ -192,7 +192,7 @@ export async function runBenchmarks( if (profile) { const traced = await traceBenchmarks({ ...runOptions, - rounds: sampling?.sampleSize ?? PROFILE_ROUNDS, + rounds: sampling.sampleSize ?? PROFILE_ROUNDS, tracesDir: path.join(outputDir, 'traces'), }); if (reporter === 'json') { diff --git a/packages/benchmark/src/runner/upload.ts b/packages/benchmark/src/runner/upload.ts index ab9e8505c9..75e4c59070 100644 --- a/packages/benchmark/src/runner/upload.ts +++ b/packages/benchmark/src/runner/upload.ts @@ -1,24 +1,44 @@ /* eslint-disable no-console -- progress belongs in the CI log. */ +import * as path from 'node:path'; import { readFile } from 'node:fs/promises'; import chalk from 'chalk'; import { postToDashboard } from '../ciApi'; import { getCiMetadata } from '../ciReport'; import { syncPrComment } from '../syncPrComment'; import { benchmarkRunReportSchema, benchmarkRunUploadSchema } from '../runReport'; -import type { BenchmarkRunReport } from '../runReport'; +import { resultsPathOf } from './outputDir'; + +export interface UploadReportOptions { + /** The harness package directory, where a run writes its report by default. */ + harnessDir: string; + /** The report to upload. Defaults to the one a run writes, `.benchmark/results/report.json`. */ + file?: string; + /** The dashboard timeline to upload into, in place of the branch's. */ + timeline?: string; +} /** - * Uploads a report and refreshes the pull request comment. + * Uploads a report that a `benchmark` run wrote, and refreshes the pull request comment. * - * Validates before sending because the server stores `report` without inspecting it. A failure to - * refresh the comment does not fail the run: the numbers are already measured, written and - * uploaded, and losing them to a comment API hiccup would waste the whole run. + * The dashboard files the report under the commit CI is building, so a report measured on another + * commit is refused rather than filed under the wrong one. Validates before sending because the + * server stores `report` without inspecting it. A failure to refresh the comment does not fail the + * upload: the report has landed, and the next sync shows it. */ -export async function publishRunReport( - report: BenchmarkRunReport, - timeline: string | undefined, -): Promise { +export async function uploadReportFile(options: UploadReportOptions): Promise { + const file = options.file ? path.resolve(options.file) : resultsPathOf(options.harnessDir); + let text: string; + try { + text = await readFile(file, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + throw new Error(`No report at ${file}: the run failed before writing one.`); + } + throw error; + } + const report = benchmarkRunReportSchema.parse(JSON.parse(text)); + const metadata = await getCiMetadata(); if (!metadata.repo) { console.warn( @@ -26,12 +46,17 @@ export async function publishRunReport( ); return; } + if (report.head.sha !== metadata.commitSha) { + throw new Error( + `${file} was measured on ${report.head.sha}, but CI is building ${metadata.commitSha}. Upload a report from this commit's run.`, + ); + } const upload = benchmarkRunUploadSchema.parse({ version: 2, reportType: 'benchmark', ...metadata, - timeline, + timeline: options.timeline, report, }); const responseText = await postToDashboard( @@ -55,20 +80,3 @@ export async function publishRunReport( ); } } - -/** - * Uploads a report file that `benchmark` wrote, and refreshes the pull request comment. - * - * The dashboard files the report under the commit CI is building, so a report measured on another - * commit is refused rather than filed under the wrong one. - */ -export async function uploadReportFile(file: string, timeline: string | undefined): Promise { - const report = benchmarkRunReportSchema.parse(JSON.parse(await readFile(file, 'utf8'))); - const { commitSha } = await getCiMetadata(); - if (commitSha && report.head.sha && report.head.sha !== commitSha) { - throw new Error( - `${file} was measured on ${report.head.sha}, but CI is building ${commitSha}. Upload a report from this commit's run.`, - ); - } - await publishRunReport(report, timeline); -} From 4a7f65ecd7b8d226273b2901f8b65d91bdac0da9 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:47:38 +0200 Subject: [PATCH 165/173] [benchmark] Serve the builds with sirv Replaces the hand-rolled static server; every repository running the CLI already has sirv through Vitest. --- packages/benchmark/package.json | 1 + .../benchmark/src/runner/runInterleaved.ts | 23 +++++-- .../benchmark/src/runner/serveDirectory.ts | 64 ------------------- pnpm-lock.yaml | 3 + 4 files changed, 20 insertions(+), 71 deletions(-) delete mode 100644 packages/benchmark/src/runner/serveDirectory.ts diff --git a/packages/benchmark/package.json b/packages/benchmark/package.json index 5dced1a71c..f883ade765 100644 --- a/packages/benchmark/package.json +++ b/packages/benchmark/package.json @@ -32,6 +32,7 @@ "es-toolkit": "^1.52.0", "execa": "^10.1.0", "globby": "^16.2.4", + "sirv": "^3.0.2", "yaml": "^2.9.0", "yargs": "^18.1.0", "zod": "^4.6.5" diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index fe9bd3b549..545b560f7c 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -1,8 +1,11 @@ /* eslint-disable no-console */ import { addAbortListener } from 'node:events'; +import * as http from 'node:http'; +import type { AddressInfo } from 'node:net'; import chalk from 'chalk'; import { chromium } from '@playwright/test'; +import sirv from 'sirv'; import type { Browser, CDPSession, Page } from '@playwright/test'; import { BENCHMARK_LAUNCH_ARGS, BENCHMARK_VIEWPORT } from '../launchArgs'; import { compareBenchmark, formatDuration, formatRoundCount } from '../runReport'; @@ -11,7 +14,6 @@ import { DEFAULT_SAMPLING, differencesSettled } from '../sampling'; import type { SamplingOptions } from '../sampling'; import type { BenchFile } from './benchFiles'; import type { ResolvedRef } from './refs'; -import { serveDirectory } from './serveDirectory'; import type { BenchPage } from '../page/page'; /** @@ -461,14 +463,21 @@ export async function withPlannedRun( options: RunInterleavedOptions, drive: (run: PlannedRun) => Promise, ): Promise { - const [server, browser] = await Promise.all([ - serveDirectory(options.buildsDir), + // Each build is served under its own path. In `dev` mode, every page load reads the build from + // disk and the browser caches nothing (`no-store`). + const server = http.createServer(sirv(options.buildsDir, { dev: true })); + const [, browser] = await Promise.all([ + new Promise((resolve) => { + server.listen(0, '127.0.0.1', resolve); + }), chromium.launch({ headless: true, args: [...BENCHMARK_LAUNCH_ARGS, ...options.launchArgs], }), ]); + const origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + // Closing the browser ends whatever a page is doing at once, rather than when its sample would. const closeOnAbort = options.signal && addAbortListener(options.signal, () => void browser.close()); @@ -479,9 +488,7 @@ export async function withPlannedRun( const { testNamePattern } = options; const planned = ( await Promise.all( - options.benchFiles.map((benchFile) => - planBenchFile(browser, server.origin, benchFile, options), - ), + options.benchFiles.map((benchFile) => planBenchFile(browser, origin, benchFile, options)), ) ) .flat() @@ -493,7 +500,9 @@ export async function withPlannedRun( } finally { closeOnAbort?.[Symbol.dispose](); await browser.close(); - await server.close(); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); } } diff --git a/packages/benchmark/src/runner/serveDirectory.ts b/packages/benchmark/src/runner/serveDirectory.ts deleted file mode 100644 index ff41cc5010..0000000000 --- a/packages/benchmark/src/runner/serveDirectory.ts +++ /dev/null @@ -1,64 +0,0 @@ -import * as http from 'node:http'; -import * as path from 'node:path'; -import { createReadStream } from 'node:fs'; -import { stat } from 'node:fs/promises'; -import type { AddressInfo } from 'node:net'; - -const CONTENT_TYPES: Record = { - '.html': 'text/html; charset=utf-8', - '.js': 'text/javascript; charset=utf-8', - '.mjs': 'text/javascript; charset=utf-8', - '.css': 'text/css; charset=utf-8', - '.json': 'application/json; charset=utf-8', - '.map': 'application/json; charset=utf-8', - '.svg': 'image/svg+xml', - '.png': 'image/png', - '.jpg': 'image/jpeg', - '.woff2': 'font/woff2', - '.wasm': 'application/wasm', -}; - -export interface StaticServer { - /** `http://127.0.0.1:`, with no trailing slash. */ - origin: string; - close: () => Promise; -} - -/** - * Serves a directory of built pages on a free local port. Nothing is cached by the server; each - * build is served under its own path, so each side of a comparison warms its own browser cache. - */ -export async function serveDirectory(root: string): Promise { - const server = http.createServer(async (request, response) => { - const pathname = decodeURIComponent(new URL(request.url ?? '/', 'http://localhost').pathname); - const file = path.join(root, pathname.endsWith('/') ? `${pathname}index.html` : pathname); - if (path.relative(root, file).startsWith('..')) { - response.writeHead(403).end(); - return; - } - try { - if (!(await stat(file)).isFile()) { - throw new Error('Not a file'); - } - } catch { - response.writeHead(404).end(); - return; - } - response.writeHead(200, { - 'Content-Type': CONTENT_TYPES[path.extname(file)] ?? 'application/octet-stream', - }); - createReadStream(file).pipe(response); - }); - - await new Promise((resolve) => { - server.listen(0, '127.0.0.1', resolve); - }); - const { port } = server.address() as AddressInfo; - return { - origin: `http://127.0.0.1:${port}`, - close: () => - new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }), - }; -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 01192bbdb1..c7e1b92478 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -549,6 +549,9 @@ importers: react-dom: specifier: ^18.0.0 || ^19.0.0 version: 19.3.0(react@19.3.0) + sirv: + specifier: ^3.0.2 + version: 3.0.2 yaml: specifier: ^2.9.0 version: 2.9.1 From 59670d8526b9afcacd58678fe57935f7842d48a2 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:56:24 +0200 Subject: [PATCH 166/173] [benchmark] Build the version 2 upload from the shared CI envelope ciReportUploadSchema moves to a browser-safe module, so the run report's upload extends it instead of repeating its fields. --- packages/benchmark/src/ciReport.ts | 22 ++---------------- .../benchmark/src/ciReportUploadSchema.ts | 23 +++++++++++++++++++ packages/benchmark/src/runReport/schema.ts | 18 ++++++--------- 3 files changed, 32 insertions(+), 31 deletions(-) create mode 100644 packages/benchmark/src/ciReportUploadSchema.ts diff --git a/packages/benchmark/src/ciReport.ts b/packages/benchmark/src/ciReport.ts index 5ca1f917ea..858592b883 100644 --- a/packages/benchmark/src/ciReport.ts +++ b/packages/benchmark/src/ciReport.ts @@ -1,6 +1,7 @@ import { execa } from 'execa'; import { z } from 'zod/v4'; import envCi from 'env-ci'; +import { ciReportUploadSchema } from './ciReportUploadSchema'; interface CiInfo { isCi: boolean; @@ -32,26 +33,7 @@ async function getGitInfo(): Promise { }; } -/** - * Creates a CI report upload schema for a specific report type. - * Common fields (commitSha, repo, branch, prNumber) are shared across all report types. - */ -export function ciReportUploadSchema( - type: string, - version: number, - reportSchema: ReportSchema, -) { - return z.object({ - version: z.literal(version), - timestamp: z.number(), - commitSha: z.string().regex(/^[0-9a-f]{40}$/, 'Must be a 40-character hex string'), - repo: z.string().includes('/', 'Must be in owner/repo format'), - reportType: z.literal(type), - prNumber: z.number().int().positive().optional(), - branch: z.string(), - report: reportSchema, - }); -} +export { ciReportUploadSchema }; const renderStatsSchema = z.object({ id: z.string(), diff --git a/packages/benchmark/src/ciReportUploadSchema.ts b/packages/benchmark/src/ciReportUploadSchema.ts new file mode 100644 index 0000000000..79b83566a4 --- /dev/null +++ b/packages/benchmark/src/ciReportUploadSchema.ts @@ -0,0 +1,23 @@ +import { z } from 'zod/v4'; + +/** + * Creates a CI report upload schema for a specific report type: the envelope every report type + * shares (commit, repository, branch, pull request) around its `report`. Browser-safe, so the + * dashboard can read with the same schemas the uploaders validate with. + */ +export function ciReportUploadSchema( + type: string, + version: number, + reportSchema: ReportSchema, +) { + return z.object({ + version: z.literal(version), + timestamp: z.number(), + commitSha: z.string().regex(/^[0-9a-f]{40}$/, 'Must be a 40-character hex string'), + repo: z.string().includes('/', 'Must be in owner/repo format'), + reportType: z.literal(type), + prNumber: z.number().int().positive().optional(), + branch: z.string(), + report: reportSchema, + }); +} diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts index a897b7de89..f6b3aa5e58 100644 --- a/packages/benchmark/src/runReport/schema.ts +++ b/packages/benchmark/src/runReport/schema.ts @@ -1,5 +1,6 @@ import { z } from 'zod/v4'; import type { MetricDefinition } from '../types'; +import { ciReportUploadSchema } from '../ciReportUploadSchema'; /** * The report the `benchmark` CLI produces, uploads and the dashboard reads: version 2 of the `benchmark` @@ -101,17 +102,13 @@ export const benchmarkRunReportSchema = z.object({ /** * The version 2 `benchmark` upload, which the dashboard stores as the commit's `benchmark.json`: the - * same envelope as every CI report around this report. Declared here, browser-safe, so the runner - * validates with it and the dashboard reads with its type. + * envelope every CI report shares around this report, plus what orders it in a timeline. */ -export const benchmarkRunUploadSchema = z.object({ - version: z.literal(2), - timestamp: z.number(), - commitSha: z.string().regex(/^[0-9a-f]{40}$/, 'Must be a 40-character hex string'), - repo: z.string().includes('/', 'Must be in owner/repo format'), - reportType: z.literal('benchmark'), - prNumber: z.number().int().positive().optional(), - branch: z.string(), +export const benchmarkRunUploadSchema = ciReportUploadSchema( + 'benchmark', + 2, + benchmarkRunReportSchema, +).extend({ /** When the commit was made, in milliseconds: orders it in its timeline. */ commitTimestamp: z.number().optional(), /** @@ -119,7 +116,6 @@ export const benchmarkRunUploadSchema = z.object({ * such as the last release. The dashboard stores it as `benchmark@.json`. */ timeline: z.string().optional(), - report: benchmarkRunReportSchema, }); export type BenchmarkRunReport = z.infer; From 3a0c1ca028cae698181a50a8f76d56c787857360 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:56:24 +0200 Subject: [PATCH 167/173] [benchmark] Tidy the run analysis Look metric names up as own keys, decide which side is worse once, drop a check severity already implies, and keep median with the other statistics. --- .../src/runReport/analyzeRun.test.ts | 9 +--- .../benchmark/src/runReport/analyzeRun.ts | 44 ++++++++++--------- packages/benchmark/src/stats.test.ts | 16 ++++++- packages/benchmark/src/stats.ts | 7 +++ 4 files changed, 46 insertions(+), 30 deletions(-) diff --git a/packages/benchmark/src/runReport/analyzeRun.test.ts b/packages/benchmark/src/runReport/analyzeRun.test.ts index 183321958e..eec6d1b893 100644 --- a/packages/benchmark/src/runReport/analyzeRun.test.ts +++ b/packages/benchmark/src/runReport/analyzeRun.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { analyzeRun, compareSamples, findRegressions, median, tCritical } from './analyzeRun'; +import { analyzeRun, compareSamples, findRegressions, tCritical } from './analyzeRun'; import type { BenchmarkAnalysis } from './analyzeRun'; import type { BenchmarkRunReport, RunBenchmark, RunMetricDefinition } from './schema'; @@ -45,13 +45,6 @@ describe('tCritical', () => { }); }); -describe('median', () => { - it('takes the middle value, or the mean of the middle two', () => { - expect(median([3, 1, 2])).toBe(2); - expect(median([4, 1, 3, 2])).toBe(2.5); - }); -}); - describe('compareSamples', () => { it('resolves a small shift that per-round drift hides from the variant means', () => { const { base, shifted } = drifting(2); diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index a35dc42196..0961843ca1 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -1,4 +1,4 @@ -import { calculateMean, calculateSampleStdDev, quantile } from '../stats'; +import { calculateMean, calculateSampleStdDev, median } from '../stats'; import { baseMetricName } from '../metricCore'; import type { BenchmarkRunReport, @@ -197,13 +197,6 @@ function intervalAround(mean: number, stdDev: number, count: number): Interval { return { low: mean - halfWidth, high: mean + halfWidth }; } -export function median(values: number[]): number { - return quantile( - [...values].sort((left, right) => left - right), - 0.5, - ); -} - function summarizeSamples(values: number[]): SampleSummary { return { count: values.length, median: median(values) }; } @@ -215,7 +208,11 @@ function definitionOf( metrics: Record, metric: string, ): RunMetricDefinition { - return metrics[metric] ?? metrics[baseMetricName(metric)] ?? DEFAULT_DEFINITION; + if (Object.hasOwn(metrics, metric)) { + return metrics[metric]; + } + const base = baseMetricName(metric); + return Object.hasOwn(metrics, base) ? metrics[base] : DEFAULT_DEFINITION; } /** @@ -230,8 +227,8 @@ function alarmOverrideOf( if (!alarms) { return undefined; } - const key = metric in alarms ? metric : baseMetricName(metric); - return key in alarms ? alarms[key] : undefined; + const key = Object.hasOwn(alarms, metric) ? metric : baseMetricName(metric); + return Object.hasOwn(alarms, key) ? alarms[key] : undefined; } /** @@ -252,15 +249,20 @@ function inAlarmUnits( * nearest zero — against the `error` band. Without a band, which only a discrete metric's alarm * keeps, every confirmed change for the worse alarms. */ -function severityOf(change: Change, interval: Interval, alarm: RunMetricAlarm): Severity { +function severityOf( + change: Change, + interval: Interval, + error: number | undefined, + worseSign: 1 | -1, +): Severity { if (change !== 'worse') { return 'none'; } - if (alarm.error === undefined) { + if (error === undefined) { return 'error'; } - const worseningAtLeast = alarm.direction === 'higherIsBetter' ? -interval.high : interval.low; - return worseningAtLeast >= alarm.error ? 'error' : 'none'; + const worseningAtLeast = worseSign === 1 ? interval.low : -interval.high; + return worseningAtLeast >= error ? 'error' : 'none'; } /** The per-round differences `subject − against`, over the rounds both have. */ @@ -300,17 +302,17 @@ export function compareSamples( const noise = (stdDev / reference) * 100; const banded = inAlarmUnits({ absolute, relative }, definition); const precision = (banded.high - banded.low) / 2; + // Which way is worse: an increase, unless the alarm says higher is better. + const worseSign = (alarm ?? definition.alarm)?.direction === 'higherIsBetter' ? -1 : 1; // Only the worse side's edge matters: an improvement never raises an alarm, however large. - const worseEdge = - alarm?.direction === 'higherIsBetter' ? -(alarm.error ?? 0) : (alarm?.error ?? 0); + const worseEdge = worseSign * (alarm?.error ?? 0); let change: Change = 'undetected'; if (rounds > 0 && differences.every((difference) => difference === 0)) { change = 'unchanged'; } else if (absolute.low > 0 || absolute.high < 0) { const increased = absolute.low > 0; - const lowerIsBetter = (alarm ?? definition.alarm)?.direction !== 'higherIsBetter'; - change = increased === lowerIsBetter ? 'worse' : 'better'; + change = increased === (worseSign === 1) ? 'worse' : 'better'; } return { @@ -323,7 +325,7 @@ export function compareSamples( precision, settled: alarm === null || !(banded.low < worseEdge && worseEdge < banded.high), change, - severity: alarm ? severityOf(change, banded, alarm) : 'none', + severity: alarm ? severityOf(change, banded, alarm.error, worseSign) : 'none', }; } @@ -430,7 +432,7 @@ export interface AlarmedChange { * for the worse, past its `error` band. */ export function isRegression(comparison: MetricComparison): boolean { - return comparison.alarm !== null && comparison.severity !== 'none'; + return comparison.severity !== 'none'; } /** Whether a change is one for the better on an alarmed metric: what the code under test sped up. */ diff --git a/packages/benchmark/src/stats.test.ts b/packages/benchmark/src/stats.test.ts index cd6dd8b94f..8c4251b7dd 100644 --- a/packages/benchmark/src/stats.test.ts +++ b/packages/benchmark/src/stats.test.ts @@ -1,5 +1,12 @@ import { describe, it, expect } from 'vitest'; -import { calculateMean, calculateStdDev, quantile, isOutlier, aggregateSamples } from './stats'; +import { + calculateMean, + calculateStdDev, + median, + quantile, + isOutlier, + aggregateSamples, +} from './stats'; describe('calculateMean', () => { it('returns the mean of values', () => { @@ -101,3 +108,10 @@ describe('aggregateSamples', () => { expect(result.outliers).toBe(0); }); }); + +describe('median', () => { + it('takes the middle value, or the mean of the middle two', () => { + expect(median([3, 1, 2])).toBe(2); + expect(median([4, 1, 3, 2])).toBe(2.5); + }); +}); diff --git a/packages/benchmark/src/stats.ts b/packages/benchmark/src/stats.ts index 3d42eb3374..fb0b375708 100644 --- a/packages/benchmark/src/stats.ts +++ b/packages/benchmark/src/stats.ts @@ -16,6 +16,13 @@ export function calculateSampleStdDev(values: number[], mean: number): number { return Math.sqrt(squaredDiffs.reduce((sum, v) => sum + v, 0) / (values.length - 1)); } +export function median(values: number[]): number { + return quantile( + values.toSorted((first, second) => first - second), + 0.5, + ); +} + export function quantile(sorted: number[], q: number): number { const pos = (sorted.length - 1) * q; const base = Math.floor(pos); From d5b3a2d985768a94fe662f092cbb6f8099ee3a71 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:00:31 +0200 Subject: [PATCH 168/173] [benchmark] Simplify the server lifecycle and the run analysis Close the static server even when the browser fails to launch, take median from es-toolkit, orient a comparison's interval once instead of flipping signs in three places, share the own-or-base metric lookup, and keep commitTimestamp in the shared CI envelope. --- packages/benchmark/src/ciReport.ts | 4 +- .../benchmark/src/ciReportUploadSchema.ts | 6 +- .../benchmark/src/runReport/analyzeRun.ts | 46 ++++++------- packages/benchmark/src/runReport/schema.ts | 4 +- .../benchmark/src/runner/runInterleaved.ts | 65 +++++++++---------- packages/benchmark/src/stats.test.ts | 16 +---- packages/benchmark/src/stats.ts | 7 -- 7 files changed, 62 insertions(+), 86 deletions(-) diff --git a/packages/benchmark/src/ciReport.ts b/packages/benchmark/src/ciReport.ts index 858592b883..6524749252 100644 --- a/packages/benchmark/src/ciReport.ts +++ b/packages/benchmark/src/ciReport.ts @@ -3,6 +3,8 @@ import { z } from 'zod/v4'; import envCi from 'env-ci'; import { ciReportUploadSchema } from './ciReportUploadSchema'; +export { ciReportUploadSchema } from './ciReportUploadSchema'; + interface CiInfo { isCi: boolean; commit?: string; @@ -33,8 +35,6 @@ async function getGitInfo(): Promise { }; } -export { ciReportUploadSchema }; - const renderStatsSchema = z.object({ id: z.string(), phase: z.enum(['mount', 'update', 'nested-update']), diff --git a/packages/benchmark/src/ciReportUploadSchema.ts b/packages/benchmark/src/ciReportUploadSchema.ts index 79b83566a4..83148c7df2 100644 --- a/packages/benchmark/src/ciReportUploadSchema.ts +++ b/packages/benchmark/src/ciReportUploadSchema.ts @@ -2,8 +2,8 @@ import { z } from 'zod/v4'; /** * Creates a CI report upload schema for a specific report type: the envelope every report type - * shares (commit, repository, branch, pull request) around its `report`. Browser-safe, so the - * dashboard can read with the same schemas the uploaders validate with. + * shares (commit, repository, branch, pull request) around its `report`. Kept apart from + * `ciReport.ts`, which runs git, so the browser-safe run report schema can build on it. */ export function ciReportUploadSchema( type: string, @@ -18,6 +18,8 @@ export function ciReportUploadSchema( reportType: z.literal(type), prNumber: z.number().int().positive().optional(), branch: z.string(), + /** When the commit was made, in milliseconds: orders it in its timeline. */ + commitTimestamp: z.number().optional(), report: reportSchema, }); } diff --git a/packages/benchmark/src/runReport/analyzeRun.ts b/packages/benchmark/src/runReport/analyzeRun.ts index 0961843ca1..73697edb9c 100644 --- a/packages/benchmark/src/runReport/analyzeRun.ts +++ b/packages/benchmark/src/runReport/analyzeRun.ts @@ -1,4 +1,5 @@ -import { calculateMean, calculateSampleStdDev, median } from '../stats'; +import { median } from 'es-toolkit/math'; +import { calculateMean, calculateSampleStdDev } from '../stats'; import { baseMetricName } from '../metricCore'; import type { BenchmarkRunReport, @@ -203,16 +204,22 @@ function summarizeSamples(values: number[]): SampleSummary { const DEFAULT_DEFINITION: RunMetricDefinition = { kind: 'scalar' }; +/** The key `record` holds for a metric: its own, else its base metric's for a `name#id` sub-series. */ +function keyOf(record: Record, metric: string): string | undefined { + if (Object.hasOwn(record, metric)) { + return metric; + } + const base = baseMetricName(metric); + return Object.hasOwn(record, base) ? base : undefined; +} + /** A metric's definition; a `name#id` sub-series is defined by its base metric. */ function definitionOf( metrics: Record, metric: string, ): RunMetricDefinition { - if (Object.hasOwn(metrics, metric)) { - return metrics[metric]; - } - const base = baseMetricName(metric); - return Object.hasOwn(metrics, base) ? metrics[base] : DEFAULT_DEFINITION; + const key = keyOf(metrics, metric); + return key === undefined ? DEFAULT_DEFINITION : metrics[key]; } /** @@ -227,8 +234,8 @@ function alarmOverrideOf( if (!alarms) { return undefined; } - const key = Object.hasOwn(alarms, metric) ? metric : baseMetricName(metric); - return Object.hasOwn(alarms, key) ? alarms[key] : undefined; + const key = keyOf(alarms, metric); + return key === undefined ? undefined : alarms[key]; } /** @@ -249,20 +256,14 @@ function inAlarmUnits( * nearest zero — against the `error` band. Without a band, which only a discrete metric's alarm * keeps, every confirmed change for the worse alarms. */ -function severityOf( - change: Change, - interval: Interval, - error: number | undefined, - worseSign: 1 | -1, -): Severity { +function severityOf(change: Change, worsening: Interval, error: number | undefined): Severity { if (change !== 'worse') { return 'none'; } if (error === undefined) { return 'error'; } - const worseningAtLeast = worseSign === 1 ? interval.low : -interval.high; - return worseningAtLeast >= error ? 'error' : 'none'; + return worsening.low >= error ? 'error' : 'none'; } /** The per-round differences `subject − against`, over the rounds both have. */ @@ -302,17 +303,18 @@ export function compareSamples( const noise = (stdDev / reference) * 100; const banded = inAlarmUnits({ absolute, relative }, definition); const precision = (banded.high - banded.low) / 2; - // Which way is worse: an increase, unless the alarm says higher is better. + // Which way is worse: an increase, unless the alarm says higher is better. `worsening` is the + // interval in that orientation, so a positive bound is a change for the worse. const worseSign = (alarm ?? definition.alarm)?.direction === 'higherIsBetter' ? -1 : 1; + const worsening = worseSign === 1 ? banded : { low: -banded.high, high: -banded.low }; // Only the worse side's edge matters: an improvement never raises an alarm, however large. - const worseEdge = worseSign * (alarm?.error ?? 0); + const worseEdge = alarm?.error ?? 0; let change: Change = 'undetected'; if (rounds > 0 && differences.every((difference) => difference === 0)) { change = 'unchanged'; } else if (absolute.low > 0 || absolute.high < 0) { - const increased = absolute.low > 0; - change = increased === (worseSign === 1) ? 'worse' : 'better'; + change = (absolute.low > 0 ? 1 : -1) === worseSign ? 'worse' : 'better'; } return { @@ -323,9 +325,9 @@ export function compareSamples( relative, noise, precision, - settled: alarm === null || !(banded.low < worseEdge && worseEdge < banded.high), + settled: alarm === null || !(worsening.low < worseEdge && worseEdge < worsening.high), change, - severity: alarm ? severityOf(change, banded, alarm.error, worseSign) : 'none', + severity: alarm ? severityOf(change, worsening, alarm.error) : 'none', }; } diff --git a/packages/benchmark/src/runReport/schema.ts b/packages/benchmark/src/runReport/schema.ts index f6b3aa5e58..1ad562d2a7 100644 --- a/packages/benchmark/src/runReport/schema.ts +++ b/packages/benchmark/src/runReport/schema.ts @@ -102,15 +102,13 @@ export const benchmarkRunReportSchema = z.object({ /** * The version 2 `benchmark` upload, which the dashboard stores as the commit's `benchmark.json`: the - * envelope every CI report shares around this report, plus what orders it in a timeline. + * envelope every CI report shares around this report, plus the timeline it goes into. */ export const benchmarkRunUploadSchema = ciReportUploadSchema( 'benchmark', 2, benchmarkRunReportSchema, ).extend({ - /** When the commit was made, in milliseconds: orders it in its timeline. */ - commitTimestamp: z.number().optional(), /** * The timeline the upload goes into, in place of its branch's: a run against another baseline, * such as the last release. The dashboard stores it as `benchmark@.json`. diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index 545b560f7c..ee1526aedd 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -1,6 +1,6 @@ /* eslint-disable no-console */ -import { addAbortListener } from 'node:events'; +import { addAbortListener, once } from 'node:events'; import * as http from 'node:http'; import type { AddressInfo } from 'node:net'; import chalk from 'chalk'; @@ -463,46 +463,41 @@ export async function withPlannedRun( options: RunInterleavedOptions, drive: (run: PlannedRun) => Promise, ): Promise { - // Each build is served under its own path. In `dev` mode, every page load reads the build from - // disk and the browser caches nothing (`no-store`). + // In `dev` mode, every page load reads the build from disk and the browser caches nothing + // (`no-store`). const server = http.createServer(sirv(options.buildsDir, { dev: true })); - const [, browser] = await Promise.all([ - new Promise((resolve) => { - server.listen(0, '127.0.0.1', resolve); - }), - chromium.launch({ + await once(server.listen(0, '127.0.0.1'), 'listening'); + try { + const origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + const browser = await chromium.launch({ headless: true, args: [...BENCHMARK_LAUNCH_ARGS, ...options.launchArgs], - }), - ]); - - const origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; - - // Closing the browser ends whatever a page is doing at once, rather than when its sample would. - const closeOnAbort = - options.signal && addAbortListener(options.signal, () => void browser.close()); - try { - options.signal?.throwIfAborted(); - // Every file is listed before any is measured, so a name pattern that matches nothing fails - // before the run spends anything. Nothing is measured yet, so the files list at once. - const { testNamePattern } = options; - const planned = ( - await Promise.all( - options.benchFiles.map((benchFile) => planBenchFile(browser, origin, benchFile, options)), + }); + // Closing the browser ends whatever a page is doing at once, rather than when its sample would. + const closeOnAbort = + options.signal && addAbortListener(options.signal, () => void browser.close()); + try { + options.signal?.throwIfAborted(); + // Every file is listed before any is measured, so a name pattern that matches nothing fails + // before the run spends anything. Nothing is measured yet, so the files list at once. + const { testNamePattern } = options; + const planned = ( + await Promise.all( + options.benchFiles.map((benchFile) => planBenchFile(browser, origin, benchFile, options)), + ) ) - ) - .flat() - .filter((plan) => !testNamePattern || testNamePattern.test(plan.entry.name)); - if (planned.length === 0) { - throw new Error(`No benchmark name matches ${testNamePattern}.`); + .flat() + .filter((plan) => !testNamePattern || testNamePattern.test(plan.entry.name)); + if (planned.length === 0) { + throw new Error(`No benchmark name matches ${testNamePattern}.`); + } + return await drive({ browser, planned }); + } finally { + closeOnAbort?.[Symbol.dispose](); + await browser.close(); } - return await drive({ browser, planned }); } finally { - closeOnAbort?.[Symbol.dispose](); - await browser.close(); - await new Promise((resolve, reject) => { - server.close((error) => (error ? reject(error) : resolve())); - }); + await server[Symbol.asyncDispose](); } } diff --git a/packages/benchmark/src/stats.test.ts b/packages/benchmark/src/stats.test.ts index 8c4251b7dd..cd6dd8b94f 100644 --- a/packages/benchmark/src/stats.test.ts +++ b/packages/benchmark/src/stats.test.ts @@ -1,12 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { - calculateMean, - calculateStdDev, - median, - quantile, - isOutlier, - aggregateSamples, -} from './stats'; +import { calculateMean, calculateStdDev, quantile, isOutlier, aggregateSamples } from './stats'; describe('calculateMean', () => { it('returns the mean of values', () => { @@ -108,10 +101,3 @@ describe('aggregateSamples', () => { expect(result.outliers).toBe(0); }); }); - -describe('median', () => { - it('takes the middle value, or the mean of the middle two', () => { - expect(median([3, 1, 2])).toBe(2); - expect(median([4, 1, 3, 2])).toBe(2.5); - }); -}); diff --git a/packages/benchmark/src/stats.ts b/packages/benchmark/src/stats.ts index fb0b375708..3d42eb3374 100644 --- a/packages/benchmark/src/stats.ts +++ b/packages/benchmark/src/stats.ts @@ -16,13 +16,6 @@ export function calculateSampleStdDev(values: number[], mean: number): number { return Math.sqrt(squaredDiffs.reduce((sum, v) => sum + v, 0) / (values.length - 1)); } -export function median(values: number[]): number { - return quantile( - values.toSorted((first, second) => first - second), - 0.5, - ); -} - export function quantile(sorted: number[], q: number): number { const pos = (sorted.length - 1) * q; const base = Math.floor(pos); From a0053e3ebee528fcfa74cddedf8384cf4930bd90 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:15:06 +0200 Subject: [PATCH 169/173] [benchmark] Upload this repository's benchmark report to production The version 2 report from the benchmark CLI replaces the Vitest job's upload. --- .circleci/config.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 15d94f512e..c56dfab21a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -110,9 +110,7 @@ jobs: command: pnpm release:build - run: name: Run benchmarks - # TEMPORARY: `test_benchmark_run` uploads this commit's `benchmark.json` as version 2 - # instead, so the pull request comment shows the new report. Restore - # `BENCHMARK_UPLOAD: 'true'` here before merging. + # Not uploaded: `test_benchmark_run` uploads the commit's `benchmark.json`, as version 2. command: pnpm -F ./test/performance benchmark test_benchmark_run: @@ -149,11 +147,6 @@ jobs: # Also after a failed run that wrote its report, such as one where every benchmark errored, # so the comment says what happened. when: always - environment: - # TEMPORARY: uploads in place of `test_benchmark`, since both write the commit's one - # `benchmark.json`, and to this pull request's dashboard preview, since production does - # not read version 2 reports yet. Remove this step's environment before merging. - CI_REPORT_API_URL: https://code-infra-dashboard-pr-1898.onrender.com command: pnpm test:benchmark upload - save_cache: name: Save the packed-baseline cache From dbaf2af21573a83d952ebb5527a2b33b5550f4c1 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:31:38 +0200 Subject: [PATCH 170/173] [benchmark] Remove the Vitest benchmark suite The benchmark CLI job takes over the test_benchmark name, which branch rules require. --- .circleci/config.yml | 18 ------- AGENTS.md | 2 +- package.json | 1 - pnpm-lock.yaml | 22 -------- test/performance/.gitignore | 1 - test/performance/package.json | 17 ------ test/performance/tests/counter.bench.tsx | 28 ---------- .../tests/element-timing.bench.tsx | 47 ----------------- test/performance/tests/example.bench.tsx | 20 ------- .../tests/imperative-metric.bench.tsx | 38 -------------- test/performance/tests/metric.bench.tsx | 44 ---------------- .../tests/react-recording.bench.tsx | 52 ------------------- test/performance/tsconfig.json | 11 ---- test/performance/vitest.config.ts | 3 -- tsconfig.json | 1 - 15 files changed, 1 insertion(+), 304 deletions(-) delete mode 100644 test/performance/.gitignore delete mode 100644 test/performance/package.json delete mode 100644 test/performance/tests/counter.bench.tsx delete mode 100644 test/performance/tests/element-timing.bench.tsx delete mode 100644 test/performance/tests/example.bench.tsx delete mode 100644 test/performance/tests/imperative-metric.bench.tsx delete mode 100644 test/performance/tests/metric.bench.tsx delete mode 100644 test/performance/tests/react-recording.bench.tsx delete mode 100644 test/performance/tsconfig.json delete mode 100644 test/performance/vitest.config.ts diff --git a/.circleci/config.yml b/.circleci/config.yml index 5e8f2c5ea5..d7cc630035 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -99,22 +99,6 @@ jobs: - code-infra/upload-size-snapshot test_benchmark: - <<: *default-job - executor: - name: code-infra/mui-node-browser - playwright-img-version: 'v1.64.0-noble' - steps: - - checkout - - code-infra/install-deps - - run: - name: Build packages - command: pnpm release:build - - run: - name: Run benchmarks - # Not uploaded: `test_benchmark_run` uploads the commit's `benchmark.json`, as version 2. - command: pnpm -F ./test/performance benchmark - - test_benchmark_run: <<: *default-job executor: name: code-infra/mui-node-browser @@ -225,8 +209,6 @@ workflows: <<: *default-context - test_benchmark: <<: *default-context - - test_benchmark_run: - <<: *default-context - test_browser: <<: *default-context # Started nightly on master by a scheduled pipeline that sets `workflow` to `benchmark-aa`. diff --git a/AGENTS.md b/AGENTS.md index 275436d598..65b8f15f98 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -88,7 +88,7 @@ Applies to the whole repository. - **CRITICAL**: When running pnpm commands for workspace packages, always use the `-F` flag followed by the package name. - **Example**: `pnpm -F @mui/internal-bundle-size-checker add micromatch` -- Private packages without a `name` field in `package.json` must be filtered by their relative path (for example, `pnpm -F ./test/performance add `). +- Private packages without a `name` field in `package.json` must be filtered by their relative path (for example, `pnpm -F ./test/benchmark add `). - **Do NOT use `cd` to navigate into package directories** for workspace operations. - **Do NOT manually edit package.json files to add/remove dependencies** - always use `pnpm -F add ` or `pnpm -F remove ` to keep the order deterministic. - **ALWAYS run `pnpm dedupe`** after installing a dependency. diff --git a/package.json b/package.json index de5c0265a8..53f68e3d5c 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,6 @@ "docs:lint": "pnpm -F \"./docs\" run lint", "docs:lib": "pnpm -F \"./packages/docs-infra\" run build", "test:benchmark": "pnpm -F ./test/benchmark benchmark", - "test:performance": "pnpm -F ./test/performance benchmark", "org-versions": "tsx scripts/orgVersions.mts" }, "dependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0bccf27ba1..fb29af78e8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1289,28 +1289,6 @@ importers: specifier: workspace:* version: link:../../packages/bundle-size-checker - test/performance: - dependencies: - react: - specifier: 19.3.0 - version: 19.3.0 - react-dom: - specifier: 19.3.0 - version: 19.3.0(react@19.3.0) - devDependencies: - '@mui/internal-benchmark': - specifier: workspace:* - version: link:../../packages/benchmark/build - '@types/react': - specifier: 19.3.0 - version: 19.3.0 - '@vitest/browser-playwright': - specifier: 5.0.3 - version: 5.0.3(playwright@1.64.0)(vite@8.3.4(@types/node@22.20.5)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1))(vitest@5.0.3) - vitest: - specifier: 5.0.3 - version: 5.0.3(@types/node@22.20.5)(@vitest/browser-playwright@5.0.3)(@vitest/coverage-v8@5.0.3)(@vitest/ui@5.0.3)(jsdom@30.1.2)(vite@8.3.4(@types/node@22.20.5)(esbuild@0.28.2)(jiti@2.7.0)(terser@5.51.2)(tsx@4.23.15)(yaml@2.9.1)) - packages: '@-xun/debug@2.0.2': diff --git a/test/performance/.gitignore b/test/performance/.gitignore deleted file mode 100644 index ca1b347921..0000000000 --- a/test/performance/.gitignore +++ /dev/null @@ -1 +0,0 @@ -benchmarks/ diff --git a/test/performance/package.json b/test/performance/package.json deleted file mode 100644 index ac1b402b44..0000000000 --- a/test/performance/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "private": true, - "description": "Performance benchmark workspace for MUI packages", - "scripts": { - "benchmark": "vitest run" - }, - "dependencies": { - "react": "19.3.0", - "react-dom": "19.3.0" - }, - "devDependencies": { - "@mui/internal-benchmark": "workspace:*", - "@types/react": "19.3.0", - "@vitest/browser-playwright": "5.0.3", - "vitest": "5.0.3" - } -} diff --git a/test/performance/tests/counter.bench.tsx b/test/performance/tests/counter.bench.tsx deleted file mode 100644 index d7fa8189a0..0000000000 --- a/test/performance/tests/counter.bench.tsx +++ /dev/null @@ -1,28 +0,0 @@ -import * as React from 'react'; -import { benchmark } from '@mui/internal-benchmark'; - -function simulateSlowdown(ms: number) { - const jitter = ms * 0.1 * (2 * Math.random() - 1); - const end = performance.now() + ms + jitter; - while (performance.now() < end) { - // burn CPU - } -} - -function Counter() { - const [count, setCount] = React.useState(0); - simulateSlowdown(2); - return ( - - ); -} - -benchmark( - 'Counter click', - () => , - async () => { - document.querySelector('button')?.click(); - }, -); diff --git a/test/performance/tests/element-timing.bench.tsx b/test/performance/tests/element-timing.bench.tsx deleted file mode 100644 index f5b0cf39b7..0000000000 --- a/test/performance/tests/element-timing.bench.tsx +++ /dev/null @@ -1,47 +0,0 @@ -import * as React from 'react'; -import { benchmark, ElementTiming } from '@mui/internal-benchmark'; - -const rows = Array.from({ length: 200 }, (_, i) => ({ - id: i, - name: `Row ${i}`, - value: Math.sqrt(i).toFixed(4), -})); - -function DataGrid() { - return ( - - - - - - - - - {rows.map((row, index) => ( - - - - - ))} - -
    - Name - - Value
    - {row.name} - {index === 0 ? : null} - {row.value}
    - ); -} - -benchmark( - 'DataGrid mount with paint timing', - () => , - async ({ waitForElementTiming }) => { - await Promise.all([waitForElementTiming('grid-header'), waitForElementTiming('grid-body')]); - }, - { - runs: 10, - warmupRuns: 5, - }, -); diff --git a/test/performance/tests/example.bench.tsx b/test/performance/tests/example.bench.tsx deleted file mode 100644 index 4421d5e8db..0000000000 --- a/test/performance/tests/example.bench.tsx +++ /dev/null @@ -1,20 +0,0 @@ -import * as React from 'react'; -import { benchmark } from '@mui/internal-benchmark'; - -function HeavyList({ count }: { count: number }) { - const items = Array.from({ length: count }, (_, i) => i); - return ( -
      - {items.map((i) => ( -
    • - Item {i} — {Math.sqrt(i).toFixed(4)} -
      • - ))} -
    - ); -} - -benchmark('HeavyList mount', () => , { - runs: 10, - warmupRuns: 5, -}); diff --git a/test/performance/tests/imperative-metric.bench.tsx b/test/performance/tests/imperative-metric.bench.tsx deleted file mode 100644 index 1fc2c833ad..0000000000 --- a/test/performance/tests/imperative-metric.bench.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import * as React from 'react'; -import { benchmark, ScalarMetric } from '@mui/internal-benchmark'; - -function simulateSlowdown(ms: number) { - const end = performance.now() + ms; - while (performance.now() < end) { - // burn CPU - } -} - -function Widget() { - return
    Widget
    ; -} - -const updateWork = new ScalarMetric({ - name: 'imperative_update', - format: { style: 'unit', unit: 'millisecond', maximumFractionDigits: 3 }, -}); - -// Recording stays paused for the whole benchmark: the interaction mutates the DOM imperatively (no -// React re-render) and measures only a custom metric. The harness must not require a React render -// here — there is no active recording window to validate. -benchmark( - 'Widget imperative update (metric only)', - () => , - async () => { - const node = document.querySelector('[data-state]'); - updateWork.time(); - node?.setAttribute('data-state', 'active'); - simulateSlowdown(1); - updateWork.timeEnd(); - }, - { - runs: 10, - warmupRuns: 5, - reactRecordingPaused: true, - }, -); diff --git a/test/performance/tests/metric.bench.tsx b/test/performance/tests/metric.bench.tsx deleted file mode 100644 index ad5ff0899c..0000000000 --- a/test/performance/tests/metric.bench.tsx +++ /dev/null @@ -1,44 +0,0 @@ -import { it } from 'vitest'; -import { ScalarMetric, DiscreteMetric } from '@mui/internal-benchmark'; - -function fib(n: number): number { - return n < 2 ? n : fib(n - 1) + fib(n - 2); -} - -// A scalar timing metric with an alarm: regressions past 10% warn, past 25% error. -const duration = new ScalarMetric({ - name: 'fib_duration', - format: { style: 'unit', unit: 'millisecond', maximumFractionDigits: 3 }, - alarm: { direction: 'lowerIsBetter', warn: 0.1, error: 0.25 }, -}); - -// A discrete count metric (informational), compared as an exact integer. -const evenResults = new DiscreteMetric({ name: 'fib_even_results' }); - -it('custom scalar + discrete metrics', () => { - for (let run = 0; run < 50; run += 1) { - duration.time(); - const result = fib(22); - duration.timeEnd(); - - evenResults.record(result % 2 === 0 ? 1 : 0); - } -}); - -// A second scalar metric demonstrating labeled sub-series via `time`/`timeEnd` labels. -const phases = new ScalarMetric({ - name: 'fib_phase', - format: { style: 'unit', unit: 'millisecond', maximumFractionDigits: 3 }, -}); - -it('sub-series via labels', () => { - for (let run = 0; run < 50; run += 1) { - phases.time('small'); - fib(18); - phases.timeEnd('small'); // -> fib_phase#small - - phases.time('large'); - fib(24); - phases.timeEnd('large'); // -> fib_phase#large - } -}); diff --git a/test/performance/tests/react-recording.bench.tsx b/test/performance/tests/react-recording.bench.tsx deleted file mode 100644 index ed2d30bb31..0000000000 --- a/test/performance/tests/react-recording.bench.tsx +++ /dev/null @@ -1,52 +0,0 @@ -import * as React from 'react'; -import * as ReactDOM from 'react-dom'; -import { benchmark, ElementTiming, ScalarMetric } from '@mui/internal-benchmark'; - -function simulateSlowdown(ms: number) { - const end = performance.now() + ms; - while (performance.now() < end) { - // burn CPU - } -} - -function Counter() { - const [count, setCount] = React.useState(0); - simulateSlowdown(2); - return ( - - ); -} - -// A custom metric recorded inside the benchmark. Warmup iterations are excluded automatically, so -// this yields exactly `runs` samples without the interaction needing to know about warmup. -const clickWork = new ScalarMetric({ - name: 'click_work', - format: { style: 'unit', unit: 'millisecond', maximumFractionDigits: 3 }, -}); - -// Start with React recording paused so the mount is excluded. Await the mount paint (which lands in -// its own frame while paused), resume, then measure only the click's re-render and the paint it -// produces — so the report shows a single `bench:update` render and a single `bench:paint#clicked`. -benchmark( - 'Counter click (interaction only)', - () => , - async ({ resumeReactRecording, waitForElementTiming }) => { - await waitForElementTiming('ready'); // mount paint — recorded while paused, so excluded - resumeReactRecording(); - clickWork.time(); - ReactDOM.flushSync(() => { - document.querySelector('button')?.click(); - }); - clickWork.timeEnd(); // wraps the synchronous re-render - await waitForElementTiming('clicked'); // interaction paint — recorded - }, - { - runs: 10, - warmupRuns: 5, - reactRecordingPaused: true, - }, -); diff --git a/test/performance/tsconfig.json b/test/performance/tsconfig.json deleted file mode 100644 index 223f8b95e3..0000000000 --- a/test/performance/tsconfig.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "composite": false, - "jsx": "react-jsx", - "module": "es2022", - "moduleResolution": "bundler", - "resolveJsonModule": true - }, - "include": ["tests", "vitest.config.ts"] -} diff --git a/test/performance/vitest.config.ts b/test/performance/vitest.config.ts deleted file mode 100644 index 424c5e785a..0000000000 --- a/test/performance/vitest.config.ts +++ /dev/null @@ -1,3 +0,0 @@ -import { createBenchmarkVitestConfig } from '@mui/internal-benchmark/vitest'; - -export default createBenchmarkVitestConfig(); diff --git a/tsconfig.json b/tsconfig.json index ce12c69e43..df3489d328 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,7 +25,6 @@ { "path": "tsconfig.node.json" }, { "path": "scripts" }, { "path": "docs" }, - { "path": "test/performance" }, { "path": "test/benchmark" } ], "files": [] From d74fcc832ba22a32eb37915a793a558d5f5f0d34 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 20 Oct 2026 10:50:24 +0200 Subject: [PATCH 171/173] [benchmark] Use the US spelling of color --- .../src/components/BenchmarkRunHistory.tsx | 4 ++-- apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx | 4 ++-- packages/benchmark/README.md | 4 ++-- packages/benchmark/src/cli/index.ts | 2 +- packages/benchmark/src/runReport/format.ts | 6 +++--- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx index c987befff1..3bdbbf4965 100644 --- a/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx +++ b/apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx @@ -25,7 +25,7 @@ import ErrorDisplay from './ErrorDisplay'; * read off the chart directly, rather than inferred from a noisy trend. */ -// An `unchanged` comparison sits at zero, so it has no bar to colour. +// An `unchanged` comparison sits at zero, so it has no bar to color. const CHANGE_COLORS: Record, string> = { worse: 'var(--mui-palette-error-main)', better: 'var(--mui-palette-success-main)', @@ -110,7 +110,7 @@ export default function BenchmarkRunHistory({ repo }: BenchmarkRunHistoryProps) (name): name is string => name != null && metricNames.includes(name), ) ?? null; - // One series per kind of change, so each bar takes its change's colour. A commit's bar is the + // One series per kind of change, so each bar takes its change's color. A commit's bar is the // midpoint of its confidence interval; the tooltip gives the interval itself. const series = React.useMemo(() => { const selected = points.map((point) => diff --git a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx index 57ddaabb8c..874aa22556 100644 --- a/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx +++ b/apps/code-infra-dashboard/src/views/BenchmarkRunDetails.tsx @@ -39,7 +39,7 @@ import Heading from '../components/Heading'; import ReportHeader from '../components/ReportHeader'; import ErrorDisplay from '../components/ErrorDisplay'; -/** A cell's colours: a tinted background where something needs a look, so the eye finds it. */ +/** A cell's colors: a tinted background where something needs a look, so the eye finds it. */ function toneSx(tone: Tone): SxProps { if (tone === 'none') { return { color: 'text.secondary' }; @@ -59,7 +59,7 @@ const COLUMN_WIDTHS: Record = { precision: 110, }; -/** A table cell's colour: a comparison by its change, a precision by whether a regression could hide. */ +/** A table cell's color: a comparison by its change, a precision by whether a regression could hide. */ function cellSx(row: BenchmarkTableRow, column: BenchmarkTableColumn): SxProps | undefined { if (column.comparison === undefined) { return undefined; diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index 265b134972..cc5ab7189a 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -241,7 +241,7 @@ You can also `record()` or `time()` from inside a `benchmark()` render function How a regression is judged depends on which report the repository uploads: - **Version 1** (this Vitest reporter) compares the two runs' means. A scalar metric's change must pass a band; with neither band set, `error` is the dashboard's ±20% noise band. A discrete metric flags any change in its mean. -- **Version 2** ([the `benchmark` CLI](#the-benchmark-cli)) first needs the metric's confidence interval to lie wholly on the worse side; the `error` band then applies to the end of that interval nearest zero, so a change alarms only once it is confidently that large. Without an `error` band, a scalar metric uses 5% and a discrete one alarms on any confirmed change in its count. Version 2 has no `warn`: the dashboard colours every confirmed change for the worse, and the PR comment shows only alarms. A benchmark can override a metric's alarm for itself; see [Choosing what alarms](#choosing-what-alarms). +- **Version 2** ([the `benchmark` CLI](#the-benchmark-cli)) first needs the metric's confidence interval to lie wholly on the worse side; the `error` band then applies to the end of that interval nearest zero, so a change alarms only once it is confidently that large. Without an `error` band, a scalar metric uses 5% and a discrete one alarms on any confirmed change in its count. Version 2 has no `warn`: the dashboard colors every confirmed change for the worse, and the PR comment shows only alarms. A benchmark can override a metric's alarm for itself; see [Choosing what alarms](#choosing-what-alarms). Alarms are evaluated against the baseline when the PR comment is generated, not during the local `vitest run` — a regression never fails the test suite locally. In a version 1 PR comment, `error`-band regressions surface as failures and `warn`-band regressions as warnings. @@ -585,7 +585,7 @@ benchmark grid -t "scroll$" --sample-size 20 --timeout 0 `--reporter json` prints the run's analysis to stdout as JSON — per benchmark and metric, each variant's median and each comparison's interval, verdict and severity, plus the regressions and how long it all took — with everything else a run prints sent to stderr. `summarizeRun` from -`@mui/internal-benchmark/runReport` produces the same from a report. `NO_COLOR` turns colour off. +`@mui/internal-benchmark/runReport` produces the same from a report. `NO_COLOR` turns color off. `--profile` records instead of measuring: each sample runs in a freshly loaded page of its own, with a performance trace around it, saved under `.benchmark/traces/` to open in DevTools' Performance diff --git a/packages/benchmark/src/cli/index.ts b/packages/benchmark/src/cli/index.ts index 85a67568a6..422c5b5ab9 100755 --- a/packages/benchmark/src/cli/index.ts +++ b/packages/benchmark/src/cli/index.ts @@ -144,7 +144,7 @@ const uploadCommand: CommandModule<{}, { file?: string; timeline?: string }> = { }; // chalk honours FORCE_COLOR and --no-color, not the NO_COLOR convention (https://no-color.org): -// set to anything but an empty string, it turns colour off. +// set to anything but an empty string, it turns color off. if (process.env.NO_COLOR) { chalk.level = 0; } diff --git a/packages/benchmark/src/runReport/format.ts b/packages/benchmark/src/runReport/format.ts index f3254b4390..806b7f040e 100644 --- a/packages/benchmark/src/runReport/format.ts +++ b/packages/benchmark/src/runReport/format.ts @@ -43,11 +43,11 @@ export function formatPercent(interval: Interval): string { return `${signed(interval.low)} – ${signed(interval.high)}`; } -/** The colour a renderer draws the eye with; a `Severity` is one too. */ +/** The color a renderer draws the eye with; a `Severity` is one too. */ export type Tone = 'error' | 'warning' | 'success' | 'none'; /** - * The colour a comparison draws the eye with: `error` for a regression that fails the check, + * The color a comparison draws the eye with: `error` for a regression that fails the check, * `warning` for any other change for the worse, `success` for one for the better, and `none` when * nothing moved. */ @@ -65,7 +65,7 @@ export function changeTone({ change, severity }: MetricComparison): Tone { const MANY_ROUNDS = 1000; /** - * The colour a comparison's precision draws the eye with: `error` when a regression the alarm would + * The color a comparison's precision draws the eye with: `error` when a regression the alarm would * raise — one past its `error` band — could hide in the interval, `none` otherwise. */ export function precisionTone({ alarm, precision }: MetricComparison): Tone { From 76f09983edaf032b7bb87f5993af7d1eedfe4c1d Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:08:58 +0200 Subject: [PATCH 172/173] [benchmark] Don't call a fixed sample size a timeout A timeout of 0 asks for exactly sampleSize rounds, so stopping there unsettled is not reported as timed out. --- packages/benchmark/src/runner/runInterleaved.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/benchmark/src/runner/runInterleaved.ts b/packages/benchmark/src/runner/runInterleaved.ts index ee1526aedd..0126845a8a 100644 --- a/packages/benchmark/src/runner/runInterleaved.ts +++ b/packages/benchmark/src/runner/runInterleaved.ts @@ -347,7 +347,9 @@ async function runBenchFile( return { benchmark: { ...plan.entry, error, durationMs }, metrics: {} }; } const { sampleSize, timeout } = plan.sampling; - if (rounds.length > sampleSize || !settled) { + // A `timeout` of 0 asks for exactly `sampleSize` rounds: stopping there unsettled is the plan. + const timedOut = !settled && timeout > 0; + if (rounds.length > sampleSize || timedOut) { const outcome = settled ? 'settled' : `timed out after ${timeout} min`; console.log( chalk.dim( @@ -358,7 +360,7 @@ async function runBenchFile( return { benchmark: { ...benchmarkOf(progress), - sampling: { sampleSize, timedOut: !settled }, + sampling: { sampleSize, timedOut }, durationMs: Math.round(durationMs), }, metrics: progress.metrics ?? {}, From c51e84ecc9ec44477ae5b3ebbf775d1d2403b939 Mon Sep 17 00:00:00 2001 From: Janpot <2109932+Janpot@users.noreply.github.com> Date: Tue, 20 Oct 2026 11:08:58 +0200 Subject: [PATCH 173/173] [benchmark] Sample this repository's benchmarks for 10 rounds They check that the CLI works end to end rather than guard a library, so a few rounds are enough; the libs-sort comparison now samples the same way. --- test/benchmark/src/_shared/sampling.ts | 6 +++--- test/benchmark/src/libs/sort/sort.bench.tsx | 15 ++++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/test/benchmark/src/_shared/sampling.ts b/test/benchmark/src/_shared/sampling.ts index e21c0491e9..289f58ed6e 100644 --- a/test/benchmark/src/_shared/sampling.ts +++ b/test/benchmark/src/_shared/sampling.ts @@ -1,7 +1,7 @@ import type { SamplingOptions } from '@mui/internal-benchmark/page'; /** - * Sampling for this smoke-test harness, which should stay quick: a benchmark whose alarmed - * difference doesn't settle against its band stops after a minute instead of three. + * Sampling for this harness, which checks that the CLI works end to end rather than guarding a + * library: a few rounds, never more, so the CI job stays as quick as the others. */ -export const SMOKE_SAMPLING: SamplingOptions = { timeout: 1 }; +export const SMOKE_SAMPLING: SamplingOptions = { sampleSize: 10, timeout: 0 }; diff --git a/test/benchmark/src/libs/sort/sort.bench.tsx b/test/benchmark/src/libs/sort/sort.bench.tsx index 2c547ea032..aa38f3610f 100644 --- a/test/benchmark/src/libs/sort/sort.bench.tsx +++ b/test/benchmark/src/libs/sort/sort.bench.tsx @@ -3,14 +3,19 @@ import { compare, reactBenchmark } from '@mui/internal-benchmark/page'; import { AlphaSorted } from './sort.alpha'; import { BetaSorted } from './sort.beta'; import { OursSorted } from './sort.ours'; +import { SMOKE_SAMPLING } from '../../_shared/sampling'; /** * Three sorts compared with each other rather than across builds — the shape a comparison between * libraries takes. This repository has no competitor libraries to install, so the "libraries" here * are three sorts with genuinely different constant factors. `ours` is the reference. */ -compare('libs-sort', [ - reactBenchmark('ours', () => ), - reactBenchmark('alpha', () => ), - reactBenchmark('beta', () => ), -]); +compare( + 'libs-sort', + [ + reactBenchmark('ours', () => ), + reactBenchmark('alpha', () => ), + reactBenchmark('beta', () => ), + ], + SMOKE_SAMPLING, +);