[benchmark] v2

.circleci/config.yml

@@ -67,6 +67,11 @@ jobs:
           # make sure the netlify ignore command has the correct dependencies
           name: '`pnpm update-netlify-ignore` changes committed?'
           command: pnpm code-infra netlify-ignore ./apps/code-infra-dashboard --check
+      - run:
+          # A package that publishes from `build/` is linked there, not at its source, so a consumer
+          # of one sees no types at all until it is built.
+          name: Build packages
+          command: pnpm release:build
       - run:
           name: Tests TypeScript
           command: pnpm typescript
@@ -75,6 +80,11 @@ jobs:
     steps:
       - checkout
       - code-infra/install-deps
+      - run:
+          # A package that publishes from `build/` is linked there, not at its source, so an import
+          # from one fails to resolve until it is built.
+          name: Build packages
+          command: pnpm release:build
       - run:
           name: 'Run tests'
           command: pnpm test
@@ -92,18 +102,71 @@ jobs:
     <<: *default-job
     executor:
       name: code-infra/mui-node-browser
-      playwright-img-version: 'v1.64.0-noble'
+      playwright-img-version: 'v1.62.1-noble'
+    # 2 vCPUs starve V8's background compilers, which makes the same code's speed vary from sample to
+    # sample; see the benchmark package's README.
+    resource_class: medium+
     steps:
       - checkout
       - code-infra/install-deps
       - run:
           name: Build packages
           command: pnpm release:build
       - run:
-          name: Run benchmarks
-          environment:
-            BENCHMARK_UPLOAD: 'true'
-          command: pnpm -F ./test/performance benchmark
+          name: Resolve the baseline commit
+          command: |
+            set -e
+            BASE_SHA=$(pnpm code-infra baseline --base-branch master)
+            echo "Baseline commit: $BASE_SHA"
+            echo "$BASE_SHA" > .benchmark-baseline-sha
+      - restore_cache:
+          name: Restore the packed-baseline cache
+          keys:
+            - benchmark-run-cache-v1-{{ checksum ".benchmark-baseline-sha" }}
+      - run:
+          name: Run the benchmark smoke test
+          no_output_timeout: 30m
+          command: pnpm test:benchmark --baseline "$(cat .benchmark-baseline-sha)"
+      - run:
+          name: Upload the benchmark report
+          # Also after a failed run that wrote its report, such as one where every benchmark errored,
+          # so the comment says what happened.
+          when: always
+          command: pnpm test:benchmark upload
+      - save_cache:
+          name: Save the packed-baseline cache
+          key: benchmark-run-cache-v1-{{ checksum ".benchmark-baseline-sha" }}
+          paths:
+            - test/benchmark/.benchmark/packed
+  # The commit against itself: any regression it reports is a false alarm, so over many nights the
+  # job's failure rate is the benchmarks' false-alarm rate. An alarm needs a change confidently past
+  # its error band, so expect close to 0%; a rate that climbs means noise or bias in a benchmark.
+  benchmark_aa:
+    <<: *default-job
+    executor:
+      name: code-infra/mui-node-browser
+      playwright-img-version: 'v1.62.1-noble'
+    resource_class: medium+
+    steps:
+      - checkout
+      - code-infra/install-deps
+      - run:
+          name: Build packages
+          command: pnpm release:build
+      - run:
+          name: Benchmark the commit against itself
+          no_output_timeout: 30m
+          command: pnpm --silent -F ./test/benchmark benchmark --baseline HEAD --reporter json > aa.json
+      - store_artifacts:
+          path: aa.json
+      - run:
+          name: Fail on a false alarm
+          command: |
+            node -e "
+              const { regressions } = JSON.parse(require('fs').readFileSync('aa.json', 'utf8'));
+              for (const { benchmark, metric } of regressions) console.error('False alarm: ' + benchmark + ' · ' + metric);
+              process.exit(regressions.length > 0 ? 1 : 0);
+            "
 
   test_browser:
     <<: *default-job
@@ -148,3 +211,10 @@ workflows:
           <<: *default-context
       - test_browser:
           <<: *default-context
+  # Started nightly on master by a scheduled pipeline that sets `workflow` to `benchmark-aa`.
+  benchmark-aa:
+    when:
+      equal: [benchmark-aa, << pipeline.parameters.workflow >>]
+    jobs:
+      - benchmark_aa:
+          <<: *default-context

.gitignore

@@ -63,3 +63,10 @@ next-env.d.ts
 # Claude Code worktrees
 .claude/worktrees/
 .claude/cache/
+
+# Everything the benchmark harness writes: built pages per ref, packed workspace tarballs, the
+# isolated installs, the run reports, and the pages generated per benchmark file. Both directories
+# ignore themselves from the inside too, but are listed here because ESLint is fed from the root
+# .gitignore and never reads nested ones.
+/test/benchmark/.benchmark/
+/test/benchmark/src/__bench__/

AGENTS.md

@@ -88,7 +88,7 @@ Applies to the whole repository.
 
 - **CRITICAL**: When running pnpm commands for workspace packages, always use the `-F` flag followed by the package name.
 - **Example**: `pnpm -F @mui/internal-bundle-size-checker add micromatch`
-- Private packages without a `name` field in `package.json` must be filtered by their relative path (for example, `pnpm -F ./test/performance add <dependency>`).
+- Private packages without a `name` field in `package.json` must be filtered by their relative path (for example, `pnpm -F ./test/benchmark add <dependency>`).
 - **Do NOT use `cd` to navigate into package directories** for workspace operations.
 - **Do NOT manually edit package.json files to add/remove dependencies** - always use `pnpm -F <workspace> add <dependency>` or `pnpm -F <workspace> remove <dependency>` to keep the order deterministic.
 - **ALWAYS run `pnpm dedupe`** after installing a dependency.

apps/code-infra-dashboard/app/(dashboard)/benchmark-run/[owner]/[repo]/page.tsx

@@ -0,0 +1,16 @@
+import * as React from 'react';
+import type { Metadata } from 'next';
+import BenchmarkRunDetails from '@/views/BenchmarkRunDetails';
+
+export async function generateMetadata({
+  params,
+}: {
+  params: Promise<{ owner: string; repo: string }>;
+}): Promise<Metadata> {
+  const { owner, repo } = await params;
+  return { title: `Benchmark run - ${owner}/${repo}` };
+}
+
+export default function BenchmarkRunPage() {
+  return <BenchmarkRunDetails />;
+}

apps/code-infra-dashboard/src/components/BenchmarkRunHistory.tsx

@@ -0,0 +1,223 @@
+import * as React from 'react';
+import NextLink from 'next/link';
+import Autocomplete from '@mui/material/Autocomplete';
+import Box from '@mui/material/Box';
+import Button from '@mui/material/Button';
+import Link from '@mui/material/Link';
+import Paper from '@mui/material/Paper';
+import TextField from '@mui/material/TextField';
+import Typography from '@mui/material/Typography';
+import { BarChartPro } from '@mui/x-charts-pro/BarChartPro';
+import { analyzeRun, formatChange, formatComparison } from '@mui/internal-benchmark/runReport';
+import type {
+  BenchmarkRunReport,
+  Change,
+  MetricComparison,
+} from '@mui/internal-benchmark/runReport';
+import { isBenchmarkRunUpload } from '@/utils/fetchCiReport';
+import { useMasterCommits } from '../hooks/useMasterCommits';
+import { useCiReports } from '../hooks/useCiReports';
+import ErrorDisplay from './ErrorDisplay';
+
+/**
+ * Master history of the `benchmark` CLI reports. Every master commit is benchmarked against its parent
+ * in the same run, so each bar is that commit's own paired change — which commit moved a number is
+ * read off the chart directly, rather than inferred from a noisy trend.
+ */
+
+// An `unchanged` comparison sits at zero, so it has no bar to color.
+const CHANGE_COLORS: Record<Exclude<Change, 'unchanged'>, string> = {
+  worse: 'var(--mui-palette-error-main)',
+  better: 'var(--mui-palette-success-main)',
+  undetected: 'var(--mui-palette-grey-500)',
+};
+
+/** Per benchmark, per metric: a commit's change against its parent. */
+type CommitComparisons = Map<string, Map<string, MetricComparison>>;
+
+interface CommitPoint {
+  sha: string;
+  date: Date;
+  comparisons: CommitComparisons;
+}
+
+// A report is analysed once, however often the list of loaded reports changes around it. Keyed by
+// the report object, which the query cache keeps stable for as long as it holds the report.
+const comparisonsByReport = new WeakMap<BenchmarkRunReport, CommitComparisons>();
+
+function comparisonsOf(report: BenchmarkRunReport): CommitComparisons {
+  let comparisons = comparisonsByReport.get(report);
+  if (!comparisons) {
+    comparisons = new Map();
+    for (const { benchmark, metrics } of analyzeRun(report)) {
+      if (benchmark.kind === 'baseline') {
+        comparisons.set(
+          benchmark.name,
+          new Map(metrics.map(({ metric, comparisons: [comparison] }) => [metric, comparison])),
+        );
+      }
+    }
+    comparisonsByReport.set(report, comparisons);
+  }
+  return comparisons;
+}
+
+interface BenchmarkRunHistoryProps {
+  repo: string;
+}
+
+export default function BenchmarkRunHistory({ repo }: BenchmarkRunHistoryProps) {
+  const { commits, isLoading, isFetchingNextPage, hasNextPage, error, fetchNextPage } =
+    useMasterCommits(repo);
+  const { reports, isLoading: reportsLoading } = useCiReports(repo, commits, 'benchmark.json');
+
+  const points = React.useMemo(
+    () =>
+      commits.flatMap(({ timestamp, commit }): CommitPoint[] => {
+        const upload = reports[commit.sha];
+        // Version 1 reports are `DailyBenchmarkChart`'s.
+        if (!upload || !isBenchmarkRunUpload(upload)) {
+          return [];
+        }
+        return [
+          { sha: commit.sha, date: new Date(timestamp), comparisons: comparisonsOf(upload.report) },
+        ];
+      }),
+    [commits, reports],
+  );
+
+  const benchmarkNames = React.useMemo(
+    () => [...new Set(points.flatMap((point) => [...point.comparisons.keys()]))].sort(),
+    [points],
+  );
+  const [userBenchmark, setUserBenchmark] = React.useState<string | null>(null);
+  const benchmark = userBenchmark ?? benchmarkNames[0] ?? null;
+
+  const metricNames = React.useMemo(
+    () =>
+      benchmark === null
+        ? []
+        : [
+            ...new Set(
+              points.flatMap((point) => [...(point.comparisons.get(benchmark)?.keys() ?? [])]),
+            ),
+          ],
+    [points, benchmark],
+  );
+  const [userMetric, setUserMetric] = React.useState<string | null>(null);
+  const metric =
+    [userMetric, 'render', metricNames[0]].find(
+      (name): name is string => name != null && metricNames.includes(name),
+    ) ?? null;
+
+  // One series per kind of change, so each bar takes its change's color. A commit's bar is the
+  // midpoint of its confidence interval; the tooltip gives the interval itself.
+  const series = React.useMemo(() => {
+    const selected = points.map((point) =>
+      benchmark === null || metric === null
+        ? undefined
+        : point.comparisons.get(benchmark)?.get(metric),
+    );
+    return (['worse', 'better', 'undetected'] as const satisfies Change[]).map((change) => ({
+      type: 'bar' as const,
+      stack: 'change',
+      label: formatChange(change),
+      color: CHANGE_COLORS[change],
+      data: selected.map((comparison) =>
+        comparison?.change === change
+          ? (comparison.relative.low + comparison.relative.high) / 2
+          : null,
+      ),
+      valueFormatter: (_value: number | null, { dataIndex }: { dataIndex: number }) => {
+        const comparison = selected[dataIndex];
+        return comparison?.change === change ? formatComparison(comparison) : null;
+      },
+    }));
+  }, [points, benchmark, metric]);
+
+  const [selectedSha, setSelectedSha] = React.useState<string | null>(null);
+
+  // A repository still on version 1 reports has nothing to draw here.
+  if (!error && !isLoading && !reportsLoading && points.length === 0) {
+    return null;
+  }
+
+  return (
+    <Paper elevation={2} sx={{ p: 3, mt: 3 }}>
+      <Typography variant="h6" component="h2" gutterBottom>
+        Benchmark runs
+      </Typography>
+
+      {error ? (
+        <ErrorDisplay title="Error loading benchmark history" error={error} />
+      ) : (
+        <React.Fragment>
+          <Typography variant="body2" color="text.secondary" sx={{ mb: 2 }}>
+            Each master commit against its parent, measured in the same run: the midpoint of the 99%
+            confidence interval on the paired change. Click a bar to open that run.
+          </Typography>
+
+          <Box sx={{ display: 'flex', gap: 2, mb: 2, flexWrap: 'wrap' }}>
+            <Autocomplete
+              options={benchmarkNames}
+              value={benchmark}
+              onChange={(_event, value) => setUserBenchmark(value)}
+              size="small"
+              sx={{ minWidth: 320 }}
+              renderInput={(inputParams) => <TextField {...inputParams} label="Benchmark" />}
+            />
+            <Autocomplete
+              options={metricNames}
+              value={metric}
+              onChange={(_event, value) => setUserMetric(value)}
+              size="small"
+              sx={{ minWidth: 200 }}
+              renderInput={(inputParams) => <TextField {...inputParams} label="Metric" />}
+            />
+          </Box>
+
+          <BarChartPro
+            xAxis={[
+              {
+                data: points.map((point) => point.date),
+                scaleType: 'band',
+                valueFormatter: (date: Date, context: { location: string }) =>
+                  context.location === 'tick'
+                    ? date.toLocaleDateString()
+                    : `${date.toLocaleString()} (${
+                        points.find((point) => point.date === date)?.sha.slice(0, 7) ?? ''
+                      })`,
+              },
+            ]}
+            yAxis={[{ width: 60, valueFormatter: (value: number) => `${value.toFixed(1)}%` }]}
+            series={series}
+            onAxisClick={(_event, data) => {
+              if (data) {
+                setSelectedSha(points[data.dataIndex]?.sha ?? null);
+              }
+            }}
+            loading={isLoading || reportsLoading}
+            height={300}
+            skipAnimation
+            grid={{ horizontal: true }}
+          />
+
+          <Box sx={{ mt: 2, display: 'flex', justifyContent: 'space-between', gap: 2 }}>
+            <Typography variant="body2">
+              {selectedSha ? (
+                <Link component={NextLink} href={`/benchmark-run/${repo}?sha=${selectedSha}`}>
+                  Open the run for {selectedSha.slice(0, 7)}
+                </Link>
+              ) : null}
+            </Typography>
+            {hasNextPage && (
+              <Button size="small" onClick={() => fetchNextPage()} disabled={isFetchingNextPage}>
+                {isFetchingNextPage ? 'Loading…' : 'Load older commits'}
+              </Button>
+            )}
+          </Box>
+        </React.Fragment>
+      )}
+    </Paper>
+  );
+}