eval-refresh.yml

name: Refresh eval results

# Two schedules, because everything weekly costs $279 a month against a $200
# budget, and cadence is a better lever than dropping scenarios: the cheap
# experiments carry the signal and the expensive ones carry the headline.
#
# Weekly, about $185 a month: the frontier agents plus the weak pair. The weak
# pair is the only source of failures in the suite and the only place a skills
# difference has been observable, and it is the most sensitive regression
# detector because it sits on the pass/fail boundary rather than passing
# everything.
#
# Monthly adds the `-no-skills` twins of the two expensive agents. Their delta
# has measured zero on every scenario, four times, and will not move week to
# week; a month is soon enough to notice if it ever does.
#
# Costs are measured rather than estimated: $20.53 for a fourteen-scenario
# Claude pass, $7.40 for gpt-5.6 (which resolves to gpt-5.6-sol), about $4 for
# the weak model, pennies for judging.
on:
  schedule:
    # Weekly: frontier agents and the weak pair.
    - cron: '0 6 * * 1'
    # Monthly: the full matrix, adding the no-skills twins.
    - cron: '0 8 1 * *'
  workflow_dispatch:
    inputs:
      experiments:
        description: "Comma-separated experiment names to run (blank to auto-discover from experiment_suite)"
        required: false
        default: ""
      eval:
        description: "Optional comma-separated eval ids to run"
        required: false
        default: ""
      suite:
        description: "Comma-separated eval suites to run"
        required: true
        default: "regression"
      experiment_suite:
        description: "Comma-separated experiment suites to run"
        required: true
        default: "regression"
      runs:
        description: "Attempts per experiment/eval pair"
        required: true
        default: "2"
      timeout_sec:
        description: "Timeout per attempt in seconds"
        required: true
        default: "720"
      merge:
        description: "Merge into existing results instead of overwriting (graft new experiment/eval pairs)"
        type: boolean
        required: false
        default: false

permissions:
  contents: write
  # `publish-results` downloads each matrix job's artifact with `gh run download`,
  # which lists the run's artifacts through the API. Without this the listing is a
  # 403 and the job fails after every eval has already run and been paid for.
  # `read` for that artifact listing; `write` so a matrix job can cancel the
  # run when the provider stops answering, instead of letting every remaining
  # job fail the same way.
  actions: write

# Scoring runs against one shared Hookdeck project, so two workflow runs must
# never overlap. Queue rather than cancel: a cancelled run leaves the project
# dirty for the next one.
concurrency:
  group: eval-refresh
  cancel-in-progress: false

jobs:
  prepare:
    runs-on: ubuntu-latest
    outputs:
      pairs: ${{ steps.discover.outputs.pairs }}
      runs: ${{ steps.inputs.outputs.runs }}
      timeout_sec: ${{ steps.inputs.outputs.timeout_sec }}
      do_merge: ${{ steps.inputs.outputs.do_merge }}
    steps:
      - name: Prepare inputs
        id: inputs
        shell: bash
        run: |
          set -euo pipefail

          # A scheduled run has no inputs, so the two crons carry their own
          # experiment sets. Weekly is the frontier agents and the weak pair;
          # the monthly one adds the no-skills twins for a full matrix.
          if [ "${{ github.event_name }}" = "schedule" ]; then
            if [ "${{ github.event.schedule }}" = "0 8 1 * *" ]; then
              experiments="claude-code-sonnet-5,claude-code-sonnet-5-no-skills,codex-gpt-5.6,codex-gpt-5.6-no-skills,codex-gpt-5.4-mini,codex-gpt-5.4-mini-no-skills"
            else
              experiments="claude-code-sonnet-5,codex-gpt-5.6,codex-gpt-5.4-mini,codex-gpt-5.4-mini-no-skills"
            fi
            eval_id=""
            suite="benchmark"
            experiment_suite="benchmark,no-skills"
            runs="1"
            timeout_sec="900"
            # Merge, never overwrite. The weekly run covers four experiments and
            # the monthly covers six, so an overwriting weekly deletes the
            # -no-skills twins the monthly produced and the published page loses
            # the skills comparison for three weeks in four. It did exactly that
            # on 17 August, dropping the scoreboard from six columns to four.
            do_merge="true"
          else
            experiments="${{ inputs.experiments }}"
            eval_id="${{ inputs.eval }}"
            suite="${{ inputs.suite }}"
            experiment_suite="${{ inputs.experiment_suite }}"
            runs="${{ inputs.runs }}"
            timeout_sec="${{ inputs.timeout_sec }}"
            do_merge="${{ inputs.merge }}"
          fi

          suite_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$suite")"
          experiment_suite_json="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$experiment_suite")"

          {
            echo "experiments_override=$experiments"
            echo "eval=$eval_id"
            echo "suite=$suite_json"
            echo "experiment_suite=$experiment_suite_json"
            echo "runs=$runs"
            echo "timeout_sec=$timeout_sec"
            echo "do_merge=$do_merge"
          } >> "$GITHUB_OUTPUT"

      - name: Checkout
        uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
        with:
          # skills/ symlinks into the agent-skills submodule; without it a
          # skills experiment installs nothing and silently scores as baseline.
          submodules: recursive

      - name: Install pnpm
        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0

      - name: Setup Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version-file: .node-version
          cache: pnpm

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Discover eval pairs
        id: discover
        shell: bash
        run: |
          set -euo pipefail

          touch .env
          suite_json='${{ steps.inputs.outputs.suite }}'
          eval_ids="${{ steps.inputs.outputs.eval }}"
          if [ -n "$eval_ids" ]; then
            matching=()
            while IFS= read -r id; do
              [ -d "evals/$id" ] && matching+=("$id")
            done < <(jq -Rr 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | .[]' <<< "$eval_ids")
          else
            matching=()
            for dir in evals/*/; do
              [ -d "$dir" ] || continue
              id=$(basename "$dir")
              prompt="$dir/PROMPT.md"
              [ -f "$prompt" ] || continue
              suite_val=$(sed -n 's/^suite:[[:space:]]*//p' "$prompt" | head -n 1)
              if jq -e --arg s "$suite_val" 'index($s) != null' <<< "$suite_json" > /dev/null 2>&1; then
                matching+=("$id")
              fi
            done
          fi

          if [ "${#matching[@]}" -eq 0 ]; then
            echo "No evals matched the requested suites: $suite_json" >&2
            exit 1
          fi

          pairs='[]'
          experiments_override="${{ steps.inputs.outputs.experiments_override }}"
          for id in "${matching[@]}"; do
            eval_suite=$(sed -n 's/^suite:[[:space:]]*//p' "evals/$id/PROMPT.md" | head -n 1)
            case "$eval_suite" in
              benchmark) experiment_suites=(benchmark no-skills) ;;
              regression) experiment_suites=(regression) ;;
              *) continue ;;
            esac

            for experiment_suite in "${experiment_suites[@]}"; do
              if ! jq -e --arg suite "$experiment_suite" 'index($suite) != null' \
                <<< '${{ steps.inputs.outputs.experiment_suite }}' > /dev/null; then
                continue
              fi

              # Always ask which experiments belong to this suite, then narrow
              # to the override if there is one. Using the override directly
              # emitted every named experiment once per suite, so a four-name
              # weekly list produced eight pairs per scenario, half of them
              # labelled with a suite the experiment is not in.
              suite_experiments="$(pnpm --silent eval -- list --experiment-suite "$experiment_suite" --eval "$id")"
              if [ -n "$experiments_override" ]; then
                wanted="$(jq -Rc 'split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))' <<< "$experiments_override")"
                experiments_json="$(jq -c --argjson wanted "$wanted" '[.[] | select(. as $e | $wanted | index($e))]' <<< "$suite_experiments")"
              else
                experiments_json="$suite_experiments"
              fi

              while IFS= read -r experiment; do
                pairs="$(jq -c \
                  --arg eval_id "$id" \
                  --arg experiment "$experiment" \
                  --arg experiment_suite "$experiment_suite" \
                  --arg eval_suite "$eval_suite" \
                  '. + [{eval_id: $eval_id, experiment: $experiment, experiment_suite: $experiment_suite, eval_suite: $eval_suite}]' \
                  <<< "$pairs")"
              done < <(jq -r '.[]' <<< "$experiments_json")
            done
          done

          if [ "$(jq 'length' <<< "$pairs")" -eq 0 ]; then
            echo "No experiment and eval pairs matched" >&2
            exit 1
          fi

          echo "pairs=$pairs" >> "$GITHUB_OUTPUT"

      # A dead provider key is the cheapest possible thing to detect and the
      # most expensive thing to discover halfway through. On 13 August the
      # OpenAI credit balance hit zero and every one of the thirty-seven Codex
      # jobs ran, failed, and was paid for in wall-clock time before anyone
      # looked. One request per provider, before the matrix starts, costs
      # nothing and turns six hours of failure into a thirty-second one.
      #
      # Both checks make a real inference call, costing a token or two.
      # `/v1/models` is not sufficient and this was measured, not assumed: with
      # the credit balance at zero it still answers 200, and every listed model
      # comes back. Only an actual completion returns the 429
      # `credit_balance_exhausted` that stops a run. A liveness check that
      # cannot see the failure it exists to catch is worse than none, because
      # it reads as evidence.
      - name: Check provider keys are live
        shell: bash
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
        run: |
          set -uo pipefail
          failed=0

          openai="$(curl -sS -o /tmp/openai.json -w '%{http_code}' \
            -X POST https://api.openai.com/v1/responses \
            -H "Authorization: Bearer ${OPENAI_API_KEY}" \
            -H "Content-Type: application/json" \
            -d '{"model":"gpt-5.4-mini","input":"hi","max_output_tokens":16}' || echo 000)"
          if [ "$openai" != "200" ]; then
            echo "::error::OpenAI key check returned HTTP ${openai}. $(head -c 300 /tmp/openai.json 2>/dev/null)"
            failed=1
          fi

          # Anthropic has no free liveness endpoint, so spend one token. A
          # credit or auth problem answers 400/401/429 rather than 200.
          anthropic="$(curl -sS -o /tmp/anthropic.json -w '%{http_code}' \
            -X POST https://api.anthropic.com/v1/messages \
            -H "x-api-key: ${ANTHROPIC_API_KEY}" \
            -H "anthropic-version: 2023-06-01" \
            -H "content-type: application/json" \
            -d '{"model":"claude-haiku-4-5-20251001","max_tokens":1,"messages":[{"role":"user","content":"."}]}' || echo 000)"
          if [ "$anthropic" != "200" ]; then
            echo "::error::Anthropic key check returned HTTP ${anthropic}. $(head -c 300 /tmp/anthropic.json 2>/dev/null)"
            failed=1
          fi

          if [ "$failed" -ne 0 ]; then
            echo "::error::Refusing to start the matrix with a provider key that cannot answer. Top up or rotate the key, then re-dispatch."
            exit 1
          fi
          echo "Both provider keys answered."

  run-evals:
    needs: prepare
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      # One shared project means one run at a time. Source and destination names
      # are unique per project, so concurrent pairs collide with each other and
      # each pair's reset deletes the others' resources mid-run.
      max-parallel: 1
      matrix:
        include: ${{ fromJSON(needs.prepare.outputs.pairs) }}
    env:
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
      AI_GATEWAY_API_KEY: ${{ secrets.AI_GATEWAY_API_KEY }}
      HOOKDECK_API_KEY: ${{ secrets.HOOKDECK_API_KEY }}
      HOOKDECK_WEBHOOK_SECRET: ${{ secrets.HOOKDECK_WEBHOOK_SECRET }}
      # Scenarios declaring `requires: [outpost]` need this. Absent, the run
      # on 13 August scored outpost-001 as six agent failures rather than
      # skipping it, which is a false result published against named vendors.
      OUTPOST_API_KEY: ${{ secrets.OUTPOST_API_KEY }}
      # Synthetic AWS credentials a scenario's workspace note carries as
      # `${SEED_*}` placeholders. Not real, and not committed: a key realistic
      # enough to convince an agent is realistic enough to trip push
      # protection. Absent, `benchmark-outpost-004` is skipped rather than
      # failed — an agent handed a literal placeholder is right to refuse, and
      # scoring that is the 13 August mistake in a new form.
      SEED_ACME_SQS_ACCESS_KEY: ${{ secrets.SEED_ACME_SQS_ACCESS_KEY }}
      SEED_ACME_SQS_SECRET_KEY: ${{ secrets.SEED_ACME_SQS_SECRET_KEY }}
      # For `gh run cancel` when a provider stops answering. Needs
      # `actions: write` at the workflow level, which `publish-results` did
      # not previously require.
      GH_TOKEN: ${{ github.token }}
    steps:
      - name: Checkout
        uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
        with:
          # skills/ symlinks into the agent-skills submodule; without it a
          # skills experiment installs nothing and silently scores as baseline.
          submodules: recursive

      - name: Install pnpm
        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0

      - name: Setup Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version-file: .node-version
          cache: pnpm

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Check required secrets
        # Fail here with a readable message rather than deep inside a run, where
        # a missing key surfaces as an agent that silently does nothing.
        shell: bash
        run: |
          set -euo pipefail
          missing=()
          [ -n "${HOOKDECK_API_KEY}" ] || missing+=(HOOKDECK_API_KEY)
          [ -n "${OPENAI_API_KEY}" ] || missing+=(OPENAI_API_KEY)
          [ -n "${ANTHROPIC_API_KEY}" ] || missing+=(ANTHROPIC_API_KEY)
          if [ "${#missing[@]}" -gt 0 ]; then
            echo "::error::missing repository secrets: ${missing[*]}"
            exit 1
          fi

      - name: Write eval environment
        shell: bash
        run: |
          set -euo pipefail

          {
            echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}"
            echo "OPENAI_API_KEY=${OPENAI_API_KEY}"
            echo "AI_GATEWAY_API_KEY=${AI_GATEWAY_API_KEY}"
            echo "HOOKDECK_API_KEY=${HOOKDECK_API_KEY}"
            echo "HOOKDECK_WEBHOOK_SECRET=${HOOKDECK_WEBHOOK_SECRET}"
            echo "OUTPOST_API_KEY=${OUTPOST_API_KEY}"
            echo "SEED_ACME_SQS_ACCESS_KEY=${SEED_ACME_SQS_ACCESS_KEY}"
            echo "SEED_ACME_SQS_SECRET_KEY=${SEED_ACME_SQS_SECRET_KEY}"
          } > .env

      - name: Run evals
        shell: bash
        run: |
          set -euo pipefail

          # Tee rather than pipe: `pnpm eval | tee` would report tee's exit
          # status, so a failing run would pass the job.
          run_log="$(mktemp)"
          set +e
          pnpm eval -- \
            --experiment "${{ matrix.experiment }}" \
            --experiment-suite "${{ matrix.experiment_suite }}" \
            --eval "${{ matrix.eval_id }}" \
            --runs "${{ needs.prepare.outputs.runs }}" \
            --timeout-sec "${{ needs.prepare.outputs.timeout_sec }}" \
            2>&1 | tee "$run_log"
          # With pipefail set, this is the eval's status, not tee's.
          eval_status=${PIPESTATUS[0]}
          set -e

          # Stop the whole matrix when the provider is the problem, not this
          # pair. A quota, credit or auth failure will hit every remaining job
          # identically, and `fail-fast: false` means nothing else would stop
          # them: on 13 August thirty-seven Codex jobs each ran, failed and
          # burned wall-clock after the OpenAI credit balance reached zero,
          # with the cause sitting in every log.
          #
          # Deliberately narrow. A scenario failing its checks does not fail
          # this step at all, and an ordinary one-off error should not cancel
          # five hours of other work, so only these patterns abort the run.
          if [ "$eval_status" -ne 0 ] && grep -qiE "no credits remaining|insufficient_quota|exceeded your current quota|invalid_api_key|incorrect api key|authentication_error|401 unauthorized" "$run_log"; then
            echo "::error::Provider is refusing requests (credits, quota or auth). Cancelling the run: every remaining job would fail the same way."
            gh run cancel "${{ github.run_id }}" --repo "${{ github.repository }}" || true
            exit 1
          fi

          if [ "$eval_status" -ne 0 ]; then
            exit "$eval_status"
          fi

          # Two different things produce no result file and they need opposite
          # treatment. A scenario declaring `requires:` that the environment
          # does not satisfy is *meant* to write nothing: that is the gate
          # working, and failing the job would make a correct skip look like a
          # broken one. An experiment skipped for a missing agent key, or a run
          # that died, is a misconfiguration and must fail loudly.
          if [ ! -f ".eval-runs/${{ matrix.experiment }}/${{ matrix.eval_id }}.json" ]; then
            if grep -q "SKIP .* x ${{ matrix.eval_id }} (requires " "$run_log"; then
              echo "::notice::${{ matrix.eval_id }} declares a requirement this environment does not meet; skipped without a result"
              exit 0
            fi
            echo "::error::no result written for ${{ matrix.experiment }} x ${{ matrix.eval_id }} (experiment skipped?)"
            exit 1
          fi

      - name: Upload raw results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: raw-results-${{ matrix.experiment }}__${{ matrix.eval_id }}
          path: |
            .eval-runs/${{ matrix.experiment }}/
            !.eval-runs/**/node_modules/**
          # 90 days, the GitHub maximum, and not a round number picked for
          # neatness. These files carry the transcript, the tool calls and the
          # agent's own report: the only record of *why* an agent did what it
          # did, and the thing the published row cannot tell you. They are also
          # irreplaceable, because reproducing one means paying for the run
          # again against a model that may have changed underneath us.
          #
          # The window has to exceed the run cadence or it is useless for the
          # questions worth asking. Nearly every real question is comparative
          # ("this flipped between runs, why?") and answering one needs the
          # evidence from both. Weekly runs at 3 days meant the previous run's
          # evidence expired before its successor even started. The monthly
          # -no-skills twins need 35 days to compare two consecutive runs at
          # all, which is why this is not 30 either.
          #
          # Storage is free on a public repository, so the only trade-off here
          # is against nothing.
          retention-days: 90

  publish-results:
    needs: [prepare, run-evals]
    runs-on: ubuntu-latest
    # A way to say "run, but do not publish this week".
    #
    # Publishing was gated on the matrix succeeding and nothing else, so a run
    # against a `main` we already knew was wrong would publish anyway. On 24
    # August this fired at 06:14 with the addendum defect still on `main` — the
    # one that made twelve baseline cells fail for a reason that was ours — and
    # was cancelled by hand at roughly 47 of 72 cells, minutes ahead of this
    # job. The fixes existed, reviewed, on an open PR. The cron could neither
    # know that nor be told.
    #
    # Cancelling by hand needs somebody watching at the right moment, and nobody
    # was; it came up in conversation by luck. Set the repository variable
    # `EVALS_PUBLISH` to `false` and the matrix still runs and still uploads its
    # artifacts — the evidence is kept — while `results/` is left alone.
    #
    # Deliberately a variable rather than a secret: it is configuration, it
    # should be visible in the run log, and anyone reading a skipped publish
    # should be able to see why without repository admin.
    if: vars.EVALS_PUBLISH != 'false'
    steps:
      - name: Checkout
        uses: actions/checkout@9f698171ed81b15d1823a05fc7211befd50c8ae0 # v6.0.3
        with:
          # skills/ symlinks into the agent-skills submodule; without it a
          # skills experiment installs nothing and silently scores as baseline.
          submodules: recursive

      - name: Install pnpm
        uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0

      - name: Setup Node.js
        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
        with:
          node-version-file: .node-version
          cache: pnpm

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Download raw results
        # actions/download-artifact only nests a downloaded artifact under a
        # directory named after it when there's more than one to disambiguate;
        # with exactly one matching pair it dumps the contents flat into
        # `path`, silently breaking the per-experiment glob below. Naming the
        # destination directory ourselves side-steps that inconsistency.
        env:
          GH_TOKEN: ${{ github.token }}
        shell: bash
        run: |
          set -euo pipefail

          mkdir -p downloaded-results
          pairs='${{ needs.prepare.outputs.pairs }}'
          while IFS= read -r name; do
            gh run download ${{ github.run_id }} --name "$name" --dir "downloaded-results/$name"
          done < <(jq -r '.[] | "raw-results-\(.experiment)__\(.eval_id)"' <<< "$pairs" | sort -u)

      - name: Export results
        shell: bash
        run: |
          set -euo pipefail

          mkdir -p .eval-runs
          pairs='${{ needs.prepare.outputs.pairs }}'
          while IFS= read -r experiment; do
            mkdir -p ".eval-runs/$experiment"
            for artifact_dir in "downloaded-results/raw-results-${experiment}__"*/; do
              [ -d "$artifact_dir" ] && cp -R "$artifact_dir"/. ".eval-runs/$experiment"/
            done
          done < <(jq -r '.[].experiment' <<< "$pairs" | sort -u)

          if jq -e 'any(.[]; .eval_suite == "benchmark")' <<< "$pairs" > /dev/null; then
            export_args=(--suite benchmark --output apps/web/src/data/eval-results.json)
            if [ "${{ needs.prepare.outputs.do_merge }}" = "true" ]; then
              export_args+=(--merge)
            fi
            pnpm --filter @hookdeck-evals/framework export-results -- "${export_args[@]}"
          fi

          if jq -e 'any(.[]; .eval_suite == "regression")' <<< "$pairs" > /dev/null; then
            export_args=(--suite regression --output apps/web/src/data/regression-eval-results.json)
            if [ "${{ needs.prepare.outputs.do_merge }}" = "true" ]; then
              export_args+=(--merge)
            fi
            pnpm --filter @hookdeck-evals/framework export-results -- "${export_args[@]}"
          fi

      - name: Publish snapshot
        # `apps/web/src/data/eval-results.json` is the preview app's own input.
        # Anything outside this repo reading it is coupled to where our app
        # keeps its fixtures. `results/` is the contract: latest.json for the
        # most recent run, runs/ for the history, index.json for what exists.
        #
        # History is kept because it cannot be reconstructed. A row records
        # whether an agent passed, not what it did, so overwriting latest.json
        # destroys the previous answer. Snapshots are what make "did this get
        # better or worse" answerable at all.
        shell: bash
        run: |
          set -euo pipefail
          pnpm --filter @hookdeck-evals/framework exec tsx scripts/publish-snapshot.ts \
            --run-id "${{ github.run_id }}" \
            --published-at "$(date -u +%Y-%m-%dT%H:%M:%S.000Z)"

      - name: Upload exported results
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: eval-results-json
          path: apps/web/src/data/*eval-results.json
          if-no-files-found: error
          # Deliberately short, unlike the raw results above. This artifact is
          # a copy of something the next step commits to the repository and
          # publishes to results/, so it is a convenience for inspecting a run
          # in flight rather than a record. Nothing is lost when it expires.
          retention-days: 7

      - name: Commit exported results
        # Results are committed to the dispatched branch. Supabase's version
        # opened a PR through a GitHub App and self-merged it on the schedule;
        # that needs an app we have not set up, and there is no schedule to
        # serve. Restore it in Phase 3 if the scoreboard needs it.
        shell: bash
        run: |
          set -euo pipefail

          git config user.name "github-actions[bot]"
          # github-actions[bot]'s noreply email uses its public user ID: https://github.com/actions/checkout#push-a-commit-using-the-built-in-token
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"

          for result_file in apps/web/src/data/*eval-results.json; do
            [ -f "$result_file" ] && git add "$result_file"
          done
          # The published contract and its history. Staged explicitly rather
          # than with `git add -A`, so a stray file a run happens to leave in
          # the working tree cannot ride along into a published commit.
          git add results/latest.json results/index.json results/runs

          if git diff --cached --quiet; then
            echo "No eval result changes to commit"
            exit 0
          fi

          git commit -m "chore: refresh eval results"

          # A full run takes tens of minutes, so the branch has usually moved by
          # the time results are ready and a plain push is rejected as non
          # fast-forward. Rebase onto whatever landed meanwhile and try again:
          # the only thing being committed is an exported JSON file, so there is
          # nothing for a rebase to conflict with except another results commit,
          # and the newer export is the one to keep.
          branch="${{ github.ref_name }}"
          for attempt in 1 2 3; do
            if git push origin "HEAD:$branch"; then
              exit 0
            fi
            echo "push rejected (attempt $attempt), rebasing onto origin/$branch"
            git fetch origin "$branch"
            git rebase -X ours "origin/$branch" || {
              git rebase --abort || true
              echo "::error::could not rebase onto origin/$branch"
              exit 1
            }
          done

          echo "::error::exhausted push attempts"
          exit 1