ci.yml

# Continuous integration.
#
# Runs on pull requests, on main, and on demand. Deliberately does NOT run the
# container acceptance suite on every commit — that suite builds an image and
# publishes several real streams into it, which takes minutes. It runs on a
# schedule and before a release instead, where its cost buys something.
name: ci

on:
  # main only, NOT ["**"]. A PR branch fires both `push` and `pull_request`, so
  # matching every branch here produced two runs per commit and left the
  # concurrency group below to kill one of them. Those cancellations were
  # harmless but they attach cancelled contexts to the head commit, which the
  # PR reports as UNSTABLE and `gh pr checks` prints as "fail" -- indis-
  # tinguishable at a glance from a real regression, and it cost someone a
  # pointless seventeen-job rerun to find that out.
  push:
    branches: [main]
  pull_request:
  # The escape hatch for the case the old ["**"] was protecting: a branch with
  # no PR open now gets no automatic CI, so run it here by hand. A draft PR
  # does the same thing and keeps running on every push.
  workflow_dispatch:
  # The container suite is the only thing that catches base-image drift: an
  # Alpine branch republishing FFmpeg at a new -r revision makes the pinned
  # build fail, and that failure is the prompt to bump on purpose. Nothing in
  # this repository changes when that happens, so only a timer finds it.
  schedule:
    - cron: "17 6 * * 1"

# One run per branch. A push that supersedes another cancels it rather than
# leaving both to finish -- which matters here because these suites publish real
# streams and wait on them, so a superseded run can sit for its full 20-minute
# ceiling consuming a runner nobody is waiting on.
concurrency:
  # Still needed, but for a different reason than it used to be. The `push`
  # trigger above no longer matches PR branches, so there are no push/pull
  # twins left to deduplicate. What remains is the ordinary case: pushing twice
  # to a branch with an open PR fires two `pull_request` runs, and the older one
  # should die rather than sit for its full 20-minute ceiling publishing real
  # streams nobody is waiting on.
  #
  # head_ref || ref_name rather than github.ref: head_ref is the source branch
  # on a pull_request and empty on a push, ref_name is the bare branch name on a
  # push. Together they give one stable key per branch across both events, which
  # github.ref does not -- it is refs/heads/BRANCH on a push and
  # refs/pull/N/merge on a pull_request.
  group: ci-${{ github.head_ref || github.ref_name }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  # Does this pull request contain anything any suite in here could break?
  #
  # THE EXPENSIVE HALF OF THIS WORKFLOW RAN ON MARKDOWN. Three documentation-only
  # PRs on 2026-08-14 each fired all 28 checks -- the full acceptance matrix,
  # three-OS builds, Docker and browser suites -- to validate files no job reads.
  # One of them was a single file. See #350.
  #
  # WHAT THIS GATES, and #378 is the second half of it. #351/#357 stopped the
  # acceptance matrix; the three-OS test matrix, the go job and the cross-compile
  # kept running, which is most of the twenty minutes #350 was actually
  # complaining about. All five now read this output:
  #
  #	go                go build, vet, test
  #	crossplatform     test: ubuntu-latest / macos-latest / windows-latest
  #	ui                ui typecheck, lint, build
  #	cross             cross-compile all release targets
  #	acceptance        acceptance: <thirteen suites>
  #
  # Every name in the right-hand column is a required status context in the
  # branch-protection ruleset. That is the whole reason for the shape below.
  #
  # WHY THIS GATES STEPS AND NOT JOBS. A leg's context has to be added to the
  # ruleset by hand or it reports without gating anything. A workflow-level
  # `paths:` filter stops the workflow running at all, so a required check never
  # reports and the pull request stays pending forever with no way to merge it.
  #
  # A JOB-LEVEL `if:` ON A MATRIX IS THE SAME BUG WEARING A DISGUISE, and #351
  # shipped it. A skipped ordinary job does report, and a skip does satisfy the
  # requirement -- but a skipped MATRIX job never expands its matrix, so the
  # per-leg contexts are never created at all. The checks list showed one entry
  # named literally `acceptance: ${{ matrix.suite }}`, the required
  # contexts were absent rather than skipped, and #349 -- a documentation-only
  # pull request, the exact case this was built for -- was unmergeable. Fifteen
  # green checks and no way in.
  #
  # So the matrix always expands and every leg reports. What is conditional is
  # the WORK: each step carries the `if:`, the job costs a runner allocation and
  # nothing else, and the required context reports success either way.
  #
  # ONE SHAPE FOR ALL FIVE, INCLUDING THE THREE THAT HAVE NO MATRIX. `go`, `ui`
  # and `cross` are ordinary jobs today, and a job-level `if:` on them would be
  # correct today: a skipped ordinary job reports skipped and satisfies its
  # requirement. It is not written that way, because the difference between the
  # safe spelling and the outage is one `strategy:` block that nobody would think
  # to connect to branch protection while adding it -- a Go-version matrix on
  # `go`, a Node-version matrix on `ui`. The rule is therefore flat, has no
  # exceptions to remember, and is enforced rather than remembered:
  # internal/testenv/docsgate_test.go fails if this output is ever read from a
  # job-level `if:`. The price is four runner allocations that do nothing on a
  # documentation-only PR, against twenty minutes of compute they replace.
  #
  # IT FAILS TOWARD RUNNING. `code` is false only when EVERY changed path is
  # documentation; anything unrecognised makes it true. A new top-level directory
  # gets the full matrix until somebody decides otherwise, which is the right
  # direction for a mistake to point.
  #
  # THIS FILE COUNTS AS CODE, for the reason the container-suites job already
  # gives about itself: a change to how the suites run must run the suites.
  changes:
    name: which changes
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      code: ${{ steps.detect.outputs.code }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
      - id: detect
        env:
          EVENT: ${{ github.event_name }}
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          # Anything that is not a pull request runs everything. A push to main
          # and a scheduled run have no diff to reason about.
          if [ "$EVENT" != "pull_request" ]; then
            echo "code=true" >> "$GITHUB_OUTPUT"
            echo "not a pull request: running everything"
            exit 0
          fi
          CHANGED="$(git diff --name-only "$BASE_SHA" "$HEAD_SHA")"
          echo "changed files:"; echo "$CHANGED" | sed 's/^/  /'
          # Every line documentation? Then no suite can be affected by it.
          if [ -n "$CHANGED" ] && ! grep -qvE '^(docs/|[^/]*\.md$)' <<<"$CHANGED"; then
            echo "code=false" >> "$GITHUB_OUTPUT"
            echo "documentation only: skipping the acceptance matrix"
            exit 0
          fi
          echo "code=true" >> "$GITHUB_OUTPUT"
          echo "contains code: running everything"

  go:
    name: go build, vet, test
    # The documentation gate, #378. On every STEP below, never on this job --
    # see the `changes` job for why that distinction is load-bearing and for
    # what it cost the one time it was got wrong.
    needs: changes
    runs-on: ubuntu-latest
    # Above the 15m per-package go test timeout below, so Go's diagnostic
    # panic wins the race against this.
    timeout-minutes: 25
    env:
      # #187. The comment below the checkout has said since the beginning that a
      # missing FFmpeg makes those tests SKIP, "which is worse, because the run
      # goes green having checked nothing" -- and then nothing in this file made
      # that impossible. This does. With it set, internal/testenv.FFmpegBinary
      # fails and names the binary instead of skipping, so deleting the install
      # step above turns the upload gate's tests red rather than silent.
      #
      # It is set on THIS job only. The crossplatform job installs FFmpeg from
      # three different package managers and its post-install checks now assert
      # ffprobe as well, but arming a hard failure on runner images I have not
      # measured is how a gate becomes a revert; the full suite runs here, and
      # here is where the guarantee is worth having. Widening it is a small
      # change once someone has watched the three images pass with it.
      POLYEMESIS_REQUIRE_FFMPEG: "1"
    steps:
      # Says out loud why a green check did no work, so a reader of the
      # checks list is never left guessing whether this ran or no-opped.
      - name: Documentation-only change, so this job did no work
        if: needs.changes.outputs.code != 'true'
        run: |
          echo "::notice title=go build, vet, test::this check did no work -- every changed path was documentation. See the 'which changes' job."
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        if: needs.changes.outputs.code == 'true'
      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
        if: needs.changes.outputs.code == 'true'
        with:
          go-version-file: go.mod
          cache: true

      # polyemesis shells out to ffmpeg and ffprobe, and a large part of the
      # test suite asserts on real filter-graph behaviour rather than on
      # strings. Without FFmpeg present those tests do not fail — they skip,
      # which is worse, because the run goes green having checked nothing.
      # WHY A STEP TIMEOUT, when the job already has one.
      #
      # For the same reason the go test timeout below is kept under it: whichever
      # ceiling fires first decides how much you learn. A job timeout kills the
      # runner and reports "cancelled" with no indication of which step was
      # stuck -- and on 2026-08-04 an acceptance job spent its entire 20 minutes
      # inside THIS step, never ran its suite, and looked exactly like the
      # stream-waiting hangs issue #38 tracks. It was apt.
      #
      # Six minutes is far past a normal install (~30s) and far short of the
      # job budget, so it fires only on a genuinely stuck mirror.
      # KEYED ON THE PINNED ASSET, plus an epoch that can be bumped by hand.
      #
      # BtbN's `latest` release tag is ROLLING: the same asset name can serve a
      # newer build over time. A cache keyed on the name therefore FREEZES the
      # build CI runs, which is a deliberate trade and the right one here --
      # the Dockerfile pins FFMPEG_VERSION=8.1.2-r0, so a CI FFmpeg that also
      # stops moving is closer to what users get, not further from it. The
      # weekly scheduled container suite is what catches upstream drift, and it
      # builds the image rather than reading this cache.
      #
      # Bump the -v1 suffix to take a newer build on purpose.
      - name: Cache FFmpeg
        if: needs.changes.outputs.code == 'true'
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: /tmp/ffcache
          key: ffmpeg-${{ runner.os }}-n8.1-latest-linux64-gpl-8.1-v1

      - name: Install FFmpeg
        if: needs.changes.outputs.code == 'true'
        timeout-minutes: 6
        env:
          FFCACHE: /tmp/ffcache
        run: |
          # Ubuntu's own package is 6.1.1, and the Docker image ships 8.1.2.
          # Testing against 6.1.1 meant CI never exercised the FFmpeg users
          # actually get, and the two differ in ways that matter: 6.1.1 accepts
          # an 80-channel amerge where 8.1 stops at 64, and 6.1.1 cannot demux
          # multitrack FLV at all. Pin the same 8.1 line the image uses.
          set -euo pipefail
          cd /tmp
          # -c rather than a heredoc: inside a YAML block the heredoc body and
          # its terminator are indented, which Python reads as a bad indent and
          # bash does not accept as a delimiter.
          # RETRIED, because this is one un-mirrored third-party download and
          # every Linux job in this file makes it. It has hard-failed two jobs
          # in one day -- acceptance-postprod and acceptance-pull -- both by
          # sitting on a stalled connection until the 6-minute ceiling fired.
          # A stuck mirror is exactly what that ceiling is for, but a single
          # attempt turns a transient stall into a red job and a manual rerun.
          #
          # curl rather than urllib: --retry understands the difference between
          # a connection that failed and one that is merely slow, and
          # --retry-all-errors covers the 5xx a release CDN returns under load.
          # The per-attempt --max-time is well inside the step ceiling, so three
          # attempts still cannot outlast it -- the ceiling keeps its meaning.
          # A CACHE HIT SKIPS THE DOWNLOAD ENTIRELY, and that is the point.
          #
          # This artefact failed SEVEN times in one day across Linux and Windows
          # -- 503s and stalls from the release CDN -- while already wrapped in
          # `curl --retry 3 --retry-all-errors` AND the three-attempt loop below.
          # It has retries and still lands red, because a retry re-asks a host
          # that is refusing. The cache stops asking.
          #
          # Same treatment the Playwright browser got: a pinned artefact need not
          # be fetched on every run after the first.
          if [ -x "$FFCACHE/ffmpeg" ] && [ -x "$FFCACHE/ffprobe" ]; then
            echo "ffmpeg restored from cache"
          else
            for attempt in 1 2 3; do
              # --proto/--proto-redir pin the whole exchange to https. -L
              # follows redirects, and without these curl will happily follow an
              # https -> http hop and fetch the binary CI is about to run over
              # plaintext. SonarCloud githubactions:S6506 flagged the copy of
              # this block added for the acceptance job; the other two had the
              # same hole and are fixed here too, because leaving a known-bad
              # copy beside a fixed one is how the fix gets reverted later by
              # someone copying the wrong neighbour.
              if curl -fsSL --proto '=https' --proto-redir '=https' \
                   --retry 3 --retry-all-errors --retry-delay 5 \
                   --connect-timeout 20 --max-time 90 \
                   -o /tmp/ff.tar.xz \
                   "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-n8.1-latest-linux64-gpl-8.1.tar.xz"; then
                break
              fi
              echo "ffmpeg download attempt $attempt failed"
              [ "$attempt" = 3 ] && exit 1
              sleep 5
            done
            mkdir -p ffbuild && tar xf ff.tar.xz --strip-components=1 -C ffbuild
            mkdir -p "$FFCACHE"
            cp ffbuild/bin/ffmpeg ffbuild/bin/ffprobe "$FFCACHE/"
          fi
          sudo install -m 0755 "$FFCACHE/ffmpeg"  /usr/local/bin/ffmpeg
          sudo install -m 0755 "$FFCACHE/ffprobe" /usr/local/bin/ffprobe
          # Fail loudly rather than silently testing a build without SRT.
          ffmpeg -hide_banner -protocols | tr ' ' '\n' | grep -qx srt
          ffmpeg -hide_banner -version | head -1
          # ffprobe TOO, and not because the install could half-succeed. #187:
          # every test that proves the upload probe gate works starts by looking
          # ffprobe up and skipping if it is absent, so an ffprobe that stopped
          # arriving would have taken the whole gate's verification with it while
          # this step still printed an ffmpeg version and passed. The one binary
          # that was verified was the one the skips did not turn on.
          ffprobe -hide_banner -version | head -1

      # The gate is "gofmt named no files", and it has to rest on gofmt having
      # RUN. It used to be `gofmt -l ./cmd ./internal | tee /tmp/fmt && test !
      # -s /tmp/fmt`, and the default shell for a `run:` block is `bash -e {0}`
      # -- no pipefail, unlike an explicit `shell: bash`. So the pipeline's
      # status was tee's, gofmt's was discarded, and a gofmt that died on a
      # parse error wrote nothing to /tmp/fmt and left `test ! -s` to report
      # the tree as correctly formatted. Every way of failing to check produced
      # the same green as checking and finding nothing.
      #
      # A command substitution under set -e instead: gofmt's own exit status
      # now fails the step, and the emptiness of its output is a separate
      # question asked afterwards.
      - name: gofmt
        if: needs.changes.outputs.code == 'true'
        run: |
          set -euo pipefail
          unformatted=$(gofmt -l ./cmd ./internal)
          if [ -n "$unformatted" ]; then
            echo "these files are not gofmt'd:"
            echo "$unformatted"
            exit 1
          fi
          echo "gofmt: clean"
      - if: needs.changes.outputs.code == 'true'
        run: go build ./...
      - if: needs.changes.outputs.code == 'true'
        run: go vet ./...

      # THE PREFLIGHT'S OWN LIVENESS, and it runs HERE because it was not
      # running anywhere.
      #
      # internal/api forces its route coverage ledger through a SECOND m.Run with
      # the caller's -run, -skip and -count set aside, so that no test filter can
      # switch the ledger off. `make preflight-guard` proves that mechanism is
      # still wired by running the package under each of those three switches and
      # requiring the preflight's marker to print anyway.
      #
      # It was reachable only as a prerequisite of `make test`, and this workflow
      # invokes `go test` directly -- here and in the crossplatform job -- so the
      # guard existed on developer machines and nowhere else. A local-only gate
      # described in a comment as a gate is the species this whole change is
      # about. ~4s; it runs the package three times with a filter that selects
      # almost nothing.
      - name: The route-coverage preflight is still wired
        if: needs.changes.outputs.code == 'true'
        run: make preflight-guard

      # THE OTHER HALF OF THE SAME MECHANISM. #217/#223.
      #
      # The preflight runs in a SECOND m.Run rather than a first, because the
      # coverage profile is written by testing.M.after under m.afterOnce, on the
      # way out of whichever pass returns first. While the forced pass went
      # first, `go test -cover ./internal/api` reported 22.0% for zero tests, for
      # one test and for the whole suite alike -- a constant wearing a
      # percentage, which #219 already lost a round to.
      #
      # Reordering is a one-line change and reverting it is a one-line change,
      # so it needs a running gate rather than a comment. This one runs the
      # package under three different selections and requires the numbers to
      # differ. ~60s, dominated by the unfiltered probe, which is the number
      # anybody would actually quote.
      # THE STEP TIMEOUT THE FOUR BELOW GOT AND THIS ONE DID NOT, and on
      # 2026-08-14 this is the step that hung: 24 minutes against a measured
      # 98-114s, until the job's own ceiling cancelled it and named nothing.
      #
      # It can hang for a reason the note at :360 already gives about the
      # watchdog -- "a background process holding the suite's stdout ... does
      # not merely fail to report, it becomes the hang". The guard captures each
      # probe with `out="$(go test ...)"`, and command substitution blocks until
      # the pipe has no writers left, NOT until go test exits. A test that
      # leaks a child holding inherited stdout hangs the capture after the test
      # binary is gone and after Go's own timeout has already fired, so Go's
      # diagnostic cannot be the backstop here. Only this can.
      #
      # 14 rather than something tighter, so the ordering the file argues for
      # everywhere else holds: three probes at -timeout 4m is 12m of worst case,
      # and Go has to be able to win that race and print the goroutine dump
      # before this fires. Below the job's 25 either way, which is the property
      # being claimed.
      - name: internal/api coverage measures the tests, not the preflight
        if: needs.changes.outputs.code == 'true'
        timeout-minutes: 14
        run: make coverage-instrument-guard

      # -race because the engine reconciles from several goroutines and a data
      # race here shows up in production as a stream that stops for one viewer.
      #
      # -timeout because Go's default is 10 minutes PER PACKAGE and internal/db
      # runs close to it on a slow runner -- see the cross-platform job below
      # for the measurements. -race makes every one of them slower still.
      #
      # Kept BELOW timeout-minutes on purpose. Go's timeout panics with a
      # goroutine dump naming the test that was running; the job timeout just
      # kills the runner and tells you nothing. Whichever fires first decides
      # how much you learn, so Go's has to.
      # POLYEMESIS_LEDGER=strict makes internal/api's route coverage ledger run
      # its counterpart proofs from inside the ledger test as well as from their
      # own. The registry's whole value is that an excuse cannot discharge on a
      # test NAME, only on bytes that actually left the process -- and a `-run`
      # filter matching one of the two tests and not the other would leave every
      # counterpart undischarged while still printing ok. The full suite runs
      # here with no filter, so strict mode costs nothing and closes that door.
      - if: needs.changes.outputs.code == 'true'
        # 20m, RAISED FROM 15m ON EVIDENCE THAT THIS IS ACCUMULATION AND NOT
        # A HANG -- which is the only reason raising a timeout is ever the right
        # move rather than the lazy one.
        #
        # internal/api hit the 15m wall on 2026-08-16 and reported 900.034s. The
        # panic named the test that was running, which is exactly why Go's
        # timeout is kept below this job's timeout-minutes: 25, and it read
        #
        #     running tests:
        #       TestAMutationReconcilesEveryProgrammeRatherThanTheDefault (0s)
        #
        # ZERO SECONDS. Nothing was stuck; the package had simply spent fifteen
        # minutes getting there. A hang would have named a test with a large
        # number beside it, and the fix for that is never a bigger budget.
        #
        # internal/api is the largest suite in the tree -- the route ledger
        # alone drives every method-pattern pair against a live fixture -- and
        # the race detector multiplies all of it. Tests that were spending real
        # wall clock on a pacing constant were trimmed in the same change, which
        # is the part that should have been unnecessary; this is the part that
        # was overdue.
        #
        # Still five minutes below the job's own timeout, so Go keeps its
        # goroutine dump. If this needs raising again, split the package first.
        run: POLYEMESIS_LEDGER=strict go test -race -timeout 20m ./...

      # The acceptance suites' shared diagnostic helpers, tested here rather
      # than in their own job: it is pure shell, needs nothing installed and
      # finishes in about a second, so a separate runner would cost more to
      # schedule than to run.
      #
      # It earns a place in CI because the claim it guards is one a reader will
      # act on. lib-observe.sh prints "the ceiling is too low; this is not a
      # product failure" -- and a diagnostic that says that about a REAL failure
      # sends the next person to the wrong component with the authority of a
      # printed conclusion. The negative case is the one under test.
      #
      # THE FOUR SCRIPT STEPS IN THIS JOB CARRY STEP TIMEOUTS, for the reason
      # :327-334 and :393-396 give for the two smoke steps: a job timeout names
      # nothing, a step timeout names the step. Each of these four runs a shell
      # harness that starts and kills processes, so each can hang exactly the way
      # #179 hung, and behind this job's timeout-minutes: 25 a hang here costs 25
      # minutes and reports "the job was cancelled".
      #
      # HONEST ABOUT THE NUMBERS: only one of the four has a measurement --
      # test-lib-cleanup.sh at 35s, up from 20s when its SIGTERM-deaf case was
      # added. The other three were not re-measured here, because running the
      # teardown harnesses on a shared developer machine kills processes
      # belonging to whatever else is running on it. So these are blast-radius
      # ceilings, not fitted bounds: sized to be unreachable by a healthy run and
      # far below the job's 25, which is the only property being claimed.
      - if: needs.changes.outputs.code == 'true'
        run: ./scripts/test-lib-observe.sh
        timeout-minutes: 6

      # The acceptance suites' own deadline, which fires below this workflow's
      # timeout-minutes so a hung suite reports instead of being cancelled.
      # Tested here for the reason above and one that is specific to it: the
      # watchdog is a background process holding the suite's stdout, so a bug
      # in it does not merely fail to report -- it becomes the hang.
      - if: needs.changes.outputs.code == 'true'
        run: ./scripts/test-lib-watchdog.sh
        timeout-minutes: 6

      # The SBOM guard, which only ever runs for real inside a release. That is
      # exactly how it came to be wrong: it asserted a floor of 100 against a
      # document containing 437, so it could not fail on the scenario its own
      # comment described, and no one could discover that without cutting a
      # release. Running its tests on every PR is the part that makes the fix
      # a fix rather than a better guess. Pure jq and shell, about a second.
      - if: needs.changes.outputs.code == 'true'
        run: ./scripts/test-sbom-guard.sh
        timeout-minutes: 5

      # The termination guard, and its tests, alongside the SBOM pair above for
      # the same reason and one of its own. #210: after #179/#180 the class had
      # ONE gate and its jurisdiction was .github/workflows -- a fixture carrying
      # #179's body verbatim was dropped into that test's green directory and
      # passed. For scripts/ there was no gate at all, which is how
      # acceptance-mqtt.sh kept the exact shape that had just been rewritten in
      # three other suites. Found by a sweep, not by CI.
      #
      # Both halves run: the guard against the real tree, and its own red/green
      # fixtures. The guard without its fixtures is a check nobody has watched
      # fail; the fixtures without the guard oblige nothing. Pure shell, under a
      # second each.
      - if: needs.changes.outputs.code == 'true'
        run: ./scripts/termination-guard.sh
        timeout-minutes: 5
      - if: needs.changes.outputs.code == 'true'
        run: ./scripts/test-termination-guard.sh
        timeout-minutes: 5

      # The OBS container entrypoint's stop path. #208 was FILED rather than
      # fixed on the grounds that it could not be run -- "a blind edit to a
      # container entrypoint has two ways to go wrong that a review cannot see".
      # Both ways are about a process, not about OBS, so the stop logic moved
      # into scripts/obs/lib-stop.sh and this drives it with `sleep` and a
      # SIGTERM-deaf stand-in. No OBS, no Xvfb, no container, about six seconds.
      - if: needs.changes.outputs.code == 'true'
        run: ./scripts/test-obs-stop.sh
        timeout-minutes: 6

      # Outbound webhooks over a real socket. IN THIS JOB RATHER THAN THE
      # acceptance MATRIX, because it needs neither FFmpeg nor `make build` --
      # the far end is an http.Server the driver starts on a loopback port, so
      # Go alone is the whole dependency and the suite finishes in about 20
      # seconds. Scheduling it beside the FFmpeg suites would cost a six-minute
      # apt step to test a package that never touches FFmpeg.
      #
      # ON EVERY PUSH RATHER THAN WEEKLY, which is the opposite of chat-live.yml
      # and for the reason that workflow gives for its own schedule: that suite
      # measures THEIR behaviour, so its failures arrive with no commit of ours.
      # This one measures OURS, contacts nothing outside the runner, and has
      # nothing to be flaky about. Its credentialed step skips here; no
      # POLY_HOOKS_URL is configured for this workflow and none is needed --
      # 29 of its 31 checks run without one.
      - if: needs.changes.outputs.code == 'true'
        run: ./scripts/acceptance-hooks.sh
        timeout-minutes: 6

      # The acceptance suites' shared teardown. Same argument as the guard
      # above, and a sharper one: this teardown was silently leaking the ingest
      # port on every run, which made acceptance-failover fail 10 times out of
      # 20 back to back. A harness that measures itself is worse than no
      # harness, because a real regression gets dismissed as "that suite is
      # flaky". Needs lsof, which this job does not otherwise install.
      - name: ./scripts/test-lib-cleanup.sh
        if: needs.changes.outputs.code == 'true'
        timeout-minutes: 10
        run: |
          sudo apt-get install -y --no-install-recommends lsof
          ./scripts/test-lib-cleanup.sh

  # The same build and test, on the operating systems polyemesis actually ships
  # for.
  #
  # `make release` cross-compiles darwin and windows binaries and every job
  # above then runs on ubuntu, so until this existed the Windows and macOS
  # builds were proven to COMPILE and never proven to WORK. That gap matters
  # here more than in most projects: this codebase shells out to a child
  # process, manages process groups and signals, binds sockets, and builds file
  # paths that end up on an FFmpeg command line. Every one of those is a place
  # where Linux and Windows genuinely differ.
  #
  # No -race: it needs cgo, and these builds are CGO_ENABLED=0 on purpose. The
  # ubuntu job above runs the race detector, which is where a data race would
  # show up anyway -- the goroutine scheduling is not what differs across these
  # platforms.
  crossplatform:
    name: "test: ${{ matrix.os }}"
    # The documentation gate, #378. On every STEP below, never on this job --
    # see the `changes` job for why that distinction is load-bearing and for
    # what it cost the one time it was got wrong.
    needs: changes
    runs-on: ${{ matrix.os }}
    # Above the 15m per-package go test timeout below. Windows needs the room:
    # internal/db alone has been measured at 265-300s and once past 600s.
    timeout-minutes: 30
    strategy:
      # Never cancel siblings: when Windows and macOS both break, both answers
      # are wanted, and fail-fast would turn one fix into two round trips.
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
    steps:
      # Says out loud why a green check did no work, so a reader of the
      # checks list is never left guessing whether this ran or no-opped.
      - name: Documentation-only change, so this job did no work
        if: needs.changes.outputs.code != 'true'
        run: |
          echo "::notice title=test: ${{ matrix.os }}::this check did no work -- every changed path was documentation. See the 'which changes' job."
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        if: needs.changes.outputs.code == 'true'
      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
        if: needs.changes.outputs.code == 'true'
        with:
          go-version-file: go.mod
          cache: true

      # FFmpeg on every runner, for the reason the ubuntu job gives: without it
      # the measurement tests SKIP rather than fail, and the run goes green
      # having checked nothing. A cross-platform job that skipped every test
      # that touches FFmpeg would be the most misleading kind of green.
      # Same reasoning as the go job's install step.
      # KEYED ON THE PINNED ASSET, plus an epoch that can be bumped by hand.
      #
      # BtbN's `latest` release tag is ROLLING: the same asset name can serve a
      # newer build over time. A cache keyed on the name therefore FREEZES the
      # build CI runs, which is a deliberate trade and the right one here --
      # the Dockerfile pins FFMPEG_VERSION=8.1.2-r0, so a CI FFmpeg that also
      # stops moving is closer to what users get, not further from it. The
      # weekly scheduled container suite is what catches upstream drift, and it
      # builds the image rather than reading this cache.
      #
      # Bump the -v1 suffix to take a newer build on purpose.
      - name: Cache FFmpeg
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: /tmp/ffcache
          key: ffmpeg-${{ runner.os }}-n8.1-latest-linux64-gpl-8.1-v1
        # Linux only: macOS installs from brew and Windows caches its own
        # zip below. Without this the step runs on all three and stores an
        # empty directory under two keys that nothing ever reads.
        if: runner.os == 'Linux' && needs.changes.outputs.code == 'true'

      - name: Install FFmpeg (Linux)
        timeout-minutes: 6
        if: runner.os == 'Linux' && needs.changes.outputs.code == 'true'
        env:
          FFCACHE: /tmp/ffcache
        run: |
          # Ubuntu's own package is 6.1.1, and the Docker image ships 8.1.2.
          # Testing against 6.1.1 meant CI never exercised the FFmpeg users
          # actually get, and the two differ in ways that matter: 6.1.1 accepts
          # an 80-channel amerge where 8.1 stops at 64, and 6.1.1 cannot demux
          # multitrack FLV at all. Pin the same 8.1 line the image uses.
          set -euo pipefail
          cd /tmp
          # -c rather than a heredoc: inside a YAML block the heredoc body and
          # its terminator are indented, which Python reads as a bad indent and
          # bash does not accept as a delimiter.
          # RETRIED, because this is one un-mirrored third-party download and
          # every Linux job in this file makes it. It has hard-failed two jobs
          # in one day -- acceptance-postprod and acceptance-pull -- both by
          # sitting on a stalled connection until the 6-minute ceiling fired.
          # A stuck mirror is exactly what that ceiling is for, but a single
          # attempt turns a transient stall into a red job and a manual rerun.
          #
          # curl rather than urllib: --retry understands the difference between
          # a connection that failed and one that is merely slow, and
          # --retry-all-errors covers the 5xx a release CDN returns under load.
          # The per-attempt --max-time is well inside the step ceiling, so three
          # attempts still cannot outlast it -- the ceiling keeps its meaning.
          # A CACHE HIT SKIPS THE DOWNLOAD ENTIRELY, and that is the point.
          #
          # This artefact failed SEVEN times in one day across Linux and Windows
          # -- 503s and stalls from the release CDN -- while already wrapped in
          # `curl --retry 3 --retry-all-errors` AND the three-attempt loop below.
          # It has retries and still lands red, because a retry re-asks a host
          # that is refusing. The cache stops asking.
          #
          # Same treatment the Playwright browser got: a pinned artefact need not
          # be fetched on every run after the first.
          if [ -x "$FFCACHE/ffmpeg" ] && [ -x "$FFCACHE/ffprobe" ]; then
            echo "ffmpeg restored from cache"
          else
            for attempt in 1 2 3; do
              # --proto/--proto-redir pin the whole exchange to https. -L
              # follows redirects, and without these curl will happily follow an
              # https -> http hop and fetch the binary CI is about to run over
              # plaintext. SonarCloud githubactions:S6506 flagged the copy of
              # this block added for the acceptance job; the other two had the
              # same hole and are fixed here too, because leaving a known-bad
              # copy beside a fixed one is how the fix gets reverted later by
              # someone copying the wrong neighbour.
              if curl -fsSL --proto '=https' --proto-redir '=https' \
                   --retry 3 --retry-all-errors --retry-delay 5 \
                   --connect-timeout 20 --max-time 90 \
                   -o /tmp/ff.tar.xz \
                   "https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-n8.1-latest-linux64-gpl-8.1.tar.xz"; then
                break
              fi
              echo "ffmpeg download attempt $attempt failed"
              [ "$attempt" = 3 ] && exit 1
              sleep 5
            done
            mkdir -p ffbuild && tar xf ff.tar.xz --strip-components=1 -C ffbuild
            mkdir -p "$FFCACHE"
            cp ffbuild/bin/ffmpeg ffbuild/bin/ffprobe "$FFCACHE/"
          fi
          sudo install -m 0755 "$FFCACHE/ffmpeg"  /usr/local/bin/ffmpeg
          sudo install -m 0755 "$FFCACHE/ffprobe" /usr/local/bin/ffprobe
          # Fail loudly rather than silently testing a build without SRT.
          ffmpeg -hide_banner -protocols | tr ' ' '\n' | grep -qx srt
          ffmpeg -hide_banner -version | head -1
      - name: Install FFmpeg (macOS)
        if: runner.os == 'macOS' && needs.changes.outputs.code == 'true'
        # BREW CANNOT SUPPLY 8.1, AND THAT IS RECORDED RATHER THAN HIDDEN.
        # `ffmpeg` is 9.0 and there is no `ffmpeg@8` formula -- the versioned ones
        # jump from 7.1 straight to 9.0 -- and BtbN publishes no macOS build. So
        # this arm cannot have the parity Linux and Windows now share, and macOS
        # tests a MAJOR VERSION AHEAD of what the image ships.
        #
        # Left as brew deliberately: 7.1 would be behind rather than ahead, and
        # building 8.1 from source costs more than this job is worth. The version
        # is printed by the step below, so the divergence shows up in every run
        # instead of being discovered by a test failing oddly.
        run: brew install ffmpeg
      # Windows pays the same CDN tax as Linux -- this artefact failed twice
      # here today -- so it gets the same treatment. A separate key because it
      # is a different asset, and a separate step because the Linux cache above
      # is gated to runner.os == 'Linux'.
      - name: Cache FFmpeg (Windows)
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        if: runner.os == 'Windows' && needs.changes.outputs.code == 'true'
        with:
          path: ${{ runner.temp }}/ffcache
          key: ffmpeg-${{ runner.os }}-n8.1-latest-win64-gpl-8.1-v1

      - name: Install FFmpeg (Windows)
        if: runner.os == 'Windows' && needs.changes.outputs.code == 'true'
        # THE SAME BUILD THE LINUX ARM INSTALLS, not choco, for two reasons and
        # the second is the important one.
        #
        # Reliability: `choco install ffmpeg` took a 504 from
        # community.chocolatey.org and left the next step reporting "'ffmpeg' is
        # not recognized" -- a red job on an install that silently did not happen,
        # and the third distinct FFmpeg-install failure this file saw in one day.
        #
        # VERSION PARITY, which is what actually matters. The Linux arm pins n8.1
        # and says why: the image ships 8.1.2, and 6.1.1 "accepts an 80-channel
        # amerge where 8.1 stops at 64, and cannot demux multitrack FLV at all".
        # That argument is about FFmpeg, not about Linux. choco resolved to
        # whatever it held, so this job tested a version nobody ships while
        # carrying a comment explaining why that is wrong.
        #
        # BtbN publishes ffmpeg-n8.1-latest-win64-gpl-8.1.zip from the SAME
        # release the Linux arm downloads: one source, one version, one retry.
        run: |
          $ErrorActionPreference = 'Stop'
          $url = 'https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-n8.1-latest-win64-gpl-8.1.zip'
          $cache = "$env:RUNNER_TEMP\ffcache"
          # A CACHE HIT SKIPS THE DOWNLOAD. See the Linux cache step for the
          # argument: this artefact failed seven times in one day across both
          # platforms while already carrying retries, and a retry re-asks a host
          # that is refusing.
          if (Test-Path "$cache\ffmpeg.exe") {
            Write-Host 'ffmpeg restored from cache'
          } else {
            for ($i = 1; $i -le 3; $i++) {
              try { Invoke-WebRequest -Uri $url -OutFile "$env:RUNNER_TEMP\ff.zip" -TimeoutSec 90; break }
              catch { if ($i -eq 3) { throw }; Write-Host "ffmpeg download attempt $i failed"; Start-Sleep -Seconds 5 }
            }
            Expand-Archive -Path "$env:RUNNER_TEMP\ff.zip" -DestinationPath "$env:RUNNER_TEMP\ff" -Force
            $src = (Get-ChildItem "$env:RUNNER_TEMP\ff" -Filter bin -Recurse -Directory | Select-Object -First 1).FullName
            if (-not $src) { throw 'no bin/ directory in the FFmpeg archive' }
            New-Item -ItemType Directory -Force -Path $cache | Out-Null
            Copy-Item "$src\*" -Destination $cache -Force
          }
          $bin = $cache
          if (-not (Test-Path "$bin\ffmpeg.exe")) { throw 'no ffmpeg.exe after install' }
          # GITHUB_PATH, not $env:PATH: the latter dies with this step, which is
          # exactly how the choco failure stayed invisible until the NEXT step
          # could not find the binary.
          Add-Content -Path $env:GITHUB_PATH -Value $bin
      # BOTH binaries, on all three. #187: the skip helpers key off ffprobe and
      # this step only ever ran ffmpeg, so the binary that decides whether the
      # upload gate's tests run at all was the one nothing here looked at. The
      # go job additionally sets POLYEMESIS_REQUIRE_FFMPEG so a missing one is a
      # test failure rather than a skip; this job does not, deliberately -- see
      # the note on that env block.
      #
      # TWO STEPS rather than a two-line one. The default shell on the Windows
      # runner is pwsh, whose wrapper exits with the LAST command's status, so a
      # two-line block would have reported ffprobe's result and discarded
      # ffmpeg's -- the same discarded-exit-status shape as the gofmt pipeline in
      # the go job. A step boundary checks each one.
      - name: FFmpeg version
        if: needs.changes.outputs.code == 'true'
        run: ffmpeg -hide_banner -version
      - name: ffprobe version
        if: needs.changes.outputs.code == 'true'
        run: ffprobe -hide_banner -version

      - if: needs.changes.outputs.code == 'true'
        run: go build ./...
      - if: needs.changes.outputs.code == 'true'
        run: go vet ./...
      # Go's default timeout is 10 minutes PER PACKAGE, and internal/db lands
      # close enough to it on Windows that ordinary runner variance decides the
      # build. Measured on ONE unchanged tree: 264s, 297s, and once past 600s,
      # where it failed -- a commit whose identical code had passed an hour
      # earlier.
      #
      # The cause is not a slow test but a slow platform. internal/db runs
      # against modernc.org/sqlite, a pure-Go SQLite that is roughly 40x slower
      # on a Windows runner than on a developer machine: ~6s locally against
      # ~265-300s there.
      #
      # What that cost actually WAS: schema DDL plus six migrations, executed
      # once per test rather than once per package. The test helpers now build
      # one migrated database and copy the file. Measured on this runner after
      # that change, against the 265-300s above:
      #
      #   internal/db          46s
      #   internal/api         69s
      #   internal/engine      23s
      #   internal/recording    7s
      #
      # The whole windows-latest job is now under three minutes, so the ~40x
      # platform ratio that made this package a coin flip against Go's
      # per-package ceiling is gone -- it is the same pure-Go SQLite on the same
      # slow filesystem, asked to do the work once instead of 181 times.
      #
      # 15m stays. It is now ~20x the slowest package rather than ~3x, which is
      # slack rather than a reason to retune: the failure it exists for is a
      # test that HANGS, and that failure does not get less likely because the
      # suite got faster.
      #
      # 15m is well past any observed Windows run for the slowest package, and it is
      # deliberately BELOW this job's timeout-minutes. Go's timeout panics with
      # a goroutine dump naming the running test; the job timeout just kills the
      # runner and tells you nothing. Whichever fires first decides how much you
      # learn, so Go's has to.
      - if: needs.changes.outputs.code == 'true'
        run: go test -timeout 15m ./...

      # Build and RUN it. Compiling proves the code is valid for the platform;
      # it does not prove the process comes up.
      #
      # The failures this catches are the ones a compiler cannot: a data
      # directory created with a path separator the OS does not accept, a
      # listener that binds differently, an embedded asset that resolves on one
      # filesystem and not another. `make release` already cross-compiles these
      # targets, so before this the Windows binary had never been executed by
      # anything.
      #
      # bash on all three: windows-latest ships Git Bash, so one script serves
      # every runner rather than a PowerShell fork nobody maintains.
      #
      # Built in a step of its own so each half carries a ceiling that means
      # something (#198). These two things are unrelated: the build is the long
      # pole on a cold Windows cache, and the serve check is 2-4s measured
      # (n=21). Sharing one step forced its timeout to be sized for the build,
      # which put a 10-minute ceiling on a 3-second assertion and hid "the
      # Windows build cache was cold today" inside a step called "check it
      # serves". Split, the timing column says which one it was.
      #
      # -cover, so the smoke run MEASURES itself. See the covdata step at the
      # end of this job for what that is for and what it costs.
      - name: Build the smoke binary
        if: needs.changes.outputs.code == 'true'
        timeout-minutes: 8
        shell: bash
        run: go build -cover -o polyemesis-smoke ./cmd/polyemesis

      # timeout-minutes because this step BACKGROUNDS A PROCESS, and a step that
      # holds a pid can wait on something that never comes. 2 against a measured
      # 2-4s (n=21) is still 30x headroom now that the build is not inside it.
      # Deliberately below the job's timeout-minutes: 30, per the discipline at
      # :154-158 and :305-311 -- whichever timeout fires first decides how much
      # you learn, so the narrowest one has to.
      - name: Start the server and check it serves
        if: needs.changes.outputs.code == 'true'
        timeout-minutes: 2
        shell: bash
        run: |
          set -uo pipefail
          mkdir -p covdata/serve
          GOCOVERDIR=covdata/serve ./polyemesis-smoke -addr 127.0.0.1:8123 -data ./smoke-data -log warn &
          pid=$!
          # THE CLEANUP IS A TRAP, AND THE VERDICT IS EMITTED BEFORE IT RUNS.
          #
          # This step used to kill the server and then block in an unbounded
          # shell `wait` on the server pid before printing its answer. That wait
          # never returns against a `kill` Git-Bash cannot deliver to a native
          # .exe: it hung for 28 minutes and was cancelled by the job timeout,
          # discarding an answer the loop below had already decided in 2 seconds.
          # Establishing that the branch was innocent then took building both
          # refs, hashing binaries and surveying 14 runs, to recover a fact this
          # step knew and threw away.
          #
          # A trap covers more than the old inline kill did (any early exit, not
          # just the one path), and best-effort is sufficient: nothing later in
          # this job binds 8123, polyemesis-smoke is built once and never
          # rebuilt so there is no Windows sharing violation to fear, and the
          # runner VM is destroyed at the end of the job.
          trap 'kill "$pid" 2>/dev/null || true' EXIT
          ok=no
          for _ in $(seq 1 40); do
            sleep 0.5
            if curl -fsS http://127.0.0.1:8123/api/v1/health >/dev/null 2>&1; then ok=yes; break; fi
          done
          if [ "$ok" != yes ]; then
            echo "the server never answered /health on ${{ matrix.os }}"
            exit 1
          fi
          echo "server started and served /health on ${{ matrix.os }}"

      # Push an actual broadcast through it, on every platform.
      #
      # /health proves the process comes up. It does not prove the product
      # works: that needs a stream arriving, a layout being probed, two
      # destinations compiling different routing graphs, FFmpeg children being
      # spawned and supervised, and audio landing in a container. Those are the
      # parts built on process groups, signals and file paths -- precisely
      # where Windows differs -- so "it serves /health on Windows" was never
      # the claim worth making.
      #
      # The stream is injected into the relay hub rather than pushed over SRT,
      # because libsrt is not guaranteed in a runner's FFmpeg (Homebrew's has
      # none). That substitutes only the ingest hop; everything downstream is
      # the real path. SRT ingest itself is covered by the acceptance suites on
      # ubuntu.
      #
      # Verification is a measurement, not an exit status: each destination
      # gets a different pair of the three input tones, and the check reads
      # per-band energy back out of the file. A destination that silently
      # carried the wrong mix would pass any check that only asked whether
      # FFmpeg exited 0.
      #
      # timeout-minutes for the same reason as the step above: it backgrounds a
      # process. 8 against a measured 91-93s (n=21) is roughly 5x headroom, and
      # it is below the job's 30 on purpose -- a step timeout names the step
      # that hung, a job timeout names nothing.
      - name: Push a broadcast through it and measure the output
        if: needs.changes.outputs.code == 'true'
        timeout-minutes: 8
        shell: bash
        run: |
          set -uo pipefail
          rm -rf ./data
          mkdir -p covdata/bcast
          GOCOVERDIR=covdata/bcast ./polyemesis-smoke -addr 127.0.0.1:8099 -data ./data -log warn > bcast-server.log 2>&1 &
          pid=$!
          # Same shape as the step above: a trap, and the verdict before the
          # cleanup. The shell `wait` that used to sit between `rc` and the
          # report was unbounded, and on Windows it is unbounded against a kill
          # that may never land.
          #
          # THE PREVIOUS VERSION OF THIS COMMENT SAID "this step is the last in
          # the job", AND THAT WAS FALSE. `Upload broadcast artefacts on failure`
          # follows it in this same job and uploads bcast-server.log and
          # data/recordings/ -- the two paths the server still holds open. The
          # trap fires when this step ends, so on the failure path the artefacts
          # used to be read while the writer was live, and on Windows the kill
          # may never land at all. Low impact (if-no-files-found: ignore, failure
          # path only) but the justification rested on an untrue claim, so the
          # failure path below now does the work the claim assumed: verdict
          # first, then a BOUNDED stop, then the diagnostics.
          #
          # The trap stays as the backstop for every other exit path, and
          # best-effort is still sufficient there: nothing later binds 8099, the
          # binary is built once and never rebuilt so there is no Windows sharing
          # violation to fear, and the runner VM is destroyed at the end.
          trap 'kill "$pid" 2>/dev/null || true' EXIT
          go run scripts/smoketest.go
          rc=$?
          if [ "$rc" -ne 0 ]; then
            # THE VERDICT, BEFORE ANY CLEANUP. #179 is what happens when this
            # ordering is the other way round.
            echo "the broadcast smoke test failed on ${{ matrix.os }} (rc=$rc)"
            # Now quiesce the writer, BOUNDED, so the log and the recordings the
            # upload step is about to collect are not being appended to while it
            # reads them. 10s against a server that shuts down in well under one;
            # the ceiling exists because on Windows this kill may never be
            # delivered, and an unbounded wait here is the whole of #179.
            kill "$pid" 2>/dev/null || true
            gone=no
            for _ in $(seq 1 40); do
              if ! kill -0 "$pid" 2>/dev/null; then gone=yes; break; fi
              sleep 0.25
            done
            if [ "$gone" != yes ]; then
              echo "NOTE: the server was still running 10s after the kill, so the"
              echo "artefacts below and in the upload step were read from under a"
              echo "live writer and may be truncated mid-write."
            fi
            echo "--- server log ---"
            cat bcast-server.log || true
            exit 1
          fi

      # What the two runs above ACTUALLY executed in cmd/polyemesis.
      #
      # WHY THIS EXISTS. `go test -cover ./cmd/polyemesis` reports ~38%, and
      # from that number sixteen functions look untested -- main, run, watch,
      # reconcile, disconnect, stop, startMQTT, runService, reportStartup,
      # reportTLS, newTLSProvider, managerEngines among them. That number is an
      # INSTRUMENT ARTEFACT, the same class as #217 in internal/api: `go test`
      # can only see what the test binary runs, and package main's entry points
      # are by construction driven by the PROCESS, which the two steps above
      # start on all three platforms on every PR. Measured from a real
      # instrumented run (darwin/arm64, local):
      #
      #   main 66.7  run 75.7  watch 85.7  startMQTT 100  stop 100
      #   runService 100  reportStartup 67.5  reportTLS 31.2  disconnect 58.3
      #
      # Twelve of the sixteen are executed today. The gap was never in the code,
      # it was in what we were able to see, and a coverage gap you cannot see is
      # the one that gets filled with tests that assert nothing.
      #
      # THIS STEP NEVER FAILS THE JOB, and that is deliberate. It is an
      # instrument, not a gate. A ratchet here would be a threshold nobody
      # measured on three platforms yet, and the first thing it would do is fail
      # a PR for a reason unrelated to the PR. It also cannot assume data
      # exists: a cover-instrumented binary writes its profile at NORMAL EXIT,
      # and the steps above stop the server with `kill`, which Git Bash may
      # never deliver to a native .exe (the same fact the trap comments above
      # are built around). If windows-latest reports nothing, that is a real
      # answer about the platform and is printed as one rather than swallowed.
      #
      # The module-wide total is deliberately NOT printed. A bare server start
      # touches little of internal/*, so that number would be low, meaningless,
      # and immediately mistaken for the project's coverage.
      - name: Report what the smoke run executed in cmd/polyemesis
        if: always() && needs.changes.outputs.code == 'true'
        shell: bash
        run: |
          set -uo pipefail
          dirs=""
          for d in covdata/serve covdata/bcast; do
            if [ -d "$d" ] && [ -n "$(ls -A "$d" 2>/dev/null)" ]; then
              dirs="${dirs:+$dirs,}$d"
            fi
          done
          if [ -z "$dirs" ]; then
            echo "no coverage data was written on ${{ matrix.os }}."
            echo "A -cover binary flushes its profile at normal exit; if the stop"
            echo "signal was never delivered there is nothing to report. This is"
            echo "informational and does not fail the job."
            exit 0
          fi
          echo "merging coverage from: $dirs"
          go tool covdata textfmt -i="$dirs" -o=smoke-cmd.cov || {
            echo "covdata textfmt failed on ${{ matrix.os }}; informational only."
            exit 0
          }
          echo "--- cmd/polyemesis, as executed by the smoke run on ${{ matrix.os }} ---"
          go tool cover -func=smoke-cmd.cov | grep 'cmd/polyemesis' || \
            echo "(no cmd/polyemesis rows in the profile)"

      - name: Upload broadcast artefacts on failure
        if: failure() && needs.changes.outputs.code == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: broadcast-${{ matrix.os }}
          path: |
            bcast-server.log
            data/recordings/
          retention-days: 7
          if-no-files-found: ignore

  ui:
    name: ui typecheck, lint, build
    # The documentation gate, #378. On every STEP below, never on this job --
    # see the `changes` job for why that distinction is load-bearing and for
    # what it cost the one time it was got wrong.
    needs: changes
    runs-on: ubuntu-latest
    timeout-minutes: 15
    defaults:
      run:
        working-directory: ui
    steps:
      # Says out loud why a green check did no work, so a reader of the
      # checks list is never left guessing whether this ran or no-opped.
      - name: Documentation-only change, so this job did no work
        if: needs.changes.outputs.code != 'true'
        working-directory: ${{ github.workspace }}
        run: |
          echo "::notice title=ui typecheck, lint, build::this check did no work -- every changed path was documentation. See the 'which changes' job."
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        if: needs.changes.outputs.code == 'true'
      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
        if: needs.changes.outputs.code == 'true'
        with:
          # Keep in step with the Dockerfiles' node tag and @types/node major.
          node-version: 24
          cache: npm
          cache-dependency-path: ui/package-lock.json
      - if: needs.changes.outputs.code == 'true'
        run: npm ci --ignore-scripts
      - if: needs.changes.outputs.code == 'true'
        run: npx --no-install tsc -b --noEmit
      - if: needs.changes.outputs.code == 'true'
        run: npm run lint
      - if: needs.changes.outputs.code == 'true'
        run: npm run build
      # Unit tests for the pure logic the browser suite cannot enumerate --
      # platform link construction has five platforms times several missing-field
      # cases, and driving each through a real browser would cost minutes to
      # assert what a millisecond of vitest does.
      - if: needs.changes.outputs.code == 'true'
        run: npm test
      # A vulnerable direct dependency should fail the build, not sit in a
      # report nobody opens. --audit-level=high so a low-severity transitive
      # advisory does not block a hotfix.
      - if: needs.changes.outputs.code == 'true'
        run: npm audit --audit-level=high

  cross:
    name: cross-compile all release targets
    # The documentation gate, #378. On every STEP below, never on this job --
    # see the `changes` job for why that distinction is load-bearing and for
    # what it cost the one time it was got wrong.
    needs: changes
    runs-on: ubuntu-latest
    timeout-minutes: 25
    steps:
      # Says out loud why a green check did no work, so a reader of the
      # checks list is never left guessing whether this ran or no-opped.
      - name: Documentation-only change, so this job did no work
        if: needs.changes.outputs.code != 'true'
        run: |
          echo "::notice title=cross-compile all release targets::this check did no work -- every changed path was documentation. See the 'which changes' job."
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        if: needs.changes.outputs.code == 'true'
      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
        if: needs.changes.outputs.code == 'true'
        with:
          go-version-file: go.mod
          cache: true
      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
        if: needs.changes.outputs.code == 'true'
        with:
          node-version: 24
          cache: npm
          cache-dependency-path: ui/package-lock.json
      # `make release` depends on the ui target, which needs the built assets
      # for go:embed. Building them here keeps the check honest end to end.
      - if: needs.changes.outputs.code == 'true'
        run: make release
      - if: needs.changes.outputs.code == 'true'
        run: ls -lh dist/

  # Which container suites this run needs.
  #
  # A separate job rather than an `if:` on the matrix below, for two reasons.
  # The matrix context is not available to a job-level `if`, so "run only the
  # browser suite" cannot be expressed there at all. And a suite that is not
  # wanted should not EXIST as a skipped job: main's runs already carry enough
  # grey "skipping" rows that a genuinely skipped check stops being noticed.
  #
  # Cheap: a checkout and a git diff, against three jobs that each build an
  # image and publish real streams.
  container-suites:
    name: "which container suites"
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      suites: ${{ steps.pick.outputs.suites }}
    steps:
      # Full history: the diff below needs both endpoints of the PR, and the
      # default shallow fetch has neither.
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0
      - id: pick
        # Through env, never interpolated into the script. Two of these are
        # attacker-influenced on a fork PR, and `run:` is a shell.
        env:
          EVENT: ${{ github.event_name }}
          REF: ${{ github.ref }}
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          set -euo pipefail
          ALL='["acceptance-docker","acceptance-multisource","acceptance-browser"]'

          # main and the weekly schedule keep the full set. This job changes
          # what PRs get, and takes nothing away from anywhere else.
          if [ "$EVENT" = "schedule" ] || [ "$REF" = "refs/heads/main" ]; then
            echo "suites=$ALL" >> "$GITHUB_OUTPUT"
            echo "full set: main or schedule"
            exit 0
          fi

          # A PR that touches ui/ gets the browser suite and only the browser
          # suite. acceptance-docker and acceptance-multisource assert routing
          # and token handling, which no UI change reaches -- running them here
          # would cost two image builds to prove something the PR cannot break.
          #
          # Captured to a variable and matched with a here-string rather than
          # piped into grep. Under `set -o pipefail`, `git diff | grep -q` can
          # report the PIPELINE as failed when grep matches early and exits,
          # leaving git to die of SIGPIPE. It did not reproduce at 196 files, so
          # this is a latent hazard rather than an observed bug -- but its
          # failure mode is a ui/ PR silently NOT getting the browser suite,
          # which is the exact thing this job exists to stop happening.
          # THIS FILE COUNTS AS WELL, and it was added after a change to the
          # browser suite's OWN setup merged without that suite ever running.
          #
          # The Playwright install lives in this workflow, not under ui/, so a
          # PR that rewrites how the browser is installed matched nothing here
          # and the container job reported "skipping". A broken install step
          # would have gone green on its own PR and failed on the next ui/
          # change instead, pointing at whoever touched the UI.
          #
          # The whole file rather than a line range: a job's behaviour is not
          # confined to the block that names it -- concurrency, permissions,
          # the suite matrix and the runner image all sit elsewhere in here and
          # all reach it. Matching the file is the honest boundary, and ci.yml
          # changes are rare enough that one image build is the right price for
          # never shipping an unexercised one again.
          if [ "$EVENT" = "pull_request" ]; then
            CHANGED="$(git diff --name-only "$BASE_SHA" "$HEAD_SHA")"
            if grep -qE '^(ui/|\.github/workflows/ci\.yml$)' <<<"$CHANGED"; then
              echo 'suites=["acceptance-browser"]' >> "$GITHUB_OUTPUT"
              echo "browser: this PR touches ui/ or this workflow"
              exit 0
            fi
          fi

          echo 'suites=[]' >> "$GITHUB_OUTPUT"
          echo "none: this PR touches neither ui/ nor this workflow"

  # The suites that need the shipped IMAGE rather than a host binary: SRT ingest
  # (Ubuntu's FFmpeg may lack libsrt, the image never does), the one-port
  # token-routing path, and the browser running against the real artefact.
  #
  # Expensive — each builds the image and publishes real streams into it — so
  # the full set runs on main and on a schedule rather than on every push.
  #
  # The browser suite is the exception, and #53 is why. Five guards were added
  # there for three defects that had passed tsc, oxlint, go vet, go test -race
  # and six reviews -- a nav that lost its CSS classes, a collapsed rail with no
  # accessible names, and a toggle that destroyed keyboard focus. Every one of
  # those guards contributed NOTHING to that PR's status, because this job only
  # ran after the merge. Tests that cannot fail before you merge are not a gate.
  container:
    name: "container: ${{ matrix.suite }}"
    needs: container-suites
    if: needs.container-suites.outputs.suites != '[]'
    runs-on: ubuntu-latest
    timeout-minutes: 45
    strategy:
      fail-fast: false
      matrix:
        # acceptance-docker      29 checks: routing, passthrough, persistence
        # acceptance-multisource 18 checks: two programmes, one port, by token
        # acceptance-browser     30 checks: Playwright against the image
        suite: ${{ fromJSON(needs.container-suites.outputs.suites) }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
        with:
          go-version-file: go.mod
          cache: true
      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
        with:
          node-version: 24
          cache: npm
          cache-dependency-path: ui/package-lock.json

      # THE BROWSER IS CACHED; THE SYSTEM LIBRARIES ARE NOT, and the split is
      # the whole point of this block.
      #
      # `playwright install --with-deps chromium` did two unrelated things in
      # one line: downloaded a browser build from Google's Chrome-for-Testing
      # CDN, and apt-installed the shared libraries it links against. Only the
      # first is cacheable -- the second writes to /usr, which no cache
      # restores -- so they are now separate steps.
      #
      # THE CDN IS WHAT ACTUALLY FAILED. On 2026-08-12 it answered 403:
      #
      #   Downloading Chrome for Testing 151.0.7922.34
      #   Error: Download failed: server returned code 403
      #
      # A cache hit removes that host from the path entirely, which is strictly
      # better than retrying it: the retries elsewhere in this file exist for
      # dependencies that must be fetched every time, and this one need not be.
      #
      # KEYED ON THE RESOLVED PLAYWRIGHT VERSION, not on a lockfile hash. The
      # browser build is a function of that version alone, so an unrelated
      # dependency bump must not evict a browser that is still correct. Read
      # from playwright-core, which is the package that actually pins the
      # revision; @playwright/test can float above it.
      - name: Playwright version
        if: matrix.suite == 'acceptance-browser'
        id: playwright-version
        working-directory: ui
        run: |
          v=$(node -p "require('./package-lock.json').packages['node_modules/playwright-core'].version")
          [ -n "$v" ] || { echo "could not resolve playwright-core version"; exit 1; }
          echo "version=$v" >> "$GITHUB_OUTPUT"
          echo "playwright-core $v"

      # No restore-keys. A near-miss would restore a browser for a DIFFERENT
      # Playwright, which the install below would then have to download over
      # anyway -- a partial hit here buys nothing and makes the cache's
      # contents harder to reason about.
      - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        if: matrix.suite == 'acceptance-browser'
        with:
          path: ~/.cache/ms-playwright
          key: ms-playwright-${{ runner.os }}-${{ steps.playwright-version.outputs.version }}

      - name: Install Playwright
        if: matrix.suite == 'acceptance-browser'
        working-directory: ui
        run: |
          npm ci --ignore-scripts
          # install-deps runs UNCONDITIONALLY: it is apt writing to /usr, which
          # the cache above does not cover, so a cache hit still needs it.
          npx --no-install playwright install-deps chromium
          # install runs unconditionally TOO, and is not gated on cache-hit on
          # purpose. It checks the local browser registry first and skips the
          # download when the revision is already there, so a cache hit costs
          # no CDN request -- while a cache that restored incompletely still
          # self-heals instead of failing later as "chromium not found".
          npx --no-install playwright install chromium

      # The same step timeout the `acceptance` job's suite step carries at
      # :724, and missing here until now. This job's ceiling is 45 minutes,
      # so a suite that hangs costs 45 minutes and names no step -- the exact
      # cost #179 priced at 28. 35 leaves the image build and the log upload
      # their share of the job budget while still firing first.
      - run: ./scripts/${{ matrix.suite }}.sh
        timeout-minutes: 35

      - name: Upload logs
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: ${{ matrix.suite }}-logs
          path: |
            /tmp/poly-*/**/*.log
            ui/test-results/**
          if-no-files-found: ignore
          retention-days: 7

  # The measurement suites. Everything above proves the code COMPILES and that
  # its pure functions agree with themselves; these prove the product actually
  # routes audio, copies video untouched and rides a source switch — by
  # measuring the output rather than asserting on a return value.
  #
  # They ran only on a maintainer's laptop until now, which is exactly the gap
  # that let three separate bugs through in one day: a missing .gitkeep that
  # only broke a clean checkout, an FFmpeg listener colliding with the Go one
  # that hid behind a local FFmpeg without libsrt, and a README publish URL that
  # stopped working. A laptop is not a clean machine.
  #
  # A matrix rather than one job: each suite is a few minutes, they are
  # independent, and a failure names itself in the checks list instead of being
  # buried in a combined log.
  acceptance:
    name: "acceptance: ${{ matrix.suite }}"
    runs-on: ubuntu-latest
    # NO JOB-LEVEL `if:` HERE, DELIBERATELY, and #351 is why. These are required
    # checks. A skipped matrix job never expands its matrix,
    # so the per-leg contexts are not skipped -- they do not exist, and the
    # pull request can never satisfy branch protection. The `if:` is on every
    # step below instead: the matrix expands, all twelve report, and a
    # documentation-only run costs a runner allocation rather than a suite.
    needs: changes
    env:
      # See the deadline note on the suite step below. The library's 900s
      # default is for a laptop, where nothing outranks it; here it has to fire
      # before the job ceiling, and its clock starts minutes after the job's.
      POLY_WATCHDOG_SECS: 600
    # A ceiling, because GitHub's default is SIX HOURS. These suites publish
    # real streams and wait on them, so the failure mode when something does
    # not arrive is a wait, not a crash -- exactly the shape that sits there
    # burning a runner. acceptance-renditions ran 25 minutes against a ~2
    # minute local baseline before this existed.
    timeout-minutes: 20
    strategy:
      # Never cancel siblings. When two suites break, both answers are wanted —
      # fail-fast would hide the second and turn one fix into two round trips.
      fail-fast: false
      matrix:
        suite:
          - acceptance          # end to end: ingest, fan-out, persistence
          - acceptance-audio    # per-track RMS through a bandpass
          - acceptance-renditions
          # The LADDER: three tiers at once, which acceptance-renditions cannot
          # cover because every test in it uses ONE rendition and one is the
          # only number it ever counts. This counts three, and counts them going
          # up and down, which is the claim docs/ENCODING.md attaches a bill to.
          #
          # ADDING A LEG HERE ADDS A REQUIRED STATUS CHECK. The matrix is named
          # `acceptance: ${{ matrix.suite }}` and every leg of it is required by
          # branch protection, so this is not a free addition: the new context
          # has to be added to the ruleset or pull requests will show it as an
          # extra check that nothing waits for.
          #
          # It belongs in the matrix rather than in a workflow of its own for
          # the same reason acceptance-renditions does: it needs nothing but
          # FFmpeg and the built binary, it contacts no external service, and
          # the thing it guards -- how many encodes an operator is paying for --
          # is broken by ordinary changes to the engine's reconcile. A suite
          # that only runs on dispatch is a suite that reports a regression
          # after it has shipped. Measured at 75s locally, which is inside the
          # spread of the legs already here.
          - acceptance-ladder
          - acceptance-encoders
          - acceptance-tls
          - acceptance-pull     # dial-out ingest
          - acceptance-playlist-phase0 # scheduled file broadcast, no encoder
          - acceptance-synth    # silence tier and the failover slate
          - acceptance-failover # switch without restarting a destination
          - acceptance-postprod
          # A recording is finalised on SIGTERM rather than truncated. The unit
          # file has promised this since it was written and nothing checked it;
          # on Linux it was false, and the failure is silent -- a truncated
          # Matroska is exactly the size it was when the process died, so only
          # a decode tells you.
          - acceptance-recording-stop
          # Retained MQTT telemetry. Brings up eclipse-mosquitto as a container
          # rather than installing a broker on the runner, so it needs no
          # apt step of its own -- Docker is already present on ubuntu-latest.
          - acceptance-mqtt
    steps:
      # EVERY STEP CARRIES THE GATE. There is no way to end a job early from a
      # step without failing it, so "documentation only" has to be spelled out
      # on each one rather than checked once at the top.
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        if: needs.changes.outputs.code == 'true'
      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
        if: needs.changes.outputs.code == 'true'
        with:
          go-version-file: go.mod
          cache: true
      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
        if: needs.changes.outputs.code == 'true'
        with:
          node-version: 24
          cache: npm
          cache-dependency-path: ui/package-lock.json

      # Says out loud why a green check did no work, so a reader of the checks
      # list is never left guessing whether the suite ran or silently no-opped.
      - name: Documentation-only change, so this suite did not run
        if: needs.changes.outputs.code != 'true'
        run: |
          echo "::notice title=${{ matrix.suite }} skipped::every changed path was documentation, so this suite could not be affected by it. See the 'which changes' job."

      - name: Install FFmpeg
        if: needs.changes.outputs.code == 'true'
        timeout-minutes: 40
        run: |
          # STILL apt's 6.1.1, and that is now a KNOWN GAP rather than an
          # oversight -- see the issue linked below.
          #
          # Moving this job to the cached, pinned 8.1 the other jobs use is
          # correct on both counts it was attempted for: it removes an uncached
          # third-party fetch that has hit the 6-minute ceiling four times, and
          # it stops the acceptance suites testing an FFmpeg users never run.
          #
          # It was attempted on this branch and REVERTED, because it broke
          # acceptance-failover on its first run: the mismatch destination, whose
          # content changes resolution mid-stream, produced 18154ms of media into
          # a 293-byte file. Two runs green before the switch, two red after. The
          # pin did exactly what it was added to do -- surfaced something 6.1.1
          # was hiding -- and that finding deserves its own change rather than a
          # ride on a website PR.
          #
          # RETRIES AND A PER-ACQUISITION TIMEOUT. apt's default is zero retries
          # and no per-connection timeout, so one slow mirror connection stalls
          # the whole acquisition until the step ceiling fires.
          APT_OPTS="-o Acquire::Retries=3 -o Acquire::http::Timeout=60 -o Acquire::http::ConnectionAttemptDelayMsec=1000"

          # BOUND EACH ATTEMPT WELL INSIDE THE STEP CEILING, then retry the whole
          # thing. This is the part Acquire::Retries cannot do.
          #
          # Acquire::Retries retries a FILE within one apt run, and the observed
          # failure is not a file erroring -- it is the whole acquisition sitting
          # on a stalled connection until the 6-minute step ceiling fires and
          # takes the job with it. Six occurrences now: acceptance-postprod,
          # acceptance-pull, acceptance-mqtt, acceptance-audio, and today
          # acceptance-renditions and acceptance-failover together.
          #
          # `timeout` turns "stalls for six minutes and kills the job" into
          # "stalls for 100 seconds and is retried", which fits three attempts
          # plus the install inside the ceiling with room to spare.
          #
          # NOT the cached pinned 8.1 build the other jobs use, though that would
          # solve this outright. It was tried, and it broke acceptance-failover
          # -- see the note above and issue #398. Reliability and the version
          # change are separable, and this is the half that carries no risk.
          # THE BUDGET HAS TO FIT THE WORK, AND IT NEVER DID. This step failed
          # 8 of the last 9 times anything failed in this repository, and the
          # reading that it was "a flaky mirror" was wrong -- see #430. The
          # install fetches 62.8 MB of archives, and every attempt was bounded at
          # 100 seconds. That needs ~700 KB/s sustained; a mirror merely under
          # load cannot manage it, so the timeout was killing a working download
          # rather than rescuing a stalled one. A first pass at this cut the
          # bound to 90s, which made it strictly worse.
          #
          # So the bound is now per-operation. `update` moves a few megabytes of
          # indexes and 90s is generous for it; `install` moves 62.8 MB and gets
          # five minutes. Retrying still helps because apt keeps part-downloaded
          # archives in /var/cache/apt/archives, so each attempt RESUMES -- the
          # logs show successive attempts reaching Get:9 rather than restarting.
          #
          # Worst case is 5x90 + 5x300 = 1950s plus backoff, which is why the
          # ceiling is 40 minutes. A step that CAN take forty minutes is not one
          # that usually does: a healthy install finishes in under a minute, and
          # the ceiling is only ever reached by a mirror that is failing.
          #
          # AND THE MIRROR SWITCH HAS TO EDIT THE FILE THE RUNNER ACTUALLY READS.
          # GitHub's images configure apt through a Mirrorlist -- the logs show
          # `Get:1 file:/etc/apt/apt-mirrors.txt Mirrorlist` -- so rewriting
          # sources.list, which is what the first attempt did, changed nothing.
          #
          # https, NOT http, and this is not pedantry: the fallback exists to be
          # used when the primary is failing, so it runs at exactly the moment
          # nobody is watching. Fetching packages CI is about to execute over
          # plaintext is the hole githubactions:S6506 exists for, and the curl
          # block above already carries a comment about the same rule catching
          # the same mistake in its own copy. archive.ubuntu.com serves https,
          # and apt on noble speaks it without apt-transport-https.
          apt_switch_mirror() {
            echo "switching to the Ubuntu mirror pool for the remaining attempts"
            # The mirrorlist is what the runner reads; sources.list is a fallback
            # for images that do not use one.
            if [ -f /etc/apt/apt-mirrors.txt ]; then
              printf 'https://archive.ubuntu.com/ubuntu/\tpriority:1\n' \
                | sudo tee /etc/apt/apt-mirrors.txt >/dev/null
            fi
            for f in /etc/apt/sources.list /etc/apt/sources.list.d/*.list \
                     /etc/apt/sources.list.d/*.sources; do
              [ -f "$f" ] || continue
              sudo sed -i -E \
                -e 's|https?://[a-z0-9.-]*archive\.ubuntu\.com/ubuntu|https://archive.ubuntu.com/ubuntu|g' \
                "$f"
            done
            sudo timeout 90 apt-get $APT_OPTS update -qq || true
          }

          # apt_try <seconds> <apt-get args...>
          apt_try() {
            local budget="$1"; shift
            for attempt in 1 2 3 4 5; do
              if sudo timeout "$budget" apt-get $APT_OPTS "$@"; then return 0; fi
              echo "apt attempt $attempt failed or stalled; retrying"
              # Switched once, before the fourth: three attempts is enough to
              # establish that this mirror is not merely unlucky.
              [ "$attempt" = 3 ] && apt_switch_mirror
              # Backoff, because an overloaded mirror is likelier to answer in
              # twenty seconds than in five.
              sleep $(( attempt * 5 ))
            done
            return 1
          }
          apt_try 90 update -qq
          # lsof as well: three suites read the engine's relay port out of it
          # to measure the right socket, and they fail obscurely without it.
          apt_try 300 install -y --no-install-recommends ffmpeg lsof
          ffmpeg -hide_banner -version | head -1
          # Not fatal: none of these suites needs SRT (the container ones do,
          # and they run elsewhere). Logged because a suite behaving oddly is
          # much easier to read with the build's capabilities in front of you.
          ffmpeg -hide_banner -protocols | tr " " "\n" | grep -qx srt \
            && echo "libsrt: yes" || echo "libsrt: no"

      # The suites run the real binary with the UI embedded, so it has to be
      # the real build rather than `go build ./cmd/...`.
      - run: make build
        if: needs.changes.outputs.code == 'true'

      # THE THREE DEADLINES, AND WHY THEY ARE IN THIS ORDER.
      #
      # There are three things that can stop a stuck suite: lib-watchdog.sh's
      # own budget, this step timeout, and the job's timeout-minutes: 20. Only
      # the first produces a useful answer -- it prints the step it was in and
      # the tail of server.log before it kills anything. So the watchdog has to
      # fire FIRST, and the other two exist only for the case where it cannot.
      #
      # It could not, and that is the bug found while measuring #94.
      # lib-watchdog.sh justifies its 900s as "against ci.yml's
      # timeout-minutes: 20, leaving five minutes for the report". That
      # arithmetic quietly assumes the suite starts when the job does. It does
      # not: checkout, setup-go, setup-node, apt and `make build` run first,
      # and the watchdog's clock only starts at `poly_watchdog_arm`. Measured
      # over the successful acceptance jobs in the last dozen runs, setup is
      # 58-102s -- so the watchdog would fire at 16-17 minutes of job time, and
      # against an apt that is merely slow rather than hung it could not fire
      # before the ceiling at all.
      #
      # POLY_WATCHDOG_SECS is set on the job below to 600 instead. Measured
      # over the same runs, the slowest suite step that ever PASSED was
      # acceptance-failover at 206s, and most are 10-60s. 600 is ~3x the
      # slowest real run and now fits inside the ceiling with room for the
      # report and the teardown trap whatever setup did.
      #
      # This step timeout sits ABOVE the watchdog on purpose. Set equal to it
      # the two would race and the step timeout would win, killing the suite
      # before it could say what it was waiting for -- which is the failure
      # this whole change exists to stop. 12 minutes is the backstop for a
      # watchdog that itself wedges, and it still leaves the always() upload
      # below its chance to run, which a job-level cancel does not.
      - name: ./scripts/${{ matrix.suite }}.sh
        if: needs.changes.outputs.code == 'true'
        timeout-minutes: 12
        run: ./scripts/${{ matrix.suite }}.sh

      # What was the process actually doing when the deadline fired?
      #
      # SIGQUIT is the point of this step. Go's default handler dumps every
      # goroutine's stack and exits, so a server blocked on a read nobody will
      # satisfy names the function it is blocked in. Without it the artifact
      # holds the suite's own log, which by definition stops at the line before
      # the thing that never happened.
      #
      # if: failure() rather than always(): on a green run there is no process
      # left to signal and nothing to explain.
      #
      # The gate is ANDed on for completeness rather than for effect -- #357
      # missed this one step, and it was harmless only because a documentation
      # -only run has no suite step to fail, so `failure()` was never true. That
      # is a coincidence of this job's shape, not a property, and the rule in
      # internal/testenv/docsgate_test.go found it.
      - name: Diagnostics (what was it waiting for)
        if: failure() && needs.changes.outputs.code == 'true'
        # Never fail the job from here. This step exists to describe a failure
        # that already happened; a diagnostic that turns a legible suite
        # failure into a confusing one about ps flags is worse than none.
        continue-on-error: true
        timeout-minutes: 3
        run: |
          set +e
          echo "=== processes ==="
          ps -eo pid,ppid,etime,stat,args | grep -E "polyemesis|ffmpeg|ffprobe|mosquitto" | grep -v grep

          echo
          echo "=== goroutine dumps ==="
          # Every polyemesis, not just one: several suites run an ingest and a
          # destination side, and the stalled one is not reliably the first.
          pids=$(pgrep -x polyemesis)
          if [ -z "$pids" ]; then
            echo "no polyemesis process is running -- the stall is in the SCRIPT,"
            echo "not the server: a wait on a file, a port or an ffmpeg that already exited."
          else
            for pid in $pids; do
              echo "--- SIGQUIT $pid ---"
              kill -QUIT "$pid"
            done
            # The dump goes to the process's own stderr, which the suites
            # redirect into their log directory, so it lands in the artifact
            # below rather than here.
            #
            # OBSERVED, NOT ASSUMED. This used to be a bare `sleep 5` -- a fixed
            # interval standing in for an observation, in the file this round
            # edited, in the diagnostic that exists to explain hangs. SIGQUIT to
            # a Go program writes the dump and then exits, so the process no
            # longer existing IS the dump having been written; poll for that and
            # stop as soon as it is true.
            #
            # The 5s ceiling is kept as the give-up, not as the wait: a process
            # wedged in uninterruptible I/O may never take the signal, and this
            # step must not become the hang it was written to explain.
            for _ in $(seq 1 20); do
              still=""
              for pid in $pids; do
                kill -0 "$pid" 2>/dev/null && still="$still $pid"
              done
              [ -z "$still" ] && break
              sleep 0.25
            done
            if [ -n "$still" ]; then
              echo "still alive 5s after SIGQUIT:$still -- the dump for these may be"
              echo "absent or truncated in the uploaded logs."
            fi
          fi

          echo
          echo "=== listening sockets ==="
          ss -lntup 2>/dev/null | head -40

          echo
          echo "=== tail of every suite log ==="
          # Last lines only: these get large, and the useful part of a stall is
          # always the end -- the last thing that happened before nothing did.
          for f in $(find /tmp artifacts logs -maxdepth 3 -name '*.log' -newermt '-30 minutes' 2>/dev/null | head -40); do
            echo "--- $f ---"
            tail -30 "$f"
          done
          exit 0

      # The logs are where a measurement failure is actually diagnosed: the RMS
      # figures, the process output, what FFmpeg said. Losing them turns a red
      # check into a guess.
      #
      # THE TIMELINE EVIDENCE GOES WITH THEM, and #126 is why. The only occurrence
      # of that bug anybody has seen outside a deliberate measurement happened
      # here, in CI, and it left nothing behind but a count in the step output --
      # no recording to re-probe, no packet list, no ledger. So the derived
      # artifacts travel too: dts.csv is the packet list both halves of the check
      # read, seams.txt is the engine's handover ledger, and onair.mkv is the
      # recording, which is the only one of the three that can answer a question
      # nobody thought to ask before the run.
      #
      # The mkv was 13 MB on a full local run of the failover suite, measured
      # rather than guessed. That is the price of not waiting another three weeks
      # for the next occurrence. Only that suite writes one; the others match
      # nothing here and if-no-files-found already covers it.
      - name: Upload logs
        # always(), but not when the suite never ran: a documentation-only job
        # has no artifact to collect, only a warning about its absence.
        if: always() && needs.changes.outputs.code == 'true'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: ${{ matrix.suite }}-logs
          path: |
            /tmp/polyemesis-acceptance*/**/*.log
            /tmp/polyemesis-acceptance*/**/server.log
            /tmp/polyemesis-acceptance*/dts.csv
            /tmp/polyemesis-acceptance*/seams.txt
            /tmp/polyemesis-acceptance*/data/recordings/onair.mkv
          if-no-files-found: ignore
          retention-days: 7