Add staging-debug release-train workflow, validation helpers, docs, and tests

.github/workflows/staging-debug-release-train.yml

@@ -0,0 +1,141 @@
+name: ROSS staging release-train debug
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ross-staging-debug-${{ github.run_id }}
+  cancel-in-progress: false
+
+jobs:
+  debug:
+    runs-on: ubuntu-latest
+    timeout-minutes: 180
+    environment: staging-debug
+    env:
+      FLY_API_TOKEN: ${{ secrets.STAGING_FLY_API_TOKEN }}
+      FLY_ORG: ${{ vars.STAGING_FLY_ORG }}
+      ROSS_STAGING_SUPABASE_URL: ${{ secrets.STAGING_SUPABASE_URL }}
+      ROSS_STAGING_SUPABASE_PUBLISHABLE_KEY: ${{ secrets.STAGING_SUPABASE_PUBLISHABLE_KEY }}
+      ROSS_STAGING_SUPABASE_SECRET_KEY: ${{ secrets.STAGING_SUPABASE_SECRET_KEY }}
+      ROSS_STAGING_S3_ENDPOINT_URL: ${{ secrets.STAGING_S3_ENDPOINT_URL }}
+      ROSS_STAGING_S3_REGION: ${{ secrets.STAGING_S3_REGION }}
+      ROSS_STAGING_S3_ACCESS_KEY_ID: ${{ secrets.STAGING_S3_ACCESS_KEY_ID }}
+      ROSS_STAGING_S3_SECRET_ACCESS_KEY: ${{ secrets.STAGING_S3_SECRET_ACCESS_KEY }}
+      ROSS_PRODUCTION_API_APP: ${{ vars.PRODUCTION_API_APP }}
+      ROSS_PRODUCTION_WEB_APP: ${{ vars.PRODUCTION_WEB_APP }}
+      ROSS_PRODUCTION_WORKER_APP: ${{ vars.PRODUCTION_WORKER_APP }}
+      ROSS_PRODUCTION_SUPABASE_URL: ${{ vars.PRODUCTION_SUPABASE_URL }}
+      ROSS_PRODUCTION_S3_ENDPOINT_URL: ${{ vars.PRODUCTION_S3_ENDPOINT_URL }}
+    steps:
+      - uses: actions/checkout@v7
+        with: { fetch-depth: 0 }
+      - uses: ./.github/actions/setup-ross-node
+      - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1
+        with: { version: 0.4.49 }
+
+      - name: Establish isolated names and fail closed on production overlap
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p artifacts/staging-debug/{commands,diagnostics}
+          node scripts/validate-staging-debug.mjs | tee artifacts/staging-debug/isolation.txt >> "$GITHUB_ENV"
+          env | sed -nE '/^(API|WEB|WORKER)_APP=/p' >> artifacts/staging-debug/isolation.txt
+
+      - name: Run complete repository gate
+        run: npm run install:all && npm run check
+
+      - name: Provision ephemeral staging applications
+        shell: bash
+        run: |
+          set -euo pipefail
+          for app in "$API_APP" "$WEB_APP" "$WORKER_APP"; do
+            flyctl apps create "$app" --org "$FLY_ORG" --yes 2>&1 | tee "artifacts/staging-debug/commands/provision-${app}.log"
+          done
+
+      - name: Build candidate images in the isolated registry namespaces
+        shell: bash
+        run: |
+          set -euo pipefail
+          export RELEASE_IMAGE_API_APP="$API_APP" RELEASE_IMAGE_WEB_APP="$WEB_APP" RELEASE_IMAGE_WORKER_APP="$WORKER_APP"
+          export RELEASE_RUNTIME_API_APP="$API_APP" RELEASE_RUNTIME_WEB_APP="$WEB_APP" RELEASE_REHEARSAL_API_APP="$API_APP"
+          export RELEASE_SIGNUPS_ENABLED=false
+          export ROSS_SUPABASE_URL="$ROSS_STAGING_SUPABASE_URL" ROSS_SUPABASE_PUBLISHABLE_KEY="$ROSS_STAGING_SUPABASE_PUBLISHABLE_KEY"
+          export ROSS_RELEASE_ID="staging-debug-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          echo "ROSS_STAGING_DEBUG_RELEASE_ID=$ROSS_RELEASE_ID" >> "$GITHUB_ENV"
+          bash scripts/build-release-train-images.sh
+
+      - name: Configure staging-only service dependencies
+        shell: bash
+        run: |
+          set -euo pipefail
+          worker_secret="$(openssl rand -hex 32)"; echo "::add-mask::$worker_secret"
+          signing_secret="$(openssl rand -hex 32)"; echo "::add-mask::$signing_secret"
+          flyctl secrets set --stage --app "$WORKER_APP" "FILE_WORKER_SHARED_SECRET=$worker_secret" "FILE_WORKER_STORAGE_ORIGINS=$ROSS_STAGING_S3_ENDPOINT_URL"
+          flyctl secrets set --stage --app "$API_APP" "SUPABASE_URL=$ROSS_STAGING_SUPABASE_URL" "SUPABASE_SECRET_KEY=$ROSS_STAGING_SUPABASE_SECRET_KEY" "R2_ENDPOINT_URL=$ROSS_STAGING_S3_ENDPOINT_URL" "R2_REGION=$ROSS_STAGING_S3_REGION" "R2_ACCESS_KEY_ID=$ROSS_STAGING_S3_ACCESS_KEY_ID" "R2_SECRET_ACCESS_KEY=$ROSS_STAGING_S3_SECRET_ACCESS_KEY" "R2_BUCKET_NAME=ross-staging-debug" "FILE_WORKER_URL=http://${WORKER_APP}.flycast" "FILE_WORKER_SHARED_SECRET=$worker_secret" "DOWNLOAD_SIGNING_SECRET=$signing_secret" "USER_API_KEYS_ENCRYPTION_SECRET=$signing_secret" "MCP_CONNECTORS_ENCRYPTION_SECRET=$signing_secret" "ROSS_ENV=staging" "ROSS_HOSTED_MODE=controlled-beta" "HOSTED_MODEL_PROVIDERS=openai" "ROSS_RUNTIME_RELEASE_ID=$ROSS_STAGING_DEBUG_RELEASE_ID" "ROSS_DISABLE_DOCUMENT_SCAN_DISPATCHER=true" "ROSS_UPLOAD_SCAN_REQUIRED=false" "CORS_ALLOWED_ORIGINS=https://${WEB_APP}.fly.dev" "FRONTEND_URL=https://${WEB_APP}.fly.dev" "API_PUBLIC_URL=https://${API_APP}.fly.dev"
+          flyctl secrets set --stage --app "$WEB_APP" "ROSS_RUNTIME_API_BASE_URL=https://${API_APP}.fly.dev" "ROSS_RUNTIME_APP_URL=https://${WEB_APP}.fly.dev" "ROSS_RUNTIME_SIGNUPS_ENABLED=false" "ROSS_RUNTIME_ENVIRONMENT=staging-debug" "ROSS_RUNTIME_RELEASE_ID=$ROSS_STAGING_DEBUG_RELEASE_ID"
+
+      - name: Deploy and diagnose worker stage
+        shell: bash
+        run: |
+          set -euo pipefail
+          bash scripts/fly-deploy-with-retry.sh . --config deploy/fly/rehearsal-file-worker.toml --app "$WORKER_APP" --image "$CANDIDATE_WORKER_IMAGE" --ha=false --yes --flycast --no-public-ips 2>&1 | tee artifacts/staging-debug/commands/worker.log
+          flyctl status --app "$WORKER_APP" --json > artifacts/staging-debug/diagnostics/worker-candidate-status.json
+          flyctl logs --app "$WORKER_APP" --no-tail > artifacts/staging-debug/diagnostics/worker-candidate.log 2>&1 || true
+      - name: Deploy and diagnose API stage
+        shell: bash
+        run: |
+          set -euo pipefail
+          bash scripts/fly-deploy-with-retry.sh . --config deploy/fly/rehearsal-api.toml --app "$API_APP" --image "$CANDIDATE_API_IMAGE" --ha=false --yes --flycast --no-public-ips 2>&1 | tee artifacts/staging-debug/commands/api.log
+          flyctl status --app "$API_APP" --json > artifacts/staging-debug/diagnostics/api-candidate-status.json
+          flyctl logs --app "$API_APP" --no-tail > artifacts/staging-debug/diagnostics/api-candidate.log 2>&1 || true
+      - name: Deploy and diagnose web stage
+        shell: bash
+        run: |
+          set -euo pipefail
+          bash scripts/fly-deploy-with-retry.sh . --config deploy/fly/rehearsal-frontend.toml --app "$WEB_APP" --image "$CANDIDATE_WEB_IMAGE" --ha=false --yes --flycast --no-public-ips 2>&1 | tee artifacts/staging-debug/commands/web.log
+          flyctl status --app "$WEB_APP" --json > artifacts/staging-debug/diagnostics/web-candidate-status.json
+          flyctl logs --app "$WEB_APP" --no-tail > artifacts/staging-debug/diagnostics/web-candidate.log 2>&1 || true
+
+      - name: Run exact complete release-train integration probe
+        run: node scripts/run-staging-debug-probe.mjs staging-debug 2>&1 | tee artifacts/staging-debug/diagnostics/full-probe-before-failure.log
+
+      - name: Inject a genuine deployment failure and restore baseline
+        shell: bash
+        run: bash scripts/staging-debug-lifecycle.sh inject-failure-and-rollback
+
+      - name: Verify full integration recovery after rollback
+        shell: bash
+        run: |
+          set -euo pipefail
+          node scripts/run-staging-debug-probe.mjs staging-debug 2>&1 | tee artifacts/staging-debug/diagnostics/full-probe-after-rollback.log
+          printf '{"status":"passed","genuineDeploymentFailure":"observed","rollback":"verified","fullProbe":"passed","productionPromotion":false}\n' > artifacts/staging-debug/result.json
+
+      - name: Collect failure diagnostics
+        if: failure()
+        shell: bash
+        run: |
+          for app in "${API_APP:-}" "${WEB_APP:-}" "${WORKER_APP:-}"; do
+            [ -n "$app" ] || continue
+            flyctl status --app "$app" --json > "artifacts/staging-debug/diagnostics/${app}-failure-status.json" 2>&1 || true
+            flyctl logs --app "$app" --no-tail > "artifacts/staging-debug/diagnostics/${app}-failure.log" 2>&1 || true
+          done
+
+      - name: Destroy all ephemeral staging resources
+        if: always()
+        shell: bash
+        run: bash scripts/staging-debug-lifecycle.sh cleanup
+
+      - name: Upload complete staging-debug evidence
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: ross-staging-debug-${{ github.run_id }}-${{ github.run_attempt }}
+          path: artifacts/staging-debug
+          if-no-files-found: error
+          retention-days: 30
+
+# Intentionally no production environment, production credentials, promotion input, or promotion job.

config/release-manifest.v1.json

@@ -10,6 +10,7 @@
     ".github/workflows/final-controlled-beta-evidence.yml",
     ".github/workflows/refresh-release-manifest.yml",
     ".github/workflows/release-candidate.yml",
+    ".github/workflows/staging-debug-release-train.yml",
     ".github/workflows/verify-and-deploy-public-beta.yml",
     ".github/workflows/verify-ontario-sources.yml",
     "backend/migrations/20260718_01_document_scan_pipeline.sql",
@@ -70,6 +71,7 @@
     "docs/privacy/subprocessor-inventory.json",
     "docs/evaluation/capability-limitations-v1.md",
     "docs/deployment/release-train-v1.md",
+    "docs/staging-debug-release-train.md",
     "frontend/package-lock.json",
     "frontend/package.json",
     "frontend/src/app/components/projects/ProjectDocumentsView.tsx",
@@ -101,21 +103,27 @@
     "scripts/lib/release-readiness.mjs",
     "scripts/lib/live-source-observer.mjs",
     "scripts/lib/release-train.mjs",
+    "scripts/lib/release-train-probe.mjs",
+    "scripts/lib/staging-debug.mjs",
     "scripts/build-release-train-images.sh",
     "scripts/fly-deploy-with-retry.sh",
     "scripts/fly-release-train.mjs",
     "scripts/observe-legal-sources.mjs",
     "scripts/preflight-fly-images.sh",
     "scripts/release-train-image-ref.mjs",
+    "scripts/run-staging-debug-probe.mjs",
+    "scripts/staging-debug-lifecycle.sh",
     "scripts/run-backup-restore-exercise.sh",
     "scripts/lib/professional-validation.mjs",
     "scripts/lib/release-identifier.mjs",
     "scripts/lib/final-completion.mjs",
     "scripts/validate-release-id.mjs",
+    "scripts/validate-staging-debug.mjs",
     "tests/baseline/ross-ci-toolchain.test.mjs",
     "tests/baseline/ross-delivery-d.test.mjs",
     "tests/baseline/ross-production-readiness.test.mjs",
     "tests/baseline/ross-release-train.test.mjs",
+    "tests/baseline/ross-staging-debug.test.mjs",
     "website/package-lock.json",
     "website/package.json",
     "website/app/site-config.ts",

docs/staging-debug-release-train.md

@@ -0,0 +1,30 @@
+# Staging release-train debugging
+
+Use **ROSS staging release-train debug** to reproduce a release failure without
+touching the public-beta deployment. The workflow creates three run-scoped Fly
+apps, uses only secrets from the protected `staging-debug` environment, disables
+sign-ups and scan dispatch, and rejects missing or production-equal data origins.
+
+Configure `STAGING_FLY_API_TOKEN`, a dedicated staging Supabase project, and a
+dedicated staging S3-compatible bucket/endpoint in that environment. Set
+`STAGING_FLY_ORG`; do not copy production credentials into any `STAGING_*`
+secret. Configure all five non-secret `PRODUCTION_*` comparison variables for
+the three production app names, Supabase URL, and storage endpoint. Validation
+fails closed if any comparison identifier is absent or equals its staging
+counterpart. Environment approval should be limited to release operators.
+
+The job runs the complete repository gate, builds immutable image digests,
+deploys worker, API, and web separately, and captures command output plus Fly
+status and logs after every stage. It runs the exact complete integration probe
+used by the release train, then attempts a deliberately invalid web deployment
+whose unreachable service port must make `flyctl deploy` fail. The job asserts
+that nonzero result, records diagnostics, rolls Fly back to the recorded
+known-good release version, and reruns the complete probe to verify recovery.
+Its final `always()` path
+collects failure diagnostics, destroys every run-scoped app, and uploads the
+evidence for 30 days. A cleanup failure fails the job and requires an operator
+to destroy the names recorded in `isolation.txt`.
+
+The workflow has read-only repository permission and deliberately contains no
+production environment, production secret, promotion input, tag, release, or
+deployment step. It must never be repurposed for production promotion.

reports/release-manifest-v1.json

@@ -3,7 +3,7 @@
   "releaseId": "ross-public-beta-20260717-rc1",
   "generatedAt": "2026-07-26T10:39:57.000Z",
   "algorithm": "sha256",
-  "artifactCount": 119,
+  "artifactCount": 127,
   "artifacts": [
     {
       "path": ".github/actions/setup-ross-node/action.yml",
@@ -40,6 +40,11 @@
       "sha256": "b155ce7cc7a54f884e20f6a164929d8851652793725db8e2e0df18bbc7cf0511",
       "sizeBytes": 3027
     },
+    {
+      "path": ".github/workflows/staging-debug-release-train.yml",
+      "sha256": "ba54e3a0b98324906bc93e4411e04b4a7ffcb8399401f028b7dae3f77ab3fdb9",
+      "sizeBytes": 8501
+    },
     {
       "path": ".github/workflows/verify-and-deploy-public-beta.yml",
       "sha256": "69253489fffdf6b220f6ffea764c30e46510dd97730fb23e85400c9eaeba6c11",
@@ -340,6 +345,11 @@
       "sha256": "d6a62d767e79c915fbeecd1754351e376a251f4eaf5a516ec023ff9b83eb4b29",
       "sizeBytes": 5556
     },
+    {
+      "path": "docs/staging-debug-release-train.md",
+      "sha256": "dff475f33b4255cc07853a6ea26063bd5990a1317e02c164bdd73456d42bc300",
+      "sizeBytes": 1892
+    },
     {
       "path": "frontend/package-lock.json",
       "sha256": "78c5149b7d30f1d3e39e545a995e8bf537a1368b302fd8ca452ddaffbe811fef",
@@ -495,10 +505,20 @@
       "sha256": "7b08133586b241fa727c0a255ec6f1873c2ba7f3fdc85b195e8c21dc0544cfdb",
       "sizeBytes": 4775
     },
+    {
+      "path": "scripts/lib/release-train-probe.mjs",
+      "sha256": "a9a238424999ea97a1f2fbd1d9b04ee8fe044da053403e6c2d46f94e0ce1f4a0",
+      "sizeBytes": 5182
+    },
+    {
+      "path": "scripts/lib/staging-debug.mjs",
+      "sha256": "f496440b11fba669ddedca3091f51a73857adf705b0b974a569a780588654fdd",
+      "sizeBytes": 2822
+    },
     {
       "path": "scripts/build-release-train-images.sh",
-      "sha256": "6a3c18fd86d91861b0062c39e9206fd82cbf367e56952bcb4b7327cfb19fec57",
-      "sizeBytes": 4760
+      "sha256": "777328e5428fa2c580854a957d1c64c46eb82164636e077ccb87e7d5c716bd02",
+      "sizeBytes": 5794
     },
     {
       "path": "scripts/fly-deploy-with-retry.sh",
@@ -507,8 +527,8 @@
     },
     {
       "path": "scripts/fly-release-train.mjs",
-      "sha256": "731807a55f5e21267502071b679dc3a30f56c9c7fde0781efc4e68af0b37b2ff",
-      "sizeBytes": 32016
+      "sha256": "71338c98da92f12dfc244c3eb49d74b00cde169e5b8cdfbe5597ec75954199d0",
+      "sizeBytes": 26940
     },
     {
       "path": "scripts/observe-legal-sources.mjs",
@@ -525,6 +545,16 @@
       "sha256": "17cef8f7cf2b22e4bcaf19c0766fbbfef916941e654ad1156a87ab7047e92b2e",
       "sizeBytes": 1896
     },
+    {
+      "path": "scripts/run-staging-debug-probe.mjs",
+      "sha256": "c49dee35e02aea23f421f2a0244703edef7f9a6632932a40e7261a97b9eee2d3",
+      "sizeBytes": 2207
+    },
+    {
+      "path": "scripts/staging-debug-lifecycle.sh",
+      "sha256": "08dae021d6dacde7f648e0aa830838ed34a81a43c6dcfdd8f3ffea6c7da49202",
+      "sizeBytes": 3178
+    },
     {
       "path": "scripts/run-backup-restore-exercise.sh",
       "sha256": "f2d0ee4cb495214f03db3e479ba3ff58de499d999abed62f72cf5c84bafd0a63",
@@ -550,6 +580,11 @@
       "sha256": "5eca3fbee3ddd6245497243e5f2690d351956af3d9d025afd5e2fadb10766cde",
       "sizeBytes": 301
     },
+    {
+      "path": "scripts/validate-staging-debug.mjs",
+      "sha256": "48b0646e15b8b173718943cf1ce961b2404fe765818e2cb533e3ed1c477dea6e",
+      "sizeBytes": 1000
+    },
     {
       "path": "tests/baseline/ross-ci-toolchain.test.mjs",
       "sha256": "31a3b311d6d68099452a3c4d6d7c35aadd7f2c6300a28fb1d4752ffbadc97ed6",
@@ -567,8 +602,13 @@
     },
     {
       "path": "tests/baseline/ross-release-train.test.mjs",
-      "sha256": "012ab7b4d6b7b1513a0362d300c2333a477142564cb5a8acc6ea912e6779b4b4",
-      "sizeBytes": 27789
+      "sha256": "2406c9046010025257a3bd9586f7821010a8e1b35717a66f7c4fbd12ed712720",
+      "sizeBytes": 27604
+    },
+    {
+      "path": "tests/baseline/ross-staging-debug.test.mjs",
+      "sha256": "0a5a43bd6874d79b5d9dc487c4ce90ff2ac47cc8a151829a873145b1ce4fbd94",
+      "sizeBytes": 9422
     },
     {
       "path": "website/package-lock.json",