From e11dadd1c1b529e61e42c18aafad3fc4dbf764d1 Mon Sep 17 00:00:00 2001 From: ranade-oss Date: Sun, 26 Jul 2026 16:26:57 -0400 Subject: [PATCH 1/5] Fix staging rollback and diagnostics --- .../workflows/staging-debug-release-train.yml | 150 ++++++++++++++++++ docs/staging-debug-release-train.md | 25 +++ scripts/lib/staging-debug.mjs | 35 ++++ scripts/validate-staging-debug.mjs | 22 +++ tests/baseline/ross-staging-debug.test.mjs | 28 ++++ 5 files changed, 260 insertions(+) create mode 100644 .github/workflows/staging-debug-release-train.yml create mode 100644 docs/staging-debug-release-train.md create mode 100644 scripts/lib/staging-debug.mjs create mode 100644 scripts/validate-staging-debug.mjs create mode 100644 tests/baseline/ross-staging-debug.test.mjs diff --git a/.github/workflows/staging-debug-release-train.yml b/.github/workflows/staging-debug-release-train.yml new file mode 100644 index 0000000000..9cc8fe2e9f --- /dev/null +++ b/.github/workflows/staging-debug-release-train.yml @@ -0,0 +1,150 @@ +name: ROSS staging release-train debug + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ross-staging-debug-${{ github.run_id }} + cancel-in-progress: false + +jobs: + debug: + runs-on: ubuntu-latest + timeout-minutes: 180 + environment: staging-debug + env: + FLY_API_TOKEN: ${{ secrets.STAGING_FLY_API_TOKEN }} + FLY_ORG: ${{ vars.STAGING_FLY_ORG }} + ROSS_STAGING_SUPABASE_URL: ${{ secrets.STAGING_SUPABASE_URL }} + ROSS_STAGING_SUPABASE_PUBLISHABLE_KEY: ${{ secrets.STAGING_SUPABASE_PUBLISHABLE_KEY }} + ROSS_STAGING_SUPABASE_SECRET_KEY: ${{ secrets.STAGING_SUPABASE_SECRET_KEY }} + ROSS_STAGING_S3_ENDPOINT_URL: ${{ secrets.STAGING_S3_ENDPOINT_URL }} + ROSS_STAGING_S3_REGION: ${{ secrets.STAGING_S3_REGION }} + ROSS_STAGING_S3_ACCESS_KEY_ID: ${{ secrets.STAGING_S3_ACCESS_KEY_ID }} + ROSS_STAGING_S3_SECRET_ACCESS_KEY: ${{ secrets.STAGING_S3_SECRET_ACCESS_KEY }} + steps: + - uses: actions/checkout@v7 + with: { fetch-depth: 0 } + - uses: ./.github/actions/setup-ross-node + - uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 + with: { version: 0.4.49 } + + - name: Establish isolated names and fail closed on production overlap + shell: bash + run: | + set -euo pipefail + mkdir -p artifacts/staging-debug/{commands,diagnostics} + node scripts/validate-staging-debug.mjs | tee artifacts/staging-debug/isolation.txt >> "$GITHUB_ENV" + env | sed -nE '/^(API|WEB|WORKER)_APP=/p' >> artifacts/staging-debug/isolation.txt + + - name: Run complete repository gate + run: npm run install:all && npm run check + + - name: Provision ephemeral staging applications + shell: bash + run: | + set -euo pipefail + for app in "$API_APP" "$WEB_APP" "$WORKER_APP"; do + flyctl apps create "$app" --org "$FLY_ORG" --yes 2>&1 | tee "artifacts/staging-debug/commands/provision-${app}.log" + done + + - name: Build candidate images in the isolated registry namespaces + shell: bash + run: | + export PROD_API_APP="$API_APP" PROD_WEB_APP="$WEB_APP" PROD_WORKER_APP="$WORKER_APP" STAGE_API_APP="$API_APP" + export ROSS_SUPABASE_URL="$ROSS_STAGING_SUPABASE_URL" ROSS_SUPABASE_PUBLISHABLE_KEY="$ROSS_STAGING_SUPABASE_PUBLISHABLE_KEY" + export ROSS_RELEASE_ID="staging-debug-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + bash scripts/build-release-train-images.sh + + - name: Configure staging-only service dependencies + shell: bash + run: | + set -euo pipefail + worker_secret="$(openssl rand -hex 32)"; echo "::add-mask::$worker_secret" + signing_secret="$(openssl rand -hex 32)"; echo "::add-mask::$signing_secret" + flyctl secrets set --stage --app "$WORKER_APP" "FILE_WORKER_SHARED_SECRET=$worker_secret" "FILE_WORKER_STORAGE_ORIGINS=$ROSS_STAGING_S3_ENDPOINT_URL" + flyctl secrets set --stage --app "$API_APP" "SUPABASE_URL=$ROSS_STAGING_SUPABASE_URL" "SUPABASE_SECRET_KEY=$ROSS_STAGING_SUPABASE_SECRET_KEY" "R2_ENDPOINT_URL=$ROSS_STAGING_S3_ENDPOINT_URL" "R2_REGION=$ROSS_STAGING_S3_REGION" "R2_ACCESS_KEY_ID=$ROSS_STAGING_S3_ACCESS_KEY_ID" "R2_SECRET_ACCESS_KEY=$ROSS_STAGING_S3_SECRET_ACCESS_KEY" "R2_BUCKET_NAME=ross-staging-debug" "FILE_WORKER_URL=http://${WORKER_APP}.flycast" "FILE_WORKER_SHARED_SECRET=$worker_secret" "DOWNLOAD_SIGNING_SECRET=$signing_secret" "USER_API_KEYS_ENCRYPTION_SECRET=$signing_secret" "MCP_CONNECTORS_ENCRYPTION_SECRET=$signing_secret" "ROSS_ENV=staging" "ROSS_HOSTED_MODE=controlled-beta" "HOSTED_MODEL_PROVIDERS=openai" "ROSS_DISABLE_DOCUMENT_SCAN_DISPATCHER=true" "ROSS_UPLOAD_SCAN_REQUIRED=false" "CORS_ALLOWED_ORIGINS=https://${WEB_APP}.fly.dev" "FRONTEND_URL=https://${WEB_APP}.fly.dev" "API_PUBLIC_URL=https://${API_APP}.fly.dev" + flyctl secrets set --stage --app "$WEB_APP" "ROSS_RUNTIME_API_BASE_URL=https://${API_APP}.fly.dev" "ROSS_RUNTIME_APP_URL=https://${WEB_APP}.fly.dev" "ROSS_RUNTIME_SIGNUPS_ENABLED=false" "ROSS_RUNTIME_ENVIRONMENT=staging-debug" + + - name: Deploy and diagnose worker stage + shell: bash + run: | + set -euo pipefail + bash scripts/fly-deploy-with-retry.sh . --config deploy/fly/rehearsal-file-worker.toml --app "$WORKER_APP" --image "$CANDIDATE_WORKER_IMAGE" --ha=false --yes --flycast --no-public-ips 2>&1 | tee artifacts/staging-debug/commands/worker.log + flyctl status --app "$WORKER_APP" --json > artifacts/staging-debug/diagnostics/worker-candidate-status.json + flyctl logs --app "$WORKER_APP" --no-tail > artifacts/staging-debug/diagnostics/worker-candidate.log 2>&1 || true + - name: Deploy and diagnose API stage + shell: bash + run: | + set -euo pipefail + bash scripts/fly-deploy-with-retry.sh . --config deploy/fly/rehearsal-api.toml --app "$API_APP" --image "$CANDIDATE_API_IMAGE" --ha=false --yes --flycast --no-public-ips 2>&1 | tee artifacts/staging-debug/commands/api.log + flyctl status --app "$API_APP" --json > artifacts/staging-debug/diagnostics/api-candidate-status.json + flyctl logs --app "$API_APP" --no-tail > artifacts/staging-debug/diagnostics/api-candidate.log 2>&1 || true + - name: Deploy and diagnose web stage + shell: bash + run: | + set -euo pipefail + bash scripts/fly-deploy-with-retry.sh . --config deploy/fly/rehearsal-frontend.toml --app "$WEB_APP" --image "$CANDIDATE_WEB_IMAGE" --ha=false --yes --flycast --no-public-ips 2>&1 | tee artifacts/staging-debug/commands/web.log + flyctl status --app "$WEB_APP" --json > artifacts/staging-debug/diagnostics/web-candidate-status.json + flyctl logs --app "$WEB_APP" --no-tail > artifacts/staging-debug/diagnostics/web-candidate.log 2>&1 || true + + - name: Inject a staging web release failure and test real Fly rollback + shell: bash + run: | + set -euo pipefail + baseline_version="$(flyctl releases --app "$WEB_APP" --json | tee artifacts/staging-debug/diagnostics/web-releases-before-failure.json | jq -er '.[0].Version // .[0].version')" + printf '%s\n' "$baseline_version" > artifacts/staging-debug/diagnostics/web-rollback-target.txt + + # Create an actual bad Fly release using the same immutable image. The + # runtime probe makes the injected failure observable before rollback. + flyctl secrets set --stage --app "$WEB_APP" ROSS_RUNTIME_ENVIRONMENT=forced-debug-failure + flyctl deploy . --config deploy/fly/rehearsal-frontend.toml --app "$WEB_APP" --image "$CANDIDATE_WEB_IMAGE" --ha=false --yes --flycast --no-public-ips 2>&1 | tee artifacts/staging-debug/commands/web-forced-failure.log + machine_id="$(flyctl machine list --app "$WEB_APP" --json | jq -er '.[0].id // .[0].ID')" + flyctl machine start "$machine_id" --app "$WEB_APP" || true + flyctl machine wait "$machine_id" --app "$WEB_APP" --state started --wait-timeout 2m + probe='fetch("http://127.0.0.1:3000/api/runtime-config").then(r=>r.json()).then(x=>{console.log(JSON.stringify(x));if(x.environment!=="forced-debug-failure")process.exit(1)})' + flyctl ssh console --app "$WEB_APP" --machine "$machine_id" --command "node -e '$probe'" | tee artifacts/staging-debug/diagnostics/web-forced-failure-probe.json + + flyctl releases rollback "$baseline_version" --app "$WEB_APP" --yes 2>&1 | tee artifacts/staging-debug/commands/web-rollback.log + machine_id="$(flyctl machine list --app "$WEB_APP" --json | jq -er '.[0].id // .[0].ID')" + flyctl machine start "$machine_id" --app "$WEB_APP" || true + flyctl machine wait "$machine_id" --app "$WEB_APP" --state started --wait-timeout 2m + probe='fetch("http://127.0.0.1:3000/api/runtime-config").then(r=>r.json()).then(x=>{console.log(JSON.stringify(x));if(x.environment!=="staging-debug")process.exit(1)})' + flyctl ssh console --app "$WEB_APP" --machine "$machine_id" --command "node -e '$probe'" | tee artifacts/staging-debug/diagnostics/web-rollback-probe.json + flyctl releases --app "$WEB_APP" --json > artifacts/staging-debug/diagnostics/web-releases-after-rollback.json + printf '{"status":"passed","failureInjection":"observed","rollback":"verified","productionPromotion":false}\n' > artifacts/staging-debug/result.json + + - name: Collect failure diagnostics + if: failure() + shell: bash + run: | + for app in "${API_APP:-}" "${WEB_APP:-}" "${WORKER_APP:-}"; do + [ -n "$app" ] || continue + flyctl status --app "$app" --json > "artifacts/staging-debug/diagnostics/${app}-failure-status.json" 2>&1 || true + flyctl logs --app "$app" --no-tail > "artifacts/staging-debug/diagnostics/${app}-failure.log" 2>&1 || true + done + + - name: Destroy all ephemeral staging resources + if: always() + shell: bash + run: | + failed=0 + for app in "${WORKER_APP:-}" "${API_APP:-}" "${WEB_APP:-}"; do + [ -n "$app" ] || continue + flyctl apps destroy "$app" --yes > "artifacts/staging-debug/commands/cleanup-${app}.log" 2>&1 || failed=1 + done + [ "$failed" -eq 0 ] || { echo 'Ephemeral cleanup failed; operator action required.' >&2; exit 1; } + + - name: Upload complete staging-debug evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: ross-staging-debug-${{ github.run_id }}-${{ github.run_attempt }} + path: artifacts/staging-debug + if-no-files-found: error + retention-days: 30 + +# Intentionally no production environment, production credentials, promotion input, or promotion job. diff --git a/docs/staging-debug-release-train.md b/docs/staging-debug-release-train.md new file mode 100644 index 0000000000..368f0b1aa7 --- /dev/null +++ b/docs/staging-debug-release-train.md @@ -0,0 +1,25 @@ +# Staging release-train debugging + +Use **ROSS staging release-train debug** to reproduce a release failure without +touching the public-beta deployment. The workflow creates three run-scoped Fly +apps, uses only secrets from the protected `staging-debug` environment, disables +sign-ups and scan dispatch, and rejects missing or production-equal data origins. + +Configure `STAGING_FLY_API_TOKEN`, a dedicated staging Supabase project, and a +dedicated staging S3-compatible bucket/endpoint in that environment. Set +`STAGING_FLY_ORG`; do not copy production credentials into any `STAGING_*` +secret. Environment approval should be limited to release operators. + +The job runs the complete repository gate, builds immutable image digests, +deploys worker, API, and web separately, and captures command output plus Fly +status and logs after every stage. It then creates a real, deliberately invalid +web runtime release, proves that failure through the deployed runtime-config +endpoint, rolls Fly back to the recorded known-good release version, and probes +the deployed endpoint again to verify recovery. Its final `always()` path +collects failure diagnostics, destroys every run-scoped app, and uploads the +evidence for 30 days. A cleanup failure fails the job and requires an operator +to destroy the names recorded in `isolation.txt`. + +The workflow has read-only repository permission and deliberately contains no +production environment, production secret, promotion input, tag, release, or +deployment step. It must never be repurposed for production promotion. diff --git a/scripts/lib/staging-debug.mjs b/scripts/lib/staging-debug.mjs new file mode 100644 index 0000000000..2ed3e84340 --- /dev/null +++ b/scripts/lib/staging-debug.mjs @@ -0,0 +1,35 @@ +const APP_PATTERN = /^[a-z0-9][a-z0-9-]{2,61}[a-z0-9]$/; + +export function stagingDebugNames(runId, attempt = "1") { + if (!/^[1-9][0-9]*$/.test(String(runId)) || !/^[1-9][0-9]*$/.test(String(attempt))) { + throw new Error("GitHub run ID and attempt must be positive integers."); + } + const suffix = `debug-${runId}-${attempt}`; + return { + api: `ross-api-${suffix}`, + web: `ross-web-${suffix}`, + worker: `ross-worker-${suffix}`, + }; +} + +export function assertIsolatedStaging({ apps, productionApps = [], resources }) { + const values = Object.values(apps); + if (values.length !== 3 || new Set(values).size !== 3) { + throw new Error("Debug API, web, and worker apps must be distinct."); + } + for (const app of values) { + if (!APP_PATTERN.test(app) || !/-debug-[1-9][0-9]*-[1-9][0-9]*$/.test(app)) { + throw new Error(`${app} is not an ephemeral staging-debug app.`); + } + if (productionApps.includes(app)) throw new Error(`${app} is a production app.`); + } + for (const name of ["supabaseUrl", "storageEndpoint"]) { + const value = resources[name]; + if (!String(value ?? "").trim()) throw new Error(`Missing isolated staging resource: ${name}.`); + } + if ((resources.productionSupabaseUrl && resources.supabaseUrl === resources.productionSupabaseUrl) || + (resources.productionStorageEndpoint && resources.storageEndpoint === resources.productionStorageEndpoint)) { + throw new Error("Staging data resources must not equal production resources."); + } + return { apps, resources }; +} diff --git a/scripts/validate-staging-debug.mjs b/scripts/validate-staging-debug.mjs new file mode 100644 index 0000000000..6167fa1a06 --- /dev/null +++ b/scripts/validate-staging-debug.mjs @@ -0,0 +1,22 @@ +#!/usr/bin/env node +import { assertIsolatedStaging, stagingDebugNames } from "./lib/staging-debug.mjs"; + +const required = (name) => { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`Required staging-debug value is missing: ${name}`); + return value; +}; +const apps = stagingDebugNames(required("GITHUB_RUN_ID"), required("GITHUB_RUN_ATTEMPT")); +assertIsolatedStaging({ + apps, + productionApps: ["ross-ranadeoss-api", "ross-ranadeoss-public", "ross-ranadeoss-file-worker"], + resources: { + supabaseUrl: required("ROSS_STAGING_SUPABASE_URL"), + storageEndpoint: required("ROSS_STAGING_S3_ENDPOINT_URL"), + productionSupabaseUrl: process.env.ROSS_PRODUCTION_SUPABASE_URL, + productionStorageEndpoint: process.env.ROSS_PRODUCTION_S3_ENDPOINT_URL, + }, +}); +for (const [component, app] of Object.entries(apps)) { + console.log(`${component.toUpperCase()}_APP=${app}`); +} diff --git a/tests/baseline/ross-staging-debug.test.mjs b/tests/baseline/ross-staging-debug.test.mjs new file mode 100644 index 0000000000..4017b56fe0 --- /dev/null +++ b/tests/baseline/ross-staging-debug.test.mjs @@ -0,0 +1,28 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { assertIsolatedStaging, stagingDebugNames } from "../../scripts/lib/staging-debug.mjs"; + +test("staging debug names are run-scoped and reject production overlap", () => { + const apps = stagingDebugNames("123", "2"); + assert.deepEqual(apps, { api: "ross-api-debug-123-2", web: "ross-web-debug-123-2", worker: "ross-worker-debug-123-2" }); + assert.throws(() => assertIsolatedStaging({ apps: { ...apps, api: "ross-ranadeoss-api" }, productionApps: ["ross-ranadeoss-api"], resources: { supabaseUrl: "s", storageEndpoint: "b" } }), /production|ephemeral/); +}); + +test("staging debug requires data resources isolated from production", () => { + assert.doesNotThrow(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), resources: { supabaseUrl: "stage-db", storageEndpoint: "stage-store" } })); + assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), resources: { supabaseUrl: "same", productionSupabaseUrl: "same", storageEndpoint: "stage", productionStorageEndpoint: "prod" } }), /must not equal production/); +}); + +test("debug workflow is diagnostic, cleanup-safe, and cannot promote", () => { + const workflow = readFileSync(new URL("../../.github/workflows/staging-debug-release-train.yml", import.meta.url), "utf8"); + assert.match(workflow, /Run complete repository gate/); + assert.match(workflow, /Collect failure diagnostics[\s\S]*if: failure\(\)/); + assert.match(workflow, /Destroy all ephemeral staging resources[\s\S]*if: always\(\)/); + assert.match(workflow, /Upload complete staging-debug evidence[\s\S]*if: always\(\)/); + assert.match(workflow, /set -euo pipefail[\s\S]*fly-deploy-with-retry/); + assert.match(workflow, /releases rollback "\$baseline_version"/); + assert.match(workflow, /forced-debug-failure[\s\S]*web-rollback-probe/); + assert.match(workflow, /STAGING_SUPABASE_URL/); + assert.doesNotMatch(workflow, /promote_public|fly-release-train\.mjs promote|environment: public-beta|ROSS_SUPABASE_SECRET_KEY/); +}); From 75202a052bfa4d15368c4246055d4a4f602b730a Mon Sep 17 00:00:00 2001 From: ranade-oss Date: Sun, 26 Jul 2026 16:47:05 -0400 Subject: [PATCH 2/5] Separate staging image build namespaces --- .../workflows/staging-debug-release-train.yml | 5 +- scripts/build-release-train-images.sh | 48 +++++++++++++------ tests/baseline/ross-staging-debug.test.mjs | 9 +++- 3 files changed, 46 insertions(+), 16 deletions(-) diff --git a/.github/workflows/staging-debug-release-train.yml b/.github/workflows/staging-debug-release-train.yml index 9cc8fe2e9f..9a0a1dbe5a 100644 --- a/.github/workflows/staging-debug-release-train.yml +++ b/.github/workflows/staging-debug-release-train.yml @@ -54,7 +54,10 @@ jobs: - name: Build candidate images in the isolated registry namespaces shell: bash run: | - export PROD_API_APP="$API_APP" PROD_WEB_APP="$WEB_APP" PROD_WORKER_APP="$WORKER_APP" STAGE_API_APP="$API_APP" + set -euo pipefail + export RELEASE_IMAGE_API_APP="$API_APP" RELEASE_IMAGE_WEB_APP="$WEB_APP" RELEASE_IMAGE_WORKER_APP="$WORKER_APP" + export RELEASE_RUNTIME_API_APP="$API_APP" RELEASE_RUNTIME_WEB_APP="$WEB_APP" RELEASE_REHEARSAL_API_APP="$API_APP" + export RELEASE_SIGNUPS_ENABLED=false export ROSS_SUPABASE_URL="$ROSS_STAGING_SUPABASE_URL" ROSS_SUPABASE_PUBLISHABLE_KEY="$ROSS_STAGING_SUPABASE_PUBLISHABLE_KEY" export ROSS_RELEASE_ID="staging-debug-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" bash scripts/build-release-train-images.sh diff --git a/scripts/build-release-train-images.sh b/scripts/build-release-train-images.sh index dd689892c3..841cc3a546 100644 --- a/scripts/build-release-train-images.sh +++ b/scripts/build-release-train-images.sh @@ -7,10 +7,6 @@ cd "$ROOT" required=( FLY_API_TOKEN - PROD_API_APP - PROD_WEB_APP - PROD_WORKER_APP - STAGE_API_APP ROSS_SUPABASE_URL ROSS_SUPABASE_PUBLISHABLE_KEY ROSS_RELEASE_ID @@ -25,6 +21,30 @@ for name in "${required[@]}"; do fi done +# Production releases retain the historical defaults. Staging-debug callers +# must supply isolated registry and runtime app names explicitly, without +# masquerading as production through PROD_* variables. +image_api_app="${RELEASE_IMAGE_API_APP:-${PROD_API_APP:-}}" +image_web_app="${RELEASE_IMAGE_WEB_APP:-${PROD_WEB_APP:-}}" +image_worker_app="${RELEASE_IMAGE_WORKER_APP:-${PROD_WORKER_APP:-}}" +runtime_api_app="${RELEASE_RUNTIME_API_APP:-${PROD_API_APP:-}}" +runtime_web_app="${RELEASE_RUNTIME_WEB_APP:-${PROD_WEB_APP:-}}" +rehearsal_api_app="${RELEASE_REHEARSAL_API_APP:-${STAGE_API_APP:-}}" +for item in \ + "RELEASE_IMAGE_API_APP:${image_api_app}" \ + "RELEASE_IMAGE_WEB_APP:${image_web_app}" \ + "RELEASE_IMAGE_WORKER_APP:${image_worker_app}" \ + "RELEASE_RUNTIME_API_APP:${runtime_api_app}" \ + "RELEASE_RUNTIME_WEB_APP:${runtime_web_app}" \ + "RELEASE_REHEARSAL_API_APP:${rehearsal_api_app}"; do + name="${item%%:*}" + value="${item#*:}" + if [ -z "$value" ]; then + echo "Required release image value is missing: ${name}" >&2 + exit 2 + fi +done + mkdir -p artifacts/release-train-build flyctl auth docker >/dev/null @@ -37,14 +57,14 @@ fi short_sha="${GITHUB_SHA:0:12}" label_base="ross-${short_sha}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" -api_tag="registry.fly.io/${PROD_API_APP}:${label_base}-api" -web_tag="registry.fly.io/${PROD_WEB_APP}:${label_base}-web" -worker_tag="registry.fly.io/${PROD_WORKER_APP}:${label_base}-worker" +api_tag="registry.fly.io/${image_api_app}:${label_base}-api" +web_tag="registry.fly.io/${image_web_app}:${label_base}-web" +worker_tag="registry.fly.io/${image_worker_app}:${label_base}-worker" build_api() { flyctl deploy . \ --config deploy/fly/api.toml \ - --app "$PROD_API_APP" \ + --app "$image_api_app" \ --build-only \ --push \ --remote-only \ @@ -56,7 +76,7 @@ build_api() { build_worker() { flyctl deploy . \ --config deploy/fly/file-worker.toml \ - --app "$PROD_WORKER_APP" \ + --app "$image_worker_app" \ --build-only \ --push \ --remote-only \ @@ -67,7 +87,7 @@ build_worker() { build_web() { flyctl deploy . \ --config deploy/fly/frontend.toml \ - --app "$PROD_WEB_APP" \ + --app "$image_web_app" \ --build-only \ --push \ --remote-only \ @@ -75,15 +95,15 @@ build_web() { --image-label "${label_base}-web" \ --build-arg "NEXT_PUBLIC_SUPABASE_URL=${ROSS_SUPABASE_URL}" \ --build-arg "NEXT_PUBLIC_SUPABASE_PUBLISHABLE_DEFAULT_KEY=${ROSS_SUPABASE_PUBLISHABLE_KEY}" \ - --build-arg "NEXT_PUBLIC_API_BASE_URL=https://${PROD_API_APP}.fly.dev" \ - --build-arg "NEXT_PUBLIC_REHEARSAL_API_BASE_URL=https://${STAGE_API_APP}.fly.dev" \ - --build-arg "NEXT_PUBLIC_ROSS_APP_URL=https://${PROD_WEB_APP}.fly.dev" \ + --build-arg "NEXT_PUBLIC_API_BASE_URL=https://${runtime_api_app}.fly.dev" \ + --build-arg "NEXT_PUBLIC_REHEARSAL_API_BASE_URL=https://${rehearsal_api_app}.fly.dev" \ + --build-arg "NEXT_PUBLIC_ROSS_APP_URL=https://${runtime_web_app}.fly.dev" \ --build-arg "NEXT_PUBLIC_ROSS_WEBSITE_URL=https://ross-ontario.augustmaat.chatgpt.site" \ --build-arg "NEXT_PUBLIC_ROSS_HOSTED_MODE=controlled-beta" \ --build-arg "NEXT_PUBLIC_ROSS_DATA_BOUNDARY_VERSION=2026-07-17-public-beta" \ --build-arg "NEXT_PUBLIC_ROSS_TERMS_VERSION=2026-07-17-public-beta" \ --build-arg "NEXT_PUBLIC_ROSS_PRIVACY_VERSION=2026-07-17-public-beta" \ - --build-arg "NEXT_PUBLIC_ROSS_SIGNUPS_ENABLED=true" \ + --build-arg "NEXT_PUBLIC_ROSS_SIGNUPS_ENABLED=${RELEASE_SIGNUPS_ENABLED:-true}" \ --build-arg "ROSS_BUILD_RELEASE_ID=${ROSS_RELEASE_ID}" } diff --git a/tests/baseline/ross-staging-debug.test.mjs b/tests/baseline/ross-staging-debug.test.mjs index 4017b56fe0..229012e070 100644 --- a/tests/baseline/ross-staging-debug.test.mjs +++ b/tests/baseline/ross-staging-debug.test.mjs @@ -24,5 +24,12 @@ test("debug workflow is diagnostic, cleanup-safe, and cannot promote", () => { assert.match(workflow, /releases rollback "\$baseline_version"/); assert.match(workflow, /forced-debug-failure[\s\S]*web-rollback-probe/); assert.match(workflow, /STAGING_SUPABASE_URL/); - assert.doesNotMatch(workflow, /promote_public|fly-release-train\.mjs promote|environment: public-beta|ROSS_SUPABASE_SECRET_KEY/); + assert.doesNotMatch(workflow, /promote_public|fly-release-train\.mjs promote|environment: public-beta|ROSS_SUPABASE_SECRET_KEY|PROD_[A-Z_]+=/); +}); + +test("image builds accept explicit isolated namespaces without production aliases", () => { + const build = readFileSync(new URL("../../scripts/build-release-train-images.sh", import.meta.url), "utf8"); + assert.match(build, /RELEASE_IMAGE_API_APP/); + assert.match(build, /RELEASE_RUNTIME_WEB_APP/); + assert.match(build, /RELEASE_SIGNUPS_ENABLED/); }); From b6ddf77c579c8334ab1041a7f4aa16692426ae65 Mon Sep 17 00:00:00 2001 From: ranade-oss Date: Sun, 26 Jul 2026 17:09:32 -0400 Subject: [PATCH 3/5] Address staging debug review findings --- .../workflows/staging-debug-release-train.yml | 53 ++++-- config/release-manifest.v1.json | 7 + docs/staging-debug-release-train.md | 15 +- reports/release-manifest-v1.json | 49 ++++- scripts/fly-release-train.mjs | 168 +----------------- scripts/lib/release-train-probe.mjs | 165 +++++++++++++++++ scripts/lib/staging-debug.mjs | 8 + scripts/run-staging-debug-probe.mjs | 58 ++++++ scripts/validate-staging-debug.mjs | 10 +- tests/baseline/ross-release-train.test.mjs | 27 ++- tests/baseline/ross-staging-debug.test.mjs | 23 ++- 11 files changed, 365 insertions(+), 218 deletions(-) create mode 100644 scripts/lib/release-train-probe.mjs create mode 100644 scripts/run-staging-debug-probe.mjs diff --git a/.github/workflows/staging-debug-release-train.yml b/.github/workflows/staging-debug-release-train.yml index 9a0a1dbe5a..c3bc5062f1 100644 --- a/.github/workflows/staging-debug-release-train.yml +++ b/.github/workflows/staging-debug-release-train.yml @@ -25,6 +25,11 @@ jobs: ROSS_STAGING_S3_REGION: ${{ secrets.STAGING_S3_REGION }} ROSS_STAGING_S3_ACCESS_KEY_ID: ${{ secrets.STAGING_S3_ACCESS_KEY_ID }} ROSS_STAGING_S3_SECRET_ACCESS_KEY: ${{ secrets.STAGING_S3_SECRET_ACCESS_KEY }} + ROSS_PRODUCTION_API_APP: ${{ vars.PRODUCTION_API_APP }} + ROSS_PRODUCTION_WEB_APP: ${{ vars.PRODUCTION_WEB_APP }} + ROSS_PRODUCTION_WORKER_APP: ${{ vars.PRODUCTION_WORKER_APP }} + ROSS_PRODUCTION_SUPABASE_URL: ${{ vars.PRODUCTION_SUPABASE_URL }} + ROSS_PRODUCTION_S3_ENDPOINT_URL: ${{ vars.PRODUCTION_S3_ENDPOINT_URL }} steps: - uses: actions/checkout@v7 with: { fetch-depth: 0 } @@ -60,6 +65,7 @@ jobs: export RELEASE_SIGNUPS_ENABLED=false export ROSS_SUPABASE_URL="$ROSS_STAGING_SUPABASE_URL" ROSS_SUPABASE_PUBLISHABLE_KEY="$ROSS_STAGING_SUPABASE_PUBLISHABLE_KEY" export ROSS_RELEASE_ID="staging-debug-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "ROSS_STAGING_DEBUG_RELEASE_ID=$ROSS_RELEASE_ID" >> "$GITHUB_ENV" bash scripts/build-release-train-images.sh - name: Configure staging-only service dependencies @@ -69,8 +75,8 @@ jobs: worker_secret="$(openssl rand -hex 32)"; echo "::add-mask::$worker_secret" signing_secret="$(openssl rand -hex 32)"; echo "::add-mask::$signing_secret" flyctl secrets set --stage --app "$WORKER_APP" "FILE_WORKER_SHARED_SECRET=$worker_secret" "FILE_WORKER_STORAGE_ORIGINS=$ROSS_STAGING_S3_ENDPOINT_URL" - flyctl secrets set --stage --app "$API_APP" "SUPABASE_URL=$ROSS_STAGING_SUPABASE_URL" "SUPABASE_SECRET_KEY=$ROSS_STAGING_SUPABASE_SECRET_KEY" "R2_ENDPOINT_URL=$ROSS_STAGING_S3_ENDPOINT_URL" "R2_REGION=$ROSS_STAGING_S3_REGION" "R2_ACCESS_KEY_ID=$ROSS_STAGING_S3_ACCESS_KEY_ID" "R2_SECRET_ACCESS_KEY=$ROSS_STAGING_S3_SECRET_ACCESS_KEY" "R2_BUCKET_NAME=ross-staging-debug" "FILE_WORKER_URL=http://${WORKER_APP}.flycast" "FILE_WORKER_SHARED_SECRET=$worker_secret" "DOWNLOAD_SIGNING_SECRET=$signing_secret" "USER_API_KEYS_ENCRYPTION_SECRET=$signing_secret" "MCP_CONNECTORS_ENCRYPTION_SECRET=$signing_secret" "ROSS_ENV=staging" "ROSS_HOSTED_MODE=controlled-beta" "HOSTED_MODEL_PROVIDERS=openai" "ROSS_DISABLE_DOCUMENT_SCAN_DISPATCHER=true" "ROSS_UPLOAD_SCAN_REQUIRED=false" "CORS_ALLOWED_ORIGINS=https://${WEB_APP}.fly.dev" "FRONTEND_URL=https://${WEB_APP}.fly.dev" "API_PUBLIC_URL=https://${API_APP}.fly.dev" - flyctl secrets set --stage --app "$WEB_APP" "ROSS_RUNTIME_API_BASE_URL=https://${API_APP}.fly.dev" "ROSS_RUNTIME_APP_URL=https://${WEB_APP}.fly.dev" "ROSS_RUNTIME_SIGNUPS_ENABLED=false" "ROSS_RUNTIME_ENVIRONMENT=staging-debug" + flyctl secrets set --stage --app "$API_APP" "SUPABASE_URL=$ROSS_STAGING_SUPABASE_URL" "SUPABASE_SECRET_KEY=$ROSS_STAGING_SUPABASE_SECRET_KEY" "R2_ENDPOINT_URL=$ROSS_STAGING_S3_ENDPOINT_URL" "R2_REGION=$ROSS_STAGING_S3_REGION" "R2_ACCESS_KEY_ID=$ROSS_STAGING_S3_ACCESS_KEY_ID" "R2_SECRET_ACCESS_KEY=$ROSS_STAGING_S3_SECRET_ACCESS_KEY" "R2_BUCKET_NAME=ross-staging-debug" "FILE_WORKER_URL=http://${WORKER_APP}.flycast" "FILE_WORKER_SHARED_SECRET=$worker_secret" "DOWNLOAD_SIGNING_SECRET=$signing_secret" "USER_API_KEYS_ENCRYPTION_SECRET=$signing_secret" "MCP_CONNECTORS_ENCRYPTION_SECRET=$signing_secret" "ROSS_ENV=staging" "ROSS_HOSTED_MODE=controlled-beta" "HOSTED_MODEL_PROVIDERS=openai" "ROSS_RUNTIME_RELEASE_ID=$ROSS_STAGING_DEBUG_RELEASE_ID" "ROSS_DISABLE_DOCUMENT_SCAN_DISPATCHER=true" "ROSS_UPLOAD_SCAN_REQUIRED=false" "CORS_ALLOWED_ORIGINS=https://${WEB_APP}.fly.dev" "FRONTEND_URL=https://${WEB_APP}.fly.dev" "API_PUBLIC_URL=https://${API_APP}.fly.dev" + flyctl secrets set --stage --app "$WEB_APP" "ROSS_RUNTIME_API_BASE_URL=https://${API_APP}.fly.dev" "ROSS_RUNTIME_APP_URL=https://${WEB_APP}.fly.dev" "ROSS_RUNTIME_SIGNUPS_ENABLED=false" "ROSS_RUNTIME_ENVIRONMENT=staging-debug" "ROSS_RUNTIME_RELEASE_ID=$ROSS_STAGING_DEBUG_RELEASE_ID" - name: Deploy and diagnose worker stage shell: bash @@ -94,31 +100,38 @@ jobs: flyctl status --app "$WEB_APP" --json > artifacts/staging-debug/diagnostics/web-candidate-status.json flyctl logs --app "$WEB_APP" --no-tail > artifacts/staging-debug/diagnostics/web-candidate.log 2>&1 || true - - name: Inject a staging web release failure and test real Fly rollback + - name: Run exact complete release-train integration probe + run: node scripts/run-staging-debug-probe.mjs staging-debug 2>&1 | tee artifacts/staging-debug/diagnostics/full-probe-before-failure.log + + - name: Inject a genuine deployment failure and restore baseline shell: bash run: | set -euo pipefail baseline_version="$(flyctl releases --app "$WEB_APP" --json | tee artifacts/staging-debug/diagnostics/web-releases-before-failure.json | jq -er '.[0].Version // .[0].version')" printf '%s\n' "$baseline_version" > artifacts/staging-debug/diagnostics/web-rollback-target.txt - # Create an actual bad Fly release using the same immutable image. The - # runtime probe makes the injected failure observable before rollback. - flyctl secrets set --stage --app "$WEB_APP" ROSS_RUNTIME_ENVIRONMENT=forced-debug-failure - flyctl deploy . --config deploy/fly/rehearsal-frontend.toml --app "$WEB_APP" --image "$CANDIDATE_WEB_IMAGE" --ha=false --yes --flycast --no-public-ips 2>&1 | tee artifacts/staging-debug/commands/web-forced-failure.log - machine_id="$(flyctl machine list --app "$WEB_APP" --json | jq -er '.[0].id // .[0].ID')" - flyctl machine start "$machine_id" --app "$WEB_APP" || true - flyctl machine wait "$machine_id" --app "$WEB_APP" --state started --wait-timeout 2m - probe='fetch("http://127.0.0.1:3000/api/runtime-config").then(r=>r.json()).then(x=>{console.log(JSON.stringify(x));if(x.environment!=="forced-debug-failure")process.exit(1)})' - flyctl ssh console --app "$WEB_APP" --machine "$machine_id" --command "node -e '$probe'" | tee artifacts/staging-debug/diagnostics/web-forced-failure-probe.json + failure_config=deploy/fly/staging-debug-forced-failure.toml + trap 'rm -f "$failure_config"' EXIT + cp deploy/fly/rehearsal-frontend.toml "$failure_config" + sed -i 's/internal_port = 3000/internal_port = 9/' "$failure_config" + cp "$failure_config" artifacts/staging-debug/diagnostics/forced-failure.toml + if flyctl deploy . --config "$failure_config" --app "$WEB_APP" --image "$CANDIDATE_WEB_IMAGE" --ha=false --yes --flycast --no-public-ips > artifacts/staging-debug/commands/web-forced-failure.log 2>&1; then + echo "The deliberately invalid deployment unexpectedly succeeded." >&2 + exit 1 + fi + echo '{"expectedDeploymentFailureObserved":true}' > artifacts/staging-debug/diagnostics/forced-failure-result.json + flyctl status --app "$WEB_APP" --json > artifacts/staging-debug/diagnostics/web-after-failed-deploy-status.json 2>&1 || true + flyctl logs --app "$WEB_APP" --no-tail > artifacts/staging-debug/diagnostics/web-after-failed-deploy.log 2>&1 || true flyctl releases rollback "$baseline_version" --app "$WEB_APP" --yes 2>&1 | tee artifacts/staging-debug/commands/web-rollback.log - machine_id="$(flyctl machine list --app "$WEB_APP" --json | jq -er '.[0].id // .[0].ID')" - flyctl machine start "$machine_id" --app "$WEB_APP" || true - flyctl machine wait "$machine_id" --app "$WEB_APP" --state started --wait-timeout 2m - probe='fetch("http://127.0.0.1:3000/api/runtime-config").then(r=>r.json()).then(x=>{console.log(JSON.stringify(x));if(x.environment!=="staging-debug")process.exit(1)})' - flyctl ssh console --app "$WEB_APP" --machine "$machine_id" --command "node -e '$probe'" | tee artifacts/staging-debug/diagnostics/web-rollback-probe.json flyctl releases --app "$WEB_APP" --json > artifacts/staging-debug/diagnostics/web-releases-after-rollback.json - printf '{"status":"passed","failureInjection":"observed","rollback":"verified","productionPromotion":false}\n' > artifacts/staging-debug/result.json + + - name: Verify full integration recovery after rollback + shell: bash + run: | + set -euo pipefail + node scripts/run-staging-debug-probe.mjs staging-debug 2>&1 | tee artifacts/staging-debug/diagnostics/full-probe-after-rollback.log + printf '{"status":"passed","genuineDeploymentFailure":"observed","rollback":"verified","fullProbe":"passed","productionPromotion":false}\n' > artifacts/staging-debug/result.json - name: Collect failure diagnostics if: failure() @@ -137,6 +150,10 @@ jobs: failed=0 for app in "${WORKER_APP:-}" "${API_APP:-}" "${WEB_APP:-}"; do [ -n "$app" ] || continue + if ! flyctl status --app "$app" >/dev/null 2>&1; then + printf 'App %s was not provisioned; nothing to destroy.\n' "$app" > "artifacts/staging-debug/commands/cleanup-${app}.log" + continue + fi flyctl apps destroy "$app" --yes > "artifacts/staging-debug/commands/cleanup-${app}.log" 2>&1 || failed=1 done [ "$failed" -eq 0 ] || { echo 'Ephemeral cleanup failed; operator action required.' >&2; exit 1; } diff --git a/config/release-manifest.v1.json b/config/release-manifest.v1.json index 7374f10a7c..6c3dbce268 100644 --- a/config/release-manifest.v1.json +++ b/config/release-manifest.v1.json @@ -10,6 +10,7 @@ ".github/workflows/final-controlled-beta-evidence.yml", ".github/workflows/refresh-release-manifest.yml", ".github/workflows/release-candidate.yml", + ".github/workflows/staging-debug-release-train.yml", ".github/workflows/verify-and-deploy-public-beta.yml", ".github/workflows/verify-ontario-sources.yml", "backend/migrations/20260718_01_document_scan_pipeline.sql", @@ -70,6 +71,7 @@ "docs/privacy/subprocessor-inventory.json", "docs/evaluation/capability-limitations-v1.md", "docs/deployment/release-train-v1.md", + "docs/staging-debug-release-train.md", "frontend/package-lock.json", "frontend/package.json", "frontend/src/app/components/projects/ProjectDocumentsView.tsx", @@ -101,21 +103,26 @@ "scripts/lib/release-readiness.mjs", "scripts/lib/live-source-observer.mjs", "scripts/lib/release-train.mjs", + "scripts/lib/release-train-probe.mjs", + "scripts/lib/staging-debug.mjs", "scripts/build-release-train-images.sh", "scripts/fly-deploy-with-retry.sh", "scripts/fly-release-train.mjs", "scripts/observe-legal-sources.mjs", "scripts/preflight-fly-images.sh", "scripts/release-train-image-ref.mjs", + "scripts/run-staging-debug-probe.mjs", "scripts/run-backup-restore-exercise.sh", "scripts/lib/professional-validation.mjs", "scripts/lib/release-identifier.mjs", "scripts/lib/final-completion.mjs", "scripts/validate-release-id.mjs", + "scripts/validate-staging-debug.mjs", "tests/baseline/ross-ci-toolchain.test.mjs", "tests/baseline/ross-delivery-d.test.mjs", "tests/baseline/ross-production-readiness.test.mjs", "tests/baseline/ross-release-train.test.mjs", + "tests/baseline/ross-staging-debug.test.mjs", "website/package-lock.json", "website/package.json", "website/app/site-config.ts", diff --git a/docs/staging-debug-release-train.md b/docs/staging-debug-release-train.md index 368f0b1aa7..dce04a97e6 100644 --- a/docs/staging-debug-release-train.md +++ b/docs/staging-debug-release-train.md @@ -8,14 +8,19 @@ sign-ups and scan dispatch, and rejects missing or production-equal data origins Configure `STAGING_FLY_API_TOKEN`, a dedicated staging Supabase project, and a dedicated staging S3-compatible bucket/endpoint in that environment. Set `STAGING_FLY_ORG`; do not copy production credentials into any `STAGING_*` -secret. Environment approval should be limited to release operators. +secret. Configure all five non-secret `PRODUCTION_*` comparison variables for +the three production app names, Supabase URL, and storage endpoint. Validation +fails closed if any comparison identifier is absent or equals its staging +counterpart. Environment approval should be limited to release operators. The job runs the complete repository gate, builds immutable image digests, deploys worker, API, and web separately, and captures command output plus Fly -status and logs after every stage. It then creates a real, deliberately invalid -web runtime release, proves that failure through the deployed runtime-config -endpoint, rolls Fly back to the recorded known-good release version, and probes -the deployed endpoint again to verify recovery. Its final `always()` path +status and logs after every stage. It runs the exact complete integration probe +used by the release train, then attempts a deliberately invalid web deployment +whose unreachable service port must make `flyctl deploy` fail. The job asserts +that nonzero result, records diagnostics, rolls Fly back to the recorded +known-good release version, and reruns the complete probe to verify recovery. +Its final `always()` path collects failure diagnostics, destroys every run-scoped app, and uploads the evidence for 30 days. A cleanup failure fails the job and requires an operator to destroy the names recorded in `isolation.txt`. diff --git a/reports/release-manifest-v1.json b/reports/release-manifest-v1.json index de6a8273f8..ed7c6e955b 100644 --- a/reports/release-manifest-v1.json +++ b/reports/release-manifest-v1.json @@ -3,7 +3,7 @@ "releaseId": "ross-public-beta-20260717-rc1", "generatedAt": "2026-07-26T10:39:57.000Z", "algorithm": "sha256", - "artifactCount": 119, + "artifactCount": 126, "artifacts": [ { "path": ".github/actions/setup-ross-node/action.yml", @@ -40,6 +40,11 @@ "sha256": "b155ce7cc7a54f884e20f6a164929d8851652793725db8e2e0df18bbc7cf0511", "sizeBytes": 3027 }, + { + "path": ".github/workflows/staging-debug-release-train.yml", + "sha256": "57224d82cd3a095b1ecf1048e1cf28ecdb8d5dce65366b38a21ce1c87ea36f0b", + "sizeBytes": 10651 + }, { "path": ".github/workflows/verify-and-deploy-public-beta.yml", "sha256": "69253489fffdf6b220f6ffea764c30e46510dd97730fb23e85400c9eaeba6c11", @@ -340,6 +345,11 @@ "sha256": "d6a62d767e79c915fbeecd1754351e376a251f4eaf5a516ec023ff9b83eb4b29", "sizeBytes": 5556 }, + { + "path": "docs/staging-debug-release-train.md", + "sha256": "dff475f33b4255cc07853a6ea26063bd5990a1317e02c164bdd73456d42bc300", + "sizeBytes": 1892 + }, { "path": "frontend/package-lock.json", "sha256": "78c5149b7d30f1d3e39e545a995e8bf537a1368b302fd8ca452ddaffbe811fef", @@ -495,10 +505,20 @@ "sha256": "7b08133586b241fa727c0a255ec6f1873c2ba7f3fdc85b195e8c21dc0544cfdb", "sizeBytes": 4775 }, + { + "path": "scripts/lib/release-train-probe.mjs", + "sha256": "a9a238424999ea97a1f2fbd1d9b04ee8fe044da053403e6c2d46f94e0ce1f4a0", + "sizeBytes": 5182 + }, + { + "path": "scripts/lib/staging-debug.mjs", + "sha256": "a3bf56de870458f1142eaf1b843f47bfa78dbcd9b3fe75372c663fbf2f8c93b4", + "sizeBytes": 2022 + }, { "path": "scripts/build-release-train-images.sh", - "sha256": "6a3c18fd86d91861b0062c39e9206fd82cbf367e56952bcb4b7327cfb19fec57", - "sizeBytes": 4760 + "sha256": "777328e5428fa2c580854a957d1c64c46eb82164636e077ccb87e7d5c716bd02", + "sizeBytes": 5794 }, { "path": "scripts/fly-deploy-with-retry.sh", @@ -507,8 +527,8 @@ }, { "path": "scripts/fly-release-train.mjs", - "sha256": "731807a55f5e21267502071b679dc3a30f56c9c7fde0781efc4e68af0b37b2ff", - "sizeBytes": 32016 + "sha256": "71338c98da92f12dfc244c3eb49d74b00cde169e5b8cdfbe5597ec75954199d0", + "sizeBytes": 26940 }, { "path": "scripts/observe-legal-sources.mjs", @@ -525,6 +545,11 @@ "sha256": "17cef8f7cf2b22e4bcaf19c0766fbbfef916941e654ad1156a87ab7047e92b2e", "sizeBytes": 1896 }, + { + "path": "scripts/run-staging-debug-probe.mjs", + "sha256": "c49dee35e02aea23f421f2a0244703edef7f9a6632932a40e7261a97b9eee2d3", + "sizeBytes": 2207 + }, { "path": "scripts/run-backup-restore-exercise.sh", "sha256": "f2d0ee4cb495214f03db3e479ba3ff58de499d999abed62f72cf5c84bafd0a63", @@ -550,6 +575,11 @@ "sha256": "5eca3fbee3ddd6245497243e5f2690d351956af3d9d025afd5e2fadb10766cde", "sizeBytes": 301 }, + { + "path": "scripts/validate-staging-debug.mjs", + "sha256": "48b0646e15b8b173718943cf1ce961b2404fe765818e2cb533e3ed1c477dea6e", + "sizeBytes": 1000 + }, { "path": "tests/baseline/ross-ci-toolchain.test.mjs", "sha256": "31a3b311d6d68099452a3c4d6d7c35aadd7f2c6300a28fb1d4752ffbadc97ed6", @@ -567,8 +597,13 @@ }, { "path": "tests/baseline/ross-release-train.test.mjs", - "sha256": "012ab7b4d6b7b1513a0362d300c2333a477142564cb5a8acc6ea912e6779b4b4", - "sizeBytes": 27789 + "sha256": "2406c9046010025257a3bd9586f7821010a8e1b35717a66f7c4fbd12ed712720", + "sizeBytes": 27604 + }, + { + "path": "tests/baseline/ross-staging-debug.test.mjs", + "sha256": "807796a1bcfc66609a301c8c4160a68b17bfe593a4b408571d18a6160603f9a2", + "sizeBytes": 3704 }, { "path": "website/package-lock.json", diff --git a/scripts/fly-release-train.mjs b/scripts/fly-release-train.mjs index 7010d021cf..feec31d150 100644 --- a/scripts/fly-release-train.mjs +++ b/scripts/fly-release-train.mjs @@ -9,6 +9,7 @@ import { writeFileSync, } from "node:fs"; import { resolve } from "node:path"; +import { deployedReleaseTrainProbe } from "./lib/release-train-probe.mjs"; import { assertReleaseTrainAppNames, extractDigestImageRef, @@ -368,173 +369,8 @@ function verifySet(targetApps, images) { verifyImage(targetApps.web, images.web); } -const deployedProbe = ` -const [ - apiNetwork, - webNetwork, - workerNetwork, - publicApi, - publicWeb, - expectedEnvironment, - expectedRelease, - expectedSignups, - full -] = process.argv.slice(1); -const expect = (condition, message) => { - if (!condition) throw new Error(message); -}; -const wait = (milliseconds) => - new Promise((resolve) => setTimeout(resolve, milliseconds)); -const describeError = (error) => { - const parts = [ - error instanceof Error ? error.message : String(error), - error?.cause?.code, - error?.cause?.message, - ].filter(Boolean); - return [...new Set(parts)].join(": "); -}; -const retryableStatus = new Set([408, 425, 429, 500, 502, 503, 504]); -const request = async (label, url, options = {}) => { - let lastError = null; - for (let attempt = 1; attempt <= 12; attempt += 1) { - try { - const response = await fetch(url, { - ...options, - signal: AbortSignal.timeout(10000), - }); - if (!retryableStatus.has(response.status) || attempt === 12) { - return response; - } - lastError = new Error("HTTP " + response.status); - await response.body?.cancel(); - } catch (error) { - lastError = error; - } - if (attempt < 12) await wait(5000); - } - throw new Error( - label + " could not reach " + url + " after 12 attempts: " + - describeError(lastError), - ); -}; -const json = async (response, label) => { - expect(response.ok, label + " returned HTTP " + response.status); - return response.json(); -}; -const expectStatus = async (response, expected, label) => { - if (response.status === expected) return; - let detail = ""; - try { - detail = (await response.text()).trim().slice(0, 500); - } catch {} - throw new Error( - label + " expected HTTP " + expected + " but received HTTP " + - response.status + (detail ? ": " + detail : ""), - ); -}; -(async () => { - const health = await json( - await request("API health", apiNetwork + "/health"), - "API health", - ); - expect(health.ok === true && health.service === "ross-api", "API health contract failed"); - const login = await request("Web login", webNetwork + "/login", { - redirect: "manual", - }); - expect(login.status >= 200 && login.status < 400, "Web login route failed"); - const workerHealth = await json( - await request("Worker health", workerNetwork + "/health"), - "Worker health", - ); - expect( - workerHealth.ok === true && workerHealth.service === "ross-file-worker", - "Worker health contract failed", - ); - if (full !== "true") return; - expect(health.releaseId === expectedRelease, "API release identity mismatch"); - - const runtime = await json( - await request( - "Web runtime configuration", - webNetwork + "/api/runtime-config", - ), - "Web runtime configuration", - ); - expect(runtime.apiBaseUrl === publicApi, "Runtime API origin mismatch"); - expect(runtime.appUrl === publicWeb, "Runtime app origin mismatch"); - expect(runtime.environment === expectedEnvironment, "Runtime environment mismatch"); - expect(runtime.releaseId === expectedRelease, "Web release identity mismatch"); - expect( - runtime.signupsEnabled === (expectedSignups === "true"), - "Runtime signup policy mismatch", - ); - - const allowed = await request( - "Allowed CORS origin", - apiNetwork + "/health", - { headers: { Origin: publicWeb } }, - ); - expect(allowed.ok, "Allowed CORS origin failed"); - expect( - allowed.headers.get("access-control-allow-origin") === publicWeb, - "Allowed CORS origin was not echoed", - ); - const denied = await request( - "Denied CORS origin", - apiNetwork + "/health", - { headers: { Origin: "https://untrusted.example" } }, - ); - expect(denied.status === 403, "Untrusted CORS origin was not denied"); - - const protectedRead = await request( - "Protected document read", - apiNetwork + "/single-documents", - { headers: { Origin: publicWeb } }, - ); - await expectStatus(protectedRead, 401, "Authentication guard"); - const protectedUpload = await request( - "Protected document upload", - apiNetwork + "/single-documents", - { - method: "POST", - headers: { - Origin: publicWeb, - "Content-Type": "application/octet-stream", - "X-ROSS-Data-Boundary": "synthetic-or-non-confidential", - }, - body: new Uint8Array(), - }, - ); - await expectStatus(protectedUpload, 401, "Upload authentication guard"); - - const settings = await request( - "Supabase settings", - process.env.SUPABASE_URL.replace(/\\/$/, "") + "/auth/v1/settings", - { headers: { apikey: process.env.SUPABASE_SECRET_KEY } }, - ); - expect(settings.ok, "Supabase authentication configuration failed"); - - const workerAuth = await request( - "Worker authentication", - workerNetwork + "/process", - { - method: "POST", - headers: { - Authorization: "Bearer " + process.env.FILE_WORKER_SHARED_SECRET, - "Content-Type": "application/json", - }, - body: "{}", - }, - ); - expect(workerAuth.status === 400, "API-to-worker shared secret wiring failed"); -})().catch((error) => { - console.error(error instanceof Error ? error.message : String(error)); - process.exit(1); -}); -`; - function runRemoteProbe(app, machineId, args) { - const encoded = Buffer.from(deployedProbe).toString("base64"); + const encoded = Buffer.from(deployedReleaseTrainProbe).toString("base64"); const command = [ `node -e "eval(Buffer.from('${encoded}','base64').toString())"`, ...args.map((value) => JSON.stringify(value)), diff --git a/scripts/lib/release-train-probe.mjs b/scripts/lib/release-train-probe.mjs new file mode 100644 index 0000000000..7dd6bb7a30 --- /dev/null +++ b/scripts/lib/release-train-probe.mjs @@ -0,0 +1,165 @@ +export const deployedReleaseTrainProbe = ` +const [ + apiNetwork, + webNetwork, + workerNetwork, + publicApi, + publicWeb, + expectedEnvironment, + expectedRelease, + expectedSignups, + full +] = process.argv.slice(1); +const expect = (condition, message) => { + if (!condition) throw new Error(message); +}; +const wait = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); +const describeError = (error) => { + const parts = [ + error instanceof Error ? error.message : String(error), + error?.cause?.code, + error?.cause?.message, + ].filter(Boolean); + return [...new Set(parts)].join(": "); +}; +const retryableStatus = new Set([408, 425, 429, 500, 502, 503, 504]); +const request = async (label, url, options = {}) => { + let lastError = null; + for (let attempt = 1; attempt <= 12; attempt += 1) { + try { + const response = await fetch(url, { + ...options, + signal: AbortSignal.timeout(10000), + }); + if (!retryableStatus.has(response.status) || attempt === 12) { + return response; + } + lastError = new Error("HTTP " + response.status); + await response.body?.cancel(); + } catch (error) { + lastError = error; + } + if (attempt < 12) await wait(5000); + } + throw new Error( + label + " could not reach " + url + " after 12 attempts: " + + describeError(lastError), + ); +}; +const json = async (response, label) => { + expect(response.ok, label + " returned HTTP " + response.status); + return response.json(); +}; +const expectStatus = async (response, expected, label) => { + if (response.status === expected) return; + let detail = ""; + try { + detail = (await response.text()).trim().slice(0, 500); + } catch {} + throw new Error( + label + " expected HTTP " + expected + " but received HTTP " + + response.status + (detail ? ": " + detail : ""), + ); +}; +(async () => { + const health = await json( + await request("API health", apiNetwork + "/health"), + "API health", + ); + expect(health.ok === true && health.service === "ross-api", "API health contract failed"); + const login = await request("Web login", webNetwork + "/login", { + redirect: "manual", + }); + expect(login.status >= 200 && login.status < 400, "Web login route failed"); + const workerHealth = await json( + await request("Worker health", workerNetwork + "/health"), + "Worker health", + ); + expect( + workerHealth.ok === true && workerHealth.service === "ross-file-worker", + "Worker health contract failed", + ); + if (full !== "true") return; + expect(health.releaseId === expectedRelease, "API release identity mismatch"); + + const runtime = await json( + await request( + "Web runtime configuration", + webNetwork + "/api/runtime-config", + ), + "Web runtime configuration", + ); + expect(runtime.apiBaseUrl === publicApi, "Runtime API origin mismatch"); + expect(runtime.appUrl === publicWeb, "Runtime app origin mismatch"); + expect(runtime.environment === expectedEnvironment, "Runtime environment mismatch"); + expect(runtime.releaseId === expectedRelease, "Web release identity mismatch"); + expect( + runtime.signupsEnabled === (expectedSignups === "true"), + "Runtime signup policy mismatch", + ); + + const allowed = await request( + "Allowed CORS origin", + apiNetwork + "/health", + { headers: { Origin: publicWeb } }, + ); + expect(allowed.ok, "Allowed CORS origin failed"); + expect( + allowed.headers.get("access-control-allow-origin") === publicWeb, + "Allowed CORS origin was not echoed", + ); + const denied = await request( + "Denied CORS origin", + apiNetwork + "/health", + { headers: { Origin: "https://untrusted.example" } }, + ); + expect(denied.status === 403, "Untrusted CORS origin was not denied"); + + const protectedRead = await request( + "Protected document read", + apiNetwork + "/single-documents", + { headers: { Origin: publicWeb } }, + ); + await expectStatus(protectedRead, 401, "Authentication guard"); + const protectedUpload = await request( + "Protected document upload", + apiNetwork + "/single-documents", + { + method: "POST", + headers: { + Origin: publicWeb, + "Content-Type": "application/octet-stream", + "X-ROSS-Data-Boundary": "synthetic-or-non-confidential", + }, + body: new Uint8Array(), + }, + ); + await expectStatus(protectedUpload, 401, "Upload authentication guard"); + + const settings = await request( + "Supabase settings", + process.env.SUPABASE_URL.replace(/\\/$/, "") + "/auth/v1/settings", + { headers: { apikey: process.env.SUPABASE_SECRET_KEY } }, + ); + expect(settings.ok, "Supabase authentication configuration failed"); + + const workerAuth = await request( + "Worker authentication", + workerNetwork + "/process", + { + method: "POST", + headers: { + Authorization: "Bearer " + process.env.FILE_WORKER_SHARED_SECRET, + "Content-Type": "application/json", + }, + body: "{}", + }, + ); + expect(workerAuth.status === 400, "API-to-worker shared secret wiring failed"); +})().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); +`; + diff --git a/scripts/lib/staging-debug.mjs b/scripts/lib/staging-debug.mjs index 2ed3e84340..a8d922f34f 100644 --- a/scripts/lib/staging-debug.mjs +++ b/scripts/lib/staging-debug.mjs @@ -27,6 +27,14 @@ export function assertIsolatedStaging({ apps, productionApps = [], resources }) const value = resources[name]; if (!String(value ?? "").trim()) throw new Error(`Missing isolated staging resource: ${name}.`); } + for (const name of ["productionSupabaseUrl", "productionStorageEndpoint"]) { + if (!String(resources[name] ?? "").trim()) { + throw new Error(`Missing production comparison identifier: ${name}.`); + } + } + if (productionApps.length !== 3 || productionApps.some((app) => !String(app).trim())) { + throw new Error("All production app comparison identifiers are required."); + } if ((resources.productionSupabaseUrl && resources.supabaseUrl === resources.productionSupabaseUrl) || (resources.productionStorageEndpoint && resources.storageEndpoint === resources.productionStorageEndpoint)) { throw new Error("Staging data resources must not equal production resources."); diff --git a/scripts/run-staging-debug-probe.mjs b/scripts/run-staging-debug-probe.mjs new file mode 100644 index 0000000000..9dae83798a --- /dev/null +++ b/scripts/run-staging-debug-probe.mjs @@ -0,0 +1,58 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { deployedReleaseTrainProbe } from "./lib/release-train-probe.mjs"; + +const required = (name) => { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`Required staging probe value is missing: ${name}`); + return value; +}; +const apps = { + api: required("API_APP"), + web: required("WEB_APP"), + worker: required("WORKER_APP"), +}; +const expectedEnvironment = process.argv[2] ?? "staging-debug"; +const expectedRelease = required("ROSS_STAGING_DEBUG_RELEASE_ID"); + +function fly(args, { capture = false } = {}) { + const result = spawnSync("flyctl", args, { + encoding: "utf8", + stdio: capture ? ["ignore", "pipe", "pipe"] : "inherit", + }); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new Error(`flyctl ${args.join(" ")} failed (${result.status}): ${(result.stderr || result.stdout || "").trim().slice(-1000)}`); + } + return result.stdout; +} + +function start(app) { + const machines = JSON.parse(fly(["machine", "list", "--app", app, "--json"], { capture: true })); + if (!machines.length) throw new Error(`${app} has no deployed Machine.`); + for (const machine of machines) { + const id = machine.id ?? machine.ID; + const state = machine.state ?? machine.State; + if (state !== "started") fly(["machine", "start", id, "--app", app]); + fly(["machine", "wait", id, "--app", app, "--state", "started", "--wait-timeout", "2m"]); + } + return machines[0].id ?? machines[0].ID; +} + +start(apps.worker); +start(apps.web); +const apiMachine = start(apps.api); +const encoded = Buffer.from(deployedReleaseTrainProbe).toString("base64"); +const args = [ + `http://${apps.api}.flycast`, + `http://${apps.web}.flycast`, + `http://${apps.worker}.flycast`, + `https://${apps.api}.fly.dev`, + `https://${apps.web}.fly.dev`, + expectedEnvironment, + expectedRelease, + "false", + "true", +]; +const command = [`node -e "eval(Buffer.from('${encoded}','base64').toString())"`, ...args.map(JSON.stringify)].join(" "); +fly(["ssh", "console", "--app", apps.api, "--machine", apiMachine, "--command", command]); diff --git a/scripts/validate-staging-debug.mjs b/scripts/validate-staging-debug.mjs index 6167fa1a06..f47ffe7b27 100644 --- a/scripts/validate-staging-debug.mjs +++ b/scripts/validate-staging-debug.mjs @@ -9,12 +9,16 @@ const required = (name) => { const apps = stagingDebugNames(required("GITHUB_RUN_ID"), required("GITHUB_RUN_ATTEMPT")); assertIsolatedStaging({ apps, - productionApps: ["ross-ranadeoss-api", "ross-ranadeoss-public", "ross-ranadeoss-file-worker"], + productionApps: [ + required("ROSS_PRODUCTION_API_APP"), + required("ROSS_PRODUCTION_WEB_APP"), + required("ROSS_PRODUCTION_WORKER_APP"), + ], resources: { supabaseUrl: required("ROSS_STAGING_SUPABASE_URL"), storageEndpoint: required("ROSS_STAGING_S3_ENDPOINT_URL"), - productionSupabaseUrl: process.env.ROSS_PRODUCTION_SUPABASE_URL, - productionStorageEndpoint: process.env.ROSS_PRODUCTION_S3_ENDPOINT_URL, + productionSupabaseUrl: required("ROSS_PRODUCTION_SUPABASE_URL"), + productionStorageEndpoint: required("ROSS_PRODUCTION_S3_ENDPOINT_URL"), }, }); for (const [component, app] of Object.entries(apps)) { diff --git a/tests/baseline/ross-release-train.test.mjs b/tests/baseline/ross-release-train.test.mjs index 419fd63e5b..ac71252a60 100644 --- a/tests/baseline/ross-release-train.test.mjs +++ b/tests/baseline/ross-release-train.test.mjs @@ -19,18 +19,12 @@ import { nextPublicReleaseId, validateDigestImageRef, } from "../../scripts/lib/release-train.mjs"; +import { deployedReleaseTrainProbe } from "../../scripts/lib/release-train-probe.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); const read = (path) => readFileSync(resolve(root, path), "utf8"); const image = (app, character) => `registry.fly.io/${app}@sha256:${character.repeat(64)}`; -const deployedProbeSource = () => { - const match = read("scripts/fly-release-train.mjs").match( - /const deployedProbe = `([\s\S]*?)`;\n\nfunction runRemoteProbe/, - ); - assert.ok(match, "The exact embedded deployment probe must be extractable."); - return Function(`"use strict"; return \`${match[1]}\`;`)(); -}; test("release IDs are generated from the Toronto date and never reused", () => { assert.equal( @@ -178,6 +172,7 @@ test("the frontend uses public runtime configuration for staging parity", () => test("rehearsal is private, read-only, and cannot dispatch production jobs", () => { const train = read("scripts/fly-release-train.mjs"); + const probe = deployedReleaseTrainProbe; const rehearsalApi = read("deploy/fly/rehearsal-api.toml"); const rehearsalWeb = read("deploy/fly/rehearsal-frontend.toml"); const rehearsalWorker = read( @@ -198,9 +193,9 @@ test("rehearsal is private, read-only, and cannot dispatch production jobs", () ), /\.internal/, ); - assert.match(train, /AbortSignal\.timeout\(10000\)/); - assert.match(train, /after 12 attempts/); - assert.match(train, /retryableStatus = new Set\(\[408, 425, 429, 500, 502, 503, 504\]\)/); + assert.match(probe, /AbortSignal\.timeout\(10000\)/); + assert.match(probe, /after 12 attempts/); + assert.match(probe, /retryableStatus = new Set\(\[408, 425, 429, 500, 502, 503, 504\]\)/); assert.match(rehearsalApi, /force_https = false/); assert.match(rehearsalWeb, /force_https = false/); for (const config of [rehearsalApi, rehearsalWeb, rehearsalWorker]) { @@ -212,12 +207,12 @@ test("rehearsal is private, read-only, and cannot dispatch production jobs", () assert.match(train, /"--machine",\s+machineId/); assert.match(train, /ROSS_DISABLE_DOCUMENT_SCAN_DISPATCHER: "true"/); assert.match( - train, + probe, /"X-ROSS-Data-Boundary": "synthetic-or-non-confidential"/, ); - assert.match(train, /expectStatus\(protectedUpload, 401/); - assert.match(train, /expected HTTP " \+ expected \+ " but received HTTP "/); - assert.match(train, /workerAuth\.status === 400/); + assert.match(probe, /expectStatus\(protectedUpload, 401/); + assert.match(probe, /expected HTTP " \+ expected \+ " but received HTTP "/); + assert.match(probe, /workerAuth\.status === 400/); assert.match(train, /observe-legal-sources\.mjs/); assert.match(train, /verifyProductionSecrets\(\)/); assert.match(train, /class ExpectedRehearsalFailure extends Error/); @@ -236,7 +231,7 @@ test("rehearsal is private, read-only, and cannot dispatch production jobs", () ); }); -test("the exact embedded full probe executes every read-only contract", async () => { +test("the exact shared full probe executes every read-only contract", async () => { const apiNetwork = "http://ross-ranadeoss-api-rehearsal.flycast"; const webNetwork = "http://ross-ranadeoss-web-rehearsal.flycast"; const workerNetwork = "http://ross-ranadeoss-worker-rehearsal.flycast"; @@ -304,7 +299,7 @@ test("the exact embedded full probe executes every read-only contract", async () throw new Error(`Unexpected probe request: ${url}`); }; - await runInNewContext(deployedProbeSource(), { + await runInNewContext(deployedReleaseTrainProbe, { AbortSignal, Error, Promise, diff --git a/tests/baseline/ross-staging-debug.test.mjs b/tests/baseline/ross-staging-debug.test.mjs index 229012e070..9ba625db20 100644 --- a/tests/baseline/ross-staging-debug.test.mjs +++ b/tests/baseline/ross-staging-debug.test.mjs @@ -10,8 +10,11 @@ test("staging debug names are run-scoped and reject production overlap", () => { }); test("staging debug requires data resources isolated from production", () => { - assert.doesNotThrow(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), resources: { supabaseUrl: "stage-db", storageEndpoint: "stage-store" } })); - assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), resources: { supabaseUrl: "same", productionSupabaseUrl: "same", storageEndpoint: "stage", productionStorageEndpoint: "prod" } }), /must not equal production/); + const productionApps = ["prod-api", "prod-web", "prod-worker"]; + const resources = { supabaseUrl: "stage-db", storageEndpoint: "stage-store", productionSupabaseUrl: "prod-db", productionStorageEndpoint: "prod-store" }; + assert.doesNotThrow(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources })); + assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources: { ...resources, productionSupabaseUrl: "" } }), /Missing production comparison/); + assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources: { ...resources, supabaseUrl: "prod-db" } }), /must not equal production/); }); test("debug workflow is diagnostic, cleanup-safe, and cannot promote", () => { @@ -21,12 +24,26 @@ test("debug workflow is diagnostic, cleanup-safe, and cannot promote", () => { assert.match(workflow, /Destroy all ephemeral staging resources[\s\S]*if: always\(\)/); assert.match(workflow, /Upload complete staging-debug evidence[\s\S]*if: always\(\)/); assert.match(workflow, /set -euo pipefail[\s\S]*fly-deploy-with-retry/); + assert.match(workflow, /if flyctl deploy[\s\S]*unexpectedly succeeded/); + assert.match(workflow, /expectedDeploymentFailureObserved/); assert.match(workflow, /releases rollback "\$baseline_version"/); - assert.match(workflow, /forced-debug-failure[\s\S]*web-rollback-probe/); + assert.match(workflow, /run-staging-debug-probe\.mjs staging-debug/g); + assert.match(workflow, /Provision ephemeral staging applications[\s\S]*for app in "\$API_APP" "\$WEB_APP" "\$WORKER_APP"/); + assert.match(workflow, /Destroy all ephemeral staging resources[\s\S]*if: always\(\)[\s\S]*for app in "\$\{WORKER_APP:-\}" "\$\{API_APP:-\}" "\$\{WEB_APP:-\}"/); + assert.match(workflow, /if ! flyctl status --app "\$app"[\s\S]*was not provisioned; nothing to destroy/); assert.match(workflow, /STAGING_SUPABASE_URL/); assert.doesNotMatch(workflow, /promote_public|fly-release-train\.mjs promote|environment: public-beta|ROSS_SUPABASE_SECRET_KEY|PROD_[A-Z_]+=/); }); +test("staging uses the exact complete release-train integration probe", () => { + const runner = readFileSync(new URL("../../scripts/run-staging-debug-probe.mjs", import.meta.url), "utf8"); + assert.match(runner, /import \{ deployedReleaseTrainProbe \}/); + assert.match(runner, /"true",\n\];/); + assert.match(runner, /apps\.worker/); + assert.match(runner, /apps\.api/); + assert.match(runner, /apps\.web/); +}); + test("image builds accept explicit isolated namespaces without production aliases", () => { const build = readFileSync(new URL("../../scripts/build-release-train-images.sh", import.meta.url), "utf8"); assert.match(build, /RELEASE_IMAGE_API_APP/); From 9c70b278eea12b9c8a0ed657b072997a2886c3a7 Mon Sep 17 00:00:00 2001 From: ranade-oss Date: Sun, 26 Jul 2026 17:26:29 -0400 Subject: [PATCH 4/5] Test staging failure and partial cleanup --- .../workflows/staging-debug-release-train.yml | 33 +------ config/release-manifest.v1.json | 1 + reports/release-manifest-v1.json | 15 ++- scripts/run-staging-debug-probe.mjs | 0 scripts/staging-debug-lifecycle.sh | 77 +++++++++++++++ scripts/validate-staging-debug.mjs | 0 tests/baseline/ross-staging-debug.test.mjs | 96 +++++++++++++++++-- 7 files changed, 180 insertions(+), 42 deletions(-) mode change 100644 => 100755 scripts/run-staging-debug-probe.mjs create mode 100755 scripts/staging-debug-lifecycle.sh mode change 100644 => 100755 scripts/validate-staging-debug.mjs diff --git a/.github/workflows/staging-debug-release-train.yml b/.github/workflows/staging-debug-release-train.yml index c3bc5062f1..08618f4fca 100644 --- a/.github/workflows/staging-debug-release-train.yml +++ b/.github/workflows/staging-debug-release-train.yml @@ -105,26 +105,7 @@ jobs: - name: Inject a genuine deployment failure and restore baseline shell: bash - run: | - set -euo pipefail - baseline_version="$(flyctl releases --app "$WEB_APP" --json | tee artifacts/staging-debug/diagnostics/web-releases-before-failure.json | jq -er '.[0].Version // .[0].version')" - printf '%s\n' "$baseline_version" > artifacts/staging-debug/diagnostics/web-rollback-target.txt - - failure_config=deploy/fly/staging-debug-forced-failure.toml - trap 'rm -f "$failure_config"' EXIT - cp deploy/fly/rehearsal-frontend.toml "$failure_config" - sed -i 's/internal_port = 3000/internal_port = 9/' "$failure_config" - cp "$failure_config" artifacts/staging-debug/diagnostics/forced-failure.toml - if flyctl deploy . --config "$failure_config" --app "$WEB_APP" --image "$CANDIDATE_WEB_IMAGE" --ha=false --yes --flycast --no-public-ips > artifacts/staging-debug/commands/web-forced-failure.log 2>&1; then - echo "The deliberately invalid deployment unexpectedly succeeded." >&2 - exit 1 - fi - echo '{"expectedDeploymentFailureObserved":true}' > artifacts/staging-debug/diagnostics/forced-failure-result.json - flyctl status --app "$WEB_APP" --json > artifacts/staging-debug/diagnostics/web-after-failed-deploy-status.json 2>&1 || true - flyctl logs --app "$WEB_APP" --no-tail > artifacts/staging-debug/diagnostics/web-after-failed-deploy.log 2>&1 || true - - flyctl releases rollback "$baseline_version" --app "$WEB_APP" --yes 2>&1 | tee artifacts/staging-debug/commands/web-rollback.log - flyctl releases --app "$WEB_APP" --json > artifacts/staging-debug/diagnostics/web-releases-after-rollback.json + run: bash scripts/staging-debug-lifecycle.sh inject-failure-and-rollback - name: Verify full integration recovery after rollback shell: bash @@ -146,17 +127,7 @@ jobs: - name: Destroy all ephemeral staging resources if: always() shell: bash - run: | - failed=0 - for app in "${WORKER_APP:-}" "${API_APP:-}" "${WEB_APP:-}"; do - [ -n "$app" ] || continue - if ! flyctl status --app "$app" >/dev/null 2>&1; then - printf 'App %s was not provisioned; nothing to destroy.\n' "$app" > "artifacts/staging-debug/commands/cleanup-${app}.log" - continue - fi - flyctl apps destroy "$app" --yes > "artifacts/staging-debug/commands/cleanup-${app}.log" 2>&1 || failed=1 - done - [ "$failed" -eq 0 ] || { echo 'Ephemeral cleanup failed; operator action required.' >&2; exit 1; } + run: bash scripts/staging-debug-lifecycle.sh cleanup - name: Upload complete staging-debug evidence if: always() diff --git a/config/release-manifest.v1.json b/config/release-manifest.v1.json index 6c3dbce268..c83697532d 100644 --- a/config/release-manifest.v1.json +++ b/config/release-manifest.v1.json @@ -112,6 +112,7 @@ "scripts/preflight-fly-images.sh", "scripts/release-train-image-ref.mjs", "scripts/run-staging-debug-probe.mjs", + "scripts/staging-debug-lifecycle.sh", "scripts/run-backup-restore-exercise.sh", "scripts/lib/professional-validation.mjs", "scripts/lib/release-identifier.mjs", diff --git a/reports/release-manifest-v1.json b/reports/release-manifest-v1.json index ed7c6e955b..cf0b69b926 100644 --- a/reports/release-manifest-v1.json +++ b/reports/release-manifest-v1.json @@ -3,7 +3,7 @@ "releaseId": "ross-public-beta-20260717-rc1", "generatedAt": "2026-07-26T10:39:57.000Z", "algorithm": "sha256", - "artifactCount": 126, + "artifactCount": 127, "artifacts": [ { "path": ".github/actions/setup-ross-node/action.yml", @@ -42,8 +42,8 @@ }, { "path": ".github/workflows/staging-debug-release-train.yml", - "sha256": "57224d82cd3a095b1ecf1048e1cf28ecdb8d5dce65366b38a21ce1c87ea36f0b", - "sizeBytes": 10651 + "sha256": "ba54e3a0b98324906bc93e4411e04b4a7ffcb8399401f028b7dae3f77ab3fdb9", + "sizeBytes": 8501 }, { "path": ".github/workflows/verify-and-deploy-public-beta.yml", @@ -550,6 +550,11 @@ "sha256": "c49dee35e02aea23f421f2a0244703edef7f9a6632932a40e7261a97b9eee2d3", "sizeBytes": 2207 }, + { + "path": "scripts/staging-debug-lifecycle.sh", + "sha256": "a3c4db1fbb71b1224a21bf20bec044d4e310a9e83c879b48d1aace0770ea2071", + "sizeBytes": 2854 + }, { "path": "scripts/run-backup-restore-exercise.sh", "sha256": "f2d0ee4cb495214f03db3e479ba3ff58de499d999abed62f72cf5c84bafd0a63", @@ -602,8 +607,8 @@ }, { "path": "tests/baseline/ross-staging-debug.test.mjs", - "sha256": "807796a1bcfc66609a301c8c4160a68b17bfe593a4b408571d18a6160603f9a2", - "sizeBytes": 3704 + "sha256": "1cc949e10de7a48b33ae426209e3fc8a8a859079c6ea9acf6f0c2f402807585c", + "sizeBytes": 7704 }, { "path": "website/package-lock.json", diff --git a/scripts/run-staging-debug-probe.mjs b/scripts/run-staging-debug-probe.mjs old mode 100644 new mode 100755 diff --git a/scripts/staging-debug-lifecycle.sh b/scripts/staging-debug-lifecycle.sh new file mode 100755 index 0000000000..aea6054025 --- /dev/null +++ b/scripts/staging-debug-lifecycle.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" +ARTIFACT_DIR="${ROSS_STAGING_DEBUG_ARTIFACT_DIR:-artifacts/staging-debug}" +mkdir -p "$ARTIFACT_DIR/commands" "$ARTIFACT_DIR/diagnostics" + +required() { + if [ -z "${!1:-}" ]; then + echo "Required staging-debug value is missing: $1" >&2 + exit 2 + fi +} + +inject_failure_and_rollback() { + required WEB_APP + required CANDIDATE_WEB_IMAGE + local baseline_version failure_config deploy_status + baseline_version="$(flyctl releases --app "$WEB_APP" --json \ + | tee "$ARTIFACT_DIR/diagnostics/web-releases-before-failure.json" \ + | jq -er '.[0].Version // .[0].version')" + printf '%s\n' "$baseline_version" > "$ARTIFACT_DIR/diagnostics/web-rollback-target.txt" + + failure_config=deploy/fly/staging-debug-forced-failure.toml + trap "rm -f '$failure_config'" EXIT + cp deploy/fly/rehearsal-frontend.toml "$failure_config" + sed -i 's/internal_port = 3000/internal_port = 9/' "$failure_config" + cp "$failure_config" "$ARTIFACT_DIR/diagnostics/forced-failure.toml" + + set +e + flyctl deploy . --config "$failure_config" --app "$WEB_APP" \ + --image "$CANDIDATE_WEB_IMAGE" --ha=false --yes --flycast \ + --no-public-ips > "$ARTIFACT_DIR/commands/web-forced-failure.log" 2>&1 + deploy_status=$? + set -e + if [ "$deploy_status" -eq 0 ]; then + echo "The deliberately invalid deployment unexpectedly succeeded." >&2 + exit 1 + fi + printf '{"expectedDeploymentFailureObserved":true,"exitCode":%d}\n' \ + "$deploy_status" > "$ARTIFACT_DIR/diagnostics/forced-failure-result.json" + flyctl status --app "$WEB_APP" --json \ + > "$ARTIFACT_DIR/diagnostics/web-after-failed-deploy-status.json" 2>&1 || true + flyctl logs --app "$WEB_APP" --no-tail \ + > "$ARTIFACT_DIR/diagnostics/web-after-failed-deploy.log" 2>&1 || true + + flyctl releases rollback "$baseline_version" --app "$WEB_APP" --yes \ + 2>&1 | tee "$ARTIFACT_DIR/commands/web-rollback.log" + flyctl releases --app "$WEB_APP" --json \ + > "$ARTIFACT_DIR/diagnostics/web-releases-after-rollback.json" +} + +cleanup() { + local failed=0 app + for app in "${WORKER_APP:-}" "${API_APP:-}" "${WEB_APP:-}"; do + [ -n "$app" ] || continue + if ! flyctl status --app "$app" >/dev/null 2>&1; then + printf 'App %s was not provisioned; nothing to destroy.\n' "$app" \ + > "$ARTIFACT_DIR/commands/cleanup-${app}.log" + continue + fi + flyctl apps destroy "$app" --yes \ + > "$ARTIFACT_DIR/commands/cleanup-${app}.log" 2>&1 || failed=1 + done + if [ "$failed" -ne 0 ]; then + echo "Ephemeral cleanup failed; operator action required." >&2 + exit 1 + fi +} + +case "${1:-}" in + inject-failure-and-rollback) inject_failure_and_rollback ;; + cleanup) cleanup ;; + *) echo "Usage: staging-debug-lifecycle.sh inject-failure-and-rollback | cleanup" >&2; exit 2 ;; +esac diff --git a/scripts/validate-staging-debug.mjs b/scripts/validate-staging-debug.mjs old mode 100644 new mode 100755 diff --git a/tests/baseline/ross-staging-debug.test.mjs b/tests/baseline/ross-staging-debug.test.mjs index 9ba625db20..dca87a9816 100644 --- a/tests/baseline/ross-staging-debug.test.mjs +++ b/tests/baseline/ross-staging-debug.test.mjs @@ -1,5 +1,8 @@ import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; +import { chmodSync, mkdtempSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; import test from "node:test"; import { assertIsolatedStaging, stagingDebugNames } from "../../scripts/lib/staging-debug.mjs"; @@ -14,7 +17,10 @@ test("staging debug requires data resources isolated from production", () => { const resources = { supabaseUrl: "stage-db", storageEndpoint: "stage-store", productionSupabaseUrl: "prod-db", productionStorageEndpoint: "prod-store" }; assert.doesNotThrow(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources })); assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources: { ...resources, productionSupabaseUrl: "" } }), /Missing production comparison/); + assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources: { ...resources, productionStorageEndpoint: "" } }), /Missing production comparison/); + assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps: [], resources }), /production app comparison/); assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources: { ...resources, supabaseUrl: "prod-db" } }), /must not equal production/); + assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources: { ...resources, storageEndpoint: "prod-store" } }), /must not equal production/); }); test("debug workflow is diagnostic, cleanup-safe, and cannot promote", () => { @@ -24,17 +30,34 @@ test("debug workflow is diagnostic, cleanup-safe, and cannot promote", () => { assert.match(workflow, /Destroy all ephemeral staging resources[\s\S]*if: always\(\)/); assert.match(workflow, /Upload complete staging-debug evidence[\s\S]*if: always\(\)/); assert.match(workflow, /set -euo pipefail[\s\S]*fly-deploy-with-retry/); - assert.match(workflow, /if flyctl deploy[\s\S]*unexpectedly succeeded/); - assert.match(workflow, /expectedDeploymentFailureObserved/); - assert.match(workflow, /releases rollback "\$baseline_version"/); + assert.match(workflow, /staging-debug-lifecycle\.sh inject-failure-and-rollback/); assert.match(workflow, /run-staging-debug-probe\.mjs staging-debug/g); assert.match(workflow, /Provision ephemeral staging applications[\s\S]*for app in "\$API_APP" "\$WEB_APP" "\$WORKER_APP"/); - assert.match(workflow, /Destroy all ephemeral staging resources[\s\S]*if: always\(\)[\s\S]*for app in "\$\{WORKER_APP:-\}" "\$\{API_APP:-\}" "\$\{WEB_APP:-\}"/); - assert.match(workflow, /if ! flyctl status --app "\$app"[\s\S]*was not provisioned; nothing to destroy/); + assert.match(workflow, /Destroy all ephemeral staging resources[\s\S]*if: always\(\)[\s\S]*staging-debug-lifecycle\.sh cleanup/); assert.match(workflow, /STAGING_SUPABASE_URL/); assert.doesNotMatch(workflow, /promote_public|fly-release-train\.mjs promote|environment: public-beta|ROSS_SUPABASE_SECRET_KEY|PROD_[A-Z_]+=/); }); +test("lifecycle observes a genuine nonzero deployment and rolls back", () => { + const fixture = fakeFlyFixture(); + const result = runLifecycle(fixture, "inject-failure-and-rollback"); + assert.equal(result.status, 0, result.stderr || result.stdout); + const evidence = JSON.parse(readFileSync(join(fixture.artifacts, "diagnostics/forced-failure-result.json"), "utf8")); + assert.deepEqual(evidence, { expectedDeploymentFailureObserved: true, exitCode: 42 }); + const calls = readFileSync(fixture.calls, "utf8"); + assert.match(calls, /"deploy"/); + assert.match(calls, /"rollback","17"/); +}); + +test("cleanup succeeds after only one of three apps was provisioned", () => { + const fixture = fakeFlyFixture("debug-worker"); + const result = runLifecycle(fixture, "cleanup"); + assert.equal(result.status, 0, result.stderr || result.stdout); + const calls = readFileSync(fixture.calls, "utf8"); + assert.match(calls, /"destroy","debug-worker"/); + assert.doesNotMatch(calls, /"destroy","debug-api"|"destroy","debug-web"/); +}); + test("staging uses the exact complete release-train integration probe", () => { const runner = readFileSync(new URL("../../scripts/run-staging-debug-probe.mjs", import.meta.url), "utf8"); assert.match(runner, /import \{ deployedReleaseTrainProbe \}/); @@ -50,3 +73,64 @@ test("image builds accept explicit isolated namespaces without production aliase assert.match(build, /RELEASE_RUNTIME_WEB_APP/); assert.match(build, /RELEASE_SIGNUPS_ENABLED/); }); + +test("release manifest governs every staging-debug and shared-probe change", () => { + const manifest = JSON.parse(readFileSync(new URL("../../config/release-manifest.v1.json", import.meta.url), "utf8")); + for (const path of [ + ".github/workflows/staging-debug-release-train.yml", + "docs/staging-debug-release-train.md", + "scripts/build-release-train-images.sh", + "scripts/fly-release-train.mjs", + "scripts/lib/release-train-probe.mjs", + "scripts/lib/staging-debug.mjs", + "scripts/run-staging-debug-probe.mjs", + "scripts/staging-debug-lifecycle.sh", + "scripts/validate-staging-debug.mjs", + "tests/baseline/ross-release-train.test.mjs", + "tests/baseline/ross-staging-debug.test.mjs", + ]) { + assert.ok(manifest.artifacts.includes(path), `${path} must be governed`); + } +}); + +function fakeFlyFixture(existingApps = "debug-web") { + const directory = mkdtempSync(join(tmpdir(), "ross-staging-debug-")); + const bin = join(directory, "bin"); + const artifacts = join(directory, "artifacts"); + const calls = join(directory, "calls.jsonl"); + mkdirSync(bin); + const fake = `#!/usr/bin/env node +const fs = require("node:fs"); +const args = process.argv.slice(2); +fs.appendFileSync(process.env.FAKE_FLY_CALLS, JSON.stringify(args) + "\\n"); +const app = args[args.indexOf("--app") + 1]; +const existing = new Set((process.env.FAKE_EXISTING_APPS || "").split(",").filter(Boolean)); +if (args[0] === "status") process.exit(existing.has(app) ? 0 : 1); +if (args[0] === "apps" && args[1] === "destroy") process.exit(0); +if (args[0] === "deploy") process.exit(42); +if (args[0] === "logs") process.exit(0); +if (args[0] === "releases" && args[1] === "rollback") process.exit(0); +if (args[0] === "releases") { process.stdout.write('[{"Version":17}]'); process.exit(0); } +process.exit(2); +`; + writeFileSync(join(bin, "flyctl"), fake); + chmodSync(join(bin, "flyctl"), 0o755); + return { directory, artifacts, calls, bin, existingApps }; +} + +function runLifecycle(fixture, command) { + return spawnSync("bash", [new URL("../../scripts/staging-debug-lifecycle.sh", import.meta.url).pathname, command], { + encoding: "utf8", + env: { + ...process.env, + PATH: `${fixture.bin}:${process.env.PATH}`, + FAKE_FLY_CALLS: fixture.calls, + FAKE_EXISTING_APPS: fixture.existingApps, + ROSS_STAGING_DEBUG_ARTIFACT_DIR: fixture.artifacts, + API_APP: "debug-api", + WEB_APP: "debug-web", + WORKER_APP: "debug-worker", + CANDIDATE_WEB_IMAGE: `registry.fly.io/debug-web@sha256:${"a".repeat(64)}`, + }, + }); +} From 1644f8e6b5de5b3ce1b7233fe4df5984691060a7 Mon Sep 17 00:00:00 2001 From: ranade-oss Date: Sun, 26 Jul 2026 17:26:35 -0400 Subject: [PATCH 5/5] Restore staging by immutable digest --- reports/release-manifest-v1.json | 12 +++--- scripts/lib/staging-debug.mjs | 21 ++++++++- scripts/staging-debug-lifecycle.sh | 35 +++++++++------ tests/baseline/ross-staging-debug.test.mjs | 50 ++++++++++++++++------ 4 files changed, 83 insertions(+), 35 deletions(-) diff --git a/reports/release-manifest-v1.json b/reports/release-manifest-v1.json index cf0b69b926..1eb020bef1 100644 --- a/reports/release-manifest-v1.json +++ b/reports/release-manifest-v1.json @@ -512,8 +512,8 @@ }, { "path": "scripts/lib/staging-debug.mjs", - "sha256": "a3bf56de870458f1142eaf1b843f47bfa78dbcd9b3fe75372c663fbf2f8c93b4", - "sizeBytes": 2022 + "sha256": "f496440b11fba669ddedca3091f51a73857adf705b0b974a569a780588654fdd", + "sizeBytes": 2822 }, { "path": "scripts/build-release-train-images.sh", @@ -552,8 +552,8 @@ }, { "path": "scripts/staging-debug-lifecycle.sh", - "sha256": "a3c4db1fbb71b1224a21bf20bec044d4e310a9e83c879b48d1aace0770ea2071", - "sizeBytes": 2854 + "sha256": "08dae021d6dacde7f648e0aa830838ed34a81a43c6dcfdd8f3ffea6c7da49202", + "sizeBytes": 3178 }, { "path": "scripts/run-backup-restore-exercise.sh", @@ -607,8 +607,8 @@ }, { "path": "tests/baseline/ross-staging-debug.test.mjs", - "sha256": "1cc949e10de7a48b33ae426209e3fc8a8a859079c6ea9acf6f0c2f402807585c", - "sizeBytes": 7704 + "sha256": "0a5a43bd6874d79b5d9dc487c4ce90ff2ac47cc8a151829a873145b1ce4fbd94", + "sizeBytes": 9422 }, { "path": "website/package-lock.json", diff --git a/scripts/lib/staging-debug.mjs b/scripts/lib/staging-debug.mjs index a8d922f34f..83c1e9a4f7 100644 --- a/scripts/lib/staging-debug.mjs +++ b/scripts/lib/staging-debug.mjs @@ -12,6 +12,20 @@ export function stagingDebugNames(runId, attempt = "1") { }; } +export function normalizeResourceUrl(value, label) { + let parsed; + try { + parsed = new URL(value); + } catch { + throw new Error(`${label} must be a valid absolute URL.`); + } + if (parsed.protocol !== "https:" || parsed.username || parsed.password || parsed.search || parsed.hash) { + throw new Error(`${label} must be a credential-free HTTPS URL without query or fragment.`); + } + const path = parsed.pathname.replace(/\/+$/, ""); + return `${parsed.origin}${path}`; +} + export function assertIsolatedStaging({ apps, productionApps = [], resources }) { const values = Object.values(apps); if (values.length !== 3 || new Set(values).size !== 3) { @@ -35,8 +49,11 @@ export function assertIsolatedStaging({ apps, productionApps = [], resources }) if (productionApps.length !== 3 || productionApps.some((app) => !String(app).trim())) { throw new Error("All production app comparison identifiers are required."); } - if ((resources.productionSupabaseUrl && resources.supabaseUrl === resources.productionSupabaseUrl) || - (resources.productionStorageEndpoint && resources.storageEndpoint === resources.productionStorageEndpoint)) { + const stagingSupabase = normalizeResourceUrl(resources.supabaseUrl, "staging Supabase URL"); + const productionSupabase = normalizeResourceUrl(resources.productionSupabaseUrl, "production Supabase URL"); + const stagingStorage = normalizeResourceUrl(resources.storageEndpoint, "staging storage URL"); + const productionStorage = normalizeResourceUrl(resources.productionStorageEndpoint, "production storage URL"); + if (stagingSupabase === productionSupabase || stagingStorage === productionStorage) { throw new Error("Staging data resources must not equal production resources."); } return { apps, resources }; diff --git a/scripts/staging-debug-lifecycle.sh b/scripts/staging-debug-lifecycle.sh index aea6054025..2dcc25d1cb 100755 --- a/scripts/staging-debug-lifecycle.sh +++ b/scripts/staging-debug-lifecycle.sh @@ -17,11 +17,9 @@ required() { inject_failure_and_rollback() { required WEB_APP required CANDIDATE_WEB_IMAGE - local baseline_version failure_config deploy_status - baseline_version="$(flyctl releases --app "$WEB_APP" --json \ - | tee "$ARTIFACT_DIR/diagnostics/web-releases-before-failure.json" \ - | jq -er '.[0].Version // .[0].version')" - printf '%s\n' "$baseline_version" > "$ARTIFACT_DIR/diagnostics/web-rollback-target.txt" + local baseline_image failure_config deploy_status + baseline_image="$(node scripts/release-train-image-ref.mjs current "$WEB_APP")" + printf '%s\n' "$baseline_image" > "$ARTIFACT_DIR/diagnostics/web-restore-image.txt" failure_config=deploy/fly/staging-debug-forced-failure.toml trap "rm -f '$failure_config'" EXIT @@ -46,23 +44,34 @@ inject_failure_and_rollback() { flyctl logs --app "$WEB_APP" --no-tail \ > "$ARTIFACT_DIR/diagnostics/web-after-failed-deploy.log" 2>&1 || true - flyctl releases rollback "$baseline_version" --app "$WEB_APP" --yes \ - 2>&1 | tee "$ARTIFACT_DIR/commands/web-rollback.log" - flyctl releases --app "$WEB_APP" --json \ - > "$ARTIFACT_DIR/diagnostics/web-releases-after-rollback.json" + bash scripts/fly-deploy-with-retry.sh . \ + --config deploy/fly/rehearsal-frontend.toml --app "$WEB_APP" \ + --image "$baseline_image" --ha=false --yes --flycast --no-public-ips \ + 2>&1 | tee "$ARTIFACT_DIR/commands/web-digest-restore.log" + node scripts/release-train-image-ref.mjs verify "$WEB_APP" "$baseline_image" \ + > "$ARTIFACT_DIR/diagnostics/web-restored-image.txt" } cleanup() { - local failed=0 app + local failed=0 app output status for app in "${WORKER_APP:-}" "${API_APP:-}" "${WEB_APP:-}"; do [ -n "$app" ] || continue - if ! flyctl status --app "$app" >/dev/null 2>&1; then + set +e + output="$(flyctl apps destroy "$app" --yes 2>&1)" + status=$? + set -e + printf '%s\n' "$output" > "$ARTIFACT_DIR/commands/cleanup-${app}.log" + if [ "$status" -eq 0 ]; then + continue + fi + if printf '%s\n' "$output" | grep -Eqi \ + 'app(lication)? (was )?not found|could not find app|does not exist|404'; then printf 'App %s was not provisioned; nothing to destroy.\n' "$app" \ > "$ARTIFACT_DIR/commands/cleanup-${app}.log" continue fi - flyctl apps destroy "$app" --yes \ - > "$ARTIFACT_DIR/commands/cleanup-${app}.log" 2>&1 || failed=1 + printf 'Failed to destroy %s (flyctl exit %d).\n' "$app" "$status" >&2 + failed=1 done if [ "$failed" -ne 0 ]; then echo "Ephemeral cleanup failed; operator action required." >&2 diff --git a/tests/baseline/ross-staging-debug.test.mjs b/tests/baseline/ross-staging-debug.test.mjs index dca87a9816..edc46be1fb 100644 --- a/tests/baseline/ross-staging-debug.test.mjs +++ b/tests/baseline/ross-staging-debug.test.mjs @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { spawnSync } from "node:child_process"; import test from "node:test"; -import { assertIsolatedStaging, stagingDebugNames } from "../../scripts/lib/staging-debug.mjs"; +import { assertIsolatedStaging, normalizeResourceUrl, stagingDebugNames } from "../../scripts/lib/staging-debug.mjs"; test("staging debug names are run-scoped and reject production overlap", () => { const apps = stagingDebugNames("123", "2"); @@ -14,13 +14,14 @@ test("staging debug names are run-scoped and reject production overlap", () => { test("staging debug requires data resources isolated from production", () => { const productionApps = ["prod-api", "prod-web", "prod-worker"]; - const resources = { supabaseUrl: "stage-db", storageEndpoint: "stage-store", productionSupabaseUrl: "prod-db", productionStorageEndpoint: "prod-store" }; + const resources = { supabaseUrl: "https://stage-db.example", storageEndpoint: "https://stage-store.example", productionSupabaseUrl: "https://prod-db.example", productionStorageEndpoint: "https://prod-store.example" }; assert.doesNotThrow(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources })); assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources: { ...resources, productionSupabaseUrl: "" } }), /Missing production comparison/); assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources: { ...resources, productionStorageEndpoint: "" } }), /Missing production comparison/); assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps: [], resources }), /production app comparison/); - assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources: { ...resources, supabaseUrl: "prod-db" } }), /must not equal production/); - assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources: { ...resources, storageEndpoint: "prod-store" } }), /must not equal production/); + assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources: { ...resources, supabaseUrl: "https://prod-db.example/" } }), /must not equal production/); + assert.throws(() => assertIsolatedStaging({ apps: stagingDebugNames("1"), productionApps, resources: { ...resources, storageEndpoint: "https://prod-store.example/" } }), /must not equal production/); + assert.equal(normalizeResourceUrl("https://EXAMPLE.test/path///", "fixture"), "https://example.test/path"); }); test("debug workflow is diagnostic, cleanup-safe, and cannot promote", () => { @@ -39,6 +40,11 @@ test("debug workflow is diagnostic, cleanup-safe, and cannot promote", () => { }); test("lifecycle observes a genuine nonzero deployment and rolls back", () => { + const lifecycle = readFileSync(new URL("../../scripts/staging-debug-lifecycle.sh", import.meta.url), "utf8"); + assert.doesNotMatch(lifecycle, /flyctl releases rollback/); + assert.match(lifecycle, /release-train-image-ref\.mjs current/); + assert.match(lifecycle, /fly-deploy-with-retry\.sh[\s\S]*--image "\$baseline_image"/); + assert.match(lifecycle, /release-train-image-ref\.mjs verify/); const fixture = fakeFlyFixture(); const result = runLifecycle(fixture, "inject-failure-and-rollback"); assert.equal(result.status, 0, result.stderr || result.stdout); @@ -46,7 +52,9 @@ test("lifecycle observes a genuine nonzero deployment and rolls back", () => { assert.deepEqual(evidence, { expectedDeploymentFailureObserved: true, exitCode: 42 }); const calls = readFileSync(fixture.calls, "utf8"); assert.match(calls, /"deploy"/); - assert.match(calls, /"rollback","17"/); + assert.doesNotMatch(calls, /"releases","rollback"/); + assert.match(calls, /"deploy",".","--config","deploy\/fly\/rehearsal-frontend\.toml"/); + assert.equal(readFileSync(join(fixture.artifacts, "diagnostics/web-restored-image.txt"), "utf8").trim(), fixture.image); }); test("cleanup succeeds after only one of three apps was provisioned", () => { @@ -55,7 +63,16 @@ test("cleanup succeeds after only one of three apps was provisioned", () => { assert.equal(result.status, 0, result.stderr || result.stdout); const calls = readFileSync(fixture.calls, "utf8"); assert.match(calls, /"destroy","debug-worker"/); - assert.doesNotMatch(calls, /"destroy","debug-api"|"destroy","debug-web"/); + assert.match(calls, /"destroy","debug-api"/); + assert.match(calls, /"destroy","debug-web"/); + assert.match(readFileSync(join(fixture.artifacts, "commands/cleanup-debug-api.log"), "utf8"), /not provisioned/); +}); + +test("cleanup fails on non-not-found Fly errors instead of leaking apps", () => { + const fixture = fakeFlyFixture("debug-worker", "debug-api"); + const result = runLifecycle(fixture, "cleanup"); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /Failed to destroy debug-api/); }); test("staging uses the exact complete release-train integration probe", () => { @@ -93,29 +110,33 @@ test("release manifest governs every staging-debug and shared-probe change", () } }); -function fakeFlyFixture(existingApps = "debug-web") { +function fakeFlyFixture(existingApps = "debug-web", destroyErrorApp = "") { const directory = mkdtempSync(join(tmpdir(), "ross-staging-debug-")); const bin = join(directory, "bin"); const artifacts = join(directory, "artifacts"); const calls = join(directory, "calls.jsonl"); mkdirSync(bin); + const image = `registry.fly.io/debug-web@sha256:${"a".repeat(64)}`; const fake = `#!/usr/bin/env node const fs = require("node:fs"); const args = process.argv.slice(2); fs.appendFileSync(process.env.FAKE_FLY_CALLS, JSON.stringify(args) + "\\n"); const app = args[args.indexOf("--app") + 1]; const existing = new Set((process.env.FAKE_EXISTING_APPS || "").split(",").filter(Boolean)); -if (args[0] === "status") process.exit(existing.has(app) ? 0 : 1); -if (args[0] === "apps" && args[1] === "destroy") process.exit(0); -if (args[0] === "deploy") process.exit(42); +if (args[0] === "apps" && args[1] === "destroy") { + const target = args[2]; + if (target === process.env.FAKE_DESTROY_ERROR_APP) { process.stderr.write("temporary Fly API timeout"); process.exit(29); } + if (!existing.has(target)) { process.stderr.write("Could not find App"); process.exit(1); } + process.exit(0); +} +if (args[0] === "deploy") process.exit(args.some((arg) => arg.endsWith("staging-debug-forced-failure.toml")) ? 42 : 0); if (args[0] === "logs") process.exit(0); -if (args[0] === "releases" && args[1] === "rollback") process.exit(0); -if (args[0] === "releases") { process.stdout.write('[{"Version":17}]'); process.exit(0); } +if (args[0] === "image" && args[1] === "show") { process.stdout.write(JSON.stringify({ Registry: "registry.fly.io", Repository: "debug-web", Digest: "sha256:${"a".repeat(64)}" })); process.exit(0); } process.exit(2); `; writeFileSync(join(bin, "flyctl"), fake); chmodSync(join(bin, "flyctl"), 0o755); - return { directory, artifacts, calls, bin, existingApps }; + return { directory, artifacts, calls, bin, existingApps, destroyErrorApp, image }; } function runLifecycle(fixture, command) { @@ -126,11 +147,12 @@ function runLifecycle(fixture, command) { PATH: `${fixture.bin}:${process.env.PATH}`, FAKE_FLY_CALLS: fixture.calls, FAKE_EXISTING_APPS: fixture.existingApps, + FAKE_DESTROY_ERROR_APP: fixture.destroyErrorApp, ROSS_STAGING_DEBUG_ARTIFACT_DIR: fixture.artifacts, API_APP: "debug-api", WEB_APP: "debug-web", WORKER_APP: "debug-worker", - CANDIDATE_WEB_IMAGE: `registry.fly.io/debug-web@sha256:${"a".repeat(64)}`, + CANDIDATE_WEB_IMAGE: fixture.image, }, }); }