diff --git a/AGENTS.md b/AGENTS.md index 268b93de..51b6b8c1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -89,7 +89,11 @@ Use `pnpm run deploy` (not bare `pnpm deploy` — that's pnpm's built-in). `wrangler deploy`. On merge to main, `.github/workflows/d1-migrations.yml` also applies remote migrations when `apps/api/migrations/**` changes (secrets: `CLOUDFLARE_API_TOKEN`, `CLOUDFLARE_ACCOUNT_ID`). Production -worker deploys normally happen via Workers Builds on push to main. +worker deploys normally happen via Workers Builds on push to main. For an +isolated branch environment, `npx wrangler preview` from an app directory — +playbook in [docs/previews.md](docs/previews.md); the Workers Builds bot's +"Branch Preview URL" on PRs is a different, legacy mechanism that runs +against production bindings. Operator runbook: [docs/ops.md](docs/ops.md). Daily retention cron on the API worker; BYO secrets use `WORKSPACE_SECRETS_KEY`; bare upload keys get `f//…`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d8af8405..39ec8c40 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -193,6 +193,13 @@ migrations when `apps/api/migrations/**` changes. Manual deploys `deploy:auth`) need `CLOUDFLARE_ACCOUNT_ID` and `CLOUDFLARE_API_TOKEN` in `.env`. Use `pnpm run deploy`, not bare `pnpm deploy` — that is pnpm's built-in. +For a branch environment, run `npx wrangler preview` from the app's directory +(Workers Previews; enabled on the account). Each app's `wrangler.jsonc` has a +`previews` block that re-points stateful bindings at preview-tier resources — +previews never touch production KV/D1/R2. Which app to preview for a given +change, per-branch overrides, migrations, seeding, and secrets are all in +[docs/previews.md](docs/previews.md). + Full setup is in [docs/deploy.md](docs/deploy.md); the operator runbook is [docs/ops.md](docs/ops.md). diff --git a/apps/api/wrangler.jsonc b/apps/api/wrangler.jsonc index b79f680a..16375c8c 100644 --- a/apps/api/wrangler.jsonc +++ b/apps/api/wrangler.jsonc @@ -234,4 +234,161 @@ "bucket_name": "uploads-default", }, ], + // Workers Previews (`npx wrangler preview` from a branch). Bindings do NOT + // inherit from the production config — a preview gets only what this block + // lists — so everything is spelled out: stateful bindings re-pointed at + // preview-tier resources (without that, a preview of an API branch would + // write to production KV/D1/R2), and the stateless/shared-safe ones + // repeated verbatim: BROWSER/MEDIA (metered, stateless), FLAGS (read-only), + // EMAIL, the rate limiters, and the AUTH service binding — service bindings + // always call the bound Worker's *production* deployment, so a preview + // verifies sessions against real auth. Cron triggers and queue consumers + // never run in Previews. Secrets start unset per Preview (set base values + // with `wrangler preview secret put `); every secret-gated path here + // fails closed or degrades, so an unset secret means a 401/no-op, not a + // fallback to production credentials. + // + // One-time setup before the first preview deploy — create the preview-tier + // resources and paste the ids below (apps/mcp's previews block shares + // REGISTRY and DB — keep it on the same ids): + // wrangler kv namespace create UPLOADS_REGISTRY_PREVIEW + // wrangler d1 create uploads-preview + // wrangler r2 bucket create uploads-preview-default + // wrangler queues create uploads-github-webhook-preview + // Migrations for the preview D1 go through wrangler.preview.jsonc — the + // `d1 migrations` command cannot see databases declared inside `previews`: + // wrangler d1 migrations apply uploads-preview --remote -c apps/api/wrangler.preview.jsonc + "previews": { + "vars": { + "WEB_ORIGIN": "https://uploads.sh", + // Not "production": previews keep dev-only CORS origin reflection for + // loopback origins, which only ever reflects localhost/127.0.0.1. + "ENVIRONMENT": "preview", + "GITHUB_APP_ID": "4346270", + "GITHUB_APP_HOME_INSTALLATION_ID": "147814297", + "CLOUDFLARE_ACCOUNT_ID": "b082600d280d44fd5da3501bc1bffe2f", + }, + "kv_namespaces": [ + // Isolated: workspace records are the product's core state. + { + "binding": "REGISTRY", + "id": "7bb5509c12f54fd7b238c75726e63c09", + }, + // Shared with production on purpose: a TTL'd cache of public GitHub + // data (titles, logins) — sharing is harmless and keeps previews warm. + { + "binding": "GITHUB_CACHE", + "id": "8076084dae50495ab883ec79c1c5968f", + }, + ], + "d1_databases": [ + { + "binding": "DB", + "database_name": "uploads-preview", + "database_id": "b8c82095-b471-4e3a-8634-56e0861e43f1", + "migrations_dir": "migrations", + }, + ], + "r2_buckets": [ + // buildinternet-dev is already the dev bucket; only the default-tier + // bucket needs a preview twin so preview uploads never land in + // uploads-default. + { + "binding": "UPLOADS", + "bucket_name": "buildinternet-dev", + }, + { + "binding": "UPLOADS_DEFAULT", + "bucket_name": "uploads-preview-default", + }, + ], + // Producer only — Previews cannot run queue consumers, so preview-enqueued + // messages simply expire. GitHub webhooks point at production anyway; this + // exists so a preview's enqueue path works without touching the prod queue. + "queues": { + "producers": [ + { + "binding": "GITHUB_WEBHOOK_QUEUE", + "queue": "uploads-github-webhook-preview", + }, + ], + }, + // Datasets auto-create on first write; keeps preview traffic out of the + // adoption metrics the /admin/metrics rollups read. + "analytics_engine_datasets": [ + { + "binding": "ANALYTICS", + "dataset": "uploads_adoption_preview", + }, + ], + // Everything below repeats the production bindings verbatim — stateless, + // account-level, or per-colo, so sharing is safe. See the notes on the + // top-level entries for what each does. + "browser": { + "binding": "BROWSER", + "remote": true, + }, + "media": { + "binding": "MEDIA", + "remote": true, + }, + "flagship": [ + { + "binding": "FLAGS", + "app_id": "8371bfe7-9767-4b4d-b75a-37b94d2724f7", + }, + ], + "send_email": [{ "name": "EMAIL" }], + "services": [ + { + "binding": "AUTH", + "service": "uploads-auth", + }, + ], + // Previews declare rate limiters via the dedicated `ratelimits` field + // (same name/namespace_id/simple values as the production `unsafe` + // entries above); the runtime binding name comes from `name`. + "ratelimits": [ + { + "name": "WRITE_LIMITER", + "namespace_id": "1001", + "simple": { + "limit": 60, + "period": 60, + }, + }, + { + "name": "INVITE_LIMITER", + "namespace_id": "1002", + "simple": { + "limit": 30, + "period": 60, + }, + }, + { + "name": "WS_CREATE_LIMITER", + "namespace_id": "1003", + "simple": { + "limit": 3, + "period": 60, + }, + }, + { + "name": "RENDER_LIMITER", + "namespace_id": "1004", + "simple": { + "limit": 10, + "period": 60, + }, + }, + { + "name": "POSTER_LIMITER", + "namespace_id": "1005", + "simple": { + "limit": 30, + "period": 60, + }, + }, + ], + }, } diff --git a/apps/api/wrangler.preview.jsonc b/apps/api/wrangler.preview.jsonc new file mode 100644 index 00000000..cdbec7bc --- /dev/null +++ b/apps/api/wrangler.preview.jsonc @@ -0,0 +1,18 @@ +// Companion config for the preview-tier D1 only. `wrangler d1 migrations` +// resolves databases from top-level `d1_databases` and cannot see the one +// declared inside the main config's `previews` block, so migrations for the +// preview database run against this file instead: +// npx wrangler d1 migrations apply uploads-preview --remote -c apps/api/wrangler.preview.jsonc +// Keep database_id in sync with the `previews` block in wrangler.jsonc. +{ + "$schema": "node_modules/wrangler/config-schema.json", + "name": "uploads-api", + "d1_databases": [ + { + "binding": "DB", + "database_name": "uploads-preview", + "database_id": "b8c82095-b471-4e3a-8634-56e0861e43f1", + "migrations_dir": "migrations", + }, + ], +} diff --git a/apps/auth/wrangler.jsonc b/apps/auth/wrangler.jsonc index 6fcda8f9..c38e71e4 100644 --- a/apps/auth/wrangler.jsonc +++ b/apps/auth/wrangler.jsonc @@ -126,4 +126,79 @@ }, ], }, + // Workers Previews (`npx wrangler preview` from a branch). Deliberately + // narrow for this worker: BETTER_AUTH_URL must equal the preview's own + // origin, which is only known after the first preview deploy prints it — + // override it per-preview (dashboard → the Preview's settings) before + // exercising auth flows. The GitHub OAuth app only registers production + // callback URLs, so social login cannot work on a preview; magic-link and + // session flows can once BETTER_AUTH_URL is set. Auth state must never be + // shared, so previews get their own D1. Bindings do NOT inherit from the + // production config — a preview gets only what this block lists — so the + // Secrets Store references, EMAIL, the API service binding (always calls + // production uploads-api), and the rate limiter are repeated verbatim. + // Note a preview CAN send real mail from noreply@uploads.sh, but only + // explicit flows (magic links) do. + // + // One-time setup (migrations go through wrangler.preview.jsonc — the + // `d1 migrations` command cannot see databases declared inside `previews`): + // wrangler d1 create uploads-auth-preview + // wrangler d1 migrations apply uploads-auth-preview --remote -c apps/auth/wrangler.preview.jsonc + "previews": { + "vars": { + "BETTER_AUTH_URL": "https://auth.uploads.sh", + "WEB_ORIGIN": "https://uploads.sh", + "ENVIRONMENT": "preview", + "STRIPE_CHECKOUT_TOS_CONSENT": "true", + }, + "d1_databases": [ + { + "binding": "DB", + "database_name": "uploads-auth-preview", + "database_id": "8f581b6a-6238-4153-a846-a94b83cac86c", + "migrations_dir": "migrations", + }, + ], + "secrets_store_secrets": [ + { + "binding": "UPL_BETTER_AUTH_SECRET", + "store_id": "a887a71cab084105b79706df23380723", + "secret_name": "UPL_BETTER_AUTH_SECRET", + }, + { + "binding": "UPL_BETTER_AUTH_API_KEY", + "store_id": "a887a71cab084105b79706df23380723", + "secret_name": "UPL_BETTER_AUTH_API_KEY", + }, + { + "binding": "UPL_GITHUB_CLIENT_ID", + "store_id": "a887a71cab084105b79706df23380723", + "secret_name": "UPL_GITHUB_CLIENT_ID", + }, + { + "binding": "UPL_GITHUB_CLIENT_SECRET", + "store_id": "a887a71cab084105b79706df23380723", + "secret_name": "UPL_GITHUB_CLIENT_SECRET", + }, + ], + "send_email": [{ "name": "EMAIL", "allowed_sender_addresses": ["noreply@uploads.sh"] }], + "services": [ + { + "binding": "API", + "service": "uploads-api", + }, + ], + // Previews declare rate limiters via the dedicated `ratelimits` field + // (same values as the production `unsafe` entry above). + "ratelimits": [ + { + "name": "AUTH_RATE_LIMITER", + "namespace_id": "1003", + "simple": { + "limit": 60, + "period": 60, + }, + }, + ], + }, } diff --git a/apps/auth/wrangler.preview.jsonc b/apps/auth/wrangler.preview.jsonc new file mode 100644 index 00000000..bfe62a28 --- /dev/null +++ b/apps/auth/wrangler.preview.jsonc @@ -0,0 +1,18 @@ +// Companion config for the preview-tier D1 only. `wrangler d1 migrations` +// resolves databases from top-level `d1_databases` and cannot see the one +// declared inside the main config's `previews` block, so migrations for the +// preview database run against this file instead: +// npx wrangler d1 migrations apply uploads-auth-preview --remote -c apps/auth/wrangler.preview.jsonc +// Keep database_id in sync with the `previews` block in wrangler.jsonc. +{ + "$schema": "node_modules/wrangler/config-schema.json", + "name": "uploads-auth", + "d1_databases": [ + { + "binding": "DB", + "database_name": "uploads-auth-preview", + "database_id": "8f581b6a-6238-4153-a846-a94b83cac86c", + "migrations_dir": "migrations", + }, + ], +} diff --git a/apps/mcp/wrangler.jsonc b/apps/mcp/wrangler.jsonc index 48a99777..76fcb309 100644 --- a/apps/mcp/wrangler.jsonc +++ b/apps/mcp/wrangler.jsonc @@ -103,4 +103,66 @@ "bucket_name": "uploads-default", }, ], + // Workers Previews (`npx wrangler preview` from a branch). Mirrors + // apps/api's previews block — the two workers must resolve the same + // registry, database, and buckets at the preview tier exactly as they do in + // production, so the ids below are the SAME ids as apps/api's previews + // block (the resources are created once, from that file's instructions). + // Bindings do NOT inherit from the production config — a preview gets only + // what this block lists — so the AUTH service binding (always calls + // production uploads-auth) and the rate limiter are repeated verbatim. + "previews": { + "vars": { + "WEB_ORIGIN": "https://uploads.sh", + "AUTH_ORIGIN": "https://auth.uploads.sh", + "GITHUB_APP_ID": "4346270", + "GITHUB_APP_HOME_INSTALLATION_ID": "147814297", + }, + "kv_namespaces": [ + { + "binding": "REGISTRY", + "id": "7bb5509c12f54fd7b238c75726e63c09", + }, + // Shared with production on purpose — see apps/api's previews block. + { + "binding": "GITHUB_CACHE", + "id": "8076084dae50495ab883ec79c1c5968f", + }, + ], + "d1_databases": [ + { + "binding": "DB", + "database_name": "uploads-preview", + "database_id": "b8c82095-b471-4e3a-8634-56e0861e43f1", + }, + ], + "r2_buckets": [ + { + "binding": "UPLOADS", + "bucket_name": "buildinternet-dev", + }, + { + "binding": "UPLOADS_DEFAULT", + "bucket_name": "uploads-preview-default", + }, + ], + "services": [ + { + "binding": "AUTH", + "service": "uploads-auth", + }, + ], + // Previews declare rate limiters via the dedicated `ratelimits` field + // (same values as the production `unsafe` entry above). + "ratelimits": [ + { + "name": "WRITE_LIMITER", + "namespace_id": "1002", + "simple": { + "limit": 60, + "period": 60, + }, + }, + ], + }, } diff --git a/apps/web/wrangler.jsonc b/apps/web/wrangler.jsonc index 5469908f..f0b58d32 100644 --- a/apps/web/wrangler.jsonc +++ b/apps/web/wrangler.jsonc @@ -57,4 +57,41 @@ "custom_domain": true, }, ], + // Workers Previews (`npx wrangler preview` from a branch — branch-scoped + // environments, distinct from the legacy `preview_urls` version URLs above). + // uploads-web holds no state of its own, so previews serve the branch's UI + // against production api/auth — session auth still applies, and that is + // exactly what UI review wants. Bindings do NOT inherit from the production + // config — a preview gets only what this block lists (wrangler warns about + // "diverged" bindings otherwise) — so vars and bindings are spelled out in + // full. For a branch that spans apps, override these origins per-preview + // (dashboard → the Preview's settings) to point at the sibling workers' + // preview URLs. + "previews": { + "vars": { + "UPLOADS_API_ORIGIN": "https://api.uploads.sh", + "UPLOADS_AUTH_ORIGIN": "https://auth.uploads.sh", + "CONSOLE_MODE": "linked-only", + }, + // The Astro Cloudflare adapter's generated config (dist/server/wrangler.json, + // via the .wrangler/deploy redirect) injects an id-less SESSION KV binding. + // Wrangler normally auto-provisions an omitted id and writes it back to + // the config file, but a generated build artifact has no user config to + // write back to, so the provisioning never engages and the preview deploy + // fails with error 10021. Previews therefore pin an explicit, + // preview-only session namespace here. + "kv_namespaces": [ + { + "binding": "SESSION", + "id": "e72efe1ae34b48aea469481595f38c3d", + }, + ], + // Same read-only flag app as production. + "flagship": [ + { + "binding": "FLAGS", + "app_id": "8371bfe7-9767-4b4d-b75a-37b94d2724f7", + }, + ], + }, } diff --git a/docs/previews.md b/docs/previews.md new file mode 100644 index 00000000..f8014924 --- /dev/null +++ b/docs/previews.md @@ -0,0 +1,103 @@ +# Workers Previews + +A Preview is a branch environment for one worker: run `npx wrangler preview` +from an app directory and the branch gets its own URL, vars, secrets, and +bindings, without touching the production deployment. Previews are enabled on +the Build Internet account (Cloudflare docs: +[Workers Previews](https://developers.cloudflare.com/workers/previews/)). + +Preview URLs are gated by Cloudflare Access. Sign in with a Cloudflare account +that is a member of the Build Internet account and you pass straight through. + +Do not confuse a Preview with the Workers Builds bot's "Branch Preview URL" +comment on PRs. That URL is the legacy aliased-version mechanism +(`preview_urls: true`) and runs against **production bindings**. Only +`npx wrangler preview` uses the preview tier described here. + +## The two layers + +**Layer 1 — the shared preview tier.** The `previews` block in each app's +`wrangler.jsonc` is the default every preview of that worker gets. All +previews of `uploads-api`, on every branch, share the same preview-tier +resources: the `uploads-preview` D1, the preview REGISTRY KV, the +`uploads-preview-default` bucket. Previews are isolated from production, not +from each other. + +**Layer 2 — per-branch overrides.** `wrangler preview` reads the `previews` +block from the current branch. A branch that needs to deviate edits its own +`previews` block — for example, pointing `DB` at a scratch database — and the +change applies only to that branch's preview. Vars and secrets can also be +overridden per-preview: `wrangler preview secret put NAME --name `, +or the Preview's settings in the dashboard. + +Bindings do not inherit from the production config: a preview gets only what +the `previews` block lists. When you add a binding to an app, add it to the +`previews` block too, or `wrangler preview` warns that the configuration has +diverged. + +## Which app to preview + +Cross-service references (service bindings, origin vars) point at production +by default, so one preview usually suffices: + +| Change | Preview | Everything else | +| --------------------- | ---------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| Web UI | `uploads-web` only | Talks to prod api/auth — realistic, signed out | +| API | `uploads-api` only | Hit its URL with curl or the CLI; writes land in the preview tier; sessions verify against prod auth | +| Cross-app (web + api) | Both | Override the web preview's `UPLOADS_API_ORIGIN` var to the api preview's URL | +| Auth | `uploads-auth`, rarely | Needs a per-preview `BETTER_AUTH_URL` override; magic-link flows only — GitHub OAuth callbacks only cover production origins | + +Platform rules that shape this: service bindings always call the bound +worker's production deployment, and cron triggers, queue consumers, and +routes never run against a preview. + +## Playbook + +**Deploying a preview.** + +```bash +cd apps/web && pnpm build # web only; the other apps bundle on deploy +npx wrangler preview # preview name defaults to the git branch +``` + +The output prints two URLs: the Preview URL (tracks the latest deployment for +that name) and an immutable per-deployment URL. Clean up with +`wrangler preview delete ` — the account caps active previews per +worker, evicting the least-recently deployed first. + +**Schema-changing branches.** The shared `uploads-preview` D1 serves every +open preview, so applying a branch's migration migrates it for all of them. +Additive migrations: fine — apply through the companion config: + +```bash +npx wrangler d1 migrations apply uploads-preview --remote -c apps/api/wrangler.preview.jsonc +``` + +(The companion `wrangler.preview.jsonc` exists because `d1 migrations` cannot +see databases declared inside a `previews` block.) Destructive or risky +migrations: give the branch a scratch database instead — in the branch's own +`previews` block, remove the shared entry's identifier. For KV, D1, and R2, +omitting the identifier (`id`, `database_name`, or `bucket_name`) makes +`wrangler preview` create a fresh resource on first run and write the +identifier back to the config. Revert the override before merge and delete +the scratch resource. + +**Seeding.** The preview tier starts empty — no workspaces, no files. Mint a +workspace against the preview api URL the same way you would against +production (see [docs/workspaces.md](workspaces.md)), or exercise public +endpoints directly. + +**Secrets.** Preview secrets start unset, and every secret-gated path fails +closed — an unset `ADMIN_TOKEN` means 401, an unset GitHub App key degrades +the comment path. Leave them unset until a branch actually needs one, then +set it per-preview (`wrangler preview secret put NAME --name `) so +the blast radius stays one branch. Avoid seeding production credentials into +the preview base config. + +## Decisions of record + +- Previews stay on workers.dev URLs. Hostnames under `uploads.sh` would + receive the `.uploads.sh`-scoped production session cookie, handing live + sessions to unreviewed branch code. Revisit tracked in + [issue #684](https://github.com/buildinternet/uploads/issues/684). +- The legacy `preview_urls: true` flags stay during the transition.