diff --git a/.changeset/fast-media-usage-backfill.md b/.changeset/fast-media-usage-backfill.md new file mode 100644 index 0000000000..b413543137 --- /dev/null +++ b/.changeset/fast-media-usage-backfill.md @@ -0,0 +1,6 @@ +--- +"emdash": patch +"@emdash-cms/cloudflare": patch +--- + +Speeds up initial Media Usage indexing on Cloudflare Workers and Node.js. diff --git a/.changeset/media-usage-activation-ui.md b/.changeset/media-usage-activation-ui.md new file mode 100644 index 0000000000..3daf08cfd2 --- /dev/null +++ b/.changeset/media-usage-activation-ui.md @@ -0,0 +1,7 @@ +--- +"emdash": minor +"@emdash-cms/admin": minor +"@emdash-cms/cloudflare": minor +--- + +Adds an admin setup flow for Media Usage. Administrators can enable it from Settings with one confirmation and follow setup and indexing progress until it is ready. diff --git a/.changeset/media-usage-progress-locale.md b/.changeset/media-usage-progress-locale.md new file mode 100644 index 0000000000..c865c1ee61 --- /dev/null +++ b/.changeset/media-usage-progress-locale.md @@ -0,0 +1,5 @@ +--- +"@emdash-cms/admin": patch +--- + +Fixes Media Usage progress counts showing raw plural syntax instead of readable text. diff --git a/.changeset/remove-media-usage-scheduled-recovery.md b/.changeset/remove-media-usage-scheduled-recovery.md new file mode 100644 index 0000000000..efd35dfaf5 --- /dev/null +++ b/.changeset/remove-media-usage-scheduled-recovery.md @@ -0,0 +1,12 @@ +--- +"emdash": minor +"@emdash-cms/cloudflare": minor +--- + +Removes scheduled Media Usage recovery APIs. Media Usage now starts when an administrator enables it and continues through the Cloudflare Queue or Node.js scheduler. This is a breaking change for Cloudflare deployments that configure `mediaUsageCron` and Node.js integrations that provide a custom `CronScheduler`. + +#### What should I do? + +On Cloudflare, remove the dedicated Media Usage Cron and the `mediaUsageCron` option. Configure `createMediaUsageFetchHandler()` and `createMediaUsageQueueHandler()` to start and continue indexing. + +If you provide a custom Node.js scheduler, replace `setMediaUsageMaintenance()` with `setContinuousMediaUsageMaintenance()` and `wakeMediaUsageMaintenance()`. Keep the Queue consumer or Node.js process running until Media Usage shows **Ready**; the general Cron no longer restarts interrupted indexing. diff --git a/demos/cloudflare/src/worker.ts b/demos/cloudflare/src/worker.ts index d154c752d9..8843215f7f 100644 --- a/demos/cloudflare/src/worker.ts +++ b/demos/cloudflare/src/worker.ts @@ -7,11 +7,21 @@ */ import handler from "@astrojs/cloudflare/entrypoints/server"; -import { createScheduledHandler, PluginBridge } from "@emdash-cms/cloudflare/worker"; +import { + createMediaUsageQueueHandler, + createMediaUsageFetchHandler, + createScheduledHandler, + type MediaUsageWakeMessage, + PluginBridge, +} from "@emdash-cms/cloudflare/worker"; export { PluginBridge }; +const resolveMediaUsageQueue = (env: Env) => env.MEDIA_USAGE_QUEUE; + export default { ...handler, + fetch: createMediaUsageFetchHandler(handler, resolveMediaUsageQueue), scheduled: createScheduledHandler(), -} satisfies ExportedHandler; + queue: createMediaUsageQueueHandler(resolveMediaUsageQueue), +} satisfies ExportedHandler; diff --git a/demos/cloudflare/worker-configuration.d.ts b/demos/cloudflare/worker-configuration.d.ts index 6603007357..6dc5705216 100644 --- a/demos/cloudflare/worker-configuration.d.ts +++ b/demos/cloudflare/worker-configuration.d.ts @@ -8,6 +8,7 @@ declare namespace Cloudflare { interface Env { MEDIA: R2Bucket; DB: D1Database; + MEDIA_USAGE_QUEUE: Queue; LOADER: WorkerLoader; AI_SEARCH: AiSearchNamespace; } diff --git a/demos/cloudflare/wrangler.jsonc b/demos/cloudflare/wrangler.jsonc index 71bbfb825c..f6814b5c0b 100644 --- a/demos/cloudflare/wrangler.jsonc +++ b/demos/cloudflare/wrangler.jsonc @@ -34,9 +34,29 @@ "bucket_name": "emdash-media", }, ], - // Cron trigger drives the AI Search reindex queue flush. + "queues": { + "producers": [ + { + "binding": "MEDIA_USAGE_QUEUE", + "queue": "emdash-demo-media-usage", + }, + ], + "consumers": [ + { + "queue": "emdash-demo-media-usage", + "max_batch_size": 1, + "max_batch_timeout": 0, + "max_retries": 3, + "max_concurrency": 1, + }, + ], + }, + "limits": { + "cpu_ms": 300000, + }, + // Cron triggers drive general maintenance. "triggers": { - "crons": ["* * * * *", "*/2 * * * *"], + "crons": ["* * * * *"], }, // Observability "observability": { diff --git a/docs/src/content/docs/deployment/cloudflare.mdx b/docs/src/content/docs/deployment/cloudflare.mdx index 8e52d3650f..da6b7c5174 100644 --- a/docs/src/content/docs/deployment/cloudflare.mdx +++ b/docs/src/content/docs/deployment/cloudflare.mdx @@ -10,7 +10,8 @@ Cloudflare Workers provides a fast, globally distributed runtime for EmDash. Thi ## Prerequisites - A Cloudflare account -- Wrangler CLI installed (`npm install -g wrangler`) +- Workers Paid when activating automatic Media Usage indexing +- Wrangler 4.123 or newer (`npx wrangler@latest`) - Authenticated with Cloudflare (`wrangler login`) ## Configure Bindings @@ -94,46 +95,84 @@ If the database is empty (no collections) and the setup wizard hasn't been compl To change the schema or content model of a site that is already deployed, see [Evolving a Deployed Site](/deployment/schema-evolution/). -## Scheduled Publishing +## Scheduled tasks and Media Usage -On Cloudflare Workers, scheduled publishing, plugin cron, and maintenance tasks run from Worker Cron Triggers. New Cloudflare templates include both required schedules automatically. When updating an existing project, configure distinct general and Media Usage lanes: +Cloudflare runs scheduled publishing, plugin tasks, and general maintenance from one Cron Trigger. Media Usage uses a Queue so indexing starts when an administrator enables it and continues after the browser closes. + +New Cloudflare templates include this configuration. When updating an existing project, add the Media Usage handlers to the Worker entry point: ```ts title="src/worker.ts" import handler, { + createMediaUsageFetchHandler, + createMediaUsageQueueHandler, createScheduledHandler, + type MediaUsageWakeMessage, PluginBridge, } from "@emdash-cms/cloudflare/worker"; export { PluginBridge }; +const resolveMediaUsageQueue = (env: Env) => env.MEDIA_USAGE_QUEUE; + export default { ...handler, + fetch: createMediaUsageFetchHandler(handler, resolveMediaUsageQueue), scheduled: createScheduledHandler(), -} satisfies ExportedHandler; + queue: createMediaUsageQueueHandler(resolveMediaUsageQueue), +} satisfies ExportedHandler; ``` -By default, `*/2 * * * *` runs Media Usage maintenance and every other expression runs general maintenance. Then add both Cron Triggers to `wrangler.jsonc`: +Configure one Cron Trigger for general maintenance and one Queue for Media Usage in `wrangler.jsonc`: ```jsonc title="wrangler.jsonc" { "triggers": { - "crons": ["* * * * *", "*/2 * * * *"], + "crons": ["* * * * *"], + }, + "limits": { + "cpu_ms": 300000, + }, + "queues": { + "producers": [ + { + "binding": "MEDIA_USAGE_QUEUE", + "queue": "my-emdash-site-media-usage", + }, + ], + "consumers": [ + { + "queue": "my-emdash-site-media-usage", + "max_batch_size": 1, + "max_batch_timeout": 0, + "max_retries": 3, + "max_concurrency": 1, + }, + ], }, } ``` -To use different schedules, set the corresponding `generalCron` or `mediaUsageCron` option in `createScheduledHandler()` and use the same expression in `wrangler.jsonc`. +Run `wrangler types` after changing the bindings so `Env.MEDIA_USAGE_QUEUE` is available to TypeScript. + +The fetch handler sends the first Queue message after Media Usage is enabled. The consumer indexes up to 1,000 entries at a time and sends another message while work remains. Keep `max_concurrency` at `1` because D1 processes writes sequentially. + +To use a different general maintenance schedule, set `generalCron` in `createScheduledHandler()` and +use the same expression in `wrangler.jsonc`. -### Enable automatic media usage indexing +### Enable Media Usage + +1. If another application writes directly to D1, pause it before setup. +2. Open **Settings → Media Usage**, select **Enable Media Usage**, then select **Turn on**. +3. Keep the Worker and Queue running until the page shows **Ready**. You can close the page while indexing continues. +4. Resume direct D1 writers when the page shows **Starting indexing**, **Indexing existing content**, or **Ready**. + +EmDash may briefly pause editing while the page shows **Setting up**. If the page shows **Needs attention**, fix the reported server problem and select **Retry setup**. -Keep `mediaUsageCron` running while you enable automatic media usage indexing. Pause all application -and direct database writes, follow [Enable automatic media usage -indexing](/reference/rest-api/#enable-automatic-media-usage-indexing), then resume writes when the -endpoint returns `active`. Existing content is indexed in the background. +API operators can use the [Media Usage activation endpoints](/reference/rest-api/#enable-automatic-media-usage-indexing). ## Deploy diff --git a/docs/src/content/docs/deployment/nodejs.mdx b/docs/src/content/docs/deployment/nodejs.mdx index 89b8196815..0b60804f73 100644 --- a/docs/src/content/docs/deployment/nodejs.mdx +++ b/docs/src/content/docs/deployment/nodejs.mdx @@ -53,20 +53,26 @@ export default defineConfig({ The server runs on `http://localhost:4321` by default. Migrations are applied on the first request. If the database is empty and setup hasn't been completed, your seed file (or the built-in default if you don't have one) is also applied on that first request. -## Scheduled Tasks +## Scheduled tasks -The built-in scheduler runs only while a Node process is running. It handles scheduled publishing, -plugin tasks, and background media indexing. +The built-in scheduler runs only while a Node.js process is running. It handles scheduled publishing, plugin tasks, and background Media Usage indexing. -Keep at least one Node process running continuously in production. If all processes stop or sleep, -scheduled tasks pause. +The scheduler continues indexing until Media Usage catches up. It yields between batches so the server can keep handling requests. -### Enable automatic media usage indexing +Keep at least one Node.js process running continuously in production. Scheduled tasks pause when every process stops or sleeps. -Keep at least one Node process running while you enable automatic media usage indexing. Pause all -application and direct database writes, follow [Enable automatic media usage -indexing](/reference/rest-api/#enable-automatic-media-usage-indexing), then resume writes when the -endpoint returns `active`. Existing content is indexed in the background. +### Enable Media Usage + +1. If another application writes directly to the SQLite database, pause it before setup. +2. Open **Settings → Media Usage**, select **Enable Media Usage**, then select **Turn on**. +3. Keep the Node.js process running until the page shows **Ready**. You can close the page while indexing continues. +4. Resume direct database writers when the page shows **Starting indexing**, **Indexing existing content**, or **Ready**. + +EmDash may briefly pause editing while the page shows **Setting up**. If the page shows **Needs attention**, fix the reported server problem and select **Retry setup**. + +An interrupted Node.js process does not restart Media Usage indexing from the general scheduler. Keep the process running until setup reaches **Ready**. + +API operators can use the [Media Usage activation endpoints](/reference/rest-api/#enable-automatic-media-usage-indexing). ## Production Storage diff --git a/docs/src/content/docs/guides/media-library.mdx b/docs/src/content/docs/guides/media-library.mdx index dafefed4da..ed4f2d0e91 100644 --- a/docs/src/content/docs/guides/media-library.mdx +++ b/docs/src/content/docs/guides/media-library.mdx @@ -14,6 +14,17 @@ Open the media library from the admin sidebar by clicking **Media**. The library EmDash media library showing image grid with upload button +## Used in + +Open a local media item's details to see which EmDash-managed entries reference it. References that EmDash has already found remain visible while existing content is still indexing. + +Administrators see a setup notice until Media Usage is enabled. Open **Settings → Media Usage**, select **Enable Media Usage**, then select **Turn on**. Indexing continues in the background if you leave the page, and the Settings page shows its progress. + + + ## Uploading Files ### From the Media Library diff --git a/docs/src/content/docs/reference/rest-api.mdx b/docs/src/content/docs/reference/rest-api.mdx index 607e52587f..39644f27f4 100644 --- a/docs/src/content/docs/reference/rest-api.mdx +++ b/docs/src/content/docs/reference/rest-api.mdx @@ -366,11 +366,14 @@ DELETE /_emdash/api/media/:id ### Enable automatic media usage indexing -Automatic media usage indexing must be enabled once for each production site. Writes must be -paused while EmDash prepares each collection so that no changes are missed. +Automatic media usage indexing must be enabled once for each production site. Pause direct database writes while EmDash prepares each collection. EmDash temporarily blocks its own content and schema writes during this step. Both endpoints require `schema:manage`. Bearer tokens also require the `admin` scope. +Administrators can run the same procedure from **Settings → Media Usage** by selecting **Enable Media Usage**, then **Turn on**. The page sends one POST and displays the server-owned setup and indexing progress. + +Before using either flow, confirm that the Cloudflare Queue consumer or Node.js process is running. + #### Check the current state ```http @@ -385,7 +388,7 @@ This request does not change anything. It returns one of these states: Status responses do not include internal lock data or raw database errors. -#### Prepare the next collection +#### Start activation ```http POST /_emdash/api/admin/media-usage/activation @@ -398,28 +401,33 @@ X-EmDash-Request: 1 } ``` -Each request prepares one collection. Send one request at a time until the state becomes `active`. +The request prepares at most one collection and wakes the configured Cloudflare Queue or Node.js scheduler. These standard drivers continue with bounded batches after the response, including when the client disconnects. Use `GET` to observe the stored state. With a standard driver, do not send another POST unless the stored status reports a failure that you have fixed. + +Custom integrations that do not wire the standard Cloudflare Queue or Node continuation can continue +sending one POST at a time. This compatibility path retains the same collection bound. Set both fields to `true`: - `writersDrained`: Application and direct database writes have stopped, and any writes already in progress have finished. -- `maintenanceReady`: Scheduled background tasks are running through `mediaUsageCron` on Cloudflare - or the built-in scheduler on Node.js. +- `maintenanceReady`: The Media Usage Queue consumer is running on Cloudflare, or the built-in scheduler is running on Node.js. #### Enable indexing in production -1. Confirm that scheduled background tasks are running: `mediaUsageCron` on Cloudflare, or the - built-in scheduler on Node.js. -2. Stop all application and direct database writes. Wait for writes already in progress to finish. +1. Confirm that the Media Usage Queue consumer is running on Cloudflare, or that the built-in + scheduler is running on Node.js. +2. Stop all direct database writers. Wait for writes already in progress to finish. EmDash fences its own writes during setup. 3. Call `GET` to check the current state. -4. Call `POST` one request at a time until the state becomes `active`. -5. Resume writes. -6. Keep scheduled background tasks running while EmDash indexes existing content. +4. With the standard Queue or Node driver, call `POST` once, then use `GET` until the state becomes + `active`. With a custom integration that does not continue activation, alternate one `POST` with + `GET` until the state becomes `active`. +5. Resume writes after the state becomes `active`. +6. Keep the Queue consumer or Node process running while EmDash indexes existing content. -If `POST` times out or returns `409` or `500`, call `GET` before sending another request. If -`lastErrorCode` is set, keep writes stopped, check the application logs, fix the problem, and try -again. Do not edit EmDash's internal database tables. +If `POST` times out or returns `409` or `500`, call `GET` before deciding what to do. If the state is +still `activating` without `lastErrorCode`, the server is continuing or another worker owns the +current batch. If `lastErrorCode` is set, keep writes stopped, check the application logs, fix the +problem, and send one confirmed POST to retry. Do not edit EmDash's internal database tables.