diff --git a/.changeset/deployment-migration-manifest.md b/.changeset/deployment-migration-manifest.md new file mode 100644 index 0000000000..5c8efe6b24 --- /dev/null +++ b/.changeset/deployment-migration-manifest.md @@ -0,0 +1,5 @@ +--- +"emdash": minor +--- + +Adds a validated, secret-free deployment migration manifest during Astro build and sync so deployment tooling can run the exact migrations bundled with the site. diff --git a/.changeset/deployment-migration-primitives.md b/.changeset/deployment-migration-primitives.md new file mode 100644 index 0000000000..8d72784772 --- /dev/null +++ b/.changeset/deployment-migration-primitives.md @@ -0,0 +1,5 @@ +--- +"emdash": minor +--- + +Adds public migration identity and exact-status APIs so deployment tooling can verify and run the same core migration set as the installed EmDash version. diff --git a/.changeset/direct-migration-adapters.md b/.changeset/direct-migration-adapters.md new file mode 100644 index 0000000000..d3c3819752 --- /dev/null +++ b/.changeset/direct-migration-adapters.md @@ -0,0 +1,5 @@ +--- +"emdash": minor +--- + +Adds deployment migration executors for the built-in SQLite, libSQL, and PostgreSQL database adapters with secret-free build metadata. diff --git a/.changeset/runtime-migration-policy.md b/.changeset/runtime-migration-policy.md new file mode 100644 index 0000000000..b6c5d181db --- /dev/null +++ b/.changeset/runtime-migration-policy.md @@ -0,0 +1,5 @@ +--- +"emdash": minor +--- + +Adds `auto`, `check`, and `manual` runtime migration modes so deployments can verify or manage core database migrations before application traffic while existing sites remain on automatic migrations by default. diff --git a/packages/core/package.json b/packages/core/package.json index 8519f0fe0a..bb55d124ea 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -83,14 +83,30 @@ "types": "./dist/db/sqlite.d.mts", "default": "./dist/db/sqlite.mjs" }, + "./db/sqlite-migrations": { + "types": "./dist/db/sqlite-migrations.d.mts", + "default": "./dist/db/sqlite-migrations.mjs" + }, "./db/libsql": { "types": "./dist/db/libsql.d.mts", "default": "./dist/db/libsql.mjs" }, + "./db/libsql-migrations": { + "types": "./dist/db/libsql-migrations.d.mts", + "default": "./dist/db/libsql-migrations.mjs" + }, "./db/postgres": { "types": "./dist/db/postgres.d.mts", "default": "./dist/db/postgres.mjs" }, + "./db/postgres-migrations": { + "types": "./dist/db/postgres-migrations.d.mts", + "default": "./dist/db/postgres-migrations.mjs" + }, + "./migrations": { + "types": "./dist/migrations/index.d.mts", + "default": "./dist/migrations/index.mjs" + }, "./database/instrumentation": { "types": "./dist/database/instrumentation.d.mts", "default": "./dist/database/instrumentation.mjs" diff --git a/packages/core/src/astro/integration/index.ts b/packages/core/src/astro/integration/index.ts index 8480753ae9..07b196690b 100644 --- a/packages/core/src/astro/integration/index.ts +++ b/packages/core/src/astro/integration/index.ts @@ -11,11 +11,21 @@ */ import { createRequire } from "node:module"; +import { fileURLToPath } from "node:url"; import type { AstroIntegration, AstroIntegrationLogger, AstroIntegrationMiddleware } from "astro"; import { validateAllowedOrigins, validateOriginShape } from "../../auth/allowed-origins.js"; +import { normalizeMigrationConfig } from "../../database/migrations/policy.js"; +import { normalizeAstroI18n } from "../../i18n/normalize.js"; import { INTERNAL_MEDIA_PREFIX } from "../../media/normalize.js"; +import { getCoreMigrationIdentity } from "../../migrations/identity.js"; +import { + createMigrationIntegrationMetadata, + MIGRATION_CONFIG_SYMBOL, +} from "../../migrations/integration-metadata.js"; +import { buildMigrationManifest } from "../../migrations/manifest-builder.js"; +import { writeMigrationManifest } from "../../migrations/manifest-writer.js"; import type { ResolvedPlugin } from "../../plugins/types.js"; import { VERSION } from "../../version.js"; import { local } from "../storage/adapters.js"; @@ -313,6 +323,7 @@ export function emdash(config: EmDashConfig = {}): AstroIntegration { const resolvedConfig: EmDashConfig = { ...config, storage: config.storage ?? DEFAULT_STORAGE, + migrations: normalizeMigrationConfig(config.migrations), }; // Validate marketplace URL @@ -421,6 +432,7 @@ export function emdash(config: EmDashConfig = {}): AstroIntegration { // i18n is populated in astro:config:setup from astroConfig.i18n const serializableConfig: Record = { database: resolvedConfig.database, + migrations: resolvedConfig.migrations, storage: resolvedConfig.storage, auth: resolvedConfig.auth, authProviders: resolvedConfig.authProviders, @@ -440,8 +452,10 @@ export function emdash(config: EmDashConfig = {}): AstroIntegration { // Captured in astro:config:setup so the astro:server:setup hook can tell // whether we're running `astro dev` (where the dev-bypass shortcut applies). let astroCommand: "dev" | "build" | "preview" | "sync" | undefined; + let normalizedI18n: ReturnType = null; + const migrationMetadata = createMigrationIntegrationMetadata(resolvedConfig.database); - return { + const integration: AstroIntegration = { name: "emdash", hooks: { "astro:config:setup": ({ @@ -467,18 +481,8 @@ export function emdash(config: EmDashConfig = {}): AstroIntegration { // Expose Astro's trailingSlash routing policy so plugins can build // URLs (sitemap/canonical/hreflang) that match what the site serves. serializableConfig.trailingSlash = astroConfig.trailingSlash; - // Extract i18n config from Astro config - // Astro locales can be strings OR { path, codes } objects — normalize to paths - if (astroConfig.i18n) { - const routing = astroConfig.i18n.routing; - serializableConfig.i18n = { - defaultLocale: astroConfig.i18n.defaultLocale, - locales: astroConfig.i18n.locales.map((l) => (typeof l === "string" ? l : l.path)), - fallback: astroConfig.i18n.fallback, - prefixDefaultLocale: - typeof routing === "object" ? (routing.prefixDefaultLocale ?? false) : false, - }; - } + normalizedI18n = normalizeAstroI18n(astroConfig.i18n); + if (normalizedI18n) serializableConfig.i18n = normalizedI18n; // Disable Astro's built-in checkOrigin -- EmDash's own CSRF // layer (checkPublicCsrf in api/csrf.ts) handles origin @@ -603,9 +607,31 @@ export function emdash(config: EmDashConfig = {}): AstroIntegration { // dev server is listening (see astro:server:setup), since the // port isn't known yet here. Nothing useful to print for build. }, - "astro:config:done": ({ config: finalConfig, logger }) => { + "astro:config:done": async ({ config: finalConfig, logger }) => { const warning = missingReactIntegrationWarning(finalConfig.integrations); if (warning) logger.warn(warning); + + if (astroCommand !== "build" && astroCommand !== "sync") return; + if (!migrationMetadata.database) { + logger.warn( + "EmDash migration manifest was not written because no database adapter is configured.", + ); + return; + } + if (!migrationMetadata.database.migrations) { + logger.warn( + "EmDash migration manifest was not written because the configured database adapter does not support deployment migrations.", + ); + return; + } + + const identity = await getCoreMigrationIdentity(); + const manifest = await buildMigrationManifest({ + identity, + i18n: normalizedI18n, + database: migrationMetadata.database, + }); + await writeMigrationManifest(fileURLToPath(finalConfig.root), manifest); }, "astro:server:setup": ({ server, logger }) => { // Print route info with absolute, clickable URLs once the server @@ -683,6 +709,12 @@ export function emdash(config: EmDashConfig = {}): AstroIntegration { }, }, }; + + Object.defineProperty(integration, MIGRATION_CONFIG_SYMBOL, { + value: migrationMetadata, + enumerable: false, + }); + return integration; } export default emdash; diff --git a/packages/core/src/astro/integration/runtime.ts b/packages/core/src/astro/integration/runtime.ts index 33003bb27f..d7637cbff0 100644 --- a/packages/core/src/astro/integration/runtime.ts +++ b/packages/core/src/astro/integration/runtime.ts @@ -10,6 +10,7 @@ import type { ManifestHookEntry, ManifestRouteEntry } from "@emdash-cms/plugin-types"; import type { AuthDescriptor, AuthProviderDescriptor } from "../../auth/types.js"; +import type { RuntimeMigrationConfig } from "../../database/migrations/policy.js"; import type { DatabaseDescriptor } from "../../db/adapters.js"; import type { MediaProviderDescriptor } from "../../media/types.js"; import type { ObjectCacheDescriptor } from "../../object-cache/types.js"; @@ -180,6 +181,8 @@ export interface EmDashConfig { * ``` */ database?: DatabaseDescriptor; + /** Core database migration behavior at runtime. Defaults to `auto`. */ + migrations?: RuntimeMigrationConfig; /** * Storage configuration (for media) */ diff --git a/packages/core/src/astro/middleware.ts b/packages/core/src/astro/middleware.ts index eab7aa1245..1ad671330d 100644 --- a/packages/core/src/astro/middleware.ts +++ b/packages/core/src/astro/middleware.ts @@ -39,6 +39,11 @@ import { flushRecorder, isInstrumentationEnabled, } from "../database/instrumentation.js"; +import { + PendingMigrationsError, + resolveRuntimeMigrationMode, + type RuntimeMigrationMode, +} from "../database/migrations/policy.js"; import { createDeferredTaskTracker } from "../deferred-tasks.js"; import { DB_INIT_DEADLINE_MS, @@ -192,7 +197,10 @@ function getPlugins(): ResolvedPlugin[] { /** * Build runtime dependencies from virtual modules */ -function buildDependencies(config: EmDashConfig): RuntimeDependencies { +function buildDependencies( + config: EmDashConfig, + migrationMode: RuntimeMigrationMode, +): RuntimeDependencies { /* eslint-disable typescript-eslint/no-unsafe-type-assertion -- The virtual:emdash/* imports above use @ts-ignore because tsgo/IDE resolution can't see virtual-modules.d.ts in every consumer setup, @@ -202,6 +210,7 @@ function buildDependencies(config: EmDashConfig): RuntimeDependencies { const sandboxModule = virtualSandboxRunnerModule as Record; return { config, + migrationMode, plugins: getPlugins(), createDialect: virtualCreateDialect as (config: Record) => unknown, // Optional: only batching backends (D1, DO) export this; undefined otherwise. @@ -229,6 +238,7 @@ function buildDependencies(config: EmDashConfig): RuntimeDependencies { */ async function getRuntime( config: EmDashConfig, + migrationMode: RuntimeMigrationMode, initTimings?: Array<{ name: string; dur: number; desc?: string }>, ): Promise { // Waiters poll rather than awaiting the initializing request's promise — @@ -243,7 +253,7 @@ async function getRuntime( holder.lock, () => holder.instance, async (isCurrentClaim) => { - const deps = buildDependencies(config); + const deps = buildDependencies(config, migrationMode); const runtime = await EmDashRuntime.create(deps, initTimings); if (isCurrentClaim()) { holder.instance = runtime; @@ -360,8 +370,9 @@ async function runOutsideRequest( config: EmDashConfig, fn: (runtime: EmDashRuntime) => Promise, ): Promise { + const migrationMode = resolveConfiguredMigrationMode(config); if (getRequestContext()) { - const runtime = await getRuntime(config); + const runtime = await getRuntime(config, migrationMode); return runOutsideRequestWithRuntime(config, runtime, fn); } @@ -374,7 +385,7 @@ async function runOutsideRequest( return runWithContext(context, async () => { const runtime = await (async () => { try { - return await getRuntime(config); + return await getRuntime(config, migrationMode); } finally { deferredTasks.settle(); await deferredTasks.settled; @@ -452,6 +463,31 @@ const NOOP_COOKIE_JAR = { */ const CRON_EVENT_URL = new URL("https://cron.emdash.internal/"); +function resolveConfiguredMigrationMode(config: EmDashConfig): RuntimeMigrationMode { + const processOverride = + typeof process !== "undefined" && process.env ? process.env.EMDASH_MIGRATIONS_MODE : undefined; + const importMetaOverride = import.meta.env.EMDASH_MIGRATIONS_MODE; + return resolveRuntimeMigrationMode(config.migrations, { + dev: import.meta.env.DEV, + override: processOverride ?? importMetaOverride, + }); +} + +function pendingMigrationsResponse(error: PendingMigrationsError): Response { + console.error("[emdash] database migrations are pending:", error.pending.join(", ")); + return migrationRequiredResponse(); +} + +function migrationRequiredResponse(): Response { + return new Response( + "Database migrations are required. Apply the deployment migration manifest and retry.", + { + status: 503, + headers: { "Retry-After": "60" }, + }, + ); +} + /** * Baseline security headers applied to all responses. * Admin routes get additional headers (strict CSP) from auth middleware. @@ -601,6 +637,8 @@ export const onRequest = defineMiddleware(async (context, next) => { const metrics = createRequestMetrics(performance.now()); const run = async (): Promise => { + const config = getConfig(); + const migrationMode = config ? resolveConfiguredMigrationMode(config) : "auto"; // Process /_emdash routes and public routes with an active session // (logged-in editors need the runtime for toolbar/visual editing on public pages) const isEmDashRoute = url.pathname.startsWith("/_emdash"); @@ -668,7 +706,7 @@ export const onRequest = defineMiddleware(async (context, next) => { // production. The build database is legitimately empty in CI and there // is no live visitor to send to the wizard at build time (session reads // are already skipped for prerender above for the same reason). - if (!isSetupVerified() && !context.isPrerendered) { + if (migrationMode === "auto" && !isSetupVerified() && !context.isPrerendered) { const t0 = performance.now(); try { const { getDb } = await import("../loader.js"); @@ -695,14 +733,13 @@ export const onRequest = defineMiddleware(async (context, next) => { // The runtime is a cached singleton — after the first request, // getRuntime() is just a null-check. This enables SEO plugins to // contribute meta tags for all visitors, not just logged-in editors. - const config = getConfig(); if (config) { // Sub-phase timings are populated only on the cold init. Warm // requests hit the cached runtime and leave this empty. const initSubTimings: Array<{ name: string; dur: number; desc?: string }> = []; const t0 = performance.now(); try { - const runtime = await getRuntime(config, initSubTimings); + const runtime = await getRuntime(config, migrationMode, initSubTimings); markSetupVerified(); const handlePublicPluginApiRoute = createPublicPluginApiRouteHandler(runtime); // eslint-disable-next-line typescript/no-unsafe-type-assertion -- partial object; getPageRuntime() only checks for the page-contribution methods @@ -717,6 +754,16 @@ export const onRequest = defineMiddleware(async (context, next) => { storage: runtime.storage, } as EmDashHandlers; } catch (error) { + if (error instanceof PendingMigrationsError) { + return pendingMigrationsResponse(error); + } + if (migrationMode === "manual" && isMissingTableError(error)) { + console.error( + "[emdash] database schema is unavailable in manual migration mode:", + error, + ); + return migrationRequiredResponse(); + } // Non-fatal — EmDashHead falls back to base SEO contributions — // but log it (throttled): a persistently failing init (e.g. a // failing migration, #1744) is otherwise invisible on the @@ -796,7 +843,6 @@ export const onRequest = defineMiddleware(async (context, next) => { } } - const config = getConfig(); if (!config) { console.error("EmDash: No configuration found"); return finalizeResponse(await next()); @@ -816,14 +862,14 @@ export const onRequest = defineMiddleware(async (context, next) => { // instance and `initSubTimings` stays empty. const initSubTimings: Array<{ name: string; dur: number; desc?: string }> = []; let t0 = performance.now(); - const runtime = await getRuntime(config, initSubTimings); + const runtime = await getRuntime(config, migrationMode, initSubTimings); timings.push({ name: "rt", dur: performance.now() - t0, desc: "Runtime init" }); // Forward any sub-phase samples so cold-start breakdown is visible // in Server-Timing. Each phase appears prefixed "rt." to distinguish // from the aggregate "rt" timing above. for (const sub of initSubTimings) timings.push(sub); - // Runtime init runs migrations, so the DB is guaranteed set up + // Runtime initialization has satisfied the effective migration policy. markSetupVerified(); // The manifest is no longer pre-loaded here. It's admin-only @@ -942,6 +988,13 @@ export const onRequest = defineMiddleware(async (context, next) => { setPluginStatus: runtime.setPluginStatus.bind(runtime), }; } catch (error) { + if (error instanceof PendingMigrationsError) { + return pendingMigrationsResponse(error); + } + if (migrationMode === "manual" && isMissingTableError(error)) { + console.error("[emdash] database schema is unavailable in manual migration mode:", error); + return migrationRequiredResponse(); + } console.error("EmDash middleware error:", error); } diff --git a/packages/core/src/astro/routes/api/auth/dev-bypass.ts b/packages/core/src/astro/routes/api/auth/dev-bypass.ts index 9239de18a6..9f5f48497d 100644 --- a/packages/core/src/astro/routes/api/auth/dev-bypass.ts +++ b/packages/core/src/astro/routes/api/auth/dev-bypass.ts @@ -24,7 +24,6 @@ import { ulid } from "ulidx"; import { apiError, apiSuccess, handleError } from "#api/error.js"; import { escapeHtml } from "#api/escape.js"; import { isSafeRedirect } from "#api/redirect.js"; -import { runMigrations } from "#db/migrations/runner.js"; const DEV_USER_EMAIL = "dev@emdash.local"; const DEV_USER_NAME = "Dev Admin"; @@ -46,9 +45,6 @@ async function handleDevBypass(context: Parameters[0]): Promise[0]): Promise[0]): Promise { const body = await parseBody(request, setupBody); if (isParseError(body)) return body; - // 1. Run core migrations - try { - await runMigrations(emdash.db); - } catch (error) { - return handleError(error, "Failed to run database migrations", "MIGRATION_ERROR"); - } - - // 2. Load seed file (user seed or built-in default) + // Load seed file (user seed or built-in default) const seed = await loadSeed(); - // 3. Override seed settings with form values + // Override seed settings with form values seed.settings = { ...seed.settings, title: body.title, tagline: body.tagline, }; - // 4. Apply seed + // Apply seed const validation = validateSeed(seed); if (!validation.valid) { return apiError("INVALID_SEED", `Invalid seed file: ${validation.errors.join(", ")}`, 400); @@ -79,7 +71,7 @@ export const POST: APIRoute = async ({ request, url, locals }) => { return handleError(error, "Failed to apply seed", "SEED_ERROR"); } - // 5. Store setup state + // Store setup state // In external auth mode, mark setup complete immediately (first user to login becomes admin) // Otherwise, setup_complete is set after admin user is created (passkey or auth provider) const authMode = getAuthMode(emdash.config); @@ -117,7 +109,7 @@ export const POST: APIRoute = async ({ request, url, locals }) => { // Non-fatal - continue anyway } - // 6. Return success with result + // Return success with result return apiSuccess({ success: true, // In external auth mode, setup is complete - redirect to admin diff --git a/packages/core/src/database/index.ts b/packages/core/src/database/index.ts index f4967537c0..ebfb169066 100644 --- a/packages/core/src/database/index.ts +++ b/packages/core/src/database/index.ts @@ -2,6 +2,12 @@ // `connection.ts`, which statically imports `better-sqlite3`. See #947. export { EmDashDatabaseError } from "./errors.js"; export type { DatabaseConfig } from "./connection.js"; -export { runMigrations, getMigrationStatus, rollbackMigration } from "./migrations/runner.js"; -export type { MigrationStatus } from "./migrations/runner.js"; +export { + runMigrations, + getMigrationStatus, + getExactMigrationStatus, + rollbackMigration, + MIGRATION_NAMES, +} from "./migrations/runner.js"; +export type { MigrationStatus, ExactMigrationStatus } from "./migrations/runner.js"; export type * from "./types.js"; diff --git a/packages/core/src/database/migrations/policy.ts b/packages/core/src/database/migrations/policy.ts new file mode 100644 index 0000000000..db6dca7563 --- /dev/null +++ b/packages/core/src/database/migrations/policy.ts @@ -0,0 +1,70 @@ +import type { Kysely } from "kysely"; + +import type { Database } from "../types.js"; +import { getExactMigrationStatus, runMigrations } from "./runner.js"; + +export type RuntimeMigrationMode = "auto" | "check" | "manual"; + +export interface RuntimeMigrationConfig { + runtime: RuntimeMigrationMode; + dev?: RuntimeMigrationMode; +} + +function parseMigrationMode(value: unknown, source: string): RuntimeMigrationMode { + if (value === "auto" || value === "check" || value === "manual") { + return value; + } + throw new Error( + `Invalid ${source} value ${JSON.stringify(value)}; expected "auto", "check", or "manual"`, + ); +} + +export function normalizeMigrationConfig(value: unknown): RuntimeMigrationConfig { + if (value === undefined) return { runtime: "auto" }; + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("Invalid migrations configuration; expected an object"); + } + + const runtimeValue = Reflect.get(value, "runtime"); + const devValue = Reflect.get(value, "dev"); + const runtime = parseMigrationMode(runtimeValue, "migrations.runtime"); + const dev = devValue === undefined ? undefined : parseMigrationMode(devValue, "migrations.dev"); + return dev === undefined ? { runtime } : { runtime, dev }; +} + +export function resolveRuntimeMigrationMode( + config: RuntimeMigrationConfig | undefined, + options: { dev: boolean; override?: unknown }, +): RuntimeMigrationMode { + if (options.override !== undefined) { + return parseMigrationMode(options.override, "EMDASH_MIGRATIONS_MODE"); + } + if (options.dev) return config?.dev ?? "auto"; + return config?.runtime ?? "auto"; +} + +export class PendingMigrationsError extends Error { + readonly pending: string[]; + + constructor(pending: readonly string[]) { + super(`Database has pending EmDash migrations: ${pending.join(", ")}`); + this.name = "PendingMigrationsError"; + this.pending = [...pending]; + } +} + +export async function enforceRuntimeMigrationPolicy( + db: Kysely, + mode: RuntimeMigrationMode, +): Promise { + if (mode === "manual") return; + if (mode === "auto") { + await runMigrations(db); + return; + } + + const status = await getExactMigrationStatus(db); + if (status.pending.length > 0) { + throw new PendingMigrationsError(status.pending); + } +} diff --git a/packages/core/src/database/migrations/runner.ts b/packages/core/src/database/migrations/runner.ts index 8f60cd7e7c..84d1cda67e 100644 --- a/packages/core/src/database/migrations/runner.ts +++ b/packages/core/src/database/migrations/runner.ts @@ -146,8 +146,11 @@ const MIGRATIONS: Readonly> = Object.freeze({ "070_collection_routable": m070, }); +/** Ordered names from the statically registered migration set. */ +export const MIGRATION_NAMES: readonly string[] = Object.freeze(Object.keys(MIGRATIONS)); + /** Total number of registered migrations. Exported for use in tests. */ -export const MIGRATION_COUNT = Object.keys(MIGRATIONS).length; +export const MIGRATION_COUNT = MIGRATION_NAMES.length; /** * Migration provider that uses statically imported migrations. @@ -164,6 +167,12 @@ export interface MigrationStatus { pending: string[]; } +export interface ExactMigrationStatus { + knownApplied: string[]; + pending: string[]; + unknownApplied: string[]; +} + /** * Thrown when another instance held the migration lock for the whole wait * window. This is NOT a migration failure — the holder may simply be slow @@ -286,8 +295,8 @@ const MIGRATION_RACE_POLL_MS = 100; * built from `MIGRATION_TABLE` so a rename cannot drift. */ const MIGRATION_TABLE_MISSING_PATTERN = new RegExp( - `(?:no such table:\\s*${escapeRegExp(MIGRATION_TABLE)}\\b` + - `|(?:relation|table)\\s+"?${escapeRegExp(MIGRATION_TABLE)}"?\\s+does(?:n't| not) exist\\b)`, + `(?:no such table:\\s*(?:[a-z][a-z0-9_]*\\.)?${escapeRegExp(MIGRATION_TABLE)}\\b` + + `|(?:relation|table)\\s+"?(?:[a-z][a-z0-9_]*\\.)?${escapeRegExp(MIGRATION_TABLE)}"?\\s+does(?:n't| not) exist\\b)`, "i", ); @@ -359,6 +368,43 @@ function deepErrorMessage(error: unknown): string { } } +/** + * Read exact migration status without invoking Kysely's migration + * introspection. Registered names retain execution order; unknown database + * records are sorted so reports remain stable across dialects. + */ +export async function getExactMigrationStatus( + db: Kysely, + options?: MigrationOptions, +): Promise { + const table = options?.migrationTableSchema + ? sql`${sql.ref(options.migrationTableSchema)}.${sql.ref(MIGRATION_TABLE)}` + : sql.ref(MIGRATION_TABLE); + + let rows: readonly { name: string }[]; + try { + const result = await sql<{ name: string }>`SELECT name FROM ${table}`.execute(db); + rows = result.rows; + } catch (error) { + if (MIGRATION_TABLE_MISSING_PATTERN.test(deepErrorMessage(error))) { + return { + knownApplied: [], + pending: [...MIGRATION_NAMES], + unknownApplied: [], + }; + } + throw error; + } + + const appliedNames = new Set(rows.map((row) => row.name)); + const knownApplied = MIGRATION_NAMES.filter((name) => appliedNames.has(name)); + const pending = MIGRATION_NAMES.filter((name) => !appliedNames.has(name)); + const knownNames = new Set(MIGRATION_NAMES); + const unknownApplied = [...appliedNames].filter((name) => !knownNames.has(name)).toSorted(); + + return { knownApplied, pending, unknownApplied }; +} + /** * Run all pending migrations. * diff --git a/packages/core/src/db/adapters.ts b/packages/core/src/db/adapters.ts index 3802e0b513..caf05961c5 100644 --- a/packages/core/src/db/adapters.ts +++ b/packages/core/src/db/adapters.ts @@ -52,6 +52,20 @@ export type ExecuteCollectionDeletionGuard = ( input: CollectionDeletionGuardInput, ) => Promise; +const ENVIRONMENT_VARIABLE_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; + +function migrationEnvironmentVariable( + value: string | undefined, + fallback: string, + optionName: string, +): string { + const name = value ?? fallback; + if (!ENVIRONMENT_VARIABLE_PATTERN.test(name)) { + throw new Error(`${optionName} must be a valid environment variable name.`); + } + return name; +} + /** * Database descriptor - serializable config for virtual modules */ @@ -59,6 +73,11 @@ export interface DatabaseDescriptor { entrypoint: string; config: unknown; type: DatabaseDialectType; + /** Deployment migration capability with configuration safe for a build artifact. */ + migrations?: { + entrypoint: string; + manifestConfig: unknown; + }; /** * When true, the adapter's runtime entrypoint MUST export a named * `createRequestScopedDb` function matching the signature declared in @@ -111,6 +130,7 @@ export interface LibsqlConfig { * Auth token for remote libSQL */ authToken?: string; + migrationAuthTokenEnv?: string; } /** @@ -128,6 +148,10 @@ export function sqlite(config: SqliteConfig): DatabaseDescriptor { entrypoint: "emdash/db/sqlite", config, type: "sqlite", + migrations: { + entrypoint: "emdash/db/sqlite-migrations", + manifestConfig: { url: config.url }, + }, }; } @@ -145,10 +169,22 @@ export function sqlite(config: SqliteConfig): DatabaseDescriptor { * ``` */ export function libsql(config: LibsqlConfig): DatabaseDescriptor { + const { migrationAuthTokenEnv, ...runtimeConfig } = config; return { entrypoint: "emdash/db/libsql", - config, + config: runtimeConfig, type: "sqlite", + migrations: { + entrypoint: "emdash/db/libsql-migrations", + manifestConfig: { + url: config.url, + authTokenEnv: migrationEnvironmentVariable( + migrationAuthTokenEnv, + "TURSO_AUTH_TOKEN", + "migrationAuthTokenEnv", + ), + }, + }, }; } @@ -164,6 +200,7 @@ export interface PostgresConfig { password?: string; ssl?: boolean; pool?: { min?: number; max?: number }; + migrationConnectionStringEnv?: string; } /** @@ -177,9 +214,20 @@ export interface PostgresConfig { * ``` */ export function postgres(config: PostgresConfig): DatabaseDescriptor { + const { migrationConnectionStringEnv, ...runtimeConfig } = config; return { entrypoint: "emdash/db/postgres", - config, + config: runtimeConfig, type: "postgres", + migrations: { + entrypoint: "emdash/db/postgres-migrations", + manifestConfig: { + connectionStringEnv: migrationEnvironmentVariable( + migrationConnectionStringEnv, + "DATABASE_URL", + "migrationConnectionStringEnv", + ), + }, + }, }; } diff --git a/packages/core/src/db/index.ts b/packages/core/src/db/index.ts index cebd97424c..c88a327e69 100644 --- a/packages/core/src/db/index.ts +++ b/packages/core/src/db/index.ts @@ -35,6 +35,8 @@ export type { export { runMigrations, getMigrationStatus, + getExactMigrationStatus, rollbackMigration, + MIGRATION_NAMES, } from "../database/migrations/runner.js"; -export type { MigrationStatus } from "../database/migrations/runner.js"; +export type { MigrationStatus, ExactMigrationStatus } from "../database/migrations/runner.js"; diff --git a/packages/core/src/db/libsql-migrations.ts b/packages/core/src/db/libsql-migrations.ts new file mode 100644 index 0000000000..9f336f1eee --- /dev/null +++ b/packages/core/src/db/libsql-migrations.ts @@ -0,0 +1,72 @@ +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { createDirectMigrationExecutor } from "../migrations/direct-executor.js"; +import type { MigrationExecutor, MigrationExecutorFactoryContext } from "../migrations/protocol.js"; +import { createMigrationTarget, requireMigrationEnvironment } from "../migrations/target.js"; +import { createDialect } from "./libsql.js"; + +export interface LibsqlMigrationManifestConfig { + url: string; + authTokenEnv: string; +} + +interface ResolvedLibsqlUrl { + connectionUrl: string; + label: string; + identity: string[]; + requiresAuthToken: boolean; +} + +function resolveLibsqlUrl(url: unknown, projectRoot: string): ResolvedLibsqlUrl { + if (typeof url !== "string" || url.length === 0) { + throw new Error("libSQL migration URL is missing."); + } + if (url.startsWith("file:")) { + const configuredPath = url.slice(5); + if (configuredPath.length === 0) throw new Error("libSQL migration URL is invalid."); + const fileUrl = pathToFileURL(resolve(projectRoot, configuredPath)); + return { + connectionUrl: fileUrl.href, + label: fileUrl.href, + identity: [fileUrl.protocol, fileUrl.pathname], + requiresAuthToken: false, + }; + } + + let parsed: URL; + try { + parsed = new URL(url); + } catch { + throw new Error("libSQL migration URL is invalid."); + } + if (parsed.username || parsed.password) { + throw new Error("libSQL migration URL must not contain credentials."); + } + const label = `${parsed.protocol}//${parsed.host}${parsed.pathname}`; + return { + connectionUrl: url, + label, + identity: [parsed.protocol, parsed.host, parsed.pathname], + requiresAuthToken: true, + }; +} + +export async function createMigrationExecutor( + manifestConfig: LibsqlMigrationManifestConfig, + context: MigrationExecutorFactoryContext, +): Promise { + const resolvedUrl = resolveLibsqlUrl(manifestConfig.url, context.projectRoot); + const authToken = resolvedUrl.requiresAuthToken + ? requireMigrationEnvironment(manifestConfig.authTokenEnv, context.env) + : undefined; + const target = await createMigrationTarget("libsql", resolvedUrl.label, resolvedUrl.identity); + return createDirectMigrationExecutor({ + target, + createDialect: () => + createDialect({ + url: resolvedUrl.connectionUrl, + authToken, + }), + }); +} diff --git a/packages/core/src/db/postgres-migrations.ts b/packages/core/src/db/postgres-migrations.ts new file mode 100644 index 0000000000..e7c42c6d22 --- /dev/null +++ b/packages/core/src/db/postgres-migrations.ts @@ -0,0 +1,59 @@ +import { createDirectMigrationExecutor } from "../migrations/direct-executor.js"; +import type { MigrationExecutor, MigrationExecutorFactoryContext } from "../migrations/protocol.js"; +import { createMigrationTarget, requireMigrationEnvironment } from "../migrations/target.js"; +import { createDialect } from "./postgres.js"; + +export interface PostgresMigrationManifestConfig { + connectionStringEnv: string; +} + +interface PostgresTargetIdentity { + host: string; + port: string; + database: string; +} + +function parseTargetIdentity(connectionString: string): PostgresTargetIdentity { + let parsed: URL; + try { + parsed = new URL(connectionString); + } catch { + throw new Error("PostgreSQL migration connection string is invalid."); + } + if (parsed.protocol !== "postgres:" && parsed.protocol !== "postgresql:") { + throw new Error("PostgreSQL migration connection string is invalid."); + } + const database = parsed.pathname.slice(1); + if (!parsed.hostname || !database) { + throw new Error("PostgreSQL migration connection string must identify a host and database."); + } + return { + host: parsed.hostname, + port: parsed.port || "5432", + database, + }; +} + +export async function createMigrationExecutor( + manifestConfig: PostgresMigrationManifestConfig, + context: MigrationExecutorFactoryContext, +): Promise { + const connectionStringEnv = + context.overrides?.databaseUrlEnv ?? manifestConfig.connectionStringEnv; + const connectionString = requireMigrationEnvironment(connectionStringEnv, context.env); + const identity = parseTargetIdentity(connectionString); + const label = `${identity.host}:${identity.port}/${identity.database}`; + const target = await createMigrationTarget("postgres", label, [ + identity.host, + identity.port, + identity.database, + ]); + return createDirectMigrationExecutor({ + target, + createDialect: () => + createDialect({ + connectionString, + pool: { min: 0, max: 1 }, + }), + }); +} diff --git a/packages/core/src/db/sqlite-migrations.ts b/packages/core/src/db/sqlite-migrations.ts new file mode 100644 index 0000000000..978604fd2b --- /dev/null +++ b/packages/core/src/db/sqlite-migrations.ts @@ -0,0 +1,37 @@ +import { isAbsolute, resolve } from "node:path"; + +import { createDirectMigrationExecutor } from "../migrations/direct-executor.js"; +import type { MigrationExecutor, MigrationExecutorFactoryContext } from "../migrations/protocol.js"; +import { createMigrationTarget } from "../migrations/target.js"; +import { createDialect } from "./sqlite.js"; + +export interface SqliteMigrationManifestConfig { + url: string; +} + +function resolveDatabasePath( + manifestConfig: SqliteMigrationManifestConfig, + context: MigrationExecutorFactoryContext, +): string { + const configuredUrl = context.overrides?.database ?? manifestConfig.url; + if (typeof configuredUrl !== "string" || configuredUrl.length === 0) { + throw new Error("SQLite migration database path is missing."); + } + const path = configuredUrl.startsWith("file:") ? configuredUrl.slice(5) : configuredUrl; + if (path.length === 0) { + throw new Error("SQLite migration database path is missing."); + } + return isAbsolute(path) ? resolve(path) : resolve(context.projectRoot, path); +} + +export async function createMigrationExecutor( + manifestConfig: SqliteMigrationManifestConfig, + context: MigrationExecutorFactoryContext, +): Promise { + const databasePath = resolveDatabasePath(manifestConfig, context); + const target = await createMigrationTarget("sqlite", databasePath, [databasePath]); + return createDirectMigrationExecutor({ + target, + createDialect: () => createDialect({ url: databasePath }), + }); +} diff --git a/packages/core/src/emdash-runtime.ts b/packages/core/src/emdash-runtime.ts index 6c3780c9bc..8490cc7004 100644 --- a/packages/core/src/emdash-runtime.ts +++ b/packages/core/src/emdash-runtime.ts @@ -26,10 +26,14 @@ import { getTrustedProxyHeaders } from "./auth/trusted-proxy.js"; import type { ContentFieldFilters } from "./content-list-query.js"; import { isSqlite } from "./database/dialect-helpers.js"; import { kyselyLogOption } from "./database/instrumentation.js"; +import { + enforceRuntimeMigrationPolicy, + PendingMigrationsError, + type RuntimeMigrationMode, +} from "./database/migrations/policy.js"; import { ConcurrentMigrationTimeoutError, MIGRATION_RACE_WAIT_MS, - runMigrations, } from "./database/migrations/runner.js"; import { AuditRepository } from "./database/repositories/audit.js"; import { ContentRepository } from "./database/repositories/content.js"; @@ -87,6 +91,7 @@ import type { } from "./plugins/types.js"; import { recordSchedulerHeartbeatSafely } from "./scheduler-health.js"; import { MAX_COLLECTION_LIST_COLUMNS, type FieldType } from "./schema/types.js"; +import { isMissingTableError } from "./utils/db-errors.js"; import { hashString } from "./utils/hash.js"; import { createInitLock, type InitLock, initWithLock } from "./utils/init-lock.js"; import { createSingleFlightCache, singleFlightCached } from "./utils/single-flight-cache.js"; @@ -347,6 +352,8 @@ export type CreateSchedulerFn = (executor: CronExecutor) => CronScheduler; */ export interface RuntimeDependencies { config: EmDashConfig; + /** Effective migration mode, resolved once by the runtime entrypoint. */ + migrationMode?: RuntimeMigrationMode; plugins: ResolvedPlugin[]; // eslint-disable-next-line @typescript-eslint/no-explicit-any createDialect: (config: any) => Dialect; @@ -1247,8 +1254,10 @@ export class EmDashRuntime { } }; - // Initialize database (connects, runs migrations if needed) - const db = await phase("rt.db", "DB init + migrations", () => EmDashRuntime.getDatabase(deps)); + // Initialize the database and enforce its configured migration policy. + const db = await phase("rt.db", "DB init + migration policy", () => + EmDashRuntime.getDatabase(deps), + ); // Resolver for the live connection, mirroring the `get db()` getter // below (which can't be used here — the runtime instance doesn't exist @@ -1311,6 +1320,16 @@ export class EmDashRuntime { // fall back to the singleton, where serialization costs nothing. let readDb = db; let readDbDisposable: Kysely | undefined; + const disposeReadDb = async () => { + const disposable = readDbDisposable; + readDbDisposable = undefined; + if (!disposable) return; + try { + await disposable.destroy(); + } catch { + // Non-fatal — the underlying binding is shared and needs no teardown. + } + }; if (ownsConfiguredDb && deps.createCoalescingDialect && deps.config.database) { try { const dialect = deps.createCoalescingDialect(deps.config.database.config); @@ -1323,6 +1342,12 @@ export class EmDashRuntime { } } const optionsRepo = new OptionsRepository(readDb); + let missingManualSchemaError: unknown; + const captureMissingManualSchema = (error: unknown) => { + if ((deps.migrationMode ?? "auto") === "manual" && isMissingTableError(error)) { + missingManualSchemaError ??= error; + } + }; const readSiteInfo = async () => { const siteOpts = await optionsRepo.getMany([ @@ -1351,14 +1376,16 @@ export class EmDashRuntime { .select(["plugin_id", "status"]) .execute(); pluginStates = new Map(states.map((s) => [s.plugin_id, s.status])); - } catch { + } catch (error) { + captureMissingManualSchema(error); // _plugin_state may not exist yet on a pre-migration db. } }), phase("rt.site", "Site info options", async () => { try { siteInfo = await readSiteInfo(); - } catch { + } catch (error) { + captureMissingManualSchema(error); // options may not exist yet on a pre-migration db. } }), @@ -1387,7 +1414,8 @@ export class EmDashRuntime { } })(); seedGate = { collectionCount: collectionCount.count, setupDone }; - } catch { + } catch (error) { + captureMissingManualSchema(error); // Leave the "already set up" default so a read failure never // triggers a seed onto a half-built db. } @@ -1396,6 +1424,10 @@ export class EmDashRuntime { } await Promise.all(coldStartReads); + if (missingManualSchemaError) { + await disposeReadDb(); + throw missingManualSchemaError; + } if ( localeCasingRepairVersion && @@ -1452,13 +1484,7 @@ export class EmDashRuntime { } // The read connection is single-use; everything below uses the singleton. - if (readDbDisposable) { - try { - await readDbDisposable.destroy(); - } catch { - // Non-fatal — the underlying binding is shared and needs no teardown. - } - } + await disposeReadDb(); const enabledPlugins = new Set(); for (const plugin of deps.plugins) { @@ -1911,13 +1937,16 @@ export class EmDashRuntime { const db = new Kysely({ dialect, log: kyselyLogOption() }); try { - await runMigrations(db); + await enforceRuntimeMigrationPolicy(db, deps.migrationMode ?? "auto"); } catch (error) { // Timing out behind another instance's in-flight migrations // is not a failure of OUR migration — the holder may just be // slow. Don't back off for it: the next request waits again // and init recovers the moment the holder finishes. - if (!(error instanceof ConcurrentMigrationTimeoutError)) { + if ( + !(error instanceof ConcurrentMigrationTimeoutError) && + !(error instanceof PendingMigrationsError) + ) { holder.failures.set(cacheKey, { at: Date.now(), message: error instanceof Error ? error.message : String(error), diff --git a/packages/core/src/i18n/normalize.ts b/packages/core/src/i18n/normalize.ts new file mode 100644 index 0000000000..dbf247452b --- /dev/null +++ b/packages/core/src/i18n/normalize.ts @@ -0,0 +1,31 @@ +import type { I18nConfig } from "./config.js"; + +export interface AstroLocaleObject { + path: string; + codes: readonly string[]; +} + +export interface AstroI18nInput { + defaultLocale: string; + locales: readonly (string | AstroLocaleObject)[]; + fallback?: Readonly>; + routing?: string | { prefixDefaultLocale?: boolean }; +} + +export function normalizeAstroI18n(config: AstroI18nInput | null | undefined): I18nConfig | null { + if (!config) return null; + + return { + defaultLocale: config.defaultLocale, + locales: config.locales.map((locale) => (typeof locale === "string" ? locale : locale.path)), + fallback: config.fallback + ? Object.fromEntries( + Object.entries(config.fallback).filter( + (entry): entry is [string, string] => entry[1] !== undefined, + ), + ) + : undefined, + prefixDefaultLocale: + typeof config.routing === "object" ? (config.routing.prefixDefaultLocale ?? false) : false, + }; +} diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index be00ac0079..4936d04156 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,8 +1,14 @@ // Database (only types and utilities - internal functions not exported) -export { EmDashDatabaseError, getMigrationStatus } from "./database/index.js"; +export { + EmDashDatabaseError, + getMigrationStatus, + getExactMigrationStatus, + MIGRATION_NAMES, +} from "./database/index.js"; export type { DatabaseConfig, MigrationStatus, + ExactMigrationStatus, Database, UserTable, MediaTable, diff --git a/packages/core/src/migrations/config-loader.ts b/packages/core/src/migrations/config-loader.ts new file mode 100644 index 0000000000..bd19c4daac --- /dev/null +++ b/packages/core/src/migrations/config-loader.ts @@ -0,0 +1,205 @@ +import { access } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { isAbsolute, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +import type { AstroI18nInput } from "../i18n/normalize.js"; +import { normalizeAstroI18n } from "../i18n/normalize.js"; +import type { CoreMigrationIdentity } from "./identity.js"; +import { + getMigrationIntegrationMetadata, + MIGRATION_CONFIG_SYMBOL, +} from "./integration-metadata.js"; +import { buildMigrationManifest } from "./manifest-builder.js"; +import type { MigrationManifestV1 } from "./manifest.js"; + +const ASTRO_CONFIG_FILENAMES = [ + "astro.config.mjs", + "astro.config.js", + "astro.config.ts", + "astro.config.mts", +] as const; + +interface EvaluatedAstroConfig { + integrations?: unknown[]; + i18n?: AstroI18nInput | null; +} + +export interface BuildMigrationManifestFromConfigOptions { + projectRoot: string; + configFile?: string; +} + +export interface MigrationConfigLoaderDependencies { + findConfigFile: (projectRoot: string, configFile?: string) => Promise; + loadConfig: (projectRoot: string, configFile: string) => Promise; + loadIdentity: (projectRoot: string) => Promise; +} + +export class MigrationConfigLoaderError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = "MigrationConfigLoaderError"; + } +} + +async function isAccessible(path: string): Promise { + try { + await access(path); + return true; + } catch { + return false; + } +} + +export async function findAstroConfigFile( + projectRoot: string, + configFile?: string, +): Promise { + const root = resolve(projectRoot); + if (configFile) { + const explicitPath = isAbsolute(configFile) ? configFile : resolve(root, configFile); + if (await isAccessible(explicitPath)) return explicitPath; + throw new MigrationConfigLoaderError(`Astro config file not found: ${explicitPath}`); + } + + for (const filename of ASTRO_CONFIG_FILENAMES) { + const candidate = resolve(root, filename); + if (await isAccessible(candidate)) return candidate; + } + + throw new MigrationConfigLoaderError(`No Astro config file found in ${root}`); +} + +function projectRequire(projectRoot: string) { + return createRequire(resolve(projectRoot, "package.json")); +} + +async function importModule(path: string): Promise { + return import(pathToFileURL(path).href); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function moduleExport(module: unknown, name: string): unknown { + if (!isRecord(module)) return undefined; + const direct = module[name]; + if (direct !== undefined) return direct; + const defaultExport = module.default; + return isRecord(defaultExport) ? defaultExport[name] : undefined; +} + +function isCoreMigrationIdentity(value: unknown): value is CoreMigrationIdentity { + return ( + isRecord(value) && + typeof value.emdashVersion === "string" && + Array.isArray(value.names) && + value.names.every((name) => typeof name === "string") && + typeof value.fingerprint === "string" + ); +} + +async function loadProjectConfig(projectRoot: string, configFile: string): Promise { + let viteEntrypoint: string; + try { + const require = projectRequire(projectRoot); + const astroEntrypoint = require.resolve("astro"); + viteEntrypoint = createRequire(astroEntrypoint).resolve("vite"); + } catch (error) { + throw new MigrationConfigLoaderError( + "Could not resolve Vite through the project's Astro installation", + { cause: error }, + ); + } + + const vite = await importModule(viteEntrypoint); + const loadConfigFromFile = moduleExport(vite, "loadConfigFromFile"); + if (typeof loadConfigFromFile !== "function") { + throw new MigrationConfigLoaderError( + "The Vite installation used by project-local Astro does not export loadConfigFromFile", + ); + } + const loaded: unknown = await Reflect.apply(loadConfigFromFile, undefined, [ + { command: "build", mode: "production" }, + configFile, + projectRoot, + "silent", + ]); + if (!isRecord(loaded) || !Object.hasOwn(loaded, "config")) { + throw new MigrationConfigLoaderError(`Vite could not load Astro config: ${configFile}`); + } + return loaded.config; +} + +async function loadProjectIdentity(projectRoot: string): Promise { + let identityEntrypoint: string; + try { + identityEntrypoint = projectRequire(projectRoot).resolve("emdash/migrations"); + } catch (error) { + throw new MigrationConfigLoaderError("Could not resolve emdash/migrations from the project", { + cause: error, + }); + } + + const module = await importModule(identityEntrypoint); + const getIdentity = moduleExport(module, "getCoreMigrationIdentity"); + if (typeof getIdentity !== "function") { + throw new MigrationConfigLoaderError( + "The project-local emdash/migrations module does not export getCoreMigrationIdentity", + ); + } + const identity: unknown = await Reflect.apply(getIdentity, undefined, []); + if (!isCoreMigrationIdentity(identity)) { + throw new MigrationConfigLoaderError( + "The project-local emdash/migrations module returned an invalid migration identity", + ); + } + return identity; +} + +const defaultDependencies: MigrationConfigLoaderDependencies = { + findConfigFile: findAstroConfigFile, + loadConfig: loadProjectConfig, + loadIdentity: loadProjectIdentity, +}; + +function asAstroConfig(value: unknown): EvaluatedAstroConfig { + if (typeof value !== "object" || value === null) { + throw new MigrationConfigLoaderError("Astro config must export an object"); + } + return value; +} + +export async function buildMigrationManifestFromConfig( + options: BuildMigrationManifestFromConfigOptions, + dependencies: MigrationConfigLoaderDependencies = defaultDependencies, +): Promise { + const projectRoot = resolve(options.projectRoot); + const configFile = await dependencies.findConfigFile(projectRoot, options.configFile); + const config = asAstroConfig(await dependencies.loadConfig(projectRoot, configFile)); + const integrations = Array.isArray(config.integrations) ? config.integrations : []; + const matchingIntegrations = integrations.filter( + (integration) => getMigrationIntegrationMetadata(integration) !== undefined, + ); + if (matchingIntegrations.length !== 1) { + throw new MigrationConfigLoaderError( + `Expected exactly one EmDash integration with migration metadata (${String(MIGRATION_CONFIG_SYMBOL)}), found ${matchingIntegrations.length}`, + ); + } + + const metadata = getMigrationIntegrationMetadata(matchingIntegrations[0]); + if (!metadata?.database) { + throw new MigrationConfigLoaderError( + "The EmDash integration does not provide database migration metadata", + ); + } + const identity = await dependencies.loadIdentity(projectRoot); + + return buildMigrationManifest({ + identity, + i18n: normalizeAstroI18n(config.i18n), + database: metadata.database, + }); +} diff --git a/packages/core/src/migrations/direct-executor.ts b/packages/core/src/migrations/direct-executor.ts new file mode 100644 index 0000000000..88ff607f23 --- /dev/null +++ b/packages/core/src/migrations/direct-executor.ts @@ -0,0 +1,141 @@ +import { type Dialect, Kysely } from "kysely"; + +import { + getExactMigrationStatus, + runMigrations, + type ExactMigrationStatus, +} from "../database/migrations/runner.js"; +import type { Database } from "../database/types.js"; +import { getI18nConfig, setI18nConfig } from "../i18n/config.js"; +import { getCoreMigrationIdentity } from "./identity.js"; +import type { + MigrationExecutor, + MigrationReport, + MigrationRequest, + MigrationTarget, +} from "./protocol.js"; + +export interface DirectMigrationExecutorOptions { + target: MigrationTarget; + createDialect: () => Dialect | Promise; +} + +function createReport( + target: MigrationTarget, + status: ExactMigrationStatus, + executed: readonly string[], +): MigrationReport { + return { + target, + knownApplied: [...status.knownApplied], + pending: [...status.pending], + unknownApplied: [...status.unknownApplied], + executed: [...executed], + }; +} + +async function verifyRequest(request: MigrationRequest): Promise { + const identity = await getCoreMigrationIdentity(); + if ( + request.artifact.emdashVersion !== identity.emdashVersion || + request.artifact.migrationSetFingerprint !== identity.fingerprint + ) { + throw new Error( + "Migration artifact does not match the loaded EmDash version and migration registry.", + ); + } +} + +async function executeRequest( + db: Kysely, + target: MigrationTarget, + request: MigrationRequest, +): Promise { + const initialStatus = await getExactMigrationStatus(db); + if (request.action === "check") { + return createReport(target, initialStatus, []); + } + + if (initialStatus.unknownApplied.length > 0) { + throw new Error( + `Cannot apply migrations with unknown applied migrations: ${initialStatus.unknownApplied.join(", ")}`, + ); + } + if (initialStatus.pending.length === 0) { + return createReport(target, initialStatus, []); + } + + const result = await runMigrations(db); + const finalStatus = await getExactMigrationStatus(db); + return createReport(target, finalStatus, result.applied); +} + +export function createDirectMigrationExecutor( + options: DirectMigrationExecutorOptions, +): MigrationExecutor { + const target = Object.freeze({ ...options.target }); + let used = false; + let disposed = false; + let activeDb: Kysely | undefined; + let closePromise: Promise | undefined; + + const closeActiveDb = (): Promise => { + if (!activeDb) return Promise.resolve(); + closePromise ??= activeDb.destroy(); + return closePromise; + }; + + return { + target, + async execute(request) { + if (disposed) { + throw new Error("Migration executor has been disposed."); + } + if (used) { + throw new Error("Migration executors are single-use."); + } + used = true; + + if (request.action !== "check" && request.action !== "apply") { + throw new Error("Unsupported migration action."); + } + await verifyRequest(request); + + const dialect = await options.createDialect(); + const db = new Kysely({ dialect }); + activeDb = db; + if (disposed) { + await closeActiveDb(); + throw new Error("Migration executor has been disposed."); + } + const previousI18n = getI18nConfig(); + let executionFailed = false; + let executionError: unknown; + let report: MigrationReport | undefined; + setI18nConfig(request.i18n); + + try { + report = await executeRequest(db, target, request); + } catch (error) { + executionFailed = true; + executionError = error; + } finally { + setI18nConfig(previousI18n); + try { + await closeActiveDb(); + } catch { + console.error("[migrations] Database close failed."); + closePromise = Promise.resolve(); + } + } + + if (executionFailed) throw executionError; + if (!report) throw new Error("Migration execution did not produce a report."); + return report; + }, + dispose() { + disposed = true; + return closeActiveDb(); + }, + }; +} diff --git a/packages/core/src/migrations/identity.ts b/packages/core/src/migrations/identity.ts new file mode 100644 index 0000000000..93b1735563 --- /dev/null +++ b/packages/core/src/migrations/identity.ts @@ -0,0 +1,46 @@ +import { MIGRATION_NAMES } from "../database/migrations/runner.js"; +import { VERSION } from "../version.js"; + +export interface CoreMigrationIdentity { + emdashVersion: string; + names: readonly string[]; + fingerprint: string; +} + +function encodeFingerprintInput(emdashVersion: string, names: readonly string[]): ArrayBuffer { + const encoded = new TextEncoder().encode(JSON.stringify({ emdashVersion, names })); + const buffer = new ArrayBuffer(encoded.byteLength); + new Uint8Array(buffer).set(encoded); + return buffer; +} + +function encodeHex(bytes: Uint8Array): string { + return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +export async function fingerprintMigrationSet( + emdashVersion: string, + names: readonly string[], +): Promise { + const digest = await crypto.subtle.digest( + "SHA-256", + encodeFingerprintInput(emdashVersion, names), + ); + return encodeHex(new Uint8Array(digest)); +} + +export async function createCoreMigrationIdentity( + emdashVersion: string, + names: readonly string[], +): Promise { + const nameSnapshot = Object.freeze([...names]); + return Object.freeze({ + emdashVersion, + names: nameSnapshot, + fingerprint: await fingerprintMigrationSet(emdashVersion, nameSnapshot), + }); +} + +export function getCoreMigrationIdentity(): Promise { + return createCoreMigrationIdentity(VERSION, MIGRATION_NAMES); +} diff --git a/packages/core/src/migrations/index.ts b/packages/core/src/migrations/index.ts new file mode 100644 index 0000000000..7ca84c32a9 --- /dev/null +++ b/packages/core/src/migrations/index.ts @@ -0,0 +1,19 @@ +export { createDirectMigrationExecutor } from "./direct-executor.js"; +export type { DirectMigrationExecutorOptions } from "./direct-executor.js"; +export { + createCoreMigrationIdentity, + fingerprintMigrationSet, + getCoreMigrationIdentity, +} from "./identity.js"; +export type { CoreMigrationIdentity } from "./identity.js"; +export type { + MigrationAction, + MigrationExecutor, + MigrationExecutorFactory, + MigrationExecutorFactoryContext, + MigrationExecutorModule, + MigrationReport, + MigrationRequest, + MigrationTarget, + MigrationTargetOverrides, +} from "./protocol.js"; diff --git a/packages/core/src/migrations/integration-metadata.ts b/packages/core/src/migrations/integration-metadata.ts new file mode 100644 index 0000000000..2b0cc1e7c0 --- /dev/null +++ b/packages/core/src/migrations/integration-metadata.ts @@ -0,0 +1,67 @@ +import type { DatabaseDescriptor } from "../db/adapters.js"; +import { validateSecretFreeExecutorConfig } from "./manifest.js"; + +export const MIGRATION_CONFIG_SYMBOL = Symbol.for("emdash:migration-config"); + +export interface MigrationIntegrationMetadata { + database?: Pick; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function isMigrationIntegrationMetadata(value: unknown): value is MigrationIntegrationMetadata { + if (!isRecord(value)) return false; + const database = value.database; + if (database === undefined) return true; + if (!isRecord(database) || (database.type !== "sqlite" && database.type !== "postgres")) { + return false; + } + + const migrations = database.migrations; + return ( + migrations === undefined || + (isRecord(migrations) && + typeof migrations.entrypoint === "string" && + Object.hasOwn(migrations, "manifestConfig")) + ); +} + +export function createMigrationIntegrationMetadata( + database?: DatabaseDescriptor, +): MigrationIntegrationMetadata { + if (!database) return {}; + + return { + database: { + type: database.type, + migrations: database.migrations + ? { + entrypoint: database.migrations.entrypoint, + manifestConfig: validateSecretFreeExecutorConfig( + database.migrations.manifestConfig, + "database.migrations.manifestConfig", + ), + } + : undefined, + }, + }; +} + +export function getMigrationIntegrationMetadata( + integration: unknown, +): MigrationIntegrationMetadata | undefined { + if ( + (typeof integration !== "object" || integration === null) && + typeof integration !== "function" + ) { + return undefined; + } + if (!Object.hasOwn(integration, MIGRATION_CONFIG_SYMBOL)) { + return undefined; + } + + const metadata: unknown = Reflect.get(integration, MIGRATION_CONFIG_SYMBOL); + return isMigrationIntegrationMetadata(metadata) ? metadata : undefined; +} diff --git a/packages/core/src/migrations/manifest-builder.ts b/packages/core/src/migrations/manifest-builder.ts new file mode 100644 index 0000000000..732d1178bd --- /dev/null +++ b/packages/core/src/migrations/manifest-builder.ts @@ -0,0 +1,55 @@ +import type { DatabaseDescriptor } from "../db/adapters.js"; +import type { I18nConfig } from "../i18n/config.js"; +import type { CoreMigrationIdentity } from "./identity.js"; +import { fingerprintMigrationSet } from "./identity.js"; +import type { MigrationManifestV1 } from "./manifest.js"; +import { MigrationManifestValidationError, validateMigrationManifest } from "./manifest.js"; + +export type ManifestDatabaseDescriptor = Pick; + +export interface BuildMigrationManifestOptions { + identity: CoreMigrationIdentity; + i18n: I18nConfig | null; + database: ManifestDatabaseDescriptor; +} + +export class UnsupportedMigrationAdapterError extends Error { + constructor() { + super("The configured database adapter does not provide a deployment migration executor"); + this.name = "UnsupportedMigrationAdapterError"; + } +} + +export async function buildMigrationManifest({ + identity, + i18n, + database, +}: BuildMigrationManifestOptions): Promise { + const identityFingerprint = await fingerprintMigrationSet(identity.emdashVersion, identity.names); + if (identityFingerprint !== identity.fingerprint) { + throw new MigrationManifestValidationError( + "loaded identity fingerprint does not match its version and ordered names", + ); + } + if (!database.migrations) { + throw new UnsupportedMigrationAdapterError(); + } + + return validateMigrationManifest( + { + schemaVersion: 1, + emdashVersion: identity.emdashVersion, + migrationSet: { + names: [...identity.names], + fingerprint: identity.fingerprint, + }, + i18n, + database: { + type: database.type, + executorEntrypoint: database.migrations.entrypoint, + executorConfig: database.migrations.manifestConfig, + }, + }, + identity, + ); +} diff --git a/packages/core/src/migrations/manifest-writer.ts b/packages/core/src/migrations/manifest-writer.ts new file mode 100644 index 0000000000..6922288103 --- /dev/null +++ b/packages/core/src/migrations/manifest-writer.ts @@ -0,0 +1,63 @@ +import { randomUUID } from "node:crypto"; +import { + mkdir as nodeMkdir, + rename as nodeRename, + unlink as nodeUnlink, + writeFile as nodeWriteFile, +} from "node:fs/promises"; +import { dirname, join } from "node:path"; + +import type { MigrationManifestV1 } from "./manifest.js"; +import { serializeMigrationManifest, validateMigrationManifest } from "./manifest.js"; + +export const MIGRATION_MANIFEST_PATH = ".emdash/migrations.json"; + +export interface ManifestWriterFileSystem { + mkdir(path: string, options: { recursive: true }): Promise; + writeFile( + path: string, + data: string, + options: { encoding: "utf8"; flag: "wx" }, + ): Promise; + rename(from: string, to: string): Promise; + unlink(path: string): Promise; +} + +const nodeFileSystem: ManifestWriterFileSystem = { + mkdir: (path, options) => nodeMkdir(path, options), + writeFile: (path, data, options) => nodeWriteFile(path, data, options), + rename: (from, to) => nodeRename(from, to), + unlink: (path) => nodeUnlink(path), +}; + +export async function writeMigrationManifest( + projectRoot: string, + manifest: MigrationManifestV1, + fileSystem: ManifestWriterFileSystem = nodeFileSystem, +): Promise { + const validated = await validateMigrationManifest(manifest); + const outputPath = join(projectRoot, MIGRATION_MANIFEST_PATH); + const outputDirectory = dirname(outputPath); + await fileSystem.mkdir(outputDirectory, { recursive: true }); + + const temporaryPath = join( + outputDirectory, + `.migrations.json.${process.pid}.${randomUUID()}.tmp`, + ); + let temporaryFileMayExist = false; + try { + temporaryFileMayExist = true; + await fileSystem.writeFile(temporaryPath, serializeMigrationManifest(validated), { + encoding: "utf8", + flag: "wx", + }); + await fileSystem.rename(temporaryPath, outputPath); + temporaryFileMayExist = false; + return outputPath; + } catch (error) { + if (temporaryFileMayExist) { + await fileSystem.unlink(temporaryPath).catch(() => undefined); + } + throw error; + } +} diff --git a/packages/core/src/migrations/manifest.ts b/packages/core/src/migrations/manifest.ts new file mode 100644 index 0000000000..8c1690ba1b --- /dev/null +++ b/packages/core/src/migrations/manifest.ts @@ -0,0 +1,197 @@ +import { z } from "zod"; + +import type { CoreMigrationIdentity } from "./identity.js"; +import { fingerprintMigrationSet } from "./identity.js"; + +const FINGERPRINT_PATTERN = /^[0-9a-f]{64}$/; +const MIGRATION_NAME_PATTERN = /^\d{3}_[a-z0-9_]+$/; +const EXECUTOR_ENTRYPOINT_PATTERN = + /^(?:@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*(?:\/[A-Za-z0-9][A-Za-z0-9._-]*)*$/; +const ENVIRONMENT_VARIABLE_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; +const ENVIRONMENT_REFERENCE_KEY_PATTERN = /(?:env|envName|envVar)$/i; +const URL_SCHEME_PATTERN = /^[A-Za-z][A-Za-z0-9+.-]*:/; +const SECRET_QUERY_KEY_PATTERN = /(?:auth|credential|key|password|secret|signature|token)/i; +const SECRET_CONFIG_KEY_PATTERN = + /(?:accessToken|apiKey|authToken|certificate|connectionString|password|privateKey|secret)/i; +const SECRET_CONFIG_EXACT_KEY_PATTERN = /^(?:ca|cert|key)$/i; +const CONNECTION_URL_PROTOCOLS = new Set(["mysql:", "postgres:", "postgresql:"]); + +type JsonPrimitive = boolean | number | string | null; +type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; + +const i18nConfigSchema = z + .object({ + defaultLocale: z.string().min(1), + locales: z.array(z.string().min(1)).min(1), + fallback: z.record(z.string(), z.string()).optional(), + prefixDefaultLocale: z.boolean().optional(), + }) + .strict(); + +export const migrationManifestV1Schema = z + .object({ + schemaVersion: z.literal(1), + emdashVersion: z.string().min(1), + migrationSet: z + .object({ + names: z + .array(z.string().regex(MIGRATION_NAME_PATTERN)) + .min(1) + .refine((names) => new Set(names).size === names.length), + fingerprint: z.string().regex(FINGERPRINT_PATTERN), + }) + .strict(), + i18n: i18nConfigSchema.nullable(), + database: z + .object({ + type: z.enum(["sqlite", "postgres"]), + executorEntrypoint: z.string().regex(EXECUTOR_ENTRYPOINT_PATTERN), + executorConfig: z.unknown(), + }) + .strict(), + }) + .strict(); + +export type MigrationManifestV1 = z.infer; + +export class MigrationManifestValidationError extends Error { + constructor(message: string) { + super(`Invalid migration manifest: ${message}`); + this.name = "MigrationManifestValidationError"; + } +} + +function isEnvironmentReferenceKey(key: string): boolean { + return ENVIRONMENT_REFERENCE_KEY_PATTERN.test(key); +} + +function assertSafeUrl(value: string, path: string): void { + if (!URL_SCHEME_PATTERN.test(value)) return; + + let url: URL; + try { + url = new URL(value); + } catch { + throw new MigrationManifestValidationError(`${path} contains an invalid URL`); + } + + if (CONNECTION_URL_PROTOCOLS.has(url.protocol)) { + throw new MigrationManifestValidationError(`${path} contains a database connection URL`); + } + if (url.username || url.password) { + throw new MigrationManifestValidationError(`${path} contains URL credentials`); + } + for (const key of url.searchParams.keys()) { + if (SECRET_QUERY_KEY_PATTERN.test(key)) { + throw new MigrationManifestValidationError(`${path} contains a credential query parameter`); + } + } +} + +function cloneSecretFreeJson(value: unknown, path: string, ancestors: Set): JsonValue { + if (value === null || typeof value === "boolean") return value; + if (typeof value === "string") { + assertSafeUrl(value, path); + return value; + } + if (typeof value === "number") { + if (Number.isFinite(value)) return value; + throw new MigrationManifestValidationError(`${path} contains a non-finite number`); + } + if (typeof value !== "object") { + throw new MigrationManifestValidationError(`${path} is not JSON-serializable`); + } + if (ancestors.has(value)) { + throw new MigrationManifestValidationError(`${path} contains a cycle`); + } + + ancestors.add(value); + try { + if (Array.isArray(value)) { + return value.map((item, index) => cloneSecretFreeJson(item, `${path}[${index}]`, ancestors)); + } + + const prototype = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) { + throw new MigrationManifestValidationError(`${path} contains a non-plain object`); + } + + const cloned: Record = {}; + for (const [key, item] of Object.entries(value)) { + const itemPath = `${path}.${key}`; + const environmentReference = isEnvironmentReferenceKey(key); + if ( + !environmentReference && + (SECRET_CONFIG_KEY_PATTERN.test(key) || SECRET_CONFIG_EXACT_KEY_PATTERN.test(key)) + ) { + throw new MigrationManifestValidationError(`${itemPath} is a credential-bearing field`); + } + if (environmentReference) { + if (typeof item !== "string" || !ENVIRONMENT_VARIABLE_PATTERN.test(item)) { + throw new MigrationManifestValidationError( + `${itemPath} must contain an environment-variable name`, + ); + } + } + cloned[key] = cloneSecretFreeJson(item, itemPath, ancestors); + } + return cloned; + } finally { + ancestors.delete(value); + } +} + +export function validateSecretFreeExecutorConfig( + value: unknown, + path = "database.executorConfig", +): unknown { + return cloneSecretFreeJson(value, path, new Set()); +} + +function identitiesMatch( + manifest: MigrationManifestV1, + expectedIdentity: CoreMigrationIdentity, +): boolean { + return ( + manifest.emdashVersion === expectedIdentity.emdashVersion && + manifest.migrationSet.fingerprint === expectedIdentity.fingerprint && + manifest.migrationSet.names.length === expectedIdentity.names.length && + manifest.migrationSet.names.every((name, index) => name === expectedIdentity.names[index]) + ); +} + +export async function validateMigrationManifest( + value: unknown, + expectedIdentity?: CoreMigrationIdentity, +): Promise { + const parsed = migrationManifestV1Schema.safeParse(value); + if (!parsed.success) { + throw new MigrationManifestValidationError("schema validation failed"); + } + + const executorConfig = validateSecretFreeExecutorConfig(parsed.data.database.executorConfig); + const manifest: MigrationManifestV1 = { + ...parsed.data, + database: { ...parsed.data.database, executorConfig }, + }; + const fingerprint = await fingerprintMigrationSet( + manifest.emdashVersion, + manifest.migrationSet.names, + ); + if (fingerprint !== manifest.migrationSet.fingerprint) { + throw new MigrationManifestValidationError( + "migrationSet fingerprint does not match its version and ordered names", + ); + } + if (expectedIdentity && !identitiesMatch(manifest, expectedIdentity)) { + throw new MigrationManifestValidationError( + "manifest does not match the loaded EmDash migration identity", + ); + } + + return manifest; +} + +export function serializeMigrationManifest(manifest: MigrationManifestV1): string { + return `${JSON.stringify(manifest, null, "\t")}\n`; +} diff --git a/packages/core/src/migrations/protocol.ts b/packages/core/src/migrations/protocol.ts new file mode 100644 index 0000000000..fd1bd858d4 --- /dev/null +++ b/packages/core/src/migrations/protocol.ts @@ -0,0 +1,59 @@ +import type { I18nConfig } from "../i18n/config.js"; + +export type MigrationAction = "check" | "apply"; + +export interface MigrationRequest { + action: MigrationAction; + i18n: I18nConfig | null; + artifact: { + emdashVersion: string; + migrationSetFingerprint: string; + }; +} + +export interface MigrationTarget { + kind: string; + label: string; + fingerprint: string; + accountId?: string; + environment?: string; + resourceId?: string; +} + +export interface MigrationReport { + target: MigrationTarget; + knownApplied: string[]; + pending: string[]; + unknownApplied: string[]; + executed: string[]; +} + +export interface MigrationExecutor { + target: MigrationTarget; + execute(request: MigrationRequest): Promise; + dispose?(): Promise; +} + +export interface MigrationTargetOverrides { + database?: string; + databaseUrlEnv?: string; + d1?: string; + accountId?: string; + wranglerConfig?: string; + wranglerEnv?: string; +} + +export interface MigrationExecutorFactoryContext { + projectRoot: string; + env: Readonly>; + overrides?: Readonly; +} + +export type MigrationExecutorFactory = ( + manifestConfig: ManifestConfig, + context: MigrationExecutorFactoryContext, +) => MigrationExecutor | Promise; + +export interface MigrationExecutorModule { + createMigrationExecutor: MigrationExecutorFactory; +} diff --git a/packages/core/src/migrations/target.ts b/packages/core/src/migrations/target.ts new file mode 100644 index 0000000000..11b2b457a8 --- /dev/null +++ b/packages/core/src/migrations/target.ts @@ -0,0 +1,35 @@ +import type { MigrationTarget } from "./protocol.js"; + +const ENVIRONMENT_VARIABLE_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; + +function encodeHex(bytes: Uint8Array): string { + return Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +export async function createMigrationTarget( + kind: string, + label: string, + identity: readonly string[], +): Promise { + const encoded = new TextEncoder().encode(JSON.stringify({ kind, identity })); + const digest = await crypto.subtle.digest("SHA-256", encoded); + return { + kind, + label, + fingerprint: encodeHex(new Uint8Array(digest)), + }; +} + +export function requireMigrationEnvironment( + name: unknown, + env: Readonly>, +): string { + if (typeof name !== "string" || !ENVIRONMENT_VARIABLE_PATTERN.test(name)) { + throw new Error("Migration credential environment variable name is invalid."); + } + const value = env[name]; + if (!value) { + throw new Error(`Migration credential environment variable ${name} is not set.`); + } + return value; +} diff --git a/packages/core/src/virtual-modules.d.ts b/packages/core/src/virtual-modules.d.ts index dd4379a53a..50e7af1065 100644 --- a/packages/core/src/virtual-modules.d.ts +++ b/packages/core/src/virtual-modules.d.ts @@ -16,6 +16,7 @@ declare module "virtual:emdash/config" { interface VirtualConfig { database?: DatabaseDescriptor; + migrations?: import("./database/migrations/policy.js").RuntimeMigrationConfig; storage?: StorageDescriptor; auth?: AuthDescriptor; authProviders?: AuthProviderDescriptor[]; diff --git a/packages/core/tests/integration/database/dialect-compat.test.ts b/packages/core/tests/integration/database/dialect-compat.test.ts index f08ac5028d..bd109a7265 100644 --- a/packages/core/tests/integration/database/dialect-compat.test.ts +++ b/packages/core/tests/integration/database/dialect-compat.test.ts @@ -10,13 +10,14 @@ import { it, expect, beforeEach, afterEach } from "vitest"; -import { MIGRATION_COUNT } from "../../../src/database/migrations/runner.js"; +import { MIGRATION_COUNT, MIGRATION_NAMES } from "../../../src/database/migrations/runner.js"; import { ContentRepository } from "../../../src/database/repositories/content.js"; import type { Database } from "../../../src/database/types.js"; import { SchemaRegistry } from "../../../src/schema/registry.js"; import { createForDialect, describeEachDialect, + getExactMigrationStatusForDialect, getMigrationStatusForDialect, runMigrationsForDialect, setupForDialect, @@ -104,6 +105,33 @@ describeEachDialect("Migrations", (dialect) => { expect(after.applied).toContain("001_initial"); expect(after.pending).toHaveLength(0); }); + + it("reports exact fresh, current, pending, and database-ahead states", async () => { + await expect(getExactMigrationStatusForDialect(ctx)).resolves.toEqual({ + knownApplied: [], + pending: MIGRATION_NAMES, + unknownApplied: [], + }); + + await runMigrationsForDialect(ctx); + await expect(getExactMigrationStatusForDialect(ctx)).resolves.toEqual({ + knownApplied: MIGRATION_NAMES, + pending: [], + unknownApplied: [], + }); + + const pendingName = MIGRATION_NAMES.at(-1)!; + await ctx.db.deleteFrom("_emdash_migrations").where("name", "=", pendingName).execute(); + await ctx.db + .insertInto("_emdash_migrations") + .values({ name: "999_future", timestamp: new Date().toISOString() }) + .execute(); + + const status = await getExactMigrationStatusForDialect(ctx); + expect(status.knownApplied).toEqual(MIGRATION_NAMES.slice(0, -1)); + expect(status.pending).toEqual([pendingName]); + expect(status.unknownApplied).toEqual(["999_future"]); + }); }); // --------------------------------------------------------------------------- diff --git a/packages/core/tests/integration/database/migrations.test.ts b/packages/core/tests/integration/database/migrations.test.ts index 5a0629f6c0..88987f0e0d 100644 --- a/packages/core/tests/integration/database/migrations.test.ts +++ b/packages/core/tests/integration/database/migrations.test.ts @@ -1,16 +1,39 @@ -import type { Kysely } from "kysely"; +import type { + Kysely, + KyselyPlugin, + PluginTransformQueryArgs, + PluginTransformResultArgs, + QueryResult, + RootOperationNode, + UnknownRow, +} from "kysely"; import { sql } from "kysely"; import { describe, it, expect, beforeEach, afterEach } from "vitest"; import { createDatabase } from "../../../src/database/connection.js"; import { runMigrations, + getExactMigrationStatus, getMigrationStatus, MIGRATION_COUNT, + MIGRATION_NAMES, } from "../../../src/database/migrations/runner.js"; import type { Database } from "../../../src/database/types.js"; import { setupTestDatabaseWithCollections } from "../../utils/test-db.js"; +class QueryCountingPlugin implements KyselyPlugin { + count = 0; + + transformQuery(args: PluginTransformQueryArgs): RootOperationNode { + this.count += 1; + return args.node; + } + + transformResult(args: PluginTransformResultArgs): Promise> { + return Promise.resolve(args.result); + } +} + describe("Database Migrations (Integration)", () => { let db: Kysely; @@ -187,6 +210,83 @@ describe("Database Migrations (Integration)", () => { expect(statusAfter.pending).toHaveLength(0); }); + describe("exact migration status", () => { + it("exports the registered migration names in execution order", async () => { + await runMigrations(db); + const rows = await db + .selectFrom("_emdash_migrations") + .select("name") + .orderBy("timestamp") + .execute(); + + expect(MIGRATION_NAMES).toEqual(rows.map((row) => row.name)); + expect(MIGRATION_NAMES).toHaveLength(MIGRATION_COUNT); + expect(Object.isFrozen(MIGRATION_NAMES)).toBe(true); + }); + + it("reports every registered migration as pending for a fresh database", async () => { + const counter = new QueryCountingPlugin(); + + await expect(getExactMigrationStatus(db.withPlugin(counter))).resolves.toEqual({ + knownApplied: [], + pending: MIGRATION_NAMES, + unknownApplied: [], + }); + expect(counter.count).toBe(1); + }); + + it("recognizes a missing schema-qualified migration table", async () => { + await sql`ATTACH DATABASE ':memory:' AS migration_status`.execute(db); + + await expect( + getExactMigrationStatus(db, { migrationTableSchema: "migration_status" }), + ).resolves.toEqual({ + knownApplied: [], + pending: MIGRATION_NAMES, + unknownApplied: [], + }); + }); + + it("reports a current database with one migration-table query", async () => { + await runMigrations(db); + const counter = new QueryCountingPlugin(); + + await expect(getExactMigrationStatus(db.withPlugin(counter))).resolves.toEqual({ + knownApplied: MIGRATION_NAMES, + pending: [], + unknownApplied: [], + }); + expect(counter.count).toBe(1); + }); + + it("reports pending and unknown names in deterministic order", async () => { + await runMigrations(db); + const pending = [MIGRATION_NAMES[1]!, MIGRATION_NAMES.at(-2)!]; + await db.deleteFrom("_emdash_migrations").where("name", "in", pending).execute(); + await db + .insertInto("_emdash_migrations") + .values([ + { name: "999_future_z", timestamp: new Date().toISOString() }, + { name: "999_future_a", timestamp: new Date().toISOString() }, + ]) + .execute(); + + const status = await getExactMigrationStatus(db); + + expect(status.knownApplied).toEqual( + MIGRATION_NAMES.filter((name) => !pending.includes(name)), + ); + expect(status.pending).toEqual(pending); + expect(status.unknownApplied).toEqual(["999_future_a", "999_future_z"]); + }); + + it("rethrows migration-table query errors other than a missing table", async () => { + await sql`CREATE TABLE _emdash_migrations (unexpected TEXT)`.execute(db); + + await expect(getExactMigrationStatus(db)).rejects.toThrow(/no such column.*name/i); + }); + }); + it("should create schema registry tables", async () => { await runMigrations(db); diff --git a/packages/core/tests/integration/runtime/create.test.ts b/packages/core/tests/integration/runtime/create.test.ts index 2c9db277ea..5006040170 100644 --- a/packages/core/tests/integration/runtime/create.test.ts +++ b/packages/core/tests/integration/runtime/create.test.ts @@ -15,6 +15,7 @@ import { Kysely, sql, SqliteDialect } from "kysely"; import { describe, expect, it, vi } from "vitest"; import { DEFAULT_COMMENT_MODERATOR_PLUGIN_ID } from "../../../src/comments/moderator.js"; +import { PendingMigrationsError } from "../../../src/database/migrations/policy.js"; import { runMigrations } from "../../../src/database/migrations/runner.js"; import { OptionsRepository } from "../../../src/database/repositories/options.js"; import type { Database as EmDashDatabase } from "../../../src/database/types.js"; @@ -409,6 +410,31 @@ describe("EmDashRuntime.create — cold boot", () => { expect(dialectCalls).toBe(1); }); + it("rechecks pending migrations without entering migration-failure backoff", async () => { + let dialectCalls = 0; + const deps: RuntimeDependencies = { + ...createDeps(), + migrationMode: "check", + createDialect: () => { + dialectCalls += 1; + return new SqliteDialect({ database: new Database(":memory:") }); + }, + }; + + await expect(EmDashRuntime.create(deps)).rejects.toBeInstanceOf(PendingMigrationsError); + await expect(EmDashRuntime.create(deps)).rejects.toBeInstanceOf(PendingMigrationsError); + expect(dialectCalls).toBe(2); + }); + + it("rejects an empty database in manual migration mode", async () => { + const deps: RuntimeDependencies = { + ...createDeps(), + migrationMode: "manual", + }; + + await expect(EmDashRuntime.create(deps)).rejects.toThrow(/no such table/i); + }); + // A per-request isolated db (playground / DO preview) must never be // auto-seeded. With an isolated, empty, not-set-up db, a broken guard would // run the gate (rt.seedcheck appears) and attempt a seed; assert neither diff --git a/packages/core/tests/unit/astro/integration/migration-manifest.test.ts b/packages/core/tests/unit/astro/integration/migration-manifest.test.ts new file mode 100644 index 0000000000..5149c38ffd --- /dev/null +++ b/packages/core/tests/unit/astro/integration/migration-manifest.test.ts @@ -0,0 +1,167 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +import type { AstroIntegration } from "astro"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { emdash } from "../../../../src/astro/integration/index.js"; +import { libsql } from "../../../../src/db/adapters.js"; +import { MIGRATION_MANIFEST_PATH } from "../../../../src/migrations/manifest-writer.js"; + +const MIGRATION_CONFIG_SYMBOL = Symbol.for("emdash:migration-config"); + +function hook(integration: AstroIntegration, name: "astro:config:setup" | "astro:config:done") { + const handler = integration.hooks[name]; + if (typeof handler !== "function") throw new Error(`Missing ${name} hook`); + return handler; +} + +describe("migration manifest integration", () => { + const projectRoots: string[] = []; + + afterEach(async () => { + await Promise.all( + projectRoots.splice(0).map((path) => rm(path, { recursive: true, force: true })), + ); + }); + + async function runHooks( + command: "dev" | "build" | "preview" | "sync", + integration: AstroIntegration, + ) { + const projectRoot = await mkdtemp(join(tmpdir(), `emdash-manifest-${command}-`)); + projectRoots.push(projectRoot); + const root = pathToFileURL(`${projectRoot}/`); + const logger = { + debug: vi.fn(), + error: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + }; + const astroConfig = { + root, + srcDir: new URL("src/", root), + security: {}, + trailingSlash: "ignore", + integrations: [{ name: "@astrojs/react", hooks: {} }], + i18n: { + defaultLocale: "english", + locales: [ + { path: "english", codes: ["en", "en-US"] }, + { path: "french", codes: ["fr", "fr-FR"] }, + ], + fallback: { french: "english" }, + routing: { prefixDefaultLocale: true }, + }, + }; + + await hook( + integration, + "astro:config:setup", + )({ + command, + config: astroConfig, + logger, + injectRoute: vi.fn(), + addMiddleware: vi.fn(), + updateConfig: vi.fn(), + } as never); + await hook( + integration, + "astro:config:done", + )({ + config: astroConfig, + logger, + } as never); + + return { + logger, + manifestPath: join(projectRoot, MIGRATION_MANIFEST_PATH), + projectRoot, + }; + } + + it("writes equivalent secret-free manifests under each build and sync project root", async () => { + const database = libsql({ + url: "libsql://public-db.example.com", + authToken: "runtime-only-secret", + }); + const build = await runHooks("build", emdash({ database })); + const sync = await runHooks("sync", emdash({ database })); + const buildOutput = await readFile(build.manifestPath, "utf8"); + const syncOutput = await readFile(sync.manifestPath, "utf8"); + + expect(syncOutput).toBe(buildOutput); + expect(build.manifestPath).toBe(join(build.projectRoot, ".emdash", "migrations.json")); + expect(buildOutput).not.toContain("runtime-only-secret"); + expect(JSON.parse(buildOutput)).toMatchObject({ + i18n: { + defaultLocale: "english", + locales: ["english", "french"], + fallback: { french: "english" }, + prefixDefaultLocale: true, + }, + database: { + type: "sqlite", + executorEntrypoint: "emdash/db/libsql-migrations", + executorConfig: { + url: "libsql://public-db.example.com", + authTokenEnv: "TURSO_AUTH_TOKEN", + }, + }, + }); + }); + + it.each(["dev", "preview"] as const)("does not write a manifest during %s", async (command) => { + const { manifestPath } = await runHooks( + command, + emdash({ database: libsql({ url: "libsql://public-db.example.com" }) }), + ); + + await expect(readFile(manifestPath, "utf8")).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it.each([ + ["missing", undefined], + [ + "unsupported", + { + type: "sqlite" as const, + entrypoint: "example/runtime", + config: { token: "runtime-only-secret" }, + }, + ], + ] as const)("warns and skips a %s database adapter", async (_kind, database) => { + const { logger, manifestPath } = await runHooks("build", emdash({ database })); + + await expect(readFile(manifestPath, "utf8")).rejects.toMatchObject({ code: "ENOENT" }); + expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining("migration manifest")); + }); + + it("attaches only secret-free migration metadata to the integration", () => { + const integration = emdash({ + database: libsql({ + url: "libsql://public-db.example.com", + authToken: "runtime-only-secret", + }), + }); + const metadata = (integration as unknown as Record)[MIGRATION_CONFIG_SYMBOL]; + const serialized = JSON.stringify(metadata); + + expect(serialized).not.toContain("runtime-only-secret"); + expect(metadata).toEqual({ + database: { + type: "sqlite", + migrations: { + entrypoint: "emdash/db/libsql-migrations", + manifestConfig: { + url: "libsql://public-db.example.com", + authTokenEnv: "TURSO_AUTH_TOKEN", + }, + }, + }, + }); + }); +}); diff --git a/packages/core/tests/unit/astro/integration/migration-policy.test.ts b/packages/core/tests/unit/astro/integration/migration-policy.test.ts new file mode 100644 index 0000000000..650534d708 --- /dev/null +++ b/packages/core/tests/unit/astro/integration/migration-policy.test.ts @@ -0,0 +1,14 @@ +import { describe, expect, it } from "vitest"; + +import { emdash } from "../../../../src/astro/integration/index.js"; + +describe("EmDash integration migration policy", () => { + it("rejects invalid runtime and development modes during integration setup", () => { + expect(() => emdash({ migrations: { runtime: "later" } as never })).toThrow( + /migrations\.runtime.*later/, + ); + expect(() => emdash({ migrations: { runtime: "auto", dev: "later" } as never })).toThrow( + /migrations\.dev.*later/, + ); + }); +}); diff --git a/packages/core/tests/unit/astro/middleware-migration-policy.test.ts b/packages/core/tests/unit/astro/middleware-migration-policy.test.ts new file mode 100644 index 0000000000..9e8a187b8b --- /dev/null +++ b/packages/core/tests/unit/astro/middleware-migration-policy.test.ts @@ -0,0 +1,117 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("astro:middleware", () => ({ + defineMiddleware: (handler: unknown) => handler, +})); + +const { VIRTUAL_CONFIG, mockRuntimeCreate, mockGetDb } = vi.hoisted(() => ({ + VIRTUAL_CONFIG: { + database: { entrypoint: "test", config: {}, type: "sqlite" }, + migrations: { runtime: "check", dev: "check" }, + }, + mockRuntimeCreate: vi.fn(), + mockGetDb: vi.fn(), +})); + +vi.mock( + "virtual:emdash/config", + () => ({ + default: VIRTUAL_CONFIG, + }), + { virtual: true }, +); +vi.mock( + "virtual:emdash/dialect", + () => ({ + createDialect: vi.fn(), + createCoalescingDialect: undefined, + createRequestScopedDb: vi.fn().mockReturnValue(null), + }), + { virtual: true }, +); +vi.mock("virtual:emdash/media-providers", () => ({ mediaProviders: [] }), { virtual: true }); +vi.mock("virtual:emdash/plugins", () => ({ plugins: [] }), { virtual: true }); +vi.mock( + "virtual:emdash/sandbox-runner", + () => ({ + createSandboxRunner: null, + sandboxBypassed: false, + sandboxEnabled: false, + }), + { virtual: true }, +); +vi.mock("virtual:emdash/sandboxed-plugins", () => ({ sandboxedPlugins: [] }), { virtual: true }); +vi.mock("virtual:emdash/storage", () => ({ createStorage: null }), { virtual: true }); +vi.mock("virtual:emdash/wait-until", () => ({ waitUntil: undefined }), { virtual: true }); +vi.mock("virtual:emdash/scheduler", () => ({ createScheduler: null }), { virtual: true }); + +vi.mock("../../../src/emdash-runtime.js", () => ({ + DB_INIT_DEADLINE_MS: 30_000, + EmDashRuntime: { create: mockRuntimeCreate }, +})); +vi.mock("../../../src/loader.js", () => ({ getDb: mockGetDb })); + +import onRequest from "../../../src/astro/middleware.js"; +import { PendingMigrationsError } from "../../../src/database/migrations/policy.js"; + +const RUNTIME_HOLDER_KEY = Symbol.for("emdash:runtime-holder"); +const SETUP_VERIFIED_KEY = Symbol.for("emdash:setup-verified"); + +function contextFor(pathname: string) { + const url = new URL(pathname, "https://example.com"); + return { + request: new Request(url), + url, + cookies: { get: vi.fn(() => undefined), set: vi.fn() }, + locals: {} as Record, + redirect: vi.fn(), + isPrerendered: false, + session: { get: vi.fn(async () => null) }, + }; +} + +describe("middleware migration check failures", () => { + beforeEach(() => { + delete (globalThis as Record)[RUNTIME_HOLDER_KEY]; + delete (globalThis as Record)[SETUP_VERIFIED_KEY]; + mockGetDb.mockReset(); + VIRTUAL_CONFIG.migrations = { runtime: "check", dev: "check" }; + mockRuntimeCreate + .mockReset() + .mockRejectedValue(new PendingMigrationsError(["059_private_migration_name"])); + vi.spyOn(console, "error").mockImplementation(() => {}); + }); + + it.each(["/", "/_emdash/api/content/posts"])( + "returns a generic retryable 503 for %s", + async (pathname) => { + const next = vi.fn(async () => new Response("route response")); + const response = await onRequest(contextFor(pathname) as never, next); + + expect(response.status).toBe(503); + expect(response.headers.get("Retry-After")).toBe("60"); + expect(await response.text()).toBe( + "Database migrations are required. Apply the deployment migration manifest and retry.", + ); + expect(next).not.toHaveBeenCalled(); + expect(mockGetDb).not.toHaveBeenCalled(); + expect(JSON.stringify([...response.headers])).not.toContain("059_private_migration_name"); + }, + ); + + it.each(["/", "/_emdash/api/setup"])( + "instructs operators when manual mode reaches an unmigrated schema at %s", + async (pathname) => { + VIRTUAL_CONFIG.migrations = { runtime: "manual", dev: "manual" }; + mockRuntimeCreate.mockRejectedValue(new Error("no such table: options")); + const next = vi.fn(async () => new Response("route response")); + + const response = await onRequest(contextFor(pathname) as never, next); + + expect(response.status).toBe(503); + expect(await response.text()).toContain("deployment migration manifest"); + expect(next).not.toHaveBeenCalled(); + expect(mockGetDb).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/packages/core/tests/unit/astro/setup-dev-bypass.test.ts b/packages/core/tests/unit/astro/setup-dev-bypass.test.ts index b791aaefff..74388fc801 100644 --- a/packages/core/tests/unit/astro/setup-dev-bypass.test.ts +++ b/packages/core/tests/unit/astro/setup-dev-bypass.test.ts @@ -49,7 +49,10 @@ const fixtureSeed: SeedFile = { vi.mock("virtual:emdash/seed", () => ({ seed: fixtureSeed, userSeed: null }), { virtual: true }); +import { GET as AUTH_GET } from "../../../src/astro/routes/api/auth/dev-bypass.js"; import { GET } from "../../../src/astro/routes/api/setup/dev-bypass.js"; +import { POST as SETUP_POST } from "../../../src/astro/routes/api/setup/index.js"; +import { MIGRATION_NAMES } from "../../../src/database/migrations/runner.js"; function makeContext(db: Kysely, search = ""): APIContext { return { @@ -59,6 +62,20 @@ function makeContext(db: Kysely, search = ""): APIContext { } as unknown as APIContext; } +function makeSetupContext(db: Kysely): APIContext { + const url = new URL("http://localhost:4321/_emdash/api/setup"); + return { + locals: { emdash: { db, storage: null, config: { migrations: { runtime: "manual" } } } }, + url, + request: new Request(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ title: "Manual Site", tagline: "", includeContent: false }), + }), + session: undefined, + } as unknown as APIContext; +} + async function countPosts(db: Kysely) { const { items } = await new ContentRepository(db).findMany("posts", {}); return items.length; @@ -126,4 +143,23 @@ describe("setup dev-bypass seed gating", () => { expect(await countPosts(db)).toBe(0); }); + + it.each([ + ["setup dev bypass", (database: Kysely) => GET(makeContext(database, "?content=0"))], + ["auth dev bypass", (database: Kysely) => AUTH_GET(makeContext(database))], + ["setup wizard", (database: Kysely) => SETUP_POST(makeSetupContext(database))], + ])("does not migrate from the %s route", async (_name, invoke) => { + const pending = MIGRATION_NAMES.at(-1)!; + await db.deleteFrom("_emdash_migrations").where("name", "=", pending).execute(); + + const response = await invoke(db); + + expect(response.status).toBe(200); + const record = await db + .selectFrom("_emdash_migrations") + .select("name") + .where("name", "=", pending) + .executeTakeFirst(); + expect(record).toBeUndefined(); + }); }); diff --git a/packages/core/tests/unit/database/migration-policy.test.ts b/packages/core/tests/unit/database/migration-policy.test.ts new file mode 100644 index 0000000000..5f541f19d1 --- /dev/null +++ b/packages/core/tests/unit/database/migration-policy.test.ts @@ -0,0 +1,123 @@ +import type { + KyselyPlugin, + PluginTransformQueryArgs, + PluginTransformResultArgs, + QueryResult, + RootOperationNode, + UnknownRow, +} from "kysely"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { + enforceRuntimeMigrationPolicy, + normalizeMigrationConfig, + PendingMigrationsError, + resolveRuntimeMigrationMode, +} from "../../../src/database/migrations/policy.js"; +import { MIGRATION_NAMES, runMigrations } from "../../../src/database/migrations/runner.js"; +import { createTestDatabase, teardownTestDatabase } from "../../utils/test-db.js"; + +class QueryCountingPlugin implements KyselyPlugin { + count = 0; + + transformQuery(args: PluginTransformQueryArgs): RootOperationNode { + this.count += 1; + return args.node; + } + + transformResult(args: PluginTransformResultArgs): Promise> { + return Promise.resolve(args.result); + } +} + +describe("runtime migration policy configuration", () => { + it("defaults production and development to auto", () => { + const config = normalizeMigrationConfig(undefined); + + expect(config).toEqual({ runtime: "auto" }); + expect(resolveRuntimeMigrationMode(config, { dev: false })).toBe("auto"); + expect(resolveRuntimeMigrationMode(config, { dev: true })).toBe("auto"); + }); + + it("uses the development override without inheriting the production mode", () => { + expect(resolveRuntimeMigrationMode({ runtime: "manual" }, { dev: true })).toBe("auto"); + expect(resolveRuntimeMigrationMode({ runtime: "manual", dev: "check" }, { dev: true })).toBe( + "check", + ); + }); + + it("gives a validated environment override precedence", () => { + expect( + resolveRuntimeMigrationMode( + { runtime: "auto", dev: "auto" }, + { dev: true, override: "manual" }, + ), + ).toBe("manual"); + expect(() => + resolveRuntimeMigrationMode({ runtime: "auto" }, { dev: false, override: "later" }), + ).toThrow(/EMDASH_MIGRATIONS_MODE.*later/); + }); + + it("rejects invalid integration modes", () => { + expect(() => normalizeMigrationConfig({ runtime: "later" })).toThrow( + /migrations\.runtime.*later/, + ); + expect(() => normalizeMigrationConfig({ runtime: "auto", dev: "later" })).toThrow( + /migrations\.dev.*later/, + ); + }); +}); + +describe("runtime migration policy execution", () => { + let db = createTestDatabase(); + + beforeEach(() => { + db = createTestDatabase(); + }); + + afterEach(async () => { + await teardownTestDatabase(db); + }); + + it("keeps auto on the existing single-query current-schema fast path", async () => { + await runMigrations(db); + const counter = new QueryCountingPlugin(); + + await enforceRuntimeMigrationPolicy(db.withPlugin(counter), "auto"); + + expect(counter.count).toBe(1); + }); + + it("checks directionally in one query and tolerates unknown applied names", async () => { + await runMigrations(db); + await db + .insertInto("_emdash_migrations") + .values({ name: "999_future", timestamp: new Date().toISOString() }) + .execute(); + const counter = new QueryCountingPlugin(); + + await enforceRuntimeMigrationPolicy(db.withPlugin(counter), "check"); + + expect(counter.count).toBe(1); + }); + + it("throws the pending names after one directional check", async () => { + await runMigrations(db); + const pending = MIGRATION_NAMES.at(-1)!; + await db.deleteFrom("_emdash_migrations").where("name", "=", pending).execute(); + const counter = new QueryCountingPlugin(); + + const result = enforceRuntimeMigrationPolicy(db.withPlugin(counter), "check"); + await expect(result).rejects.toMatchObject({ pending: [pending] }); + await expect(result).rejects.toBeInstanceOf(PendingMigrationsError); + expect(counter.count).toBe(1); + }); + + it("manual issues no migration or status query", async () => { + const counter = new QueryCountingPlugin(); + + await enforceRuntimeMigrationPolicy(db.withPlugin(counter), "manual"); + + expect(counter.count).toBe(0); + }); +}); diff --git a/packages/core/tests/unit/db/migration-adapters.test.ts b/packages/core/tests/unit/db/migration-adapters.test.ts new file mode 100644 index 0000000000..37f17e1572 --- /dev/null +++ b/packages/core/tests/unit/db/migration-adapters.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "vitest"; + +import { libsql, postgres, sqlite } from "../../../src/db/adapters.js"; + +describe("database adapter migration descriptors", () => { + it("opts SQLite into migrations with an explicit secret-free config copy", () => { + const config = { url: "file:./data.db" }; + const descriptor = sqlite(config); + + expect(descriptor).toEqual({ + entrypoint: "emdash/db/sqlite", + config: { url: "file:./data.db" }, + type: "sqlite", + migrations: { + entrypoint: "emdash/db/sqlite-migrations", + manifestConfig: { url: "file:./data.db" }, + }, + }); + expect(descriptor.config).toBe(config); + expect(descriptor.migrations?.manifestConfig).not.toBe(descriptor.config); + }); + + it("keeps the libSQL runtime token out of migration metadata", () => { + const descriptor = libsql({ + url: "libsql://example.turso.io", + authToken: "runtime-secret", + migrationAuthTokenEnv: "DEPLOY_TURSO_TOKEN", + }); + + expect(descriptor.config).toEqual({ + url: "libsql://example.turso.io", + authToken: "runtime-secret", + }); + expect(descriptor.migrations).toEqual({ + entrypoint: "emdash/db/libsql-migrations", + manifestConfig: { + url: "libsql://example.turso.io", + authTokenEnv: "DEPLOY_TURSO_TOKEN", + }, + }); + expect(JSON.stringify(descriptor.migrations)).not.toContain("runtime-secret"); + }); + + it("defaults the libSQL migration token variable without changing runtime config", () => { + const descriptor = libsql({ url: "libsql://example.turso.io" }); + + expect(descriptor.config).toEqual({ url: "libsql://example.turso.io" }); + expect(descriptor.migrations?.manifestConfig).toEqual({ + url: "libsql://example.turso.io", + authTokenEnv: "TURSO_AUTH_TOKEN", + }); + }); + + it("rejects an invalid libSQL migration environment variable name", () => { + expect(() => + libsql({ + url: "libsql://example.turso.io", + migrationAuthTokenEnv: "not valid", + }), + ).toThrow(/migrationAuthTokenEnv/); + }); + + it("keeps PostgreSQL runtime credentials out of migration metadata", () => { + const descriptor = postgres({ + connectionString: "postgres://runtime:secret@example.com/site?sslmode=require", + pool: { min: 2, max: 20 }, + migrationConnectionStringEnv: "DEPLOY_DATABASE_URL", + }); + + expect(descriptor.config).toEqual({ + connectionString: "postgres://runtime:secret@example.com/site?sslmode=require", + pool: { min: 2, max: 20 }, + }); + expect(descriptor.migrations).toEqual({ + entrypoint: "emdash/db/postgres-migrations", + manifestConfig: { connectionStringEnv: "DEPLOY_DATABASE_URL" }, + }); + expect(JSON.stringify(descriptor.migrations)).not.toContain("runtime:secret"); + }); + + it("defaults the PostgreSQL migration connection variable to DATABASE_URL", () => { + const descriptor = postgres({ host: "runtime-host", password: "runtime-secret" }); + + expect(descriptor.config).toEqual({ host: "runtime-host", password: "runtime-secret" }); + expect(descriptor.migrations?.manifestConfig).toEqual({ + connectionStringEnv: "DATABASE_URL", + }); + }); + + it("rejects an invalid PostgreSQL migration environment variable name", () => { + expect(() => postgres({ migrationConnectionStringEnv: "" })).toThrow( + /migrationConnectionStringEnv/, + ); + }); +}); diff --git a/packages/core/tests/unit/db/migration-executors.test.ts b/packages/core/tests/unit/db/migration-executors.test.ts new file mode 100644 index 0000000000..04e44ba091 --- /dev/null +++ b/packages/core/tests/unit/db/migration-executors.test.ts @@ -0,0 +1,223 @@ +import { existsSync } from "node:fs"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { MIGRATION_NAMES } from "../../../src/database/migrations/runner.js"; +import { createMigrationExecutor as createLibsqlExecutor } from "../../../src/db/libsql-migrations.js"; +import { createMigrationExecutor as createPostgresExecutor } from "../../../src/db/postgres-migrations.js"; +import { createMigrationExecutor as createSqliteExecutor } from "../../../src/db/sqlite-migrations.js"; +import { getCoreMigrationIdentity } from "../../../src/migrations/identity.js"; +import type { + MigrationExecutorFactoryContext, + MigrationRequest, +} from "../../../src/migrations/protocol.js"; + +const temporaryDirectories: string[] = []; + +async function temporaryDirectory(): Promise { + const directory = await mkdtemp(join(tmpdir(), "emdash-migrations-")); + temporaryDirectories.push(directory); + return directory; +} + +async function migrationRequest(action: MigrationRequest["action"]): Promise { + const identity = await getCoreMigrationIdentity(); + return { + action, + i18n: null, + artifact: { + emdashVersion: identity.emdashVersion, + migrationSetFingerprint: identity.fingerprint, + }, + }; +} + +function context( + projectRoot: string, + env: Readonly> = {}, +): MigrationExecutorFactoryContext { + return { projectRoot, env }; +} + +afterEach(async () => { + await Promise.all( + temporaryDirectories + .splice(0) + .map((directory) => rm(directory, { recursive: true, force: true })), + ); +}); + +describe("SQLite migration executor", () => { + it("resolves relative paths from the project and opens the database only on execution", async () => { + const projectRoot = await temporaryDirectory(); + await mkdir(join(projectRoot, "data")); + const databasePath = join(projectRoot, "data", "deployment.db"); + const executor = await createSqliteExecutor( + { url: "file:./data/deployment.db" }, + context(projectRoot), + ); + + expect(existsSync(databasePath)).toBe(false); + expect(executor.target).toEqual({ + kind: "sqlite", + label: databasePath, + fingerprint: expect.stringMatching(/^[a-f0-9]{64}$/), + }); + expect(Object.isFrozen(executor.target)).toBe(true); + + const firstReport = await executor.execute(await migrationRequest("apply")); + expect(existsSync(databasePath)).toBe(true); + expect(firstReport.executed).toEqual(MIGRATION_NAMES); + expect(firstReport.pending).toEqual([]); + + const secondExecutor = await createSqliteExecutor( + { url: "file:./data/deployment.db" }, + context(projectRoot), + ); + const secondReport = await secondExecutor.execute(await migrationRequest("apply")); + expect(secondReport.executed).toEqual([]); + expect(secondReport.knownApplied).toEqual(MIGRATION_NAMES); + }); + + it("uses the explicit database override instead of the manifest path", async () => { + const projectRoot = await temporaryDirectory(); + const overridePath = join(projectRoot, "override.db"); + const executor = await createSqliteExecutor( + { url: "manifest.db" }, + { + ...context(projectRoot), + overrides: { database: "override.db" }, + }, + ); + + expect(executor.target.label).toBe(overridePath); + await executor.execute(await migrationRequest("check")); + expect(existsSync(overridePath)).toBe(true); + expect(existsSync(join(projectRoot, "manifest.db"))).toBe(false); + }); +}); + +describe("libSQL migration executor", () => { + it("migrates a local file without reading a token and is idempotent", async () => { + const projectRoot = await temporaryDirectory(); + const databasePath = join(projectRoot, "libsql.db"); + const manifestConfig = { + url: "file:./libsql.db", + authTokenEnv: "UNSET_TURSO_TOKEN", + }; + const firstExecutor = await createLibsqlExecutor(manifestConfig, context(projectRoot)); + + expect(existsSync(databasePath)).toBe(false); + const firstReport = await firstExecutor.execute(await migrationRequest("apply")); + expect(firstReport.executed).toEqual(MIGRATION_NAMES); + expect(firstReport.pending).toEqual([]); + + const secondExecutor = await createLibsqlExecutor(manifestConfig, context(projectRoot)); + const secondReport = await secondExecutor.execute(await migrationRequest("apply")); + expect(secondReport.executed).toEqual([]); + expect(secondReport.knownApplied).toEqual(MIGRATION_NAMES); + }); + + it("fails on a missing token before constructing an executor", async () => { + const projectRoot = await temporaryDirectory(); + + await expect( + createLibsqlExecutor( + { url: "libsql://example.turso.io/site", authTokenEnv: "DEPLOY_TURSO_TOKEN" }, + context(projectRoot), + ), + ).rejects.toThrow("DEPLOY_TURSO_TOKEN"); + }); + + it("keeps credentials and URL parameters out of its safe target", async () => { + const projectRoot = await temporaryDirectory(); + const secret = "very-secret-token"; + const executor = await createLibsqlExecutor( + { + url: "libsql://example.turso.io/site?tls=1#deployment", + authTokenEnv: "DEPLOY_TURSO_TOKEN", + }, + context(projectRoot, { DEPLOY_TURSO_TOKEN: secret }), + ); + + expect(executor.target.label).toBe("libsql://example.turso.io/site"); + expect(JSON.stringify(executor.target)).not.toContain(secret); + expect(JSON.stringify(executor.target)).not.toContain("tls"); + expect(JSON.stringify(executor.target)).not.toContain("deployment"); + }); + + it("rejects URL credentials without echoing them", async () => { + const projectRoot = await temporaryDirectory(); + const url = "libsql://alice:password@example.turso.io/site"; + + let error: unknown; + try { + await createLibsqlExecutor( + { url, authTokenEnv: "DEPLOY_TURSO_TOKEN" }, + context(projectRoot, { DEPLOY_TURSO_TOKEN: "token" }), + ); + } catch (caught) { + error = caught; + } + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).not.toContain("alice"); + expect((error as Error).message).not.toContain("password"); + }); +}); + +describe("PostgreSQL migration executor", () => { + it("fails on a missing connection string before constructing an executor", async () => { + const projectRoot = await temporaryDirectory(); + + await expect( + createPostgresExecutor({ connectionStringEnv: "DEPLOY_DATABASE_URL" }, context(projectRoot)), + ).rejects.toThrow("DEPLOY_DATABASE_URL"); + }); + + it("derives target identity only from host, port, and database", async () => { + const projectRoot = await temporaryDirectory(); + const first = await createPostgresExecutor( + { connectionStringEnv: "DEPLOY_DATABASE_URL" }, + context(projectRoot, { + DEPLOY_DATABASE_URL: "postgresql://alice:first-secret@db.example:5433/site?sslmode=require", + }), + ); + const second = await createPostgresExecutor( + { connectionStringEnv: "OTHER_DATABASE_URL" }, + context(projectRoot, { + OTHER_DATABASE_URL: "postgresql://bob:second-secret@db.example:5433/site?sslmode=disable", + }), + ); + + expect(first.target).toEqual({ + kind: "postgres", + label: "db.example:5433/site", + fingerprint: expect.stringMatching(/^[a-f0-9]{64}$/), + }); + expect(second.target).toEqual(first.target); + const serialized = JSON.stringify([first.target, second.target]); + expect(serialized).not.toContain("alice"); + expect(serialized).not.toContain("bob"); + expect(serialized).not.toContain("secret"); + expect(serialized).not.toContain("sslmode"); + }); + + it("uses the connection-string environment override", async () => { + const projectRoot = await temporaryDirectory(); + const executor = await createPostgresExecutor( + { connectionStringEnv: "MANIFEST_DATABASE_URL" }, + { + ...context(projectRoot, { + OVERRIDE_DATABASE_URL: "postgresql://db.example/override", + }), + overrides: { databaseUrlEnv: "OVERRIDE_DATABASE_URL" }, + }, + ); + + expect(executor.target.label).toBe("db.example:5432/override"); + }); +}); diff --git a/packages/core/tests/unit/i18n/normalize.test.ts b/packages/core/tests/unit/i18n/normalize.test.ts new file mode 100644 index 0000000000..c917c17614 --- /dev/null +++ b/packages/core/tests/unit/i18n/normalize.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; + +import { normalizeAstroI18n } from "../../../src/i18n/normalize.js"; + +describe("normalizeAstroI18n", () => { + it("returns null when Astro i18n is not configured", () => { + expect(normalizeAstroI18n(undefined)).toBeNull(); + }); + + it("preserves string locales and a custom default locale", () => { + expect( + normalizeAstroI18n({ + defaultLocale: "fr", + locales: ["en", "fr"], + }), + ).toEqual({ + defaultLocale: "fr", + locales: ["en", "fr"], + fallback: undefined, + prefixDefaultLocale: false, + }); + }); + + it("normalizes locale objects to their configured paths", () => { + expect( + normalizeAstroI18n({ + defaultLocale: "english", + locales: [ + { path: "english", codes: ["en", "en-US"] }, + { path: "french", codes: ["fr", "fr-FR"] }, + ], + }), + ).toMatchObject({ + defaultLocale: "english", + locales: ["english", "french"], + }); + }); + + it("preserves fallback and prefixDefaultLocale routing", () => { + const fallback = { "fr-CA": "fr", fr: "en" }; + + expect( + normalizeAstroI18n({ + defaultLocale: "en", + locales: ["en", "fr", "fr-CA"], + fallback, + routing: { prefixDefaultLocale: true }, + }), + ).toEqual({ + defaultLocale: "en", + locales: ["en", "fr", "fr-CA"], + fallback, + prefixDefaultLocale: true, + }); + }); +}); diff --git a/packages/core/tests/unit/migrations/config-loader.test.ts b/packages/core/tests/unit/migrations/config-loader.test.ts new file mode 100644 index 0000000000..98011bb5a9 --- /dev/null +++ b/packages/core/tests/unit/migrations/config-loader.test.ts @@ -0,0 +1,233 @@ +import { mkdir, mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { DatabaseDescriptor } from "../../../src/db/adapters.js"; +import { + buildMigrationManifestFromConfig, + findAstroConfigFile, +} from "../../../src/migrations/config-loader.js"; +import { createCoreMigrationIdentity } from "../../../src/migrations/identity.js"; +import { + createMigrationIntegrationMetadata, + MIGRATION_CONFIG_SYMBOL, +} from "../../../src/migrations/integration-metadata.js"; + +const sqliteDatabase: DatabaseDescriptor = { + type: "sqlite", + entrypoint: "emdash/db/sqlite", + config: { secret: "runtime only" }, + migrations: { + entrypoint: "emdash/db/sqlite-migrations", + manifestConfig: { url: "file:./data.db" }, + }, +}; + +describe("migration integration metadata", () => { + it("copies only deployment-safe database fields", () => { + const metadata = createMigrationIntegrationMetadata(sqliteDatabase); + + expect(metadata).toEqual({ + database: { + type: "sqlite", + migrations: { + entrypoint: "emdash/db/sqlite-migrations", + manifestConfig: { url: "file:./data.db" }, + }, + }, + }); + expect(metadata.database).not.toHaveProperty("entrypoint"); + expect(metadata.database).not.toHaveProperty("config"); + }); + + it("rejects credential-bearing manifest configuration before attaching metadata", () => { + expect(() => + createMigrationIntegrationMetadata({ + ...sqliteDatabase, + migrations: { + entrypoint: "emdash/db/sqlite-migrations", + manifestConfig: { password: "do-not-attach" }, + }, + }), + ).toThrow("database.migrations.manifestConfig.password is a credential-bearing field"); + }); +}); + +describe("findAstroConfigFile", () => { + const tempDirectories: string[] = []; + + afterEach(async () => { + await Promise.all( + tempDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); + }); + + it("discovers Astro config using Astro's filename precedence", async () => { + const root = await mkdtemp(join(tmpdir(), "emdash-config-discovery-")); + tempDirectories.push(root); + await writeFile(join(root, "astro.config.ts"), "export default {};"); + await writeFile(join(root, "astro.config.mjs"), "export default {};"); + + await expect(findAstroConfigFile(root)).resolves.toBe(join(root, "astro.config.mjs")); + }); + + it("accepts an explicit config path relative to the project root", async () => { + const root = await mkdtemp(join(tmpdir(), "emdash-config-explicit-")); + tempDirectories.push(root); + await mkdir(join(root, "config")); + await writeFile(join(root, "config", "astro.mjs"), "export default {};"); + + await expect(findAstroConfigFile(root, "config/astro.mjs")).resolves.toBe( + join(root, "config", "astro.mjs"), + ); + }); +}); + +describe("buildMigrationManifestFromConfig", () => { + const identityPromise = createCoreMigrationIdentity("1.2.3", ["001_initial"]); + + it("evaluates explicit config, reads one metadata integration, and normalizes i18n", async () => { + const root = "/project"; + const configFile = "/project/custom.astro.config.mjs"; + const integration = { + name: "emdash", + hooks: { "astro:config:setup": vi.fn(() => Promise.reject(new Error("hook ran"))) }, + [MIGRATION_CONFIG_SYMBOL]: createMigrationIntegrationMetadata(sqliteDatabase), + }; + const loadConfig = vi.fn(async () => ({ + integrations: [integration], + i18n: { + defaultLocale: "en", + locales: ["en", { path: "fr", codes: ["fr-FR"] }], + fallback: { fr: "en", de: undefined }, + routing: { prefixDefaultLocale: true }, + }, + })); + + const manifest = await buildMigrationManifestFromConfig( + { projectRoot: root, configFile }, + { + findConfigFile: vi.fn(async () => configFile), + loadConfig, + loadIdentity: vi.fn(async () => identityPromise), + }, + ); + + expect(loadConfig).toHaveBeenCalledOnce(); + expect(integration.hooks["astro:config:setup"]).not.toHaveBeenCalled(); + expect(manifest.i18n).toEqual({ + defaultLocale: "en", + locales: ["en", "fr"], + fallback: { fr: "en" }, + prefixDefaultLocale: true, + }); + expect(manifest.database).toEqual({ + type: "sqlite", + executorEntrypoint: "emdash/db/sqlite-migrations", + executorConfig: { url: "file:./data.db" }, + }); + }); + + it.each([ + ["no", []], + [ + "multiple", + [ + { [MIGRATION_CONFIG_SYMBOL]: createMigrationIntegrationMetadata(sqliteDatabase) }, + { [MIGRATION_CONFIG_SYMBOL]: createMigrationIntegrationMetadata(sqliteDatabase) }, + ], + ], + ])("rejects config with %s metadata integrations", async (_, integrations) => { + await expect( + buildMigrationManifestFromConfig( + { projectRoot: "/project" }, + { + findConfigFile: vi.fn(async () => "/project/astro.config.mjs"), + loadConfig: vi.fn(async () => ({ integrations })), + loadIdentity: vi.fn(async () => identityPromise), + }, + ), + ).rejects.toThrow(`Expected exactly one EmDash integration with migration metadata`); + }); +}); + +describe("project-local package resolution", () => { + const tempDirectories: string[] = []; + + afterEach(async () => { + await Promise.all( + tempDirectories.splice(0).map((path) => rm(path, { force: true, recursive: true })), + ); + }); + + async function writeModule(path: string, contents: string): Promise { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, contents); + } + + it("loads Vite through project Astro and identity through project EmDash under pnpm links", async () => { + const root = await mkdtemp(join(tmpdir(), "emdash-project-resolution-")); + tempDirectories.push(root); + const modules = join(root, "node_modules"); + const astroPackage = join(modules, ".pnpm", "astro@local", "node_modules", "astro"); + const astroModules = join(modules, ".pnpm", "astro@local", "node_modules"); + const vitePackage = join(modules, ".pnpm", "vite@local", "node_modules", "vite"); + const emdashPackage = join(modules, ".pnpm", "emdash@local", "node_modules", "emdash"); + const identity = await createCoreMigrationIdentity("9.8.7-project", ["001_project"]); + await writeFile(join(root, "package.json"), '{"type":"module"}'); + await writeModule( + join(astroPackage, "package.json"), + '{"name":"astro","type":"module","exports":"./index.js"}', + ); + await writeModule(join(astroPackage, "index.js"), "export {};\n"); + await writeModule( + join(vitePackage, "package.json"), + '{"name":"vite","type":"module","exports":"./index.js"}', + ); + await writeModule( + join(vitePackage, "index.js"), + `import { pathToFileURL } from "node:url"; + export async function loadConfigFromFile(_env, configFile) { + const loaded = await import(pathToFileURL(configFile).href); + return { path: configFile, config: loaded.default, dependencies: [] }; + }`, + ); + await writeModule( + join(emdashPackage, "package.json"), + '{"name":"emdash","type":"module","exports":{"./migrations":"./migrations.js"}}', + ); + await writeModule( + join(emdashPackage, "migrations.js"), + `export async function getCoreMigrationIdentity() { return ${JSON.stringify(identity)}; }`, + ); + await mkdir(modules, { recursive: true }); + await symlink(astroPackage, join(modules, "astro"), "dir"); + await symlink(vitePackage, join(astroModules, "vite"), "dir"); + await symlink(emdashPackage, join(modules, "emdash"), "dir"); + await writeFile( + join(root, "astro.config.mjs"), + `export default { + integrations: [{ + [Symbol.for("emdash:migration-config")]: { + database: { + type: "sqlite", + migrations: { + entrypoint: "emdash/db/sqlite-migrations", + manifestConfig: { url: "file:./project.db" } + } + } + } + }], + i18n: { defaultLocale: "en", locales: ["en"] } + };`, + ); + + const manifest = await buildMigrationManifestFromConfig({ projectRoot: root }); + + expect(manifest.emdashVersion).toBe("9.8.7-project"); + expect(manifest.migrationSet.names).toEqual(["001_project"]); + expect(manifest.database.executorConfig).toEqual({ url: "file:./project.db" }); + }); +}); diff --git a/packages/core/tests/unit/migrations/direct-executor.test.ts b/packages/core/tests/unit/migrations/direct-executor.test.ts new file mode 100644 index 0000000000..44e87301b3 --- /dev/null +++ b/packages/core/tests/unit/migrations/direct-executor.test.ts @@ -0,0 +1,351 @@ +import Database from "better-sqlite3"; +import type { + CompiledQuery, + DatabaseConnection, + DatabaseIntrospector, + Dialect, + Driver, + QueryResult, +} from "kysely"; +import { SqliteAdapter, SqliteDialect, SqliteQueryCompiler } from "kysely"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { MIGRATION_NAMES } from "../../../src/database/migrations/runner.js"; +import { getI18nConfig, setI18nConfig, type I18nConfig } from "../../../src/i18n/config.js"; +import { createDirectMigrationExecutor } from "../../../src/migrations/direct-executor.js"; +import { getCoreMigrationIdentity } from "../../../src/migrations/identity.js"; +import type { MigrationRequest, MigrationTarget } from "../../../src/migrations/protocol.js"; + +const TARGET: MigrationTarget = { + kind: "sqlite", + label: "test database", + fingerprint: "target-fingerprint", +}; + +interface DialectTracker { + factoryCalls: number; + closeCalls: number; + i18nAtConnection: Array; +} + +interface TrackedDialectOptions { + setup?: (database: Database.Database) => void; + closeError?: Error; +} + +function createTrackedDialectFactory(options: TrackedDialectOptions = {}): { + tracker: DialectTracker; + createDialect: () => Dialect; +} { + const tracker: DialectTracker = { + factoryCalls: 0, + closeCalls: 0, + i18nAtConnection: [], + }; + + return { + tracker, + createDialect() { + tracker.factoryCalls += 1; + const database = new Database(":memory:"); + options.setup?.(database); + const close = database.close.bind(database); + database.close = () => { + tracker.closeCalls += 1; + close(); + if (options.closeError) throw options.closeError; + }; + + return new SqliteDialect({ + database, + onCreateConnection() { + tracker.i18nAtConnection.push(getI18nConfig()); + }, + }); + }, + }; +} + +async function migrationRequest(action: MigrationRequest["action"]): Promise { + const identity = await getCoreMigrationIdentity(); + return { + action, + i18n: null, + artifact: { + emdashVersion: identity.emdashVersion, + migrationSetFingerprint: identity.fingerprint, + }, + }; +} + +afterEach(() => { + setI18nConfig(null); + vi.restoreAllMocks(); +}); + +describe("createDirectMigrationExecutor", () => { + it("checks a fresh database and destroys its only Kysely connection", async () => { + const { tracker, createDialect } = createTrackedDialectFactory(); + const executor = createDirectMigrationExecutor({ target: TARGET, createDialect }); + + await expect(executor.execute(await migrationRequest("check"))).resolves.toEqual({ + target: TARGET, + knownApplied: [], + pending: MIGRATION_NAMES, + unknownApplied: [], + executed: [], + }); + expect(tracker.factoryCalls).toBe(1); + expect(tracker.closeCalls).toBe(1); + }); + + it("applies migrations with request i18n and restores the previous process config", async () => { + const previousI18n: I18nConfig = { + defaultLocale: "en", + locales: ["en"], + }; + const requestI18n: I18nConfig = { + defaultLocale: "fr", + locales: ["fr", "en"], + }; + setI18nConfig(previousI18n); + const { tracker, createDialect } = createTrackedDialectFactory(); + const executor = createDirectMigrationExecutor({ target: TARGET, createDialect }); + const request = await migrationRequest("apply"); + request.i18n = requestI18n; + + await expect(executor.execute(request)).resolves.toEqual({ + target: TARGET, + knownApplied: MIGRATION_NAMES, + pending: [], + unknownApplied: [], + executed: MIGRATION_NAMES, + }); + expect(tracker.i18nAtConnection).toEqual([requestI18n]); + expect(getI18nConfig()).toBe(previousI18n); + expect(tracker.factoryCalls).toBe(1); + expect(tracker.closeCalls).toBe(1); + }); + + it.each([ + ["version", { emdashVersion: "stale-version" }], + ["fingerprint", { migrationSetFingerprint: "stale-fingerprint" }], + ])("rejects an artifact %s mismatch before creating a dialect", async (_name, artifact) => { + const { tracker, createDialect } = createTrackedDialectFactory(); + const executor = createDirectMigrationExecutor({ target: TARGET, createDialect }); + const request = await migrationRequest("check"); + Object.assign(request.artifact, artifact); + + await expect(executor.execute(request)).rejects.toThrow(/artifact.*does not match/i); + expect(tracker.factoryCalls).toBe(0); + expect(tracker.closeCalls).toBe(0); + }); + + it("refuses unknown applied migrations without running known migrations", async () => { + const { tracker, createDialect } = createTrackedDialectFactory({ + setup(database) { + database.exec(` + CREATE TABLE _emdash_migrations ( + name TEXT PRIMARY KEY, + timestamp TEXT NOT NULL + ); + INSERT INTO _emdash_migrations (name, timestamp) + VALUES ('999_future', '2026-01-01T00:00:00.000Z'); + `); + }, + }); + const executor = createDirectMigrationExecutor({ target: TARGET, createDialect }); + + await expect(executor.execute(await migrationRequest("apply"))).rejects.toThrow( + /unknown applied migrations.*999_future/i, + ); + expect(tracker.factoryCalls).toBe(1); + expect(tracker.closeCalls).toBe(1); + }); + + it("refuses apply when every known migration and an unknown migration are recorded", async () => { + const { tracker, createDialect } = createTrackedDialectFactory({ + setup(database) { + database.exec(` + CREATE TABLE _emdash_migrations ( + name TEXT PRIMARY KEY, + timestamp TEXT NOT NULL + ) + `); + const insert = database.prepare( + "INSERT INTO _emdash_migrations (name, timestamp) VALUES (?, ?)", + ); + const insertAll = database.transaction((names: readonly string[]) => { + for (const name of names) insert.run(name, "2026-01-01T00:00:00.000Z"); + }); + insertAll([...MIGRATION_NAMES, "999_future"]); + }, + }); + const executor = createDirectMigrationExecutor({ target: TARGET, createDialect }); + + await expect(executor.execute(await migrationRequest("apply"))).rejects.toThrow( + /unknown applied migrations.*999_future/i, + ); + expect(tracker.closeCalls).toBe(1); + }); + + it("destroys the database and restores i18n after a status failure", async () => { + const previousI18n: I18nConfig = { defaultLocale: "de", locales: ["de"] }; + setI18nConfig(previousI18n); + const { tracker, createDialect } = createTrackedDialectFactory({ + setup(database) { + database.exec("CREATE TABLE _emdash_migrations (unexpected TEXT)"); + }, + }); + const executor = createDirectMigrationExecutor({ target: TARGET, createDialect }); + const request = await migrationRequest("check"); + request.i18n = { defaultLocale: "fr", locales: ["fr"] }; + + await expect(executor.execute(request)).rejects.toThrow(/no such column.*name/i); + expect(tracker.closeCalls).toBe(1); + expect(getI18nConfig()).toBe(previousI18n); + }); + + it("destroys the database after a migration failure", async () => { + const { tracker, createDialect } = createTrackedDialectFactory({ + setup(database) { + const prepare = database.prepare.bind(database); + database.prepare = ((source: string) => { + if (/create table if not exists [`"]?revisions/i.test(source)) { + throw new Error("injected migration failure"); + } + return prepare(source); + }) as typeof database.prepare; + }, + }); + const executor = createDirectMigrationExecutor({ target: TARGET, createDialect }); + + await expect(executor.execute(await migrationRequest("apply"))).rejects.toThrow( + /injected migration failure/i, + ); + expect(tracker.closeCalls).toBe(1); + }); + + it("does not replace an execution failure with a destroy failure", async () => { + const consoleError = vi.spyOn(console, "error").mockImplementation(() => undefined); + const { tracker, createDialect } = createTrackedDialectFactory({ + setup(database) { + database.exec("CREATE TABLE _emdash_migrations (unexpected TEXT)"); + }, + closeError: new Error("destroy failure"), + }); + const executor = createDirectMigrationExecutor({ target: TARGET, createDialect }); + + await expect(executor.execute(await migrationRequest("check"))).rejects.toThrow( + /no such column.*name/i, + ); + expect(consoleError).toHaveBeenCalledWith("[migrations] Database close failed."); + expect(JSON.stringify(consoleError.mock.calls)).not.toContain("destroy failure"); + expect(tracker.closeCalls).toBe(1); + }); + + it("returns a successful report when destroy fails", async () => { + const destroyError = new Error("destroy failure"); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => undefined); + const { tracker, createDialect } = createTrackedDialectFactory({ closeError: destroyError }); + const executor = createDirectMigrationExecutor({ target: TARGET, createDialect }); + + await expect(executor.execute(await migrationRequest("check"))).resolves.toMatchObject({ + target: TARGET, + pending: MIGRATION_NAMES, + }); + expect(consoleError).toHaveBeenCalledWith("[migrations] Database close failed."); + expect(tracker.closeCalls).toBe(1); + await expect(executor.dispose?.()).resolves.toBeUndefined(); + expect(tracker.closeCalls).toBe(1); + }); + + it("is single-use even when its first execution fails identity verification", async () => { + const { tracker, createDialect } = createTrackedDialectFactory(); + const executor = createDirectMigrationExecutor({ target: TARGET, createDialect }); + const request = await migrationRequest("check"); + request.artifact.emdashVersion = "stale-version"; + + await expect(executor.execute(request)).rejects.toThrow(/artifact.*does not match/i); + await expect(executor.execute(await migrationRequest("check"))).rejects.toThrow(/single-use/i); + expect(tracker.factoryCalls).toBe(0); + }); + + it("rejects a second execution after success without creating another dialect", async () => { + const { tracker, createDialect } = createTrackedDialectFactory(); + const executor = createDirectMigrationExecutor({ target: TARGET, createDialect }); + + await executor.execute(await migrationRequest("check")); + await expect(executor.execute(await migrationRequest("check"))).rejects.toThrow(/single-use/i); + expect(tracker.factoryCalls).toBe(1); + expect(tracker.closeCalls).toBe(1); + }); + + it("makes disposal idempotent and rejects execution after disposal", async () => { + const { tracker, createDialect } = createTrackedDialectFactory(); + const executor = createDirectMigrationExecutor({ target: TARGET, createDialect }); + + await executor.dispose?.(); + await executor.dispose?.(); + await expect(executor.execute(await migrationRequest("check"))).rejects.toThrow(/disposed/i); + expect(tracker.factoryCalls).toBe(0); + }); + + it("destroys an active database when disposed during execution", async () => { + let rejectQuery!: (error: Error) => void; + const destroy = vi.fn(async () => rejectQuery(new Error("execution interrupted"))); + const queryStarted = vi.fn(); + const connection: DatabaseConnection = { + executeQuery(_query: CompiledQuery): Promise> { + queryStarted(); + return new Promise((_resolve, reject) => { + rejectQuery = reject; + }); + }, + // eslint-disable-next-line require-yield -- the test driver does not stream + async *streamQuery(): AsyncIterableIterator> { + throw new Error("Streaming is not supported"); + }, + }; + const driver: Driver = { + init: async () => undefined, + acquireConnection: async () => connection, + beginTransaction: async () => undefined, + commitTransaction: async () => undefined, + rollbackTransaction: async () => undefined, + releaseConnection: async () => undefined, + destroy, + }; + const dialect: Dialect = { + createAdapter: () => new SqliteAdapter(), + createDriver: () => driver, + createIntrospector: (): DatabaseIntrospector => { + throw new Error("Introspection is not supported"); + }, + createQueryCompiler: () => new SqliteQueryCompiler(), + }; + const executor = createDirectMigrationExecutor({ + target: TARGET, + createDialect: () => dialect, + }); + const execution = executor.execute(await migrationRequest("check")); + await vi.waitFor(() => expect(queryStarted).toHaveBeenCalledOnce()); + + await executor.dispose?.(); + + await expect(execution).rejects.toThrow("execution interrupted"); + expect(destroy).toHaveBeenCalledOnce(); + }); + + it("snapshots its target before it can be confirmed or reported", async () => { + const mutableTarget = { ...TARGET }; + const { createDialect } = createTrackedDialectFactory(); + const executor = createDirectMigrationExecutor({ target: mutableTarget, createDialect }); + mutableTarget.label = "changed target"; + + expect(executor.target).toEqual(TARGET); + expect(Object.isFrozen(executor.target)).toBe(true); + const report = await executor.execute(await migrationRequest("check")); + expect(report.target).toBe(executor.target); + }); +}); diff --git a/packages/core/tests/unit/migrations/identity.test.ts b/packages/core/tests/unit/migrations/identity.test.ts new file mode 100644 index 0000000000..12a9c23b9c --- /dev/null +++ b/packages/core/tests/unit/migrations/identity.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; + +import { + createCoreMigrationIdentity, + fingerprintMigrationSet, +} from "../../../src/migrations/identity.js"; + +describe("fingerprintMigrationSet", () => { + it("changes when the EmDash version changes", async () => { + const names = ["001_initial", "002_media_status"]; + + await expect(fingerprintMigrationSet("1.0.0", names)).resolves.not.toBe( + await fingerprintMigrationSet("1.0.1", names), + ); + }); + + it("changes when migration order changes", async () => { + await expect( + fingerprintMigrationSet("1.0.0", ["001_initial", "002_media_status"]), + ).resolves.not.toBe( + await fingerprintMigrationSet("1.0.0", ["002_media_status", "001_initial"]), + ); + }); + + it("is deterministic and encoded as a lowercase SHA-256 digest", async () => { + const first = await fingerprintMigrationSet("1.0.0", ["001_initial"]); + const second = await fingerprintMigrationSet("1.0.0", ["001_initial"]); + + expect(second).toBe(first); + expect(first).toMatch(/^[0-9a-f]{64}$/); + }); +}); + +describe("createCoreMigrationIdentity", () => { + it("returns the supplied version and an immutable snapshot of the ordered names", async () => { + const names = ["001_initial", "002_media_status"]; + const identity = await createCoreMigrationIdentity("1.0.0", names); + + expect(identity).toEqual({ + emdashVersion: "1.0.0", + names, + fingerprint: await fingerprintMigrationSet("1.0.0", names), + }); + expect(identity.names).not.toBe(names); + expect(Object.isFrozen(identity.names)).toBe(true); + expect(Object.isFrozen(identity)).toBe(true); + }); +}); diff --git a/packages/core/tests/unit/migrations/manifest-writer.test.ts b/packages/core/tests/unit/migrations/manifest-writer.test.ts new file mode 100644 index 0000000000..9a5c327972 --- /dev/null +++ b/packages/core/tests/unit/migrations/manifest-writer.test.ts @@ -0,0 +1,79 @@ +import { access, mkdir, mkdtemp, readFile, rm, unlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { createCoreMigrationIdentity } from "../../../src/migrations/identity.js"; +import { buildMigrationManifest } from "../../../src/migrations/manifest-builder.js"; +import { + MIGRATION_MANIFEST_PATH, + writeMigrationManifest, +} from "../../../src/migrations/manifest-writer.js"; + +describe("writeMigrationManifest", () => { + const tempDirs: string[] = []; + + afterEach(async () => { + await Promise.all(tempDirs.splice(0).map((path) => rm(path, { recursive: true, force: true }))); + }); + + async function createManifest() { + return buildMigrationManifest({ + identity: await createCoreMigrationIdentity("1.2.3", ["001_initial"]), + i18n: null, + database: { + type: "sqlite", + entrypoint: "emdash/db/sqlite", + config: {}, + migrations: { + entrypoint: "emdash/db/sqlite-migrations", + manifestConfig: { url: "file:./data.db" }, + }, + }, + }); + } + + it("creates .emdash and atomically writes the validated manifest", async () => { + const projectRoot = await mkdtemp(join(tmpdir(), "emdash-manifest-writer-")); + tempDirs.push(projectRoot); + const manifest = await createManifest(); + + const outputPath = await writeMigrationManifest(projectRoot, manifest); + + expect(outputPath).toBe(join(projectRoot, MIGRATION_MANIFEST_PATH)); + expect(JSON.parse(await readFile(outputPath, "utf8"))).toEqual(manifest); + }); + + it("preserves the previous manifest and cleans up its temp file when rename fails", async () => { + const projectRoot = await mkdtemp(join(tmpdir(), "emdash-manifest-writer-failure-")); + tempDirs.push(projectRoot); + const manifest = await createManifest(); + const outputPath = join(projectRoot, MIGRATION_MANIFEST_PATH); + await mkdir(join(projectRoot, ".emdash"), { recursive: true }); + const previousManifest = `${JSON.stringify({ ...manifest, i18n: { defaultLocale: "en", locales: ["en"] } })}\n`; + await writeFile(outputPath, previousManifest, "utf8"); + let temporaryPath: string | undefined; + + await expect( + writeMigrationManifest(projectRoot, manifest, { + mkdir, + async writeFile(path, data, options) { + temporaryPath = path; + await writeFile(path, data, options); + }, + async rename() { + throw new Error("simulated interruption"); + }, + unlink, + }), + ).rejects.toThrow("simulated interruption"); + + expect(await readFile(outputPath, "utf8")).toBe(previousManifest); + expect(temporaryPath).toBeDefined(); + expect(temporaryPath).toMatch( + new RegExp(`^${join(projectRoot, ".emdash", ".migrations.json.").replaceAll(".", "\\.")}`), + ); + await expect(access(temporaryPath!)).rejects.toThrow(); + }); +}); diff --git a/packages/core/tests/unit/migrations/manifest.test.ts b/packages/core/tests/unit/migrations/manifest.test.ts new file mode 100644 index 0000000000..0d19e9f601 --- /dev/null +++ b/packages/core/tests/unit/migrations/manifest.test.ts @@ -0,0 +1,168 @@ +import { describe, expect, it } from "vitest"; + +import { createCoreMigrationIdentity } from "../../../src/migrations/identity.js"; +import { + buildMigrationManifest, + UnsupportedMigrationAdapterError, +} from "../../../src/migrations/manifest-builder.js"; +import { + MigrationManifestValidationError, + validateMigrationManifest, +} from "../../../src/migrations/manifest.js"; + +async function fixture() { + const identity = await createCoreMigrationIdentity("1.2.3", ["001_initial", "002_media"]); + const database = { + type: "sqlite" as const, + entrypoint: "emdash/db/sqlite", + config: { url: "file:./runtime.db", authToken: "runtime-only-secret" }, + migrations: { + entrypoint: "emdash/db/sqlite-migrations", + manifestConfig: { url: "file:./data.db" }, + }, + }; + const i18n = { + defaultLocale: "fr", + locales: ["en", "fr"], + fallback: { fr: "en" }, + prefixDefaultLocale: true, + }; + const manifest = await buildMigrationManifest({ identity, i18n, database }); + return { database, i18n, identity, manifest }; +} + +describe("buildMigrationManifest", () => { + it("builds the versioned manifest from explicit migration metadata", async () => { + const { manifest } = await fixture(); + + expect(manifest).toMatchObject({ + schemaVersion: 1, + emdashVersion: "1.2.3", + migrationSet: { + names: ["001_initial", "002_media"], + }, + i18n: { + defaultLocale: "fr", + locales: ["en", "fr"], + }, + database: { + type: "sqlite", + executorEntrypoint: "emdash/db/sqlite-migrations", + executorConfig: { url: "file:./data.db" }, + }, + }); + expect(JSON.stringify(manifest)).not.toContain("runtime-only-secret"); + }); + + it("never falls back to runtime database configuration", async () => { + const identity = await createCoreMigrationIdentity("1.2.3", ["001_initial"]); + const database = { + type: "postgres" as const, + entrypoint: "emdash/db/postgres", + config: { connectionString: "postgres://user:password@example.com/db" }, + }; + + await expect(buildMigrationManifest({ identity, i18n: null, database })).rejects.toBeInstanceOf( + UnsupportedMigrationAdapterError, + ); + }); + + it("rejects a malformed identity instead of emitting it", async () => { + const { database, identity } = await fixture(); + + await expect( + buildMigrationManifest({ + identity: { ...identity, fingerprint: "0".repeat(64) }, + i18n: null, + database, + }), + ).rejects.toThrow("identity fingerprint"); + }); +}); + +describe("validateMigrationManifest", () => { + it("rejects unsupported schema versions", async () => { + const { manifest } = await fixture(); + + await expect( + validateMigrationManifest({ ...manifest, schemaVersion: 2 }), + ).rejects.toBeInstanceOf(MigrationManifestValidationError); + }); + + it("rejects stale versions and migration sets", async () => { + const { manifest } = await fixture(); + const newerIdentity = await createCoreMigrationIdentity("1.3.0", [ + "001_initial", + "002_media", + "003_new", + ]); + + await expect(validateMigrationManifest(manifest, newerIdentity)).rejects.toThrow( + "does not match the loaded EmDash migration identity", + ); + }); + + it.each([ + "../executor.js", + "/tmp/executor.js", + "file:///tmp/executor.js", + "data:text/javascript,x", + ])("rejects unsafe executor entrypoint %s", async (executorEntrypoint) => { + const { manifest } = await fixture(); + + await expect( + validateMigrationManifest({ + ...manifest, + database: { ...manifest.database, executorEntrypoint }, + }), + ).rejects.toThrow("schema validation failed"); + }); + + it.each([ + "postgres://user:password@db.example.com/site", + { authToken: "top-secret-token" }, + { connectionString: "postgres://db.example.com/site" }, + { url: "libsql://user:password@db.example.com" }, + { url: "https://db.example.com?token=top-secret-token" }, + ])("rejects secret-bearing executor configuration", async (executorConfig) => { + const { manifest } = await fixture(); + + await expect( + validateMigrationManifest({ + ...manifest, + database: { ...manifest.database, executorConfig }, + }), + ).rejects.toBeInstanceOf(MigrationManifestValidationError); + }); + + it("does not include rejected credential values in errors", async () => { + const { manifest } = await fixture(); + const credential = "must-not-appear-in-errors"; + + await expect( + validateMigrationManifest({ + ...manifest, + database: { + ...manifest.database, + executorConfig: { url: `libsql://user:${credential}@db.example.com` }, + }, + }), + ).rejects.not.toThrow(credential); + }); + + it("accepts environment-variable names without resolving their values", async () => { + const { manifest } = await fixture(); + const executorConfig = { + url: "libsql://public-db.example.com", + authTokenEnv: "TURSO_AUTH_TOKEN", + connectionStringEnv: "DATABASE_URL", + }; + + await expect( + validateMigrationManifest({ + ...manifest, + database: { ...manifest.database, executorConfig }, + }), + ).resolves.toMatchObject({ database: { executorConfig } }); + }); +}); diff --git a/packages/core/tests/utils/test-db.ts b/packages/core/tests/utils/test-db.ts index 3258ddf255..5be1e77c2e 100644 --- a/packages/core/tests/utils/test-db.ts +++ b/packages/core/tests/utils/test-db.ts @@ -6,8 +6,15 @@ import { Kysely, SqliteAdapter, SqliteDialect } from "kysely"; import { Pool } from "pg"; import { describe } from "vitest"; -import { getMigrationStatus, runMigrations } from "../../src/database/migrations/runner.js"; -import type { MigrationStatus } from "../../src/database/migrations/runner.js"; +import { + getExactMigrationStatus, + getMigrationStatus, + runMigrations, +} from "../../src/database/migrations/runner.js"; +import type { + ExactMigrationStatus, + MigrationStatus, +} from "../../src/database/migrations/runner.js"; import { FailFastPostgresDialect } from "../../src/database/pg-migration-lock.js"; import type { Database as DatabaseSchema } from "../../src/database/types.js"; import { waitForDeferredTasks } from "../../src/deferred-tasks.js"; @@ -515,6 +522,12 @@ export function getMigrationStatusForDialect(ctx: DialectTestContext): Promise { + return getExactMigrationStatus(ctx.db, { migrationTableSchema: ctx.pgCtx?.schemaName }); +} + // Private alias to avoid name collision const setupTestDatabase_pg = setupTestPostgresDatabase; diff --git a/packages/core/tsdown.config.ts b/packages/core/tsdown.config.ts index 0ba1f0163e..8a5ef82ed0 100644 --- a/packages/core/tsdown.config.ts +++ b/packages/core/tsdown.config.ts @@ -85,8 +85,12 @@ export default defineConfig({ // Database adapters (config-time + runtime via virtual:emdash/dialect) "src/db/index.ts", "src/db/sqlite.ts", + "src/db/sqlite-migrations.ts", "src/db/libsql.ts", + "src/db/libsql-migrations.ts", "src/db/postgres.ts", + "src/db/postgres-migrations.ts", + "src/migrations/index.ts", // Query instrumentation (used by first-party adapters like @emdash-cms/cloudflare) "src/database/instrumentation.ts", // Fail-fast Postgres migration lock (used by @emdash-cms/cloudflare's Hyperdrive adapter)