diff --git a/.changeset/chunked-wp-import.md b/.changeset/chunked-wp-import.md new file mode 100644 index 0000000000..c092421d71 --- /dev/null +++ b/.changeset/chunked-wp-import.md @@ -0,0 +1,6 @@ +--- +"emdash": minor +"@emdash-cms/admin": minor +--- + +Fixes "Worker exceeded resource limits" when importing large WordPress sites on Cloudflare. The plugin import now runs as a sequence of small requests — content pages, then comments, then menus and site identity — with a live progress bar instead of an indefinite spinner, and media files upload in bounded batches. An interrupted import can safely be re-run: already-imported content is skipped and the import fast-forwards to where it stopped. diff --git a/packages/admin/src/components/WordPressImport.tsx b/packages/admin/src/components/WordPressImport.tsx index b4275eb443..5386f79008 100644 --- a/packages/admin/src/components/WordPressImport.tsx +++ b/packages/admin/src/components/WordPressImport.tsx @@ -8,7 +8,8 @@ import { Switch, buttonVariants, } from "@cloudflare/kumo"; -import { plural } from "@lingui/core/macro"; +import type { MessageDescriptor } from "@lingui/core"; +import { msg, plural } from "@lingui/core/macro"; import { useLingui } from "@lingui/react/macro"; import { Upload, @@ -36,7 +37,7 @@ import { analyzeWxr, prepareWxrImport, executeWxrImport, - importWxrMedia, + importWxrMediaBatched, rewriteContentUrls, probeImportUrl, analyzeWpPluginSite, @@ -50,6 +51,7 @@ import { type PrepareResult, type MediaImportResult, type MediaImportProgress, + type WpImportProgress, type RewriteUrlsResult, type AttachmentInfo, type ProbeResult, @@ -134,6 +136,7 @@ interface PostTypeSelection { type ImportAnalysis = WxrAnalysis | WpPluginAnalysis; export function WordPressImport() { + const { t } = useLingui(); const [step, setStep] = React.useState("choose"); const [urlInput, setUrlInput] = React.useState(""); const [probeResult, setProbeResult] = React.useState(null); @@ -154,17 +157,27 @@ export function WordPressImport() { const [mediaError, setMediaError] = React.useState(null); const [skipMedia, setSkipMedia] = React.useState(false); const [mediaProgress, setMediaProgress] = React.useState(null); + const [wpImportProgress, setWpImportProgress] = React.useState(null); // New state for import options const [importMenus, setImportMenus] = React.useState(true); const [importSiteTitle, setImportSiteTitle] = React.useState(true); const [importLogo, setImportLogo] = React.useState(true); - const [importSeo, setImportSeo] = React.useState(false); + const [importSeo, setImportSeo] = React.useState(true); // Author mapping state const [authorMappings, setAuthorMappings] = React.useState([]); const [emdashUsers, setEmDashUsers] = React.useState([]); + // Total posts across enabled post types, for the chunked-import progress bar + const totalSelectedPosts = React.useMemo(() => { + if (!analysis) return 0; + return analysis.postTypes.reduce( + (sum, pt) => (selections[pt.name]?.enabled ? sum + pt.count : sum), + 0, + ); + }, [analysis, selections]); + // Initialize author mappings from analysis, auto-matching by email const initializeAuthorMappings = React.useCallback( (importAnalysis: ImportAnalysis, users: UserListItem[]) => { @@ -194,7 +207,7 @@ export function WordPressImport() { const error = params.get("error"); if (error === "auth_rejected") { - setImportError("WordPress authorization was rejected"); + setImportError(t`WordPress authorization was rejected`); setStep("probe-result"); // Clean up URL window.history.replaceState({}, "", window.location.pathname); @@ -242,8 +255,6 @@ export function WordPressImport() { } }, []); // eslint-disable-line react-hooks/exhaustive-deps - const { t } = useLingui(); - // Probe mutation const probeMutation = useMutation({ mutationFn: probeImportUrl, @@ -331,10 +342,11 @@ export function WordPressImport() { }, }); - // Media import mutation + // Media import mutation. Batched: each request handles a bounded slice + // of the library, so large media sets don't exceed Worker limits. const mediaMutation = useMutation({ mutationFn: (attachments: AttachmentInfo[]) => - importWxrMedia(attachments, (progress) => { + importWxrMediaBatched(attachments, (progress) => { setMediaProgress(progress); }), onSuccess: (data) => { @@ -403,12 +415,15 @@ export function WordPressImport() { }, }); - // WordPress Plugin import mutation + // WordPress Plugin import mutation. Chunked: the client drives the + // import as a loop of bounded requests (issue #475), so progress here + // is real, not an indefinite spinner. const wpPluginImportMutation = useMutation({ mutationFn: ({ url, token, config }: { url: string; token: string; config: ImportConfig }) => - executeWpPluginImport(url, token, config), + executeWpPluginImport(url, token, config, setWpImportProgress), onSuccess: (data) => { setImportError(null); + setWpImportProgress(null); setResult(data); if (analysis && analysis.attachments.count > 0) { setStep("media"); @@ -417,6 +432,7 @@ export function WordPressImport() { } }, onError: (error) => { + setWpImportProgress(null); setImportError(error instanceof Error ? error.message : t`Failed to import from WordPress`); setStep("review"); }, @@ -475,7 +491,7 @@ export function WordPressImport() { // Check if we're on localhost - OAuth won't work, fall back to manual if (window.location.hostname === "localhost" || window.location.hostname === "127.0.0.1") { - setImportError("OAuth authorization requires HTTPS. Please use manual credentials."); + setImportError(t`OAuth authorization requires HTTPS. Please use manual credentials.`); setStep("plugin-auth"); return; } @@ -531,6 +547,10 @@ export function WordPressImport() { postTypeMappings: selections, skipExisting: true, authorMappings: authorMappingsRecord, + importMenus, + importSiteTitle, + importLogo, + importSeo, }; if (importSource.type === "wxr") { @@ -872,8 +892,35 @@ export function WordPressImport() { {step === "importing" && (
-

{t`Importing content...`}

-

{t`This may take a while for large exports.`}

+ {wpImportProgress ? ( + <> +

+ {wpImportProgress.phase === "content" && + (totalSelectedPosts > 0 + ? t`Importing content... ${wpImportProgress.processed} of ${totalSelectedPosts}` + : t`Importing content... ${wpImportProgress.processed}`)} + {wpImportProgress.phase === "comments" && + t`Importing comments... ${wpImportProgress.comments}`} + {wpImportProgress.phase === "finalize" && t`Importing menus and site settings...`} +

+ {totalSelectedPosts > 0 && wpImportProgress.phase === "content" && ( +
+
+
+ )} +

{t`Keep this tab open. The import runs in small steps and can be safely re-run if interrupted.`}

+ + ) : ( + <> +

{t`Importing content...`}

+

{t`This may take a while for large exports.`}

+ + )}
)} @@ -994,7 +1041,7 @@ function ChooseStep({
{ setKeyInput(e.target.value); @@ -1028,8 +1075,8 @@ function ChooseStep({ {/* URL input - primary path */}
-
- +
+

{t`Enter your WordPress site URL`}

@@ -1039,7 +1086,7 @@ function ChooseStep({ onUrlChange(e.target.value)} className="flex-1" @@ -1086,37 +1133,37 @@ function ChooseStep({ // ============================================================================= interface FeatureComparisonItem { - feature: string; + feature: MessageDescriptor; wxr: "full" | "partial" | "none"; - wxrNote?: string; + wxrNote?: MessageDescriptor; plugin: "full" | "partial" | "none"; - pluginNote?: string; + pluginNote?: MessageDescriptor; } const FEATURE_COMPARISON: FeatureComparisonItem[] = [ - { feature: "Posts & Pages", wxr: "full", plugin: "full" }, - { feature: "Media", wxr: "full", plugin: "full" }, - { feature: "Categories & Tags", wxr: "full", plugin: "full" }, - { feature: "Custom Taxonomies", wxr: "full", plugin: "full" }, - { feature: "Featured Images", wxr: "full", plugin: "full" }, - { feature: "Menus", wxr: "full", plugin: "full" }, + { feature: msg`Posts & Pages`, wxr: "full", plugin: "full" }, + { feature: msg`Media`, wxr: "full", plugin: "full" }, + { feature: msg`Categories & Tags`, wxr: "full", plugin: "full" }, + { feature: msg`Custom Taxonomies`, wxr: "full", plugin: "full" }, + { feature: msg`Featured Images`, wxr: "full", plugin: "full" }, + { feature: msg`Menus`, wxr: "full", plugin: "full" }, { - feature: "Site Settings", + feature: msg`Site Settings`, wxr: "partial", - wxrNote: "Partial", + wxrNote: msg`Partial`, plugin: "full", - pluginNote: "Full", + pluginNote: msg`Full`, }, - { feature: "Widgets", wxr: "none", plugin: "full" }, - { feature: "ACF Fields", wxr: "none", plugin: "full" }, + { feature: msg`Widgets`, wxr: "none", plugin: "full" }, + { feature: msg`ACF Fields`, wxr: "none", plugin: "full" }, { - feature: "Yoast/RankMath", + feature: msg`Yoast/RankMath`, wxr: "partial", - wxrNote: "Raw meta", + wxrNote: msg`Raw meta`, plugin: "full", - pluginNote: "Structured", + pluginNote: msg`Structured`, }, - { feature: "Drafts & Private", wxr: "full", plugin: "full" }, + { feature: msg`Drafts & Private`, wxr: "full", plugin: "full" }, ]; function FeatureComparison() { @@ -1137,23 +1184,26 @@ function FeatureComparison() { {FEATURE_COMPARISON.map((item) => ( - - {item.feature} + + {t(item.feature)} - + - + ))}
-
+
- -

+ +

{t`For the best import experience, install the`}{" "} {t`EmDash Exporter`}{" "} {t`plugin on your WordPress site.`} @@ -1167,14 +1217,14 @@ function FeatureComparison() { function FeatureStatus({ status, note }: { status: "full" | "partial" | "none"; note?: string }) { if (status === "full") { return ( - + ); } if (status === "partial") { return ( - + {note && {note}} @@ -1245,9 +1295,9 @@ function ProbeResultStep({ return (

{/* Detection success */} -
+
- +

{t`${bestMatch?.detected.siteTitle || "WordPress site"} detected`} @@ -1293,12 +1343,12 @@ function ProbeResultStep({ {/* EmDash Exporter plugin detected - primary option */} {hasPlugin && ( -
+
-
+
-
+
onUsernameChange(e.target.value)} - placeholder="admin" + placeholder={t`admin`} autoComplete="username" />
@@ -1444,7 +1494,7 @@ function PluginAuthStep({ type="password" value={password} onChange={(e) => onPasswordChange(e.target.value)} - placeholder="xxxx xxxx xxxx xxxx xxxx xxxx" + placeholder={t`xxxx xxxx xxxx xxxx xxxx xxxx`} autoComplete="current-password" />

@@ -1463,9 +1513,9 @@ function PluginAuthStep({

-
+
- +

{t`How to create an Application Password`}

    @@ -1703,24 +1753,27 @@ function ReviewStep({

    {t`Additional data to import.`}

- {/* Menus */} -
- onImportMenusChange(checked)} - label={ -
- -
-

{t`Menus (${navMenus.length})`}

-

- {navMenus.map((m) => m.name).join(", ")} -

+ {/* Menus — only the plugin import can import them; for WXR the + entries are informational */} + {isPluginSource && ( +
+ onImportMenusChange(checked)} + label={ +
+ +
+

{t`Menus (${navMenus.length})`}

+

+ {navMenus.map((m) => m.name).join(", ")} +

+
-
- } - /> -
+ } + /> +
+ )} {/* Categories count */} {analysis.categories > 0 && ( @@ -1814,9 +1867,9 @@ function ReviewStep({ )} {selectedCount > 0 && ( -
+
- +

{t`What will happen when you import`}

    @@ -1938,7 +1991,7 @@ function PostTypeRow({ {expanded && (
    {!canImport && schemaStatus.reason && ( -
    +
    {schemaStatus.reason}
    @@ -1953,15 +2006,15 @@ function PostTypeRow({ {field.label} ({field.type}) {status?.status === "compatible" ? ( - + {t`Exists`} ) : status?.status === "missing" ? ( - + {t`Will create`} ) : status?.status === "type_mismatch" ? ( - + {t`Type mismatch (${status.existingType})`} ) : null} @@ -2000,8 +2053,8 @@ function MediaStep({
    -
    - +
    +

    {t`Import Media Files`}

    @@ -2031,9 +2084,9 @@ function MediaStep({
    )} -
    +
    - +

    {t`What happens when you import:`}

      @@ -2107,16 +2160,11 @@ function MediaProgressStep({ {statusLabels[progress.status]} @@ -2240,14 +2288,14 @@ function CompleteStep({ className={cn( "rounded-lg border p-6 text-center", overallSuccess - ? "border-green-200 bg-green-50 dark:border-green-900/50 dark:bg-green-900/20" - : "border-yellow-200 bg-yellow-50 dark:border-yellow-900/50 dark:bg-yellow-900/20", + ? "border-kumo-success/50 bg-kumo-success-tint" + : "border-kumo-warning/50 bg-kumo-warning-tint", )} > {overallSuccess ? ( - + ) : ( - + )}

      {overallSuccess @@ -2399,8 +2447,8 @@ function AuthorMappingStep({
      -
      - +
      +

      {t`Map Authors`}

      @@ -2408,7 +2456,7 @@ function AuthorMappingStep({ {t`Assign WordPress authors to EmDash users. Posts will be attributed to the selected user.`}

      {matchedCount > 0 && ( -

      +

      {t`${matchedCount} of ${totalCount} authors matched by email`}

      @@ -2461,9 +2509,9 @@ function AuthorMappingStep({
      {emdashUsers.length === 0 && ( -
      +
      - +

      {t`No EmDash users found`}

      diff --git a/packages/admin/src/lib/api/import.ts b/packages/admin/src/lib/api/import.ts index b3b975c6f2..74443ca05f 100644 --- a/packages/admin/src/lib/api/import.ts +++ b/packages/admin/src/lib/api/import.ts @@ -149,6 +149,14 @@ export interface ImportConfig { skipExisting: boolean; /** Author mappings (WP author login -> EmDash user ID) */ authorMappings?: Record; + /** Import navigation menus (plugin import; default true) */ + importMenus?: boolean; + /** Take over site title & tagline (plugin import; default true) */ + importSiteTitle?: boolean; + /** Take over logo & favicon (plugin import; default true) */ + importLogo?: boolean; + /** Import per-post Yoast/Rank Math SEO fields (plugin import; default true) */ + importSeo?: boolean; } export interface ImportResult { @@ -171,6 +179,152 @@ export interface ImportResult { siteSettings?: string[]; } +// ============================================================================= +// Chunked plugin import (issue #475) +// +// A single /execute request importing a whole site exceeds Cloudflare +// Worker resource limits. The admin drives the import as a loop of bounded +// requests instead: one WP content page per call, then paginated comments, +// then a small finalize step. Cross-chunk state (ID maps, translation +// groups, comment roots) is accumulated here and sent with each request, +// so the server stays stateless and an aborted import can simply re-run. +// ============================================================================= + +interface WpImportCursor { + postTypeIndex: number; + page: number; +} + +/** Client-accumulated state passed back to the server with each chunk. */ +interface WpImportChunkState { + idMap: Record; + translationGroups: Record; + commentRoots: Record; +} + +interface WpImportChunkResponse { + success: boolean; + result: ImportResult; + done: boolean; + cursor?: WpImportCursor; + chunk?: Partial; +} + +/** Progress snapshot reported after every chunk. */ +export interface WpImportProgress { + phase: "content" | "comments" | "finalize"; + /** Content items imported + skipped so far (content phase) */ + processed: number; + /** Comments imported + skipped so far (comments phase) */ + comments: number; +} + +async function executeWpPluginImportChunk( + url: string, + token: string, + config: ImportConfig, + phase: WpImportProgress["phase"], + cursor: WpImportCursor | undefined, + state: WpImportChunkState, +): Promise { + const response = await apiFetch(`${API_BASE}/import/wordpress-plugin/execute`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + url, + token, + config, + phase, + cursor, + idMap: state.idMap, + translationGroups: state.translationGroups, + commentRoots: state.commentRoots, + }), + }); + return parseApiResponse(response, "Failed to import from WordPress"); +} + +/** Merge a chunk's partial result into the running aggregate. */ +function mergeImportResults(into: ImportResult, chunk: ImportResult): void { + into.imported += chunk.imported; + into.skipped += chunk.skipped; + into.errors.push(...chunk.errors); + for (const [collection, count] of Object.entries(chunk.byCollection)) { + into.byCollection[collection] = (into.byCollection[collection] || 0) + count; + } + if (chunk.taxonomyAssignments) { + into.taxonomyAssignments = (into.taxonomyAssignments ?? 0) + chunk.taxonomyAssignments; + } + if (chunk.missingTaxonomies?.length) { + into.missingTaxonomies = [ + ...new Set([...(into.missingTaxonomies ?? []), ...chunk.missingTaxonomies]), + ]; + } + if (chunk.taxonomiesCreated?.length) { + into.taxonomiesCreated = [...(into.taxonomiesCreated ?? []), ...chunk.taxonomiesCreated]; + } + if (chunk.menus) into.menus = chunk.menus; + if (chunk.comments) { + into.comments = { + imported: (into.comments?.imported ?? 0) + chunk.comments.imported, + skipped: (into.comments?.skipped ?? 0) + chunk.comments.skipped, + }; + } + if (chunk.siteSettings) into.siteSettings = chunk.siteSettings; + into.success = into.errors.length === 0; +} + +/** + * Run the full plugin import as a sequence of bounded requests: content + * pages, then comment pages, then finalize (menus + site identity). + * Each request stays well below Worker resource limits regardless of + * site size. Re-running after an abort is safe: `skipExisting` skips + * already-imported content while rebuilding the ID maps the later + * phases need. + */ +export async function executeWpPluginImport( + url: string, + token: string, + config: ImportConfig, + onProgress?: (progress: WpImportProgress) => void, +): Promise { + const aggregate: ImportResult = { + success: true, + imported: 0, + skipped: 0, + errors: [], + byCollection: {}, + }; + const state: WpImportChunkState = { idMap: {}, translationGroups: {}, commentRoots: {} }; + let comments = 0; + + const runPhase = async (phase: WpImportProgress["phase"]) => { + let cursor: WpImportCursor | undefined; + let done = false; + while (!done) { + const chunk = await executeWpPluginImportChunk(url, token, config, phase, cursor, state); + mergeImportResults(aggregate, chunk.result); + Object.assign(state.idMap, chunk.chunk?.idMap); + Object.assign(state.translationGroups, chunk.chunk?.translationGroups); + Object.assign(state.commentRoots, chunk.chunk?.commentRoots); + comments += (chunk.result.comments?.imported ?? 0) + (chunk.result.comments?.skipped ?? 0); + onProgress?.({ + phase, + processed: aggregate.imported + aggregate.skipped, + comments, + }); + done = chunk.done; + cursor = chunk.cursor; + } + }; + + await runPhase("content"); + await runPhase("comments"); + await runPhase("finalize"); + + return aggregate; +} + /** * Analyze a WordPress WXR file */ @@ -334,6 +488,41 @@ export async function importWxrMedia( return result; } +/** Attachments per media request. Bounds each Worker invocation (issue #475). */ +const MEDIA_BATCH_SIZE = 25; + +/** + * Import media in bounded batches instead of one giant request, so each + * Worker invocation stays below resource limits and an aborted run only + * loses the batch in flight (the server dedupes re-sent files by content + * hash). Progress is reported against the overall total. + */ +export async function importWxrMediaBatched( + attachments: AttachmentInfo[], + onProgress?: (progress: MediaImportProgress) => void, +): Promise { + const merged: MediaImportResult = { imported: [], failed: [], urlMap: {} }; + + for (let offset = 0; offset < attachments.length; offset += MEDIA_BATCH_SIZE) { + const batch = attachments.slice(offset, offset + MEDIA_BATCH_SIZE); + const result = await importWxrMedia( + batch, + onProgress && + ((progress) => + onProgress({ + ...progress, + current: offset + progress.current, + total: attachments.length, + })), + ); + merged.imported.push(...result.imported); + merged.failed.push(...result.failed); + Object.assign(merged.urlMap, result.urlMap); + } + + return merged; +} + // ============================================================================= // Import Source Probing // ============================================================================= @@ -458,23 +647,3 @@ export async function analyzeWpPluginSite(url: string, token: string): Promise { - const response = await apiFetch(`${API_BASE}/import/wordpress-plugin/execute`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ url, token, config }), - }); - const data = await parseApiResponse<{ result: ImportResult }>( - response, - "Failed to import from WordPress", - ); - return data.result; -} diff --git a/packages/admin/src/lib/api/index.ts b/packages/admin/src/lib/api/index.ts index 59af99f795..f4eb905825 100644 --- a/packages/admin/src/lib/api/index.ts +++ b/packages/admin/src/lib/api/index.ts @@ -272,6 +272,7 @@ export { type ImportResult, type MediaImportResult, type MediaImportProgress, + type WpImportProgress, type RewriteUrlsResult, type SourceCapabilities, type SourceAuth, @@ -283,6 +284,7 @@ export { prepareWxrImport, executeWxrImport, importWxrMedia, + importWxrMediaBatched, probeImportUrl, rewriteContentUrls, analyzeWpPluginSite, diff --git a/packages/core/src/api/schemas/import.ts b/packages/core/src/api/schemas/import.ts index c946a371b7..6b033873a0 100644 --- a/packages/core/src/api/schemas/import.ts +++ b/packages/core/src/api/schemas/import.ts @@ -19,6 +19,25 @@ export const wpPluginExecuteBody = z.object({ url: httpUrl, token: z.string().min(1), config: z.record(z.string(), z.unknown()), + // --- Chunked mode (issue #475). Absent phase = single-shot legacy run. --- + // The admin drives the import in a loop of bounded requests; cross-chunk + // state travels with the client so the server stays stateless. + phase: z.enum(["content", "comments", "finalize"]).optional(), + /** Position within the phase: post-type index + WP page (content), page (comments) */ + cursor: z + .object({ + postTypeIndex: z.number().int().min(0).default(0), + page: z.number().int().min(1).default(1), + }) + .optional(), + /** WP post ID -> created EmDash item, accumulated across content chunks */ + idMap: z + .record(z.string(), z.object({ id: z.string().min(1), collection: z.string().min(1) })) + .optional(), + /** Source translation group -> EmDash item ID, accumulated across content chunks */ + translationGroups: z.record(z.string(), z.string().min(1)).optional(), + /** WP comment ID -> EmDash root comment ID, accumulated across comment chunks */ + commentRoots: z.record(z.string(), z.string().min(1)).optional(), }); export const wpPrepareBody = z.object({ diff --git a/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts b/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts index 65517572ec..eb7e911512 100644 --- a/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts +++ b/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts @@ -15,6 +15,7 @@ import { type WxrTag, type WxrTerm, } from "emdash"; +import type { z } from "zod"; import { requirePerm } from "#api/authorize.js"; import { apiError, apiSuccess, handleError } from "#api/error.js"; @@ -28,6 +29,8 @@ import { importMenusFromPlugin } from "#import/menus.js"; import { importSiteSettings, parseSiteSettingsFromPlugin } from "#import/settings.js"; import { fetchPluginComments, + fetchPluginCommentsPage, + fetchPluginContentPage, fetchPluginMenus, fetchPluginOptions, fetchPluginTaxonomies, @@ -37,6 +40,7 @@ import type { ImportConfig, ImportResult, NormalizedItem } from "#import/types.j import { resolveImportByline, sanitizeFieldSlug } from "#import/utils.js"; import { attachPostTaxonomies, + loadTaxonomyPlanFromDb, preImportWxrTaxonomies, type TaxonomyImportPlan, } from "#import/wxr-taxonomies.js"; @@ -51,6 +55,11 @@ export const prerender = false; export interface WpPluginImportConfig extends ImportConfig { /** Author mappings (WP author login -> EmDash user ID) */ authorMappings?: Record; + /** Wizard toggles. Absent means enabled (older admins don't send them). */ + importMenus?: boolean; + importSiteTitle?: boolean; + importLogo?: boolean; + importSeo?: boolean; } export interface WpPluginImportResponse { @@ -59,6 +68,71 @@ export interface WpPluginImportResponse { error?: { message: string }; } +// ============================================================================= +// Chunked mode (issue #475) +// +// A single-invocation import of a large site exceeds Cloudflare Worker +// resource limits (CPU time, subrequests). In chunked mode the admin drives +// the import as a loop of bounded requests: one WP content page per call, +// then paginated comments, then a small finalize step (menus + site +// identity). Cross-chunk state — the WP-ID -> EmDash-ID map, translation +// groups, comment threading roots — is accumulated by the client and sent +// back with each request, so the server stays stateless and an aborted +// import simply re-runs (skipExisting rebuilds the map while skipping work). +// ============================================================================= + +/** Posts per content chunk. 50 posts ≈ a few hundred D1 ops per invocation. */ +const CONTENT_CHUNK_SIZE = 50; + +interface ChunkCursor { + postTypeIndex: number; + page: number; +} + +/** Per-chunk response payload: partial result + state the client must carry. */ +interface ChunkResponse { + success: boolean; + result: ImportResult; + done: boolean; + cursor?: ChunkCursor; + chunk?: { + idMap?: Record; + translationGroups?: Record; + commentRoots?: Record; + }; +} + +function emptyImportResult(): ImportResult { + return { success: true, imported: 0, skipped: 0, errors: [], byCollection: {} }; +} + +function parseIdMap(idMap: Record | undefined): { + contentIdMap: Map; + collectionByWpId: Map; +} { + const contentIdMap = new Map(); + const collectionByWpId = new Map(); + for (const [key, value] of Object.entries(idMap ?? {})) { + const wpId = Number(key); + if (!Number.isFinite(wpId)) continue; + contentIdMap.set(wpId, value.id); + collectionByWpId.set(wpId, value.collection); + } + return { contentIdMap, collectionByWpId }; +} + +function serializeIdMap( + contentIdMap: Map, + collectionByWpId: Map, +): Record { + const out: Record = {}; + for (const [wpId, id] of contentIdMap) { + const collection = collectionByWpId.get(wpId); + if (collection) out[String(wpId)] = { id, collection }; + } + return out; +} + export const POST: APIRoute = async ({ request, locals }) => { const { emdash, user } = locals; @@ -105,6 +179,12 @@ export const POST: APIRoute = async ({ request, locals }) => { console.log("[WP Plugin Import] Starting import for:", body.url); console.log("[WP Plugin Import] Post types:", postTypes); + // Chunked mode: one bounded unit of work per invocation (issue #475). + if (body.phase) { + const chunk = await runImportPhase(emdash, body, config, postTypes, emdashManifest); + return apiSuccess(chunk); + } + // Pre-create taxonomy defs (for custom CPT taxonomies) and terms so // per-post assignments can attach. // Non-fatal: a site whose /taxonomies call fails still gets content. @@ -132,21 +212,8 @@ export const POST: APIRoute = async ({ request, locals }) => { // Import navigation menus, resolving item references through the // WP-post-ID -> EmDash-ID map collected during the content pass. - // Non-fatal: older plugin versions have no /menus endpoint. - try { - const menus = await fetchPluginMenus(body.url, body.token); - if (menus.length > 0) { - const menuResult = await importMenusFromPlugin(menus, emdash.db, contentIdMap); - result.menus = { - created: menuResult.menusCreated, - items: menuResult.itemsCreated, - }; - for (const menuError of menuResult.errors) { - result.errors.push({ title: `Menu: ${menuError.menu}`, error: menuError.error }); - } - } - } catch (e) { - console.warn("[WP Plugin Import] Menu import failed:", e); + if (config.importMenus !== false) { + await importMenusInto(result, emdash, body.url, body.token, contentIdMap); } // Import comments into EmDash's native comments table, preserving @@ -180,7 +247,7 @@ export const POST: APIRoute = async ({ request, locals }) => { // site's options, replacing the starter template's seed placeholders. // Non-fatal: content is already imported at this point. try { - result.siteSettings = await applySiteSettings(emdash, body.url, body.token); + result.siteSettings = await applySiteSettings(emdash, body.url, body.token, config); } catch (e) { console.warn("[WP Plugin Import] Site settings import failed:", e); } @@ -200,6 +267,213 @@ export const POST: APIRoute = async ({ request, locals }) => { } }; +/** + * Import navigation menus into `result`, resolving item references through + * the WP-post-ID -> EmDash-ID map collected during the content pass. + * Non-fatal: older plugin versions have no /menus endpoint. + */ +async function importMenusInto( + result: ImportResult, + emdash: EmDashHandlers, + url: string, + token: string, + contentIdMap: Map, +): Promise { + try { + const menus = await fetchPluginMenus(url, token); + if (menus.length > 0) { + const menuResult = await importMenusFromPlugin(menus, emdash.db, contentIdMap); + result.menus = { + created: menuResult.menusCreated, + items: menuResult.itemsCreated, + }; + for (const menuError of menuResult.errors) { + result.errors.push({ title: `Menu: ${menuError.menu}`, error: menuError.error }); + } + } + } catch (e) { + console.warn("[WP Plugin Import] Menu import failed:", e); + } +} + +type ExecuteBody = z.infer; + +/** Dispatch one chunk of work for the requested phase. */ +async function runImportPhase( + emdash: EmDashHandlers, + body: ExecuteBody, + config: WpPluginImportConfig, + postTypes: string[], + manifest: EmDashManifest, +): Promise { + switch (body.phase) { + case "content": + return runContentChunk(emdash, body, config, postTypes, manifest); + case "comments": + return runCommentsChunk(emdash, body); + case "finalize": + return runFinalizePhase(emdash, body, config); + case undefined: + throw new Error("runImportPhase called without a phase"); + default: + body.phase satisfies never; + throw new Error("Unknown import phase"); + } +} + +/** + * Import one page of one post type. The first chunk additionally runs the + * taxonomy setup (def + term creation); later chunks only reload the + * lookup maps from the database. + */ +async function runContentChunk( + emdash: EmDashHandlers, + body: ExecuteBody, + config: WpPluginImportConfig, + postTypes: string[], + manifest: EmDashManifest, +): Promise { + const cursor = body.cursor ?? { postTypeIndex: 0, page: 1 }; + const postType = postTypes[cursor.postTypeIndex]; + if (!postType) { + return { success: true, result: emptyImportResult(), done: true }; + } + + const isFirstChunk = cursor.postTypeIndex === 0 && cursor.page === 1; + let taxonomyPlan: TaxonomyImportPlan | undefined; + let taxonomyDefsCreated: string[] = []; + try { + if (isFirstChunk) { + const built = await buildTaxonomyPlan(emdash, body.url, body.token, config); + taxonomyPlan = built.plan; + taxonomyDefsCreated = built.defsCreated; + } else { + taxonomyPlan = await loadTaxonomyPlanFromDb(emdash.db); + } + } catch (e) { + console.warn("[WP Plugin Import] Taxonomy pre-import failed:", e); + } + + const page = await fetchPluginContentPage({ + siteUrl: body.url, + token: body.token, + postType, + page: cursor.page, + perPage: CONTENT_CHUNK_SIZE, + includeDrafts: true, + }); + + // Seed translation-group state from earlier chunks so a translation in + // this page links to its sibling imported three chunks ago. + const translationGroupMap = new Map(Object.entries(body.translationGroups ?? {})); + + const { result, contentIdMap, collectionByWpId } = await importContent( + page.items, + config, + emdash, + manifest, + taxonomyPlan, + translationGroupMap, + ); + + if (taxonomyDefsCreated.length > 0) { + result.taxonomiesCreated = taxonomyDefsCreated; + } + + // Advance: next page of this post type, first page of the next one, + // or done. An empty last page (totalPages can shrink while paginating + // a live site) still terminates because page >= totalPages. + let next: ChunkCursor | undefined; + if (cursor.page < page.totalPages) { + next = { postTypeIndex: cursor.postTypeIndex, page: cursor.page + 1 }; + } else if (cursor.postTypeIndex + 1 < postTypes.length) { + next = { postTypeIndex: cursor.postTypeIndex + 1, page: 1 }; + } + + return { + success: true, + result, + done: next === undefined, + cursor: next, + chunk: { + idMap: serializeIdMap(contentIdMap, collectionByWpId), + translationGroups: Object.fromEntries(translationGroupMap), + }, + }; +} + +/** + * Import one page of comments (500 per page, ordered by WP comment ID so + * parents precede children across pages). Requires the accumulated idMap + * from the content phase; threading roots accumulate in `commentRoots`. + */ +async function runCommentsChunk(emdash: EmDashHandlers, body: ExecuteBody): Promise { + const page = body.cursor?.page ?? 1; + const result = emptyImportResult(); + + const { contentIdMap, collectionByWpId } = parseIdMap(body.idMap); + const rootIds = new Map(); + for (const [key, value] of Object.entries(body.commentRoots ?? {})) { + const wpId = Number(key); + if (Number.isFinite(wpId)) rootIds.set(wpId, value); + } + + const { items, totalPages } = await fetchPluginCommentsPage(body.url, body.token, page); + + if (items.length > 0) { + const commentsResult = await importCommentsFromPlugin( + items, + emdash.db, + contentIdMap, + collectionByWpId, + rootIds, + ); + result.comments = { + imported: commentsResult.imported, + skipped: commentsResult.skipped, + }; + for (const commentError of commentsResult.errors) { + result.errors.push({ title: `Comment: ${commentError.comment}`, error: commentError.error }); + } + result.success = result.errors.length === 0; + } + + const done = page >= totalPages; + const commentRoots: Record = {}; + for (const [wpId, id] of rootIds) commentRoots[String(wpId)] = id; + + return { + success: true, + result, + done, + cursor: done ? undefined : { postTypeIndex: 0, page: page + 1 }, + chunk: { commentRoots }, + }; +} + +/** Menus + site identity — small, runs as a single closing chunk. */ +async function runFinalizePhase( + emdash: EmDashHandlers, + body: ExecuteBody, + config: WpPluginImportConfig, +): Promise { + const result = emptyImportResult(); + const { contentIdMap } = parseIdMap(body.idMap); + + if (config.importMenus !== false) { + await importMenusInto(result, emdash, body.url, body.token, contentIdMap); + } + + try { + result.siteSettings = await applySiteSettings(emdash, body.url, body.token, config); + } catch (e) { + console.warn("[WP Plugin Import] Site settings import failed:", e); + } + + result.success = result.errors.length === 0; + return { success: true, result, done: true }; +} + /** Fields that should be auto-created if they don't exist */ const IMPORT_FIELDS: Array<{ slug: string; @@ -245,6 +519,8 @@ const IMPORT_FIELDS: Array<{ }, ]; +const SEO_FIELD_SLUGS = new Set(["seo_title", "seo_description"]); + /** * Coerce a WordPress meta value to an EmDash field type. WP postmeta is * stringly typed and inconsistent across posts (the same key can hold @@ -428,9 +704,22 @@ async function applySiteSettings( emdash: EmDashHandlers, url: string, token: string, + config: WpPluginImportConfig, ): Promise { + const wantTitle = config.importSiteTitle !== false; + const wantLogo = config.importLogo !== false; + if (!wantTitle && !wantLogo) return []; + const options = await fetchPluginOptions(url, token); const parsed = parseSiteSettingsFromPlugin(options); + if (!wantTitle) { + delete parsed.title; + delete parsed.tagline; + } + if (!wantLogo) { + delete parsed.logo; + delete parsed.favicon; + } const media: { logoMediaId?: string; faviconMediaId?: string } = {}; if (emdash.storage && (parsed.logo?.url || parsed.favicon?.url)) { @@ -484,11 +773,12 @@ function toWxrAssignments(item: NormalizedItem): WxrPost { /** Exported for tests (field auto-creation regression coverage). */ export async function importContent( - items: AsyncGenerator, + items: AsyncIterable | Iterable, config: WpPluginImportConfig, emdash: EmDashHandlers, manifest: EmDashManifest, taxonomyPlan: TaxonomyImportPlan | undefined, + seedTranslationGroups?: Map, ): Promise<{ result: ImportResult; contentIdMap: Map; @@ -525,8 +815,9 @@ export async function importContent( // Track source translationGroup -> EmDash item ID for translation linking. // Maps source-side translation group ID to the EmDash ID of the first item - // imported for that group (the default-locale item). - const translationGroupMap = new Map(); + // imported for that group (the default-locale item). The chunked import + // seeds this from earlier chunks so groups can span page boundaries. + const translationGroupMap = seedTranslationGroups ?? new Map(); for await (const item of items) { console.log("[WP Plugin Import] Processing item:", { @@ -565,6 +856,7 @@ export async function importContent( // is needed depends on the item — the first post may have no SEO // override while a later one does. for (const field of IMPORT_FIELDS) { + if (config.importSeo === false && SEO_FIELD_SLUGS.has(field.slug)) continue; const ensureKey = `${collection}:${field.slug}`; if (ensuredFields.has(ensureKey) || !field.check(item)) continue; ensuredFields.add(ensureKey); @@ -647,12 +939,14 @@ export async function importContent( } // Per-post SEO overrides from Yoast / Rank Math - const seo = extractSeo(item); - if (seo.title) { - data.seo_title = seo.title; - } - if (seo.description) { - data.seo_description = seo.description; + if (config.importSeo !== false) { + const seo = extractSeo(item); + if (seo.title) { + data.seo_title = seo.title; + } + if (seo.description) { + data.seo_description = seo.description; + } } // Map ACF values and custom meta onto schema fields with the same diff --git a/packages/core/src/import/comments.ts b/packages/core/src/import/comments.ts index b5236912d2..2a61e7d357 100644 --- a/packages/core/src/import/comments.ts +++ b/packages/core/src/import/comments.ts @@ -51,12 +51,17 @@ export interface CommentsImportResult { * @param db - Database connection * @param contentIdMap - WP post ID -> EmDash content ID * @param collectionMap - WP post ID -> EmDash collection slug + * @param rootIds - Optional pre-seeded WP-comment-ID -> EmDash-root-ID map. + * The chunked import passes the map accumulated from earlier pages so a + * reply in page N can thread onto a parent imported in page N-1; the + * function adds this page's entries to it. */ export async function importCommentsFromPlugin( comments: PluginComment[], db: Kysely, contentIdMap: Map, collectionMap: Map, + rootIds?: Map, ): Promise { const result: CommentsImportResult = { imported: 0, @@ -70,7 +75,7 @@ export async function importCommentsFromPlugin( // WP comment ID -> EmDash ID of its top-level ancestor. EmDash threads // are one level deep (assembleThreads nests replies under roots only), // so deeper WP threads are flattened onto their root comment. - const rootIdMap = new Map(); + const rootIdMap = rootIds ?? new Map(); // Parents must exist before children reference them; WP comment IDs // are chronological, so ID order guarantees parents come first. diff --git a/packages/core/src/import/sources/wordpress-plugin.ts b/packages/core/src/import/sources/wordpress-plugin.ts index 67b6679b59..a7391ea31e 100644 --- a/packages/core/src/import/sources/wordpress-plugin.ts +++ b/packages/core/src/import/sources/wordpress-plugin.ts @@ -538,6 +538,50 @@ export const wordpressPluginSource: ImportSource = { }, }; +/** + * Fetch a single page of content for one post type. This is the unit of + * work for the chunked import: one Worker invocation imports one page, + * keeping each request far below Cloudflare's CPU and subrequest limits + * (see issue #475). + */ +export async function fetchPluginContentPage(options: { + siteUrl: string; + token: string; + postType: string; + page: number; + perPage: number; + includeDrafts: boolean; +}): Promise<{ items: NormalizedItem[]; totalPages: number }> { + const { siteUrl, headers } = getRequestConfig({ + type: "url", + url: options.siteUrl, + token: options.token, + }); + + const response = await fetchPluginApi( + siteUrl, + "content", + { + post_type: options.postType, + status: options.includeDrafts ? "any" : "publish", + per_page: String(options.perPage), + page: String(options.page), + }, + headers, + 60000, + ); + + if (!response.ok) { + throw new Error(`Failed to fetch ${options.postType}: ${response.statusText}`); + } + + const data: PluginContentResponse = await response.json(); + return { + items: data.items.map((post) => pluginPostToNormalizedItem(post, siteUrl)), + totalPages: data.pages, + }; +} + // ============================================================================= // Helper Functions // ============================================================================= @@ -843,42 +887,56 @@ interface PluginCommentsResponse { } /** - * Fetch all comments from plugin API (added in emdash-exporter 1.2.0), - * paginating through every page. Returns an empty array when the endpoint - * doesn't exist (older plugin). + * Fetch a single page of comments from the plugin API (added in + * emdash-exporter 1.2.0). The exporter orders by comment ID ascending, so + * parents always appear before their children across pages. Returns + * `totalPages: 0` when the endpoint doesn't exist (older plugin). */ -export async function fetchPluginComments( +export async function fetchPluginCommentsPage( siteUrl: string, authToken: string, -): Promise { + page: number, +): Promise<{ items: PluginComment[]; totalPages: number }> { const normalizedSiteUrl = normalizeUrl(siteUrl); // SSRF protection: validate URL before any outbound requests validateExternalUrl(normalizedSiteUrl); + const response = await fetchPluginApi( + normalizedSiteUrl, + "comments", + { per_page: "500", page: String(page) }, + { Accept: "application/json", Authorization: `Basic ${authToken}` }, + 30000, + ); + + if (response.status === 404) { + return { items: [], totalPages: 0 }; + } + if (!response.ok) { + throw new Error(`Failed to fetch comments: ${response.statusText}`); + } + + const data: PluginCommentsResponse = await response.json(); + return { items: data.items, totalPages: data.pages }; +} + +/** + * Fetch all comments from plugin API, paginating through every page. + * Returns an empty array when the endpoint doesn't exist (older plugin). + */ +export async function fetchPluginComments( + siteUrl: string, + authToken: string, +): Promise { const comments: PluginComment[] = []; let page = 1; let totalPages = 1; while (page <= totalPages) { - const response = await fetchPluginApi( - normalizedSiteUrl, - "comments", - { per_page: "500", page: String(page) }, - { Accept: "application/json", Authorization: `Basic ${authToken}` }, - 30000, - ); - - if (response.status === 404) { - return []; - } - if (!response.ok) { - throw new Error(`Failed to fetch comments: ${response.statusText}`); - } - - const data: PluginCommentsResponse = await response.json(); - totalPages = data.pages; - comments.push(...data.items); + const result = await fetchPluginCommentsPage(siteUrl, authToken, page); + totalPages = result.totalPages; + comments.push(...result.items); page++; } diff --git a/packages/core/src/import/wxr-taxonomies.ts b/packages/core/src/import/wxr-taxonomies.ts index 94eb7c6765..c4f8678f5a 100644 --- a/packages/core/src/import/wxr-taxonomies.ts +++ b/packages/core/src/import/wxr-taxonomies.ts @@ -198,6 +198,45 @@ async function findTaxonomyDef( return null; } +/** + * Rebuild the taxonomy lookup maps from the database without creating + * anything. The chunked WP import runs term creation once (first content + * chunk); every later chunk only needs the maps to attach per-post + * assignments, and two SELECTs are far cheaper than re-walking every term + * through the ensure path. + * + * ponytail: first row wins per (name, slug), ordered by locale asc — the + * same row `findBySlug` without a locale resolves to. Per-locale term + * variants beyond that are the mirror pass's job, not the importer's. + */ +export async function loadTaxonomyPlanFromDb(db: Kysely): Promise { + const state = makeState(); + + const defs = await db + .selectFrom("_emdash_taxonomy_defs") + .select(["name", "collections"]) + .orderBy("locale", "asc") + .execute(); + for (const def of defs) { + if (!state.plan.collectionsByTaxonomy.has(def.name)) { + state.plan.collectionsByTaxonomy.set(def.name, new Set(parseDefCollections(def.collections))); + } + } + + const terms = await db + .selectFrom("taxonomies") + .select(["name", "slug", "id"]) + .orderBy("locale", "asc") + .execute(); + for (const term of terms) { + if (!state.plan.termIdByNameAndSlug.get(term.name)?.has(term.slug)) { + rememberTerm(state, term.name, term.slug, term.id); + } + } + + return state.plan; +} + /** * Find or create a term in the given taxonomy. Returns the term id. Callers * must verify the taxonomy def exists before calling — this helper assumes diff --git a/packages/core/tests/integration/wordpress-import/plugin-execute-chunked.test.ts b/packages/core/tests/integration/wordpress-import/plugin-execute-chunked.test.ts new file mode 100644 index 0000000000..d3421604ea --- /dev/null +++ b/packages/core/tests/integration/wordpress-import/plugin-execute-chunked.test.ts @@ -0,0 +1,208 @@ +/** + * Chunked WP plugin import (issue #475): a large site is imported as a + * loop of bounded requests instead of one giant Worker invocation. These + * tests cover the state that must survive chunk boundaries: + * + * - translation groups: a translation in chunk N links to its sibling + * imported in chunk N-1 via the seeded translationGroup map + * - taxonomy lookup maps: later chunks reload the plan from the DB + * instead of re-running term creation + * - comment threading: a reply in comment page N threads onto its parent + * from page N-1 via the seeded rootIds map + */ + +import type { Kysely } from "kysely"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; + +import { handleContentCreate } from "../../../src/api/handlers/content.js"; +import { handleTaxonomyCreate } from "../../../src/api/handlers/taxonomies.js"; +import { + importContent, + type WpPluginImportConfig, +} from "../../../src/astro/routes/api/import/wordpress-plugin/execute.js"; +import type { EmDashHandlers, EmDashManifest } from "../../../src/astro/types.js"; +import { TaxonomyRepository } from "../../../src/database/repositories/taxonomy.js"; +import type { Database } from "../../../src/database/types.js"; +import { importCommentsFromPlugin, type PluginComment } from "../../../src/import/comments.js"; +import type { NormalizedItem } from "../../../src/import/types.js"; +import { loadTaxonomyPlanFromDb } from "../../../src/import/wxr-taxonomies.js"; +import { setupTestDatabaseWithCollections, teardownTestDatabase } from "../../utils/test-db.js"; + +function makeItem(overrides: Partial): NormalizedItem { + return { + sourceId: 1, + postType: "post", + status: "publish", + slug: "item", + title: "Item", + content: [], + date: new Date("2026-01-01T00:00:00Z"), + ...overrides, + }; +} + +function makeConfig(): WpPluginImportConfig { + return { + postTypeMappings: { post: { collection: "post", enabled: true } }, + skipExisting: true, + }; +} + +function makeEmdash(db: Kysely): EmDashHandlers { + // ponytail: minimal stub — importContent only touches db + handleContentCreate + return { + db, + handleContentCreate: (collection: string, body: { data: Record }) => + handleContentCreate(db, collection, body), + } as unknown as EmDashHandlers; +} + +const manifest = { collections: { post: {} } } as unknown as EmDashManifest; + +describe("chunked WP plugin import — cross-chunk state", () => { + let db: Kysely; + + beforeEach(async () => { + db = await setupTestDatabaseWithCollections(); + }); + + afterEach(async () => { + await teardownTestDatabase(db); + }); + + it("links translations across chunks via the seeded translationGroup map", async () => { + const emdash = makeEmdash(db); + const translationGroups = new Map(); + + // Chunk 1: the default-locale item establishes the group mapping. + const first = await importContent( + [makeItem({ sourceId: 1, slug: "hello", locale: "en", translationGroup: "g1" })], + makeConfig(), + emdash, + manifest, + undefined, + translationGroups, + ); + expect(first.result.imported).toBe(1); + expect(translationGroups.get("g1")).toBeDefined(); + + // Chunk 2 (separate importContent call = separate invocation): the + // translation must link into the same group. + const second = await importContent( + [makeItem({ sourceId: 2, slug: "hallo", locale: "de", translationGroup: "g1" })], + makeConfig(), + emdash, + manifest, + undefined, + translationGroups, + ); + expect(second.result.imported).toBe(1); + + const rows = await db + // eslint-disable-next-line typescript/no-explicit-any -- dynamic ec_ table not in the static schema + .selectFrom("ec_post" as any) + .select(["slug", "translation_group"]) + .where("slug", "in", ["hello", "hallo"]) + .execute(); + expect(rows).toHaveLength(2); + expect(rows[0]!.translation_group).toBeTruthy(); + expect(rows[0]!.translation_group).toBe(rows[1]!.translation_group); + }); + + it("rebuilds the id maps from existing content on a resumed run", async () => { + const emdash = makeEmdash(db); + const item = makeItem({ sourceId: 7, slug: "resumed" }); + + const first = await importContent([item], makeConfig(), emdash, manifest, undefined); + expect(first.result.imported).toBe(1); + const originalId = first.contentIdMap.get(7); + expect(originalId).toBeDefined(); + + // Re-run (e.g. after the tab was closed): skipExisting skips the + // insert but must still yield the mapping comments/menus need. + const second = await importContent([item], makeConfig(), emdash, manifest, undefined); + expect(second.result.imported).toBe(0); + expect(second.result.skipped).toBe(1); + expect(second.contentIdMap.get(7)).toBe(originalId); + expect(second.collectionByWpId.get(7)).toBe("post"); + }); + + it("loadTaxonomyPlanFromDb rebuilds lookup maps without creating anything", async () => { + await handleTaxonomyCreate(db, { + name: "company", + label: "Companies", + hierarchical: false, + collections: ["post"], + }); + const repo = new TaxonomyRepository(db); + const term = await repo.create({ name: "company", slug: "acme", label: "ACME" }); + + const plan = await loadTaxonomyPlanFromDb(db); + + expect(plan.termIdByNameAndSlug.get("company")?.get("acme")).toBe(term.id); + expect([...(plan.collectionsByTaxonomy.get("company") ?? [])]).toEqual(["post"]); + // Nothing was created by the loader + expect(plan.termsCreated).toEqual({}); + expect(plan.missingTaxonomies).toEqual([]); + }); + + it("threads a reply onto a parent imported in an earlier comment chunk", async () => { + const emdash = makeEmdash(db); + const { contentIdMap, collectionByWpId } = await importContent( + [makeItem({ sourceId: 10, slug: "commented" })], + makeConfig(), + emdash, + manifest, + undefined, + ); + + const parent: PluginComment = { + id: 100, + post_id: 10, + parent_id: null, + author_name: "Alice", + author_email: "alice@example.com", + body: "First!", + date_gmt: "2026-01-02T10:00:00Z", + status: "approved", + }; + const reply: PluginComment = { + id: 200, + post_id: 10, + parent_id: 100, + author_name: "Bob", + author_email: "bob@example.com", + body: "Replying to Alice", + date_gmt: "2026-01-03T10:00:00Z", + status: "approved", + }; + + // Page 1 and page 2 as separate invocations sharing the rootIds map. + const rootIds = new Map(); + const page1 = await importCommentsFromPlugin( + [parent], + db, + contentIdMap, + collectionByWpId, + rootIds, + ); + expect(page1.imported).toBe(1); + + const page2 = await importCommentsFromPlugin( + [reply], + db, + contentIdMap, + collectionByWpId, + rootIds, + ); + expect(page2.imported).toBe(1); + + const rows = await db + .selectFrom("_emdash_comments") + .select(["author_name", "parent_id", "id"]) + .orderBy("created_at", "asc") + .execute(); + expect(rows).toHaveLength(2); + expect(rows[1]!.parent_id).toBe(rows[0]!.id); + }); +}); diff --git a/packages/core/tests/integration/wordpress-import/plugin-execute-fields.test.ts b/packages/core/tests/integration/wordpress-import/plugin-execute-fields.test.ts index c07cd5f08f..056571b88b 100644 --- a/packages/core/tests/integration/wordpress-import/plugin-execute-fields.test.ts +++ b/packages/core/tests/integration/wordpress-import/plugin-execute-fields.test.ts @@ -105,6 +105,41 @@ describe("WordPress plugin import — field auto-creation", () => { seo_description: "Custom description", }); }); + + it("skips SEO fields entirely when the importSeo toggle is off", async () => { + const config: WpPluginImportConfig = { + postTypeMappings: { post: { collection: "post", enabled: true } }, + skipExisting: false, + importSeo: false, + }; + const emdash = { + db, + handleContentCreate: (collection: string, body: { data: Record }) => + handleContentCreate(db, collection, body), + } as unknown as EmDashHandlers; + const manifest = { collections: { post: {} } } as unknown as EmDashManifest; + + const items = [ + makeItem({ + sourceId: 1, + slug: "with-seo", + title: "Post with SEO", + meta: { _yoast: { title: "Custom SEO Title", description: "Custom description" } }, + }), + ]; + + const { result } = await importContent(generate(items), config, emdash, manifest, undefined); + expect(result.errors).toEqual([]); + expect(result.imported).toBe(1); + + // Neither the field nor the value may be created + const field = await db + .selectFrom("_emdash_fields") + .select("slug") + .where("slug", "=", "seo_title") + .executeTakeFirst(); + expect(field).toBeUndefined(); + }); }); describe("WordPress plugin import — custom taxonomy def auto-creation", () => {