From 04d1b6e09f10ef2e067d87e3a1497077cb2001d7 Mon Sep 17 00:00:00 2001 From: swissky <30409887+swissky@users.noreply.github.com> Date: Sun, 5 Jul 2026 20:52:58 +0200 Subject: [PATCH 1/4] feat(import): complete WordPress plugin imports (taxonomies, SEO, ACF, menus, media pagination, rest_route fallback, migration key) - execute.ts now pre-creates taxonomy terms via the shared WXR taxonomy machinery and attaches category/tag/custom-taxonomy assignments per post - Yoast/Rank Math per-post titles and descriptions land in seo_title / seo_description fields (auto-created like other import fields) - ACF values and custom meta are written to schema fields with matching slugs (created by the prepare step); no fields are invented - Navigation menus are imported after content via the plugin's /menus endpoint, resolving item references through a WP-ID -> EmDash-ID map - analyze() paginates the media list instead of stopping at 500 items - All plugin API calls fall back to ?rest_route= when /wp-json/ 404s (plain permalinks) - Admin import screen accepts an em1. migration key generated by the EmDash Exporter plugin wizard (URL + credentials in one paste) --- .changeset/wp-plugin-import-completeness.md | 6 + .../admin/src/components/WordPressImport.tsx | 128 +++++++++- packages/admin/src/lib/api/import.ts | 6 + .../api/import/wordpress-plugin/execute.ts | 237 +++++++++++++++++- .../src/import/sources/wordpress-plugin.ts | 182 +++++++++++--- packages/core/src/import/types.ts | 6 + .../import/wordpress-plugin-source.test.ts | 172 +++++++++++++ 7 files changed, 683 insertions(+), 54 deletions(-) create mode 100644 .changeset/wp-plugin-import-completeness.md create mode 100644 packages/core/tests/unit/import/wordpress-plugin-source.test.ts diff --git a/.changeset/wp-plugin-import-completeness.md b/.changeset/wp-plugin-import-completeness.md new file mode 100644 index 0000000000..154bf5a1fe --- /dev/null +++ b/.changeset/wp-plugin-import-completeness.md @@ -0,0 +1,6 @@ +--- +"emdash": minor +"@emdash-cms/admin": minor +--- + +Improves WordPress plugin imports: taxonomy terms and assignments, Yoast/Rank Math SEO titles and descriptions, ACF values with matching schema fields, and navigation menus are now imported. Fetches the full media library instead of the first 500 files, supports sites with plain permalinks via a `?rest_route=` fallback, and the import screen accepts a migration key generated by the EmDash Exporter plugin wizard. diff --git a/packages/admin/src/components/WordPressImport.tsx b/packages/admin/src/components/WordPressImport.tsx index 021fe5abcc..9d52550625 100644 --- a/packages/admin/src/components/WordPressImport.tsx +++ b/packages/admin/src/components/WordPressImport.tsx @@ -66,6 +66,37 @@ import { CaretNext } from "./ArrowIcons.js"; const TRAILING_SLASH_REGEX = /\/$/; const WHITESPACE_REGEX = /\s/g; +/** Prefix of migration keys generated by the EmDash Exporter plugin wizard */ +const MIGRATION_KEY_PREFIX = "em1."; + +/** + * Decode a migration key from the EmDash Exporter plugin + * (`em1.` + base64url of `{ v, url, user, pass }`). + * Returns null when the string isn't a valid key. + */ +function decodeMigrationKey(key: string): { url: string; user: string; pass: string } | null { + if (!key.startsWith(MIGRATION_KEY_PREFIX)) return null; + try { + const base64 = key.slice(MIGRATION_KEY_PREFIX.length).replace(/-/g, "+").replace(/_/g, "/"); + const parsed: unknown = JSON.parse(atob(base64)); + if ( + typeof parsed === "object" && + parsed !== null && + "url" in parsed && + typeof parsed.url === "string" && + "user" in parsed && + typeof parsed.user === "string" && + "pass" in parsed && + typeof parsed.pass === "string" + ) { + return { url: parsed.url, user: parsed.user, pass: parsed.pass }; + } + } catch { + // malformed key -- treated as "not a key" by the caller + } + return null; +} + // ============================================================================ // Types // ============================================================================ @@ -390,9 +421,29 @@ export function WordPressImport() { const handleProbeUrl = (e: React.FormEvent) => { e.preventDefault(); - if (!urlInput.trim()) return; + const input = urlInput.trim(); + if (!input) return; + // A pasted migration key skips probing entirely -- it carries the + // site URL and credentials. + if (handleMigrationKey(input)) return; setStep("probing"); - probeMutation.mutate(urlInput.trim()); + probeMutation.mutate(input); + }; + + /** + * Connect using a migration key from the EmDash Exporter plugin wizard. + * Returns false when the string isn't a valid key. + */ + const handleMigrationKey = (key: string): boolean => { + const decoded = decodeMigrationKey(key.trim()); + if (!decoded) return false; + const token = btoa(`${decoded.user}:${decoded.pass.replace(WHITESPACE_REGEX, "")}`); + setImportSource({ type: "wordpress-plugin", url: decoded.url, token }); + setUrlInput(decoded.url); + setImportError(null); + setStep("analyzing-plugin"); + wpPluginAnalyzeMutation.mutate({ url: decoded.url, token }); + return true; }; const handleFileSelect = (e: React.ChangeEvent) => { @@ -691,6 +742,7 @@ export function WordPressImport() { onProbeUrl={handleProbeUrl} onFileSelect={handleFileSelect} onDrop={handleDrop} + onMigrationKey={handleMigrationKey} /> )} @@ -904,16 +956,72 @@ function ChooseStep({ onProbeUrl, onFileSelect, onDrop, + onMigrationKey, }: { urlInput: string; onUrlChange: (url: string) => void; onProbeUrl: (e: React.FormEvent) => void; onFileSelect: (e: React.ChangeEvent) => void; onDrop: (e: React.DragEvent) => void; + onMigrationKey: (key: string) => boolean; }) { const { t } = useLingui(); + const [keyInput, setKeyInput] = React.useState(""); + const [keyError, setKeyError] = React.useState(false); + + const handleKeySubmit = (e: React.FormEvent) => { + e.preventDefault(); + if (!keyInput.trim()) return; + setKeyError(!onMigrationKey(keyInput)); + }; + return (
+ {/* Migration key - fastest path, generated by the WP plugin wizard */} +
+
+
+ +
+
+

{t`Paste your migration key`}

+

+ {t`Install the EmDash Exporter plugin on your WordPress site and generate a key under Tools → EmDash Migration.`} +

+
+ { + setKeyInput(e.target.value); + setKeyError(false); + }} + className="flex-1 font-mono" + aria-label={t`Migration key`} + /> + +
+ {keyError && ( +

+ {t`This doesn't look like a valid migration key. Generate a new one in the plugin and paste the full string starting with "em1.".`} +

+ )} +
+
+
+ +
+
+
+
+
+ {t`or connect by URL`} +
+
+ {/* URL input - primary path */}
@@ -2075,6 +2183,22 @@ function CompleteStep({ }), ); } + if (result.taxonomyAssignments && result.taxonomyAssignments > 0) { + parts.push( + plural(result.taxonomyAssignments, { + one: "# taxonomy assignment", + other: "# taxonomy assignments", + }), + ); + } + if (result.menus && result.menus.created > 0) { + parts.push( + plural(result.menus.created, { + one: "# menu imported", + other: "# menus imported", + }), + ); + } if (hasContentErrors) { parts.push( plural(result.errors.length, { one: "# content error", other: "# content errors" }), diff --git a/packages/admin/src/lib/api/import.ts b/packages/admin/src/lib/api/import.ts index c01029a345..286808c04d 100644 --- a/packages/admin/src/lib/api/import.ts +++ b/packages/admin/src/lib/api/import.ts @@ -157,6 +157,12 @@ export interface ImportResult { skipped: number; errors: Array<{ title: string; error: string }>; byCollection: Record; + /** Number of taxonomy term assignments written (plugin import) */ + taxonomyAssignments?: number; + /** Source taxonomies skipped because no matching EmDash taxonomy def exists */ + missingTaxonomies?: string[]; + /** Navigation menu import summary (plugin import) */ + menus?: { created: number; items: number }; } /** diff --git a/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts b/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts index 2a383eb096..67b5cf2033 100644 --- a/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts +++ b/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts @@ -7,7 +7,14 @@ */ import type { APIRoute } from "astro"; -import { ContentRepository, SchemaRegistry } from "emdash"; +import { + ContentRepository, + SchemaRegistry, + type WxrCategory, + type WxrPost, + type WxrTag, + type WxrTerm, +} from "emdash"; import { requirePerm } from "#api/authorize.js"; import { apiError, apiSuccess, handleError } from "#api/error.js"; @@ -15,9 +22,16 @@ import { isParseError, parseBody } from "#api/parse.js"; import { wpPluginExecuteBody } from "#api/schemas.js"; import { BylineRepository } from "#db/repositories/byline.js"; import { getSource } from "#import/index.js"; +import { importMenusFromPlugin } from "#import/menus.js"; +import { fetchPluginMenus, fetchPluginTaxonomies } from "#import/sources/wordpress-plugin.js"; import { resolveAndValidateExternalUrl, SsrfError } from "#import/ssrf.js"; import type { ImportConfig, ImportResult, NormalizedItem } from "#import/types.js"; import { resolveImportByline } from "#import/utils.js"; +import { + attachPostTaxonomies, + preImportWxrTaxonomies, + type TaxonomyImportPlan, +} from "#import/wxr-taxonomies.js"; import type { FieldType } from "#schema/types.js"; import type { EmDashHandlers, EmDashManifest } from "#types"; import { slugify } from "#utils/slugify.js"; @@ -81,8 +95,17 @@ export const POST: APIRoute = async ({ request, locals }) => { console.log("[WP Plugin Import] Starting import for:", body.url); console.log("[WP Plugin Import] Post types:", postTypes); + // Pre-create taxonomy terms so per-post assignments can attach. + // Non-fatal: a site whose /taxonomies call fails still gets content. + let taxonomyPlan: TaxonomyImportPlan | undefined; + try { + taxonomyPlan = await buildTaxonomyPlan(emdash, body.url, body.token); + } catch (e) { + console.warn("[WP Plugin Import] Taxonomy pre-import failed:", e); + } + // Import content (including drafts since we have auth) - const result = await importContent( + const { result, contentIdMap } = await importContent( source.fetchContent( { type: "url", url: body.url, token: body.token }, { postTypes, includeDrafts: true }, @@ -90,8 +113,28 @@ export const POST: APIRoute = async ({ request, locals }) => { config, emdash, emdashManifest, + taxonomyPlan, ); + // Import navigation menus, resolving item references through the + // WP-post-ID -> EmDash-ID map collected during the content pass. + // Non-fatal: older plugin versions have no /menus endpoint. + try { + const menus = await fetchPluginMenus(body.url, body.token); + if (menus.length > 0) { + const menuResult = await importMenusFromPlugin(menus, emdash.db, contentIdMap); + result.menus = { + created: menuResult.menusCreated, + items: menuResult.itemsCreated, + }; + for (const menuError of menuResult.errors) { + result.errors.push({ title: `Menu: ${menuError.menu}`, error: menuError.error }); + } + } + } catch (e) { + console.warn("[WP Plugin Import] Menu import failed:", e); + } + console.log("[WP Plugin Import] Import result:", JSON.stringify(result, null, 2)); return apiSuccess({ @@ -134,14 +177,101 @@ const IMPORT_FIELDS: Array<{ type: "image", check: (item) => !!item.featuredImage, }, + { + slug: "seo_title", + label: "SEO Title", + type: "string", + check: (item) => !!extractSeo(item).title, + }, + { + slug: "seo_description", + label: "SEO Description", + type: "text", + check: (item) => !!extractSeo(item).description, + }, ]; +/** + * Pull the per-post SEO title/description out of the Yoast / Rank Math + * blobs the plugin source stashes in `item.meta`. Empty strings mean "not + * overridden for this post" (the plugin exports the raw meta values) and + * are treated as absent. + */ +function pickSeoValue(blob: unknown, key: string): string | undefined { + if (typeof blob !== "object" || blob === null) return undefined; + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- narrowed to non-null object above + const value = (blob as Record)[key]; + return typeof value === "string" && value.trim() !== "" ? value : undefined; +} + +function extractSeo(item: NormalizedItem): { title?: string; description?: string } { + const yoast = item.meta?._yoast; + const rankmath = item.meta?._rankmath; + return { + title: pickSeoValue(yoast, "title") ?? pickSeoValue(rankmath, "title"), + description: pickSeoValue(yoast, "description") ?? pickSeoValue(rankmath, "description"), + }; +} + +/** + * Fetch the site's taxonomies from the plugin API and pre-create all terms, + * reusing the WXR taxonomy machinery (same def-lookup, idempotency, and + * collection-filter semantics). + */ +async function buildTaxonomyPlan( + emdash: EmDashHandlers, + url: string, + token: string, +): Promise { + const taxonomies = await fetchPluginTaxonomies(url, token); + + const categories: WxrCategory[] = []; + const tags: WxrTag[] = []; + const terms: WxrTerm[] = []; + + for (const taxonomy of taxonomies) { + for (const term of taxonomy.terms) { + if (taxonomy.name === "category") { + categories.push({ nicename: term.slug, name: term.name, description: term.description }); + } else if (taxonomy.name === "post_tag") { + tags.push({ slug: term.slug, name: term.name, description: term.description }); + } else if (taxonomy.name !== "nav_menu" && taxonomy.name !== "post_format") { + terms.push({ + id: term.id, + taxonomy: taxonomy.name, + slug: term.slug, + name: term.name, + description: term.description, + }); + } + } + } + + return preImportWxrTaxonomies(emdash.db, [], categories, tags, terms, undefined); +} + +/** + * Adapt a NormalizedItem's taxonomy assignments to the WxrPost shape the + * shared attach helper consumes. + */ +function toWxrAssignments(item: NormalizedItem): WxrPost { + return { + categories: item.categories ?? [], + tags: item.tags ?? [], + customTaxonomies: item.customTaxonomies + ? new Map(Object.entries(item.customTaxonomies)) + : undefined, + meta: new Map(), + }; +} + async function importContent( items: AsyncGenerator, config: WpPluginImportConfig, emdash: EmDashHandlers, manifest: EmDashManifest, -): Promise { + taxonomyPlan: TaxonomyImportPlan | undefined, +): Promise<{ result: ImportResult; contentIdMap: Map }> { const result: ImportResult = { success: true, imported: 0, @@ -150,6 +280,9 @@ async function importContent( byCollection: {}, }; + // WP post ID -> EmDash content ID, used to resolve menu item references + const contentIdMap = new Map(); + // Create content repository for checking existing items const contentRepo = new ContentRepository(emdash.db); const bylineRepo = new BylineRepository(emdash.db); @@ -159,6 +292,9 @@ async function importContent( // Track which collections have had fields ensured const ensuredCollections = new Set(); + // Field slugs per collection, for mapping custom meta/ACF onto real fields + const fieldSlugsByCollection = new Map>(); + // Track source translationGroup -> EmDash item ID for translation linking. // Maps source-side translation group ID to the EmDash ID of the first item // imported for that group (the default-locale item). @@ -225,6 +361,21 @@ async function importContent( ensuredCollections.add(collection); } + // Load the collection's field slugs once, so custom meta / ACF + // values can land in matching schema fields (created by the + // prepare step or already present). + let fieldSlugs = fieldSlugsByCollection.get(collection); + if (!fieldSlugs) { + fieldSlugs = new Set(); + const collectionDef = await schemaRegistry.getCollection(collection); + if (collectionDef) { + for (const field of await schemaRegistry.listFields(collectionDef.id)) { + fieldSlugs.add(field.slug); + } + } + fieldSlugsByCollection.set(collection, fieldSlugs); + } + // Generate slug from item slug or title const slug = item.slug || slugify(item.title || `post-${item.sourceId}`); @@ -236,6 +387,11 @@ async function importContent( if (item.translationGroup) { translationGroupMap.set(item.translationGroup, existing.id); } + // Menus may reference this post even when we skip it + const wpId = Number(item.sourceId); + if (Number.isFinite(wpId)) { + contentIdMap.set(wpId, existing.id); + } result.skipped++; continue; } @@ -259,8 +415,36 @@ async function importContent( console.log("[WP Plugin Import] Adding featured_image:", item.featuredImage); } - // Note: ACF/Yoast/RankMath fields are not added automatically - // They would need matching fields in the EmDash schema + // Per-post SEO overrides from Yoast / Rank Math + const seo = extractSeo(item); + if (seo.title) { + data.seo_title = seo.title; + } + if (seo.description) { + data.seo_description = seo.description; + } + + // Map ACF values and custom meta onto schema fields with the same + // slug (typically created by the prepare step from the analysis). + // Values without a matching field are dropped -- the user controls + // the schema, we don't invent fields per meta key. + const acf = item.meta?._acf; + if (typeof acf === "object" && acf !== null) { + for (const [key, value] of Object.entries(acf)) { + if (!(key in data) && fieldSlugs.has(key) && value !== null && value !== "") { + data[key] = value; + } + } + } + if (item.meta) { + for (const [key, value] of Object.entries(item.meta)) { + // Underscore keys are WP-internal or handled above (_acf/_yoast/_rankmath) + if (key.startsWith("_")) continue; + if (!(key in data) && fieldSlugs.has(key) && value !== null && value !== "") { + data[key] = value; + } + } + } // Resolve author ID from mappings let authorId: string | undefined; @@ -314,14 +498,39 @@ async function importContent( result.imported++; result.byCollection[collection] = (result.byCollection[collection] || 0) + 1; - // Track translation group: first item in a group establishes the mapping - if (item.translationGroup && !translationGroupMap.has(item.translationGroup)) { - // eslint-disable-next-line typescript/no-unsafe-type-assertion -- handler success data includes id - const createdData = createResult.data as { id?: string } | undefined; - if (createdData?.id) { - translationGroupMap.set(item.translationGroup, createdData.id); + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- handler success data includes id + const createdData = createResult.data as { id?: string } | undefined; + const createdId = createdData?.id; + + if (createdId) { + const wpId = Number(item.sourceId); + if (Number.isFinite(wpId)) { + contentIdMap.set(wpId, createdId); + } + + // Attach category/tag/custom-taxonomy assignments + if (taxonomyPlan) { + try { + const attached = await attachPostTaxonomies( + emdash.db, + collection, + createdId, + toWxrAssignments(item), + taxonomyPlan, + ); + if (attached > 0) { + result.taxonomyAssignments = (result.taxonomyAssignments ?? 0) + attached; + } + } catch (e) { + console.warn(`[WP Plugin Import] Taxonomy attach failed for "${slug}":`, e); + } } } + + // Track translation group: first item in a group establishes the mapping + if (item.translationGroup && !translationGroupMap.has(item.translationGroup) && createdId) { + translationGroupMap.set(item.translationGroup, createdId); + } } else { result.errors.push({ title: item.title || "Untitled", @@ -340,8 +549,12 @@ async function importContent( } } + if (taxonomyPlan && taxonomyPlan.missingTaxonomies.length > 0) { + result.missingTaxonomies = taxonomyPlan.missingTaxonomies; + } + result.success = result.errors.length === 0; - return result; + return { result, contentIdMap }; } function mapStatus(wpStatus: string | undefined): string { diff --git a/packages/core/src/import/sources/wordpress-plugin.ts b/packages/core/src/import/sources/wordpress-plugin.ts index f33bce3677..1437d82127 100644 --- a/packages/core/src/import/sources/wordpress-plugin.ts +++ b/packages/core/src/import/sources/wordpress-plugin.ts @@ -8,6 +8,7 @@ import { gutenbergToPortableText } from "@emdash-cms/gutenberg-to-portable-text"; import { encodeBase64 } from "../../utils/base64.js"; +import type { PluginMenu } from "../menus.js"; import { ssrfSafeFetch, validateExternalUrl } from "../ssrf.js"; import type { ImportSource, @@ -216,6 +217,67 @@ interface PluginMediaItem { /** Pattern to remove spaces from application passwords */ const SPACE_PATTERN = /\s/g; +/** + * Build the REST API URL for a plugin endpoint. + * + * `restRoute: false` uses the pretty form (`/wp-json/emdash/v1/...`), + * `restRoute: true` uses the `?rest_route=` form that works on sites with + * plain permalinks (where `/wp-json/` doesn't exist). + */ +function pluginApiUrl( + siteUrl: string, + path: string, + params: Record = {}, + restRoute = false, +): string { + if (restRoute) { + const url = new URL(siteUrl + "/"); + url.searchParams.set("rest_route", `/emdash/v1/${path}`); + for (const [key, value] of Object.entries(params)) { + url.searchParams.set(key, value); + } + return url.toString(); + } + const url = new URL(`${siteUrl}/wp-json/emdash/v1/${path}`); + for (const [key, value] of Object.entries(params)) { + url.searchParams.set(key, value); + } + return url.toString(); +} + +/** + * Fetch a plugin API endpoint, falling back to the `?rest_route=` form when + * the pretty `/wp-json/` route 404s or is unreachable. Sites with "Plain" + * permalinks have no `/wp-json/` rewrite, so without this fallback they + * always fail with a misleading 404. + */ +async function fetchPluginApi( + siteUrl: string, + path: string, + params: Record, + headers: HeadersInit, + timeoutMs: number, +): Promise { + let pretty: Response | null = null; + try { + pretty = await ssrfSafeFetch(pluginApiUrl(siteUrl, path, params), { + headers, + signal: AbortSignal.timeout(timeoutMs), + }); + } catch { + // Network-level failure -- try the rest_route form before giving up. + } + // Any response other than 404 (including 401/403/500) is authoritative: + // the route exists, so don't mask the real error with a fallback attempt. + if (pretty && pretty.status !== 404) { + return pretty; + } + return ssrfSafeFetch(pluginApiUrl(siteUrl, path, params, true), { + headers, + signal: AbortSignal.timeout(timeoutMs), + }); +} + // ============================================================================= // Import Source // ============================================================================= @@ -235,12 +297,13 @@ export const wordpressPluginSource: ImportSource = { // SSRF protection: validate URL before any outbound requests validateExternalUrl(siteUrl); - const probeUrl = `${siteUrl}/wp-json/emdash/v1/probe`; - - const response = await ssrfSafeFetch(probeUrl, { - headers: { Accept: "application/json" }, - signal: AbortSignal.timeout(10000), - }); + const response = await fetchPluginApi( + siteUrl, + "probe", + {}, + { Accept: "application/json" }, + 10000, + ); if (!response.ok) { return null; @@ -293,10 +356,7 @@ export const wordpressPluginSource: ImportSource = { async analyze(input: SourceInput, context: ImportContext): Promise { const { siteUrl, headers } = getRequestConfig(input); - const response = await ssrfSafeFetch(`${siteUrl}/wp-json/emdash/v1/analyze`, { - headers, - signal: AbortSignal.timeout(30000), - }); + const response = await fetchPluginApi(siteUrl, "analyze", {}, headers, 30000); if (!response.ok) { const body: unknown = await response.json().catch(() => undefined); @@ -339,20 +399,24 @@ export const wordpressPluginSource: ImportSource = { }; }); - // Fetch media list for attachment info + // Fetch the full media list, paginated. Stopping after the first page + // silently capped imports at 500 attachments (wp-emdash #1). const attachments: AttachmentInfo[] = []; if (data.attachments.count > 0) { try { - // Fetch first page of media to populate attachment info - const mediaResponse = await ssrfSafeFetch( - `${siteUrl}/wp-json/emdash/v1/media?per_page=500`, - { + let page = 1; + let totalPages = 1; + while (page <= totalPages) { + const mediaResponse = await fetchPluginApi( + siteUrl, + "media", + { per_page: "500", page: String(page) }, headers, - signal: AbortSignal.timeout(30000), - }, - ); - if (mediaResponse.ok) { + 30000, + ); + if (!mediaResponse.ok) break; const mediaData: PluginMediaResponse = await mediaResponse.json(); + totalPages = mediaData.pages; for (const item of mediaData.items) { attachments.push({ id: item.id, @@ -366,6 +430,7 @@ export const wordpressPluginSource: ImportSource = { height: item.height, }); } + page++; } } catch (e) { console.warn("Failed to fetch media list:", e); @@ -410,12 +475,13 @@ export const wordpressPluginSource: ImportSource = { while (page <= totalPages) { const status = options.includeDrafts ? "any" : "publish"; - const url = `${siteUrl}/wp-json/emdash/v1/content?post_type=${postType}&status=${status}&per_page=100&page=${page}`; - - const response = await ssrfSafeFetch(url, { + const response = await fetchPluginApi( + siteUrl, + "content", + { post_type: postType, status, per_page: "100", page: String(page) }, headers, - signal: AbortSignal.timeout(60000), - }); + 60000, + ); if (!response.ok) { throw new Error(`Failed to fetch ${postType}: ${response.statusText}`); @@ -526,6 +592,15 @@ function pluginPostToNormalizedItem(post: PluginPost): NormalizedItem { post.taxonomies?.tags?.map((t) => t.slug) ?? []; + // Everything else is a custom taxonomy assignment (genre, product_cat, ...) + const customTaxonomies: Record = {}; + for (const [name, terms] of Object.entries(post.taxonomies ?? {})) { + if (["category", "categories", "post_tag", "tags"].includes(name)) continue; + if (Array.isArray(terms) && terms.length > 0) { + customTaxonomies[name] = terms.map((t) => t.slug); + } + } + // Build meta from various sources const meta: Record = { ...post.meta }; @@ -555,6 +630,7 @@ function pluginPostToNormalizedItem(post: PluginPost): NormalizedItem { author: post.author?.login, categories, tags, + customTaxonomies: Object.keys(customTaxonomies).length > 0 ? customTaxonomies : undefined, meta, featuredImage: post.featured_image?.url, locale: post.locale, @@ -589,14 +665,13 @@ export async function fetchPluginMedia( // SSRF protection: validate URL before any outbound requests validateExternalUrl(normalizedSiteUrl); - const url = `${normalizedSiteUrl}/wp-json/emdash/v1/media?per_page=${perPage}&page=${page}`; - - const response = await ssrfSafeFetch(url, { - headers: { - Accept: "application/json", - Authorization: `Basic ${authToken}`, - }, - }); + const response = await fetchPluginApi( + normalizedSiteUrl, + "media", + { per_page: String(perPage), page: String(page) }, + { Accept: "application/json", Authorization: `Basic ${authToken}` }, + 30000, + ); if (!response.ok) { throw new Error(`Failed to fetch media: ${response.statusText}`); @@ -631,14 +706,13 @@ export async function fetchPluginTaxonomies( // SSRF protection: validate URL before any outbound requests validateExternalUrl(normalizedSiteUrl); - const url = `${normalizedSiteUrl}/wp-json/emdash/v1/taxonomies`; - - const response = await ssrfSafeFetch(url, { - headers: { - Accept: "application/json", - Authorization: `Basic ${authToken}`, - }, - }); + const response = await fetchPluginApi( + normalizedSiteUrl, + "taxonomies", + {}, + { Accept: "application/json", Authorization: `Basic ${authToken}` }, + 30000, + ); if (!response.ok) { throw new Error(`Failed to fetch taxonomies: ${response.statusText}`); @@ -646,3 +720,31 @@ export async function fetchPluginTaxonomies( return response.json(); } + +/** + * Fetch navigation menus from plugin API (added in emdash-exporter 1.1.0). + * Returns an empty array when the endpoint doesn't exist (older plugin). + */ +export async function fetchPluginMenus(siteUrl: string, authToken: string): Promise { + const normalizedSiteUrl = normalizeUrl(siteUrl); + + // SSRF protection: validate URL before any outbound requests + validateExternalUrl(normalizedSiteUrl); + + const response = await fetchPluginApi( + normalizedSiteUrl, + "menus", + {}, + { Accept: "application/json", Authorization: `Basic ${authToken}` }, + 30000, + ); + + if (response.status === 404) { + return []; + } + if (!response.ok) { + throw new Error(`Failed to fetch menus: ${response.statusText}`); + } + + return response.json(); +} diff --git a/packages/core/src/import/types.ts b/packages/core/src/import/types.ts index 5f900925e1..e0e34f1f03 100644 --- a/packages/core/src/import/types.ts +++ b/packages/core/src/import/types.ts @@ -353,6 +353,12 @@ export interface ImportResult { skipped: number; errors: Array<{ title: string; error: string }>; byCollection: Record; + /** Number of taxonomy term assignments written (plugin import) */ + taxonomyAssignments?: number; + /** Source taxonomies skipped because no matching EmDash taxonomy def exists */ + missingTaxonomies?: string[]; + /** Navigation menu import summary (plugin import) */ + menus?: { created: number; items: number }; } // ============================================================================= diff --git a/packages/core/tests/unit/import/wordpress-plugin-source.test.ts b/packages/core/tests/unit/import/wordpress-plugin-source.test.ts new file mode 100644 index 0000000000..d169e494a1 --- /dev/null +++ b/packages/core/tests/unit/import/wordpress-plugin-source.test.ts @@ -0,0 +1,172 @@ +/** + * Tests for WordPress plugin import source fetch behaviour: + * - custom taxonomy assignments on normalized items + * - full media pagination in analyze() (regression: only page 1 was fetched) + * - ?rest_route= fallback for sites with plain permalinks + */ + +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; + +import { wordpressPluginSource } from "../../../src/import/sources/wordpress-plugin.js"; +import { setDefaultDnsResolver } from "../../../src/import/ssrf.js"; + +// ─── Mock fetch ────────────────────────────────────────────────────────────── + +const mockFetch = vi.fn(); +vi.stubGlobal("fetch", mockFetch); + +// Bypass DoH so the fetch mock only sees the calls these tests model. +let previousResolver: ReturnType | undefined; +beforeAll(() => { + previousResolver = setDefaultDnsResolver(async () => ["93.184.216.34"]); +}); +afterAll(() => { + setDefaultDnsResolver(previousResolver ?? null); +}); + +beforeEach(() => { + mockFetch.mockReset(); +}); + +// ─── Fixtures ──────────────────────────────────────────────────────────────── + +function makePost(overrides: Record = {}) { + return { + id: 1, + post_type: "post", + status: "publish", + slug: "hello", + title: "Hello", + content: "", + excerpt: "", + date: "2024-01-01T00:00:00", + date_gmt: "2024-01-01T00:00:00", + modified: "2024-01-01T00:00:00", + modified_gmt: "2024-01-01T00:00:00", + author: null, + parent: null, + menu_order: 0, + taxonomies: {}, + meta: {}, + ...overrides, + }; +} + +function contentResponse(items: unknown[]) { + return new Response( + JSON.stringify({ items, total: items.length, pages: 1, page: 1, per_page: 100 }), + { status: 200 }, + ); +} + +function makeAnalyzeResponse(attachmentCount: number) { + return { + site: { title: "Test Site", url: "https://example.com" }, + post_types: [], + taxonomies: [], + authors: [], + attachments: { count: attachmentCount, by_type: {} }, + }; +} + +function mediaPage(page: number, pages: number, ids: number[]) { + return new Response( + JSON.stringify({ + items: ids.map((id) => ({ + id, + url: `https://example.com/wp-content/uploads/${id}.jpg`, + filename: `${id}.jpg`, + mime_type: "image/jpeg", + title: `Image ${id}`, + alt: "", + caption: "", + description: "", + })), + total: ids.length * pages, + pages, + page, + per_page: 500, + }), + { status: 200 }, + ); +} + +// ─── Tests ─────────────────────────────────────────────────────────────────── + +describe("WordPress Plugin Source — fetch behaviour", () => { + it("maps non-category/tag taxonomies to customTaxonomies", async () => { + mockFetch.mockResolvedValueOnce( + contentResponse([ + makePost({ + taxonomies: { + category: [{ id: 1, name: "News", slug: "news" }], + post_tag: [{ id: 2, name: "Update", slug: "update" }], + genre: [ + { id: 3, name: "Sci-Fi", slug: "sci-fi" }, + { id: 4, name: "Fantasy", slug: "fantasy" }, + ], + }, + }), + ]), + ); + + const items = []; + for await (const item of wordpressPluginSource.fetchContent( + { type: "url", url: "https://example.com", token: "test-token" }, + { postTypes: ["post"] }, + )) { + items.push(item); + } + + expect(items).toHaveLength(1); + expect(items[0]!.categories).toEqual(["news"]); + expect(items[0]!.tags).toEqual(["update"]); + expect(items[0]!.customTaxonomies).toEqual({ genre: ["sci-fi", "fantasy"] }); + }); + + it("fetches every media page during analyze, not just the first", async () => { + mockFetch.mockImplementation((input: RequestInfo | URL) => { + const url = String(input); + if (url.includes("/analyze")) { + return Promise.resolve( + new Response(JSON.stringify(makeAnalyzeResponse(3)), { status: 200 }), + ); + } + if (url.includes("page=2")) { + return Promise.resolve(mediaPage(2, 2, [3])); + } + return Promise.resolve(mediaPage(1, 2, [1, 2])); + }); + + const analysis = await wordpressPluginSource.analyze( + { type: "url", url: "https://example.com", token: "test-token" }, + {}, + ); + + expect(analysis.attachments.items.map((a) => a.id)).toEqual([1, 2, 3]); + }); + + it("falls back to ?rest_route= when the pretty route 404s (plain permalinks)", async () => { + mockFetch.mockImplementation((input: RequestInfo | URL) => { + const url = String(input); + if (url.includes("/wp-json/")) { + return Promise.resolve(new Response("Not Found", { status: 404 })); + } + if (url.includes("rest_route=")) { + return Promise.resolve(contentResponse([makePost()])); + } + return Promise.resolve(new Response("Unexpected", { status: 500 })); + }); + + const items = []; + for await (const item of wordpressPluginSource.fetchContent( + { type: "url", url: "https://example.com", token: "test-token" }, + { postTypes: ["post"] }, + )) { + items.push(item); + } + + expect(items).toHaveLength(1); + expect(items[0]!.slug).toBe("hello"); + }); +}); From aa8acb0b6bd131167d763b87372b7baaf16c8273 Mon Sep 17 00:00:00 2001 From: swissky <30409887+swissky@users.noreply.github.com> Date: Sun, 5 Jul 2026 23:03:55 +0200 Subject: [PATCH 2/4] feat(import): import comments, site identity, and ACF fields from the WordPress plugin - Import approved/pending comments with authors, original dates, threading, and status into _emdash_comments (idempotent re-runs). - Take over site identity (title, tagline, logo, favicon) from /options, writing the real site:* settings keys and side-loading logo/favicon media. - Surface per-post-type custom fields (ACF and plain meta) from the plugin analysis as suggested collection fields, with sanitized slugs and inferred types; match incoming meta keys with the same sanitization on execute. - Fix field auto-creation to run per field and item instead of once per collection, so fields needed only by a later item (e.g. a lone Yoast SEO override) are still created. --- .changeset/wp-plugin-import-completeness.md | 2 +- .../admin/src/components/WordPressImport.tsx | 11 + packages/admin/src/lib/api/import.ts | 4 + .../api/import/wordpress-plugin/execute.ts | 201 +++++++++++--- packages/core/src/import/comments.ts | 154 +++++++++++ packages/core/src/import/index.ts | 9 +- packages/core/src/import/settings.ts | 254 +++++------------- .../src/import/sources/wordpress-plugin.ts | 128 +++++++++ packages/core/src/import/types.ts | 4 + packages/core/src/import/utils.ts | 28 ++ .../plugin-execute-fields.test.ts | 89 ++++++ .../tests/unit/import/comments-import.test.ts | 119 ++++++++ .../tests/unit/import/settings-import.test.ts | 79 ++++++ .../import/wordpress-plugin-source.test.ts | 49 +++- 14 files changed, 895 insertions(+), 236 deletions(-) create mode 100644 packages/core/src/import/comments.ts create mode 100644 packages/core/tests/integration/wordpress-import/plugin-execute-fields.test.ts create mode 100644 packages/core/tests/unit/import/comments-import.test.ts create mode 100644 packages/core/tests/unit/import/settings-import.test.ts diff --git a/.changeset/wp-plugin-import-completeness.md b/.changeset/wp-plugin-import-completeness.md index 154bf5a1fe..8e167e1f22 100644 --- a/.changeset/wp-plugin-import-completeness.md +++ b/.changeset/wp-plugin-import-completeness.md @@ -3,4 +3,4 @@ "@emdash-cms/admin": minor --- -Improves WordPress plugin imports: taxonomy terms and assignments, Yoast/Rank Math SEO titles and descriptions, ACF values with matching schema fields, and navigation menus are now imported. Fetches the full media library instead of the first 500 files, supports sites with plain permalinks via a `?rest_route=` fallback, and the import screen accepts a migration key generated by the EmDash Exporter plugin wizard. +Improves WordPress plugin imports: taxonomy terms and assignments, Yoast/Rank Math SEO titles and descriptions, ACF and custom meta fields (suggested as collection fields during analysis and populated on import), navigation menus, and comments (with authors, dates, threading, and approval status) are now imported. Site identity — title, tagline, logo, and favicon — is taken over from the WordPress site, replacing the starter template's placeholders. Fetches the full media library instead of the first 500 files, supports sites with plain permalinks via a `?rest_route=` fallback, and the import screen accepts a migration key generated by the EmDash Exporter plugin wizard. diff --git a/packages/admin/src/components/WordPressImport.tsx b/packages/admin/src/components/WordPressImport.tsx index 9d52550625..eea003792d 100644 --- a/packages/admin/src/components/WordPressImport.tsx +++ b/packages/admin/src/components/WordPressImport.tsx @@ -2199,6 +2199,17 @@ function CompleteStep({ }), ); } + if (result.comments && result.comments.imported > 0) { + parts.push( + plural(result.comments.imported, { + one: "# comment imported", + other: "# comments imported", + }), + ); + } + if (result.siteSettings && result.siteSettings.length > 0) { + parts.push(t`site identity applied (title, tagline, logo)`); + } if (hasContentErrors) { parts.push( plural(result.errors.length, { one: "# content error", other: "# content errors" }), diff --git a/packages/admin/src/lib/api/import.ts b/packages/admin/src/lib/api/import.ts index 286808c04d..f3e39fc545 100644 --- a/packages/admin/src/lib/api/import.ts +++ b/packages/admin/src/lib/api/import.ts @@ -163,6 +163,10 @@ export interface ImportResult { missingTaxonomies?: string[]; /** Navigation menu import summary (plugin import) */ menus?: { created: number; items: number }; + /** Comment import summary (plugin import) */ + comments?: { imported: number; skipped: number }; + /** Site settings applied from the source (plugin import) */ + siteSettings?: string[]; } /** diff --git a/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts b/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts index 67b5cf2033..ea0f577ef0 100644 --- a/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts +++ b/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts @@ -21,12 +21,19 @@ import { apiError, apiSuccess, handleError } from "#api/error.js"; import { isParseError, parseBody } from "#api/parse.js"; import { wpPluginExecuteBody } from "#api/schemas.js"; import { BylineRepository } from "#db/repositories/byline.js"; +import { importCommentsFromPlugin } from "#import/comments.js"; import { getSource } from "#import/index.js"; import { importMenusFromPlugin } from "#import/menus.js"; -import { fetchPluginMenus, fetchPluginTaxonomies } from "#import/sources/wordpress-plugin.js"; +import { importSiteSettings, parseSiteSettingsFromPlugin } from "#import/settings.js"; +import { + fetchPluginComments, + fetchPluginMenus, + fetchPluginOptions, + fetchPluginTaxonomies, +} from "#import/sources/wordpress-plugin.js"; import { resolveAndValidateExternalUrl, SsrfError } from "#import/ssrf.js"; import type { ImportConfig, ImportResult, NormalizedItem } from "#import/types.js"; -import { resolveImportByline } from "#import/utils.js"; +import { resolveImportByline, sanitizeFieldSlug } from "#import/utils.js"; import { attachPostTaxonomies, preImportWxrTaxonomies, @@ -36,6 +43,8 @@ import type { FieldType } from "#schema/types.js"; import type { EmDashHandlers, EmDashManifest } from "#types"; import { slugify } from "#utils/slugify.js"; +import { importMediaWithProgress } from "../wordpress/media.js"; + export const prerender = false; export interface WpPluginImportConfig extends ImportConfig { @@ -105,7 +114,7 @@ export const POST: APIRoute = async ({ request, locals }) => { } // Import content (including drafts since we have auth) - const { result, contentIdMap } = await importContent( + const { result, contentIdMap, collectionByWpId } = await importContent( source.fetchContent( { type: "url", url: body.url, token: body.token }, { postTypes, includeDrafts: true }, @@ -135,6 +144,42 @@ export const POST: APIRoute = async ({ request, locals }) => { console.warn("[WP Plugin Import] Menu import failed:", e); } + // Import comments into EmDash's native comments table, preserving + // authors, dates, threading, and approval status. + // Non-fatal: older plugin versions have no /comments endpoint. + try { + const comments = await fetchPluginComments(body.url, body.token); + if (comments.length > 0) { + const commentsResult = await importCommentsFromPlugin( + comments, + emdash.db, + contentIdMap, + collectionByWpId, + ); + result.comments = { + imported: commentsResult.imported, + skipped: commentsResult.skipped, + }; + for (const commentError of commentsResult.errors) { + result.errors.push({ + title: `Comment: ${commentError.comment}`, + error: commentError.error, + }); + } + } + } catch (e) { + console.warn("[WP Plugin Import] Comment import failed:", e); + } + + // Apply site identity (title, tagline, logo, favicon) from the source + // site's options, replacing the starter template's seed placeholders. + // Non-fatal: content is already imported at this point. + try { + result.siteSettings = await applySiteSettings(emdash, body.url, body.token); + } catch (e) { + console.warn("[WP Plugin Import] Site settings import failed:", e); + } + console.log("[WP Plugin Import] Import result:", JSON.stringify(result, null, 2)); return apiSuccess({ @@ -250,6 +295,57 @@ async function buildTaxonomyPlan( return preImportWxrTaxonomies(emdash.db, [], categories, tags, terms, undefined); } +/** + * Fetch the source site's options and apply its identity (title, tagline, + * logo, favicon) as EmDash site settings, overwriting seed placeholders. + * Logo/favicon files are side-loaded into media storage first; the later + * full media pass dedupes them by content hash. + * + * Returns the list of applied setting keys. + */ +async function applySiteSettings( + emdash: EmDashHandlers, + url: string, + token: string, +): Promise { + const options = await fetchPluginOptions(url, token); + const parsed = parseSiteSettingsFromPlugin(options); + + const media: { logoMediaId?: string; faviconMediaId?: string } = {}; + if (emdash.storage && (parsed.logo?.url || parsed.favicon?.url)) { + const attachments = []; + if (parsed.logo?.url) { + attachments.push({ id: parsed.logo.id, url: parsed.logo.url }); + } + if (parsed.favicon?.url && parsed.favicon.url !== parsed.logo?.url) { + attachments.push({ id: parsed.favicon.id, url: parsed.favicon.url }); + } + const mediaResult = await importMediaWithProgress( + attachments, + emdash.db, + emdash.storage, + () => {}, + ); + for (const item of mediaResult.imported) { + if (item.originalUrl === parsed.logo?.url) { + media.logoMediaId = item.mediaId; + } + if (item.originalUrl === parsed.favicon?.url) { + media.faviconMediaId = item.mediaId; + } + } + } + + const settingsResult = await importSiteSettings(parsed, emdash.db, true, media); + for (const settingError of settingsResult.errors) { + console.warn( + `[WP Plugin Import] Site setting "${settingError.setting}" failed:`, + settingError.error, + ); + } + return settingsResult.applied; +} + /** * Adapt a NormalizedItem's taxonomy assignments to the WxrPost shape the * shared attach helper consumes. @@ -265,13 +361,18 @@ function toWxrAssignments(item: NormalizedItem): WxrPost { }; } -async function importContent( +/** Exported for tests (field auto-creation regression coverage). */ +export async function importContent( items: AsyncGenerator, config: WpPluginImportConfig, emdash: EmDashHandlers, manifest: EmDashManifest, taxonomyPlan: TaxonomyImportPlan | undefined, -): Promise<{ result: ImportResult; contentIdMap: Map }> { +): Promise<{ + result: ImportResult; + contentIdMap: Map; + collectionByWpId: Map; +}> { const result: ImportResult = { success: true, imported: 0, @@ -283,14 +384,19 @@ async function importContent( // WP post ID -> EmDash content ID, used to resolve menu item references const contentIdMap = new Map(); + // WP post ID -> EmDash collection slug, used by the comment import + const collectionByWpId = new Map(); + // Create content repository for checking existing items const contentRepo = new ContentRepository(emdash.db); const bylineRepo = new BylineRepository(emdash.db); const bylineCache = new Map(); const schemaRegistry = new SchemaRegistry(emdash.db); - // Track which collections have had fields ensured - const ensuredCollections = new Set(); + // Track which (collection, field) pairs have been ensured. Keyed per + // field, not per collection: a field like seo_title may first be + // needed by the 30th item, and gating on the first item would skip it. + const ensuredFields = new Set(); // Field slugs per collection, for mapping custom meta/ACF onto real fields const fieldSlugsByCollection = new Map>(); @@ -332,33 +438,35 @@ async function importContent( } try { - // Ensure required fields exist in the collection schema (once per collection) - if (!ensuredCollections.has(collection)) { - for (const field of IMPORT_FIELDS) { - if (field.check(item)) { - const existingField = await schemaRegistry.getField(collection, field.slug); - if (!existingField) { - console.log( - `[WP Plugin Import] Creating missing field "${field.slug}" in collection "${collection}"`, - ); - try { - await schemaRegistry.createField(collection, { - slug: field.slug, - label: field.label, - type: field.type, - required: false, - }); - } catch (e) { - // Field might already exist from concurrent creation - console.log( - `[WP Plugin Import] Field "${field.slug}" creation skipped:`, - e instanceof Error ? e.message : e, - ); - } - } + // Ensure required fields exist in the collection schema. Checked + // per field and item (not once per collection): whether a field + // is needed depends on the item — the first post may have no SEO + // override while a later one does. + for (const field of IMPORT_FIELDS) { + const ensureKey = `${collection}:${field.slug}`; + if (ensuredFields.has(ensureKey) || !field.check(item)) continue; + ensuredFields.add(ensureKey); + const existingField = await schemaRegistry.getField(collection, field.slug); + if (!existingField) { + console.log( + `[WP Plugin Import] Creating missing field "${field.slug}" in collection "${collection}"`, + ); + try { + await schemaRegistry.createField(collection, { + slug: field.slug, + label: field.label, + type: field.type, + required: false, + }); + fieldSlugsByCollection.get(collection)?.add(field.slug); + } catch (e) { + // Field might already exist from concurrent creation + console.log( + `[WP Plugin Import] Field "${field.slug}" creation skipped:`, + e instanceof Error ? e.message : e, + ); } } - ensuredCollections.add(collection); } // Load the collection's field slugs once, so custom meta / ACF @@ -387,10 +495,11 @@ async function importContent( if (item.translationGroup) { translationGroupMap.set(item.translationGroup, existing.id); } - // Menus may reference this post even when we skip it + // Menus and comments may reference this post even when we skip it const wpId = Number(item.sourceId); if (Number.isFinite(wpId)) { contentIdMap.set(wpId, existing.id); + collectionByWpId.set(wpId, collection); } result.skipped++; continue; @@ -425,24 +534,29 @@ async function importContent( } // Map ACF values and custom meta onto schema fields with the same - // slug (typically created by the prepare step from the analysis). - // Values without a matching field are dropped -- the user controls - // the schema, we don't invent fields per meta key. + // (sanitized) slug — the same sanitization the analysis applied + // when suggesting the fields, so keys like `event-date` land in + // the `event_date` field the prepare step created. Values without + // a matching field are dropped -- the user controls the schema, + // we don't invent fields per meta key. + const assignMetaValue = (key: string, value: unknown) => { + if (value === null || value === "") return; + const fieldSlug = sanitizeFieldSlug(key); + if (!(fieldSlug in data) && fieldSlugs.has(fieldSlug)) { + data[fieldSlug] = value; + } + }; const acf = item.meta?._acf; if (typeof acf === "object" && acf !== null) { for (const [key, value] of Object.entries(acf)) { - if (!(key in data) && fieldSlugs.has(key) && value !== null && value !== "") { - data[key] = value; - } + assignMetaValue(key, value); } } if (item.meta) { for (const [key, value] of Object.entries(item.meta)) { // Underscore keys are WP-internal or handled above (_acf/_yoast/_rankmath) if (key.startsWith("_")) continue; - if (!(key in data) && fieldSlugs.has(key) && value !== null && value !== "") { - data[key] = value; - } + assignMetaValue(key, value); } } @@ -506,6 +620,7 @@ async function importContent( const wpId = Number(item.sourceId); if (Number.isFinite(wpId)) { contentIdMap.set(wpId, createdId); + collectionByWpId.set(wpId, collection); } // Attach category/tag/custom-taxonomy assignments @@ -554,7 +669,7 @@ async function importContent( } result.success = result.errors.length === 0; - return { result, contentIdMap }; + return { result, contentIdMap, collectionByWpId }; } function mapStatus(wpStatus: string | undefined): string { diff --git a/packages/core/src/import/comments.ts b/packages/core/src/import/comments.ts new file mode 100644 index 0000000000..b5236912d2 --- /dev/null +++ b/packages/core/src/import/comments.ts @@ -0,0 +1,154 @@ +/** + * Comment import functions + * + * Import comments from the WordPress plugin API into EmDash's native + * comments table, preserving authors, dates, threading, and status. + */ + +import type { Kysely } from "kysely"; +import { ulid } from "ulidx"; + +import type { Database } from "../database/types.js"; +import { invalidateCommentObjectCache } from "../object-cache/index.js"; + +/** + * Plugin API comment format (matches /emdash/v1/comments items) + */ +export interface PluginComment { + id: number; + post_id: number; + parent_id: number | null; + author_name: string; + author_email: string; + /** Plain-text body (the plugin strips HTML) */ + body: string; + /** ISO 8601 UTC timestamp */ + date_gmt: string; + status: "approved" | "pending"; +} + +/** + * Result of comment import operation + */ +export interface CommentsImportResult { + /** Number of comments created */ + imported: number; + /** Comments skipped (unresolvable post reference or already imported) */ + skipped: number; + /** Errors encountered */ + errors: Array<{ comment: string; error: string }>; +} + +/** + * Import comments from the plugin API. + * + * Preserves original timestamps and threading. Comments whose post was + * not imported (no entry in `contentIdMap`) are skipped. Re-running the + * import is idempotent: a comment with the same post, author email, and + * timestamp is not created twice. + * + * @param comments - Comments from the plugin API (all pages, flat) + * @param db - Database connection + * @param contentIdMap - WP post ID -> EmDash content ID + * @param collectionMap - WP post ID -> EmDash collection slug + */ +export async function importCommentsFromPlugin( + comments: PluginComment[], + db: Kysely, + contentIdMap: Map, + collectionMap: Map, +): Promise { + const result: CommentsImportResult = { + imported: 0, + skipped: 0, + errors: [], + }; + + // WP comment ID -> EmDash comment ID, for parent threading + const commentIdMap = new Map(); + + // WP comment ID -> EmDash ID of its top-level ancestor. EmDash threads + // are one level deep (assembleThreads nests replies under roots only), + // so deeper WP threads are flattened onto their root comment. + const rootIdMap = new Map(); + + // Parents must exist before children reference them; WP comment IDs + // are chronological, so ID order guarantees parents come first. + const sorted = comments.toSorted((a, b) => a.id - b.id); + + for (const comment of sorted) { + const label = `${comment.author_name || "Anonymous"} (${comment.date_gmt})`; + try { + const contentId = contentIdMap.get(comment.post_id); + const collection = collectionMap.get(comment.post_id); + if (!contentId || !collection) { + result.skipped++; + continue; + } + + const parsed = new Date(comment.date_gmt); + const createdAt = Number.isNaN(parsed.getTime()) + ? new Date().toISOString() + : parsed.toISOString(); + + // Idempotency: same post + author + timestamp + body = already + // imported (body included so same-second comments from one author + // don't collide). ponytail: one SELECT per comment — fine at + // import scale; batch with WHERE IN for six-figure comment counts. + const existing = await db + .selectFrom("_emdash_comments") + .select("id") + .where("content_id", "=", contentId) + .where("author_email", "=", comment.author_email) + .where("created_at", "=", createdAt) + .where("body", "=", comment.body) + .executeTakeFirst(); + const parentId = + comment.parent_id !== null ? (rootIdMap.get(comment.parent_id) ?? null) : null; + + if (existing) { + commentIdMap.set(comment.id, existing.id); + rootIdMap.set(comment.id, parentId ?? existing.id); + result.skipped++; + continue; + } + + const id = ulid(); + + await db + .insertInto("_emdash_comments") + .values({ + id, + collection, + content_id: contentId, + parent_id: parentId, + author_name: comment.author_name || "Anonymous", + author_email: comment.author_email, + author_user_id: null, + body: comment.body, + status: comment.status, + ip_hash: null, + user_agent: null, + moderation_metadata: null, + created_at: createdAt, + updated_at: createdAt, + }) + .execute(); + + commentIdMap.set(comment.id, id); + rootIdMap.set(comment.id, parentId ?? id); + result.imported++; + } catch (error) { + result.errors.push({ + comment: label, + error: error instanceof Error ? error.message : String(error), + }); + } + } + + if (result.imported > 0) { + invalidateCommentObjectCache(); + } + + return result; +} diff --git a/packages/core/src/import/index.ts b/packages/core/src/import/index.ts index 9f8420f7a7..6e404e020d 100644 --- a/packages/core/src/import/index.ts +++ b/packages/core/src/import/index.ts @@ -44,13 +44,20 @@ export { // Sections import export { importReusableBlocksAsSections, type SectionsImportResult } from "./sections.js"; +// Comments import +export { + importCommentsFromPlugin, + type PluginComment, + type CommentsImportResult, +} from "./comments.js"; + // Site settings import export { importSiteSettings, parseSiteSettingsFromPlugin, type SiteSettingsAnalysis, type SettingsImportResult, - type WidgetAreaAnalysis, + type SettingsMediaIds, } from "./settings.js"; // Registry diff --git a/packages/core/src/import/settings.ts b/packages/core/src/import/settings.ts index 9c6ac1e9f3..6cb9c83474 100644 --- a/packages/core/src/import/settings.ts +++ b/packages/core/src/import/settings.ts @@ -1,12 +1,20 @@ /** * Site settings import functions * - * Import site settings from WordPress (title, tagline, logo, favicon, etc.) + * Import site settings from WordPress (title, tagline, logo, favicon) + * into EmDash's site settings (the `site:*` options read by + * `getSiteSettings()` and the templates). */ import type { Kysely } from "kysely"; +import { OptionsRepository } from "../database/repositories/options.js"; import type { Database } from "../database/types.js"; +import { invalidateSiteSettingsCache } from "../settings/index.js"; +import type { SiteSettings } from "../settings/types.js"; + +/** Options key prefix used by the site settings module (`settings/index.ts`). */ +const SETTINGS_PREFIX = "site:"; /** * Site settings analysis from import source @@ -16,30 +24,10 @@ export interface SiteSettingsAnalysis { title?: string; /** Site tagline/description */ tagline?: string; - /** Custom logo */ + /** Custom logo (source URL + WP attachment ID) */ logo?: { url: string; id?: number }; - /** Favicon/site icon */ + /** Favicon/site icon (source URL + WP attachment ID) */ favicon?: { url: string; id?: number }; - /** Front page settings */ - frontPage?: { type: "posts" | "page"; pageId?: number }; - /** SEO settings (Yoast, RankMath, etc.) */ - seo?: Record; -} - -/** - * Widget area analysis - */ -export interface WidgetAreaAnalysis { - /** Widget area ID */ - id: string; - /** Widget area name */ - name: string; - /** Widget area label */ - label: string; - /** Number of widgets */ - widgetCount: number; - /** Widget summaries */ - widgets: Array<{ type: string; title?: string }>; } /** @@ -55,17 +43,33 @@ export interface SettingsImportResult { } /** - * Import site settings from analysis + * Resolved EmDash media IDs for the logo/favicon source URLs. + * The caller side-loads the files (they need storage access) and + * passes the resulting media IDs here. + */ +export interface SettingsMediaIds { + logoMediaId?: string; + faviconMediaId?: string; +} + +/** + * Import site settings from analysis into EmDash site settings. + * + * Writes the `site:*` options consumed by `getSiteSettings()` and + * invalidates the settings cache. Logo/favicon are only applied when + * the caller resolved them to EmDash media IDs. * * @param settings - Site settings analysis * @param db - Database connection - * @param overwrite - Whether to overwrite existing settings - * @returns Import result + * @param overwrite - Whether to overwrite existing settings (a fresh + * site's seed already sets title/tagline, so migrations pass true) + * @param media - Resolved media IDs for logo/favicon */ export async function importSiteSettings( settings: SiteSettingsAnalysis, db: Kysely, overwrite = false, + media: SettingsMediaIds = {}, ): Promise { const result: SettingsImportResult = { applied: [], @@ -73,155 +77,54 @@ export async function importSiteSettings( errors: [], }; - // Import title + const updates: Array<{ setting: keyof SiteSettings; value: unknown }> = []; if (settings.title) { - try { - const applied = await setOption(db, "site_title", settings.title, overwrite); - if (applied) { - result.applied.push("site_title"); - } else { - result.skipped.push("site_title"); - } - } catch (error) { - result.errors.push({ - setting: "site_title", - error: error instanceof Error ? error.message : String(error), - }); - } + updates.push({ setting: "title", value: settings.title }); } - - // Import tagline if (settings.tagline) { - try { - const applied = await setOption(db, "site_tagline", settings.tagline, overwrite); - if (applied) { - result.applied.push("site_tagline"); - } else { - result.skipped.push("site_tagline"); - } - } catch (error) { - result.errors.push({ - setting: "site_tagline", - error: error instanceof Error ? error.message : String(error), - }); - } + updates.push({ setting: "tagline", value: settings.tagline }); } - - // Import logo URL (actual media import handled separately) - if (settings.logo?.url) { - try { - const applied = await setOption(db, "site_logo_url", settings.logo.url, overwrite); - if (applied) { - result.applied.push("site_logo_url"); - } else { - result.skipped.push("site_logo_url"); - } - } catch (error) { - result.errors.push({ - setting: "site_logo_url", - error: error instanceof Error ? error.message : String(error), - }); - } + if (media.logoMediaId) { + updates.push({ setting: "logo", value: { mediaId: media.logoMediaId } }); } - - // Import favicon URL - if (settings.favicon?.url) { - try { - const applied = await setOption(db, "site_favicon_url", settings.favicon.url, overwrite); - if (applied) { - result.applied.push("site_favicon_url"); - } else { - result.skipped.push("site_favicon_url"); - } - } catch (error) { - result.errors.push({ - setting: "site_favicon_url", - error: error instanceof Error ? error.message : String(error), - }); - } + if (media.faviconMediaId) { + updates.push({ setting: "favicon", value: { mediaId: media.faviconMediaId } }); } - // Import front page settings - if (settings.frontPage) { - try { - const applied = await setOption(db, "front_page_type", settings.frontPage.type, overwrite); - if (applied) { - result.applied.push("front_page_type"); - } else { - result.skipped.push("front_page_type"); - } + if (updates.length === 0) { + return result; + } - if (settings.frontPage.pageId) { - const pageApplied = await setOption( - db, - "front_page_id", - String(settings.frontPage.pageId), - overwrite, - ); - if (pageApplied) { - result.applied.push("front_page_id"); - } else { - result.skipped.push("front_page_id"); + const options = new OptionsRepository(db); + try { + for (const { setting, value } of updates) { + try { + const key = `${SETTINGS_PREFIX}${setting}`; + if (!overwrite) { + const existing = await options.get(key); + if (existing !== null) { + result.skipped.push(setting); + continue; + } } + await options.set(key, value); + result.applied.push(setting); + } catch (error) { + result.errors.push({ + setting, + error: error instanceof Error ? error.message : String(error), + }); } - } catch (error) { - result.errors.push({ - setting: "front_page", - error: error instanceof Error ? error.message : String(error), - }); } - } - - // Import SEO settings as JSON blob - if (settings.seo && Object.keys(settings.seo).length > 0) { - try { - const applied = await setOption(db, "seo_settings", JSON.stringify(settings.seo), overwrite); - if (applied) { - result.applied.push("seo_settings"); - } else { - result.skipped.push("seo_settings"); - } - } catch (error) { - result.errors.push({ - setting: "seo_settings", - error: error instanceof Error ? error.message : String(error), - }); + } finally { + if (result.applied.length > 0) { + invalidateSiteSettingsCache(); } } return result; } -/** - * Set an option in the database - * - * @returns true if the option was set, false if skipped (already exists and !overwrite) - */ -async function setOption( - db: Kysely, - key: string, - value: string, - overwrite: boolean, -): Promise { - const existing = await db - .selectFrom("options") - .select("value") - .where("name", "=", key) - .executeTakeFirst(); - - if (existing && !overwrite) { - return false; - } - - if (existing) { - await db.updateTable("options").set({ value }).where("name", "=", key).execute(); - } else { - await db.insertInto("options").values({ name: key, value }).execute(); - } - - return true; -} - /** * Parse site settings from WordPress plugin options response */ @@ -230,52 +133,25 @@ export function parseSiteSettingsFromPlugin( ): SiteSettingsAnalysis { const settings: SiteSettingsAnalysis = {}; - // Basic settings - if (typeof options.blogname === "string") { + if (typeof options.blogname === "string" && options.blogname.trim() !== "") { settings.title = options.blogname; } - if (typeof options.blogdescription === "string") { + if (typeof options.blogdescription === "string" && options.blogdescription.trim() !== "") { settings.tagline = options.blogdescription; } - // Logo and favicon - if (typeof options.custom_logo_url === "string") { + if (typeof options.custom_logo_url === "string" && options.custom_logo_url !== "") { settings.logo = { url: options.custom_logo_url, id: typeof options.custom_logo === "number" ? options.custom_logo : undefined, }; } - if (typeof options.site_icon_url === "string") { + if (typeof options.site_icon_url === "string" && options.site_icon_url !== "") { settings.favicon = { url: options.site_icon_url, id: typeof options.site_icon === "number" ? options.site_icon : undefined, }; } - // Front page settings - if (options.show_on_front === "page") { - settings.frontPage = { - type: "page", - pageId: typeof options.page_on_front === "number" ? options.page_on_front : undefined, - }; - } else { - settings.frontPage = { type: "posts" }; - } - - // SEO settings (Yoast) - const seo: Record = {}; - if (typeof options.wpseo === "object" && options.wpseo !== null) { - seo.yoast = options.wpseo; - } - if (typeof options.wpseo_titles === "object" && options.wpseo_titles !== null) { - seo.yoast_titles = options.wpseo_titles; - } - if (typeof options.wpseo_social === "object" && options.wpseo_social !== null) { - seo.yoast_social = options.wpseo_social; - } - if (Object.keys(seo).length > 0) { - settings.seo = seo; - } - return settings; } diff --git a/packages/core/src/import/sources/wordpress-plugin.ts b/packages/core/src/import/sources/wordpress-plugin.ts index 1437d82127..e8a0230415 100644 --- a/packages/core/src/import/sources/wordpress-plugin.ts +++ b/packages/core/src/import/sources/wordpress-plugin.ts @@ -8,6 +8,7 @@ import { gutenbergToPortableText } from "@emdash-cms/gutenberg-to-portable-text"; import { encodeBase64 } from "../../utils/base64.js"; +import type { PluginComment } from "../comments.js"; import type { PluginMenu } from "../menus.js"; import { ssrfSafeFetch, validateExternalUrl } from "../ssrf.js"; import type { @@ -29,6 +30,7 @@ import { mapWpStatus, normalizeUrl, checkSchemaCompatibility, + sanitizeFieldSlug, } from "../utils.js"; // ============================================================================= @@ -390,6 +392,22 @@ export const wordpressPluginSource: ImportSource = { ? [...BASE_REQUIRED_FIELDS, FEATURED_IMAGE_FIELD] : [...BASE_REQUIRED_FIELDS]; + // Surface the post type's custom fields (ACF and plain meta) so + // the prepare step creates them — without this, execute() has no + // matching schema fields and silently drops the values. + const knownSlugs = new Set(requiredFields.map((f) => f.slug)); + for (const customField of pt.custom_fields ?? []) { + const slug = sanitizeFieldSlug(customField.key); + if (knownSlugs.has(slug)) continue; + knownSlugs.add(slug); + requiredFields.push({ + slug, + label: fieldLabelFromKey(customField.key), + type: mapInferredFieldType(customField.inferred_type), + required: false, + }); + } + return { name: pt.name, count: pt.total, @@ -521,6 +539,37 @@ export const wordpressPluginSource: ImportSource = { // Helper Functions // ============================================================================= +/** Plugin `inferred_type` values that are valid EmDash field types as-is */ +const VALID_INFERRED_TYPES = new Set([ + "string", + "text", + "number", + "integer", + "boolean", + "datetime", + "json", + "reference", +]); + +/** + * Map the plugin's inferred custom-field type to an EmDash field type. + * Unknown values fall back to string (always safe for TEXT storage). + */ +function mapInferredFieldType(inferredType: string): string { + return VALID_INFERRED_TYPES.has(inferredType) ? inferredType : "string"; +} + +const FIELD_KEY_SEPARATORS = /[_-]+/; + +/** Derive a human label from a meta key: "event_start-date" -> "Event Start Date" */ +function fieldLabelFromKey(key: string): string { + return key + .split(FIELD_KEY_SEPARATORS) + .filter(Boolean) + .map((word) => word.charAt(0).toUpperCase() + word.slice(1)) + .join(" "); +} + /** * Convert plugin i18n info to the shared I18nDetection type. * Returns undefined when no multilingual plugin is detected. @@ -748,3 +797,82 @@ export async function fetchPluginMenus(siteUrl: string, authToken: string): Prom return response.json(); } + +/** + * Fetch site options from plugin API (title, tagline, logo, favicon, ...) + */ +export async function fetchPluginOptions( + siteUrl: string, + authToken: string, +): Promise> { + const normalizedSiteUrl = normalizeUrl(siteUrl); + + // SSRF protection: validate URL before any outbound requests + validateExternalUrl(normalizedSiteUrl); + + const response = await fetchPluginApi( + normalizedSiteUrl, + "options", + {}, + { Accept: "application/json", Authorization: `Basic ${authToken}` }, + 30000, + ); + + if (!response.ok) { + throw new Error(`Failed to fetch options: ${response.statusText}`); + } + + return response.json(); +} + +/** Comments response from /emdash/v1/comments */ +interface PluginCommentsResponse { + items: PluginComment[]; + total: number; + pages: number; + page: number; + per_page: number; +} + +/** + * Fetch all comments from plugin API (added in emdash-exporter 1.2.0), + * paginating through every page. Returns an empty array when the endpoint + * doesn't exist (older plugin). + */ +export async function fetchPluginComments( + siteUrl: string, + authToken: string, +): Promise { + const normalizedSiteUrl = normalizeUrl(siteUrl); + + // SSRF protection: validate URL before any outbound requests + validateExternalUrl(normalizedSiteUrl); + + const comments: PluginComment[] = []; + let page = 1; + let totalPages = 1; + + while (page <= totalPages) { + const response = await fetchPluginApi( + normalizedSiteUrl, + "comments", + { per_page: "500", page: String(page) }, + { Accept: "application/json", Authorization: `Basic ${authToken}` }, + 30000, + ); + + if (response.status === 404) { + return []; + } + if (!response.ok) { + throw new Error(`Failed to fetch comments: ${response.statusText}`); + } + + const data: PluginCommentsResponse = await response.json(); + totalPages = data.pages; + comments.push(...data.items); + page++; + } + + return comments; +} diff --git a/packages/core/src/import/types.ts b/packages/core/src/import/types.ts index e0e34f1f03..8d18839149 100644 --- a/packages/core/src/import/types.ts +++ b/packages/core/src/import/types.ts @@ -359,6 +359,10 @@ export interface ImportResult { missingTaxonomies?: string[]; /** Navigation menu import summary (plugin import) */ menus?: { created: number; items: number }; + /** Comment import summary (plugin import) */ + comments?: { imported: number; skipped: number }; + /** Site settings applied from the source (plugin import) */ + siteSettings?: string[]; } // ============================================================================= diff --git a/packages/core/src/import/utils.ts b/packages/core/src/import/utils.ts index 6007ee2ce8..52479c97dc 100644 --- a/packages/core/src/import/utils.ts +++ b/packages/core/src/import/utils.ts @@ -6,6 +6,7 @@ import mime from "mime/lite"; +import { RESERVED_FIELD_SLUGS } from "../schema/types.js"; import type { ImportFieldDef, CollectionSchemaStatus } from "./types.js"; // ============================================================================= @@ -183,6 +184,33 @@ export function inferMetaType( return "string"; } +// ============================================================================= +// Field Slug Sanitization +// ============================================================================= + +const INVALID_FIELD_SLUG_CHARS = /[^a-z0-9_]+/g; +const LEADING_NON_ALPHA_CHARS = /^[^a-z]+/; + +/** + * Sanitize a WordPress meta/ACF key into a valid EmDash field slug + * (`/^[a-z][a-z0-9_]*$/`, max 63 chars, not reserved). + * + * Must be applied consistently on both sides of an import: once when + * creating fields from the analysis, and again when matching incoming + * meta keys onto schema fields — otherwise keys like `my-field` create + * `my_field` but never receive values. + */ +export function sanitizeFieldSlug(key: string): string { + const sanitized = key + .toLowerCase() + .replace(INVALID_FIELD_SLUG_CHARS, "_") + .replace(LEADING_NON_ALPHA_CHARS, "") + .slice(0, 63); + if (!sanitized) return "field"; + if (RESERVED_FIELD_SLUGS.includes(sanitized)) return `wp_${sanitized}`; + return sanitized; +} + // ============================================================================= // String Utilities // ============================================================================= diff --git a/packages/core/tests/integration/wordpress-import/plugin-execute-fields.test.ts b/packages/core/tests/integration/wordpress-import/plugin-execute-fields.test.ts new file mode 100644 index 0000000000..cc8c5a5723 --- /dev/null +++ b/packages/core/tests/integration/wordpress-import/plugin-execute-fields.test.ts @@ -0,0 +1,89 @@ +/** + * Regression test: import fields (seo_title, seo_description, ...) must be + * auto-created even when the first imported item of a collection doesn't + * carry them. The field-ensure pass used to run once per collection, gated + * on the first item's data — a later post with a Yoast SEO override then + * failed with `seo_title: unknown field on collection`. + */ + +import type { Kysely } from "kysely"; +import { describe, it, expect, beforeEach, afterEach } from "vitest"; + +import { handleContentCreate } from "../../../src/api/handlers/content.js"; +import { + importContent, + type WpPluginImportConfig, +} from "../../../src/astro/routes/api/import/wordpress-plugin/execute.js"; +import type { EmDashHandlers, EmDashManifest } from "../../../src/astro/types.js"; +import type { Database } from "../../../src/database/types.js"; +import type { NormalizedItem } from "../../../src/import/types.js"; +import { setupTestDatabaseWithCollections, teardownTestDatabase } from "../../utils/test-db.js"; + +function makeItem(overrides: Partial): NormalizedItem { + return { + sourceId: 1, + postType: "post", + status: "publish", + slug: "item", + title: "Item", + content: [], + date: new Date("2026-01-01T00:00:00Z"), + ...overrides, + }; +} + +async function* generate(items: NormalizedItem[]): AsyncGenerator { + for (const item of items) yield item; +} + +describe("WordPress plugin import — field auto-creation", () => { + let db: Kysely; + + beforeEach(async () => { + db = await setupTestDatabaseWithCollections(); + }); + + afterEach(async () => { + await teardownTestDatabase(db); + }); + + it("creates seo fields needed only by a later item", async () => { + const config: WpPluginImportConfig = { + postTypeMappings: { post: { collection: "post", enabled: true } }, + skipExisting: false, + }; + // ponytail: minimal stub — importContent only touches db + handleContentCreate + const emdash = { + db, + handleContentCreate: (collection: string, body: { data: Record }) => + handleContentCreate(db, collection, body), + } as unknown as EmDashHandlers; + const manifest = { collections: { post: {} } } as unknown as EmDashManifest; + + const items = [ + makeItem({ sourceId: 1, slug: "plain", title: "Plain post" }), + makeItem({ + sourceId: 2, + slug: "with-seo", + title: "Post with SEO", + meta: { _yoast: { title: "Custom SEO Title", description: "Custom description" } }, + }), + ]; + + const { result } = await importContent(generate(items), config, emdash, manifest, undefined); + + expect(result.errors).toEqual([]); + expect(result.imported).toBe(2); + + const row = await db + // eslint-disable-next-line typescript/no-explicit-any -- dynamic ec_ table not in the static schema + .selectFrom("ec_post" as any) + .select(["slug", "seo_title", "seo_description"]) + .where("slug", "=", "with-seo") + .executeTakeFirstOrThrow(); + expect(row).toMatchObject({ + seo_title: "Custom SEO Title", + seo_description: "Custom description", + }); + }); +}); diff --git a/packages/core/tests/unit/import/comments-import.test.ts b/packages/core/tests/unit/import/comments-import.test.ts new file mode 100644 index 0000000000..f352b7a03d --- /dev/null +++ b/packages/core/tests/unit/import/comments-import.test.ts @@ -0,0 +1,119 @@ +/** + * Tests for comment import from the WordPress plugin API: + * - preserves author, date, and status + * - threads replies (deep WP threads flatten onto the root comment) + * - skips comments whose post was not imported + * - idempotent re-import + */ + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { importCommentsFromPlugin, type PluginComment } from "../../../src/import/comments.js"; +import { setupTestDatabase, teardownTestDatabase } from "../../utils/test-db.js"; + +let db: Awaited>; + +beforeEach(async () => { + db = await setupTestDatabase(); +}); + +afterEach(async () => { + await teardownTestDatabase(db); +}); + +function comment(overrides: Partial & { id: number }): PluginComment { + return { + post_id: 10, + parent_id: null, + author_name: "Alice", + author_email: "alice@example.com", + body: "Nice post!", + date_gmt: "2020-05-01T12:00:00Z", + status: "approved", + ...overrides, + }; +} + +const contentIdMap = new Map([[10, "CONTENT1"]]); +const collectionMap = new Map([[10, "posts"]]); + +describe("importCommentsFromPlugin", () => { + it("imports comments with preserved author, date, and status", async () => { + const result = await importCommentsFromPlugin( + [ + comment({ id: 1 }), + comment({ id: 2, author_name: "Bob", author_email: "bob@example.com", status: "pending" }), + ], + db, + contentIdMap, + collectionMap, + ); + + expect(result.imported).toBe(2); + expect(result.errors).toEqual([]); + + const rows = await db + .selectFrom("_emdash_comments") + .selectAll() + .orderBy("created_at") + .execute(); + expect(rows).toHaveLength(2); + expect(rows[0]!.collection).toBe("posts"); + expect(rows[0]!.content_id).toBe("CONTENT1"); + expect(rows[0]!.author_name).toBe("Alice"); + expect(rows[0]!.created_at).toBe("2020-05-01T12:00:00.000Z"); + expect(rows[1]!.status).toBe("pending"); + }); + + it("threads replies and flattens deep threads onto the root", async () => { + await importCommentsFromPlugin( + [ + comment({ id: 1 }), + comment({ id: 2, parent_id: 1, date_gmt: "2020-05-02T12:00:00Z" }), + // Reply to the reply -- deeper than EmDash's 1-level threading + comment({ id: 3, parent_id: 2, date_gmt: "2020-05-03T12:00:00Z" }), + ], + db, + contentIdMap, + collectionMap, + ); + + const rows = await db + .selectFrom("_emdash_comments") + .select(["id", "parent_id", "created_at"]) + .orderBy("created_at") + .execute(); + const root = rows[0]!; + expect(root.parent_id).toBeNull(); + // Both the reply and the reply-to-reply hang off the root + expect(rows[1]!.parent_id).toBe(root.id); + expect(rows[2]!.parent_id).toBe(root.id); + }); + + it("skips comments for posts that were not imported", async () => { + const result = await importCommentsFromPlugin( + [comment({ id: 1, post_id: 999 })], + db, + contentIdMap, + collectionMap, + ); + + expect(result.imported).toBe(0); + expect(result.skipped).toBe(1); + }); + + it("is idempotent on re-import", async () => { + const comments = [ + comment({ id: 1 }), + comment({ id: 2, parent_id: 1, date_gmt: "2020-05-02T12:00:00Z" }), + ]; + await importCommentsFromPlugin(comments, db, contentIdMap, collectionMap); + const second = await importCommentsFromPlugin(comments, db, contentIdMap, collectionMap); + + expect(second.imported).toBe(0); + expect(second.skipped).toBe(2); + + const rows = await db.selectFrom("_emdash_comments").select("id").execute(); + expect(rows).toHaveLength(2); + }); +}); diff --git a/packages/core/tests/unit/import/settings-import.test.ts b/packages/core/tests/unit/import/settings-import.test.ts new file mode 100644 index 0000000000..335311b9dc --- /dev/null +++ b/packages/core/tests/unit/import/settings-import.test.ts @@ -0,0 +1,79 @@ +/** + * Tests for site settings import: + * - writes the real `site:*` options read by getSiteSettings() + * - overwrite semantics (seeded titles are replaced during migration) + * - logo/favicon applied as media references + * - parseSiteSettingsFromPlugin field mapping + */ + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { importSiteSettings, parseSiteSettingsFromPlugin } from "../../../src/import/settings.js"; +import { getSiteSettingsWithDb } from "../../../src/settings/index.js"; +import { setupTestDatabase, teardownTestDatabase } from "../../utils/test-db.js"; + +let db: Awaited>; + +beforeEach(async () => { + db = await setupTestDatabase(); +}); + +afterEach(async () => { + await teardownTestDatabase(db); +}); + +describe("importSiteSettings", () => { + it("writes settings that getSiteSettings() actually reads", async () => { + const result = await importSiteSettings({ title: "WP Blog", tagline: "From WP" }, db, true); + + expect(result.applied).toEqual(["title", "tagline"]); + const settings = await getSiteSettingsWithDb(db); + expect(settings.title).toBe("WP Blog"); + expect(settings.tagline).toBe("From WP"); + }); + + it("overwrites seed placeholders when overwrite=true, skips when false", async () => { + await importSiteSettings({ title: "My Blog" }, db, true); + + const skipped = await importSiteSettings({ title: "Real Title" }, db, false); + expect(skipped.skipped).toEqual(["title"]); + expect((await getSiteSettingsWithDb(db)).title).toBe("My Blog"); + + const overwritten = await importSiteSettings({ title: "Real Title" }, db, true); + expect(overwritten.applied).toEqual(["title"]); + expect((await getSiteSettingsWithDb(db)).title).toBe("Real Title"); + }); + + it("applies logo/favicon only via resolved media IDs", async () => { + const withoutMedia = await importSiteSettings( + { logo: { url: "https://wp.example/logo.png", id: 5 } }, + db, + true, + ); + expect(withoutMedia.applied).toEqual([]); + + await importSiteSettings({ logo: { url: "https://wp.example/logo.png", id: 5 } }, db, true, { + logoMediaId: "01MEDIA", + faviconMediaId: "02MEDIA", + }); + const settings = await getSiteSettingsWithDb(db); + expect(settings.logo?.mediaId).toBe("01MEDIA"); + expect(settings.favicon?.mediaId).toBe("02MEDIA"); + }); +}); + +describe("parseSiteSettingsFromPlugin", () => { + it("maps WP option keys and ignores empty values", () => { + const parsed = parseSiteSettingsFromPlugin({ + blogname: "WP Blog", + blogdescription: "", + custom_logo: 12, + custom_logo_url: "https://wp.example/logo.png", + }); + + expect(parsed.title).toBe("WP Blog"); + expect(parsed.tagline).toBeUndefined(); + expect(parsed.logo).toEqual({ url: "https://wp.example/logo.png", id: 12 }); + expect(parsed.favicon).toBeUndefined(); + }); +}); diff --git a/packages/core/tests/unit/import/wordpress-plugin-source.test.ts b/packages/core/tests/unit/import/wordpress-plugin-source.test.ts index d169e494a1..1e684b8e92 100644 --- a/packages/core/tests/unit/import/wordpress-plugin-source.test.ts +++ b/packages/core/tests/unit/import/wordpress-plugin-source.test.ts @@ -28,6 +28,10 @@ beforeEach(() => { mockFetch.mockReset(); }); +function toUrlString(input: RequestInfo | URL): string { + return typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; +} + // ─── Fixtures ──────────────────────────────────────────────────────────────── function makePost(overrides: Record = {}) { @@ -124,9 +128,50 @@ describe("WordPress Plugin Source — fetch behaviour", () => { expect(items[0]!.customTaxonomies).toEqual({ genre: ["sci-fi", "fantasy"] }); }); + it("surfaces custom fields (ACF/meta) as suggested fields with sanitized slugs", async () => { + const analyzeResponse = { + ...makeAnalyzeResponse(0), + post_types: [ + { + name: "event", + label: "Events", + label_singular: "Event", + total: 3, + by_status: { publish: 3 }, + supports: {}, + taxonomies: [], + custom_fields: [ + { key: "event-start_date", count: 3, inferred_type: "datetime", sample: "2026-01-01" }, + { key: "Ticket Price", count: 3, inferred_type: "number", sample: "25.50" }, + { key: "venue", count: 2, inferred_type: "weird_type", sample: "Hall A" }, + // Collides with a base field -- must not be duplicated + { key: "title", count: 3, inferred_type: "string", sample: "x" }, + ], + hierarchical: false, + has_archive: true, + }, + ], + }; + mockFetch.mockResolvedValueOnce(new Response(JSON.stringify(analyzeResponse), { status: 200 })); + + const analysis = await wordpressPluginSource.analyze( + { type: "url", url: "https://example.com", token: "test-token" }, + {}, + ); + + const fields = analysis.postTypes[0]!.requiredFields; + const bySlug = new Map(fields.map((f) => [f.slug, f])); + expect(bySlug.get("event_start_date")).toMatchObject({ type: "datetime", required: false }); + expect(bySlug.get("ticket_price")).toMatchObject({ type: "number", label: "Ticket Price" }); + // Unknown inferred types fall back to string + expect(bySlug.get("venue")).toMatchObject({ type: "string" }); + // Base fields are not duplicated + expect(fields.filter((f) => f.slug === "title")).toHaveLength(1); + }); + it("fetches every media page during analyze, not just the first", async () => { mockFetch.mockImplementation((input: RequestInfo | URL) => { - const url = String(input); + const url = toUrlString(input); if (url.includes("/analyze")) { return Promise.resolve( new Response(JSON.stringify(makeAnalyzeResponse(3)), { status: 200 }), @@ -148,7 +193,7 @@ describe("WordPress Plugin Source — fetch behaviour", () => { it("falls back to ?rest_route= when the pretty route 404s (plain permalinks)", async () => { mockFetch.mockImplementation((input: RequestInfo | URL) => { - const url = String(input); + const url = toUrlString(input); if (url.includes("/wp-json/")) { return Promise.resolve(new Response("Not Found", { status: 404 })); } From ba8c496e41cc7d518d9070a19fb51d522b124c78 Mon Sep 17 00:00:00 2001 From: swissky <30409887+swissky@users.noreply.github.com> Date: Mon, 6 Jul 2026 00:08:12 +0200 Subject: [PATCH 3/4] feat(import): ACF fields end-to-end, CPT taxonomy auto-creation, and internal link rewriting - Map plugin custom_fields into requiredFields during analysis (with sanitizeFieldSlug so hyphenated meta keys like event-date become valid field slugs) and sanitize incoming meta keys on execute so values land - Filter well-known plugin bookkeeping meta (rank_math_*, wpil_*, ...) out of field suggestions - Ensure import fields per (collection, field) instead of once per collection, so items later in the stream still get their SEO fields - Coerce meta values to the target field type; drop incoercible values instead of failing the whole item - Read created content IDs from data.item.id (was data.id, which broke the WP-ID map and skipped all comments and taxonomy assignments) - Auto-create taxonomy defs for custom CPT taxonomies during plugin import, scoped to the mapped collections (plugin now exports post_types per taxonomy) - Rewrite internal links in imported content to root-relative URLs in both plugin and WXR sources, so posts stop linking back to the old WordPress domain --- .changeset/wp-plugin-import-completeness.md | 2 +- .../admin/src/components/WordPressImport.tsx | 8 + packages/admin/src/lib/api/import.ts | 2 + .../api/import/wordpress-plugin/execute.ts | 166 +++++++++++++++-- .../src/import/sources/wordpress-plugin.ts | 12 +- packages/core/src/import/sources/wxr.ts | 5 +- packages/core/src/import/types.ts | 2 + packages/core/src/import/utils.ts | 173 ++++++++++++++++++ .../plugin-execute-fields.test.ts | 114 +++++++++++- .../unit/import/relativize-links.test.ts | 125 +++++++++++++ .../import/wordpress-plugin-source.test.ts | 11 ++ 11 files changed, 596 insertions(+), 24 deletions(-) create mode 100644 packages/core/tests/unit/import/relativize-links.test.ts diff --git a/.changeset/wp-plugin-import-completeness.md b/.changeset/wp-plugin-import-completeness.md index 8e167e1f22..b93ab22fbe 100644 --- a/.changeset/wp-plugin-import-completeness.md +++ b/.changeset/wp-plugin-import-completeness.md @@ -3,4 +3,4 @@ "@emdash-cms/admin": minor --- -Improves WordPress plugin imports: taxonomy terms and assignments, Yoast/Rank Math SEO titles and descriptions, ACF and custom meta fields (suggested as collection fields during analysis and populated on import), navigation menus, and comments (with authors, dates, threading, and approval status) are now imported. Site identity — title, tagline, logo, and favicon — is taken over from the WordPress site, replacing the starter template's placeholders. Fetches the full media library instead of the first 500 files, supports sites with plain permalinks via a `?rest_route=` fallback, and the import screen accepts a migration key generated by the EmDash Exporter plugin wizard. +Improves WordPress plugin imports: taxonomy terms and assignments (custom post type taxonomies are created as EmDash taxonomies automatically, scoped to the collections they map to), Yoast/Rank Math SEO titles and descriptions, ACF and custom meta fields (suggested as collection fields during analysis and populated on import), navigation menus, and comments (with authors, dates, threading, and approval status) are now imported. Site identity — title, tagline, logo, and favicon — is taken over from the WordPress site, replacing the starter template's placeholders. Internal links in imported content are rewritten to root-relative URLs so they stay on the new site instead of pointing back to the old WordPress domain. Fetches the full media library instead of the first 500 files, supports sites with plain permalinks via a `?rest_route=` fallback, and the import screen accepts a migration key generated by the EmDash Exporter plugin wizard. diff --git a/packages/admin/src/components/WordPressImport.tsx b/packages/admin/src/components/WordPressImport.tsx index eea003792d..bac12f5144 100644 --- a/packages/admin/src/components/WordPressImport.tsx +++ b/packages/admin/src/components/WordPressImport.tsx @@ -2183,6 +2183,14 @@ function CompleteStep({ }), ); } + if (result.taxonomiesCreated && result.taxonomiesCreated.length > 0) { + parts.push( + plural(result.taxonomiesCreated.length, { + one: "# taxonomy created", + other: "# taxonomies created", + }), + ); + } if (result.taxonomyAssignments && result.taxonomyAssignments > 0) { parts.push( plural(result.taxonomyAssignments, { diff --git a/packages/admin/src/lib/api/import.ts b/packages/admin/src/lib/api/import.ts index f3e39fc545..b3b975c6f2 100644 --- a/packages/admin/src/lib/api/import.ts +++ b/packages/admin/src/lib/api/import.ts @@ -161,6 +161,8 @@ export interface ImportResult { taxonomyAssignments?: number; /** Source taxonomies skipped because no matching EmDash taxonomy def exists */ missingTaxonomies?: string[]; + /** Custom taxonomy defs auto-created during the import (plugin import) */ + taxonomiesCreated?: string[]; /** Navigation menu import summary (plugin import) */ menus?: { created: number; items: number }; /** Comment import summary (plugin import) */ diff --git a/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts b/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts index ea0f577ef0..65517572ec 100644 --- a/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts +++ b/packages/core/src/astro/routes/api/import/wordpress-plugin/execute.ts @@ -18,6 +18,7 @@ import { import { requirePerm } from "#api/authorize.js"; import { apiError, apiSuccess, handleError } from "#api/error.js"; +import { handleTaxonomyCreate } from "#api/handlers/taxonomies.js"; import { isParseError, parseBody } from "#api/parse.js"; import { wpPluginExecuteBody } from "#api/schemas.js"; import { BylineRepository } from "#db/repositories/byline.js"; @@ -104,11 +105,15 @@ export const POST: APIRoute = async ({ request, locals }) => { console.log("[WP Plugin Import] Starting import for:", body.url); console.log("[WP Plugin Import] Post types:", postTypes); - // Pre-create taxonomy terms so per-post assignments can attach. + // Pre-create taxonomy defs (for custom CPT taxonomies) and terms so + // per-post assignments can attach. // Non-fatal: a site whose /taxonomies call fails still gets content. let taxonomyPlan: TaxonomyImportPlan | undefined; + let taxonomyDefsCreated: string[] = []; try { - taxonomyPlan = await buildTaxonomyPlan(emdash, body.url, body.token); + const built = await buildTaxonomyPlan(emdash, body.url, body.token, config); + taxonomyPlan = built.plan; + taxonomyDefsCreated = built.defsCreated; } catch (e) { console.warn("[WP Plugin Import] Taxonomy pre-import failed:", e); } @@ -180,6 +185,10 @@ export const POST: APIRoute = async ({ request, locals }) => { console.warn("[WP Plugin Import] Site settings import failed:", e); } + if (taxonomyDefsCreated.length > 0) { + result.taxonomiesCreated = taxonomyDefsCreated; + } + console.log("[WP Plugin Import] Import result:", JSON.stringify(result, null, 2)); return apiSuccess({ @@ -236,6 +245,53 @@ const IMPORT_FIELDS: Array<{ }, ]; +/** + * Coerce a WordPress meta value to an EmDash field type. WP postmeta is + * stringly typed and inconsistent across posts (the same key can hold + * "5", 5, "", or false). Returns `undefined` when the value can't + * reasonably represent the target type — the caller drops it. + */ +export function coerceToFieldType(value: unknown, fieldType: string): unknown { + switch (fieldType) { + case "integer": { + const n = typeof value === "number" ? value : Number(value); + return Number.isInteger(n) ? n : undefined; + } + case "number": { + const n = typeof value === "number" ? value : Number(value); + return Number.isFinite(n) ? n : undefined; + } + case "boolean": { + if (typeof value === "boolean") return value; + if (value === 1 || value === "1" || value === "true" || value === "yes") return true; + if (value === 0 || value === "0" || value === "false" || value === "no") return false; + return undefined; + } + case "datetime": { + if (typeof value !== "string" && typeof value !== "number") return undefined; + const d = new Date(value); + return Number.isNaN(d.getTime()) ? undefined : d.toISOString(); + } + case "json": + // Anything serializes; objects/arrays pass through as-is + return value; + case "string": + case "text": + case "url": + case "select": + case "slug": + case "reference": + if (typeof value === "string") return value; + if (typeof value === "number") return String(value); + return undefined; + default: + // Complex types (image, portableText, repeater, multiSelect, file): + // raw meta can't be coerced reliably -- pass through only if it + // already has a non-primitive shape. + return typeof value === "object" ? value : undefined; + } +} + /** * Pull the per-post SEO title/description out of the Yoast / Rank Math * blobs the plugin source stashes in `item.meta`. Empty strings mean "not @@ -263,13 +319,77 @@ function extractSeo(item: NormalizedItem): { title?: string; description?: strin * reusing the WXR taxonomy machinery (same def-lookup, idempotency, and * collection-filter semantics). */ +/** WP built-in taxonomies that either map to seeded defs or are not content taxonomies. */ +const BUILTIN_TAXONOMIES = new Set(["category", "post_tag", "nav_menu", "post_format"]); + +/** Mirrors NAME_PATTERN in the taxonomy handler -- names that fail stay in missingTaxonomies. */ +const TAXONOMY_NAME_PATTERN = /^[a-z][a-z0-9_]*$/; + +/** + * Create EmDash taxonomy defs for custom WP taxonomies (e.g. CPT taxonomies + * like `company` or `plattform`) that don't exist yet, scoped to the + * collections the enabled post-type mappings target. Returns the names of + * the defs created. Runs before term pre-import so the terms and per-post + * assignments flow through the existing machinery instead of being dropped + * as `missingTaxonomies`. + */ +export async function ensureCustomTaxonomyDefs( + db: EmDashHandlers["db"], + taxonomies: Awaited>, + config: WpPluginImportConfig, +): Promise { + const created: string[] = []; + for (const taxonomy of taxonomies) { + if (BUILTIN_TAXONOMIES.has(taxonomy.name)) continue; + if (taxonomy.terms.length === 0) continue; + if (!TAXONOMY_NAME_PATTERN.test(taxonomy.name)) continue; + + // handleTaxonomyCreate's duplicate guard is locale-scoped; check + // name-wide existence ourselves to stay idempotent on re-runs. + const existing = await db + .selectFrom("_emdash_taxonomy_defs") + .select("id") + .where("name", "=", taxonomy.name) + .executeTakeFirst(); + if (existing) continue; + + // Scope the def to the collections the WP post types map onto. + // Older plugin versions don't send post_types -> empty list = no + // collection filter, which the term machinery treats as "any". + const collections = (taxonomy.post_types ?? []) + .map((postType) => config.postTypeMappings[postType]) + .filter((mapping) => mapping?.enabled) + .map((mapping) => mapping.collection); + + const result = await handleTaxonomyCreate(db, { + name: taxonomy.name, + label: taxonomy.label, + labelSingular: taxonomy.label_singular, + hierarchical: taxonomy.hierarchical, + collections: [...new Set(collections)], + }); + if (result.success) { + created.push(taxonomy.name); + } else { + console.warn( + `[WP Plugin Import] Could not create taxonomy '${taxonomy.name}':`, + result.error.message, + ); + } + } + return created; +} + async function buildTaxonomyPlan( emdash: EmDashHandlers, url: string, token: string, -): Promise { + config: WpPluginImportConfig, +): Promise<{ plan: TaxonomyImportPlan; defsCreated: string[] }> { const taxonomies = await fetchPluginTaxonomies(url, token); + const defsCreated = await ensureCustomTaxonomyDefs(emdash.db, taxonomies, config); + const categories: WxrCategory[] = []; const tags: WxrTag[] = []; const terms: WxrTerm[] = []; @@ -292,7 +412,8 @@ async function buildTaxonomyPlan( } } - return preImportWxrTaxonomies(emdash.db, [], categories, tags, terms, undefined); + const plan = await preImportWxrTaxonomies(emdash.db, [], categories, tags, terms, undefined); + return { plan, defsCreated }; } /** @@ -398,8 +519,9 @@ export async function importContent( // needed by the 30th item, and gating on the first item would skip it. const ensuredFields = new Set(); - // Field slugs per collection, for mapping custom meta/ACF onto real fields - const fieldSlugsByCollection = new Map>(); + // Field slug -> type per collection, for mapping custom meta/ACF onto + // real fields (and coercing values to the field's type) + const fieldTypesByCollection = new Map>(); // Track source translationGroup -> EmDash item ID for translation linking. // Maps source-side translation group ID to the EmDash ID of the first item @@ -458,7 +580,7 @@ export async function importContent( type: field.type, required: false, }); - fieldSlugsByCollection.get(collection)?.add(field.slug); + fieldTypesByCollection.get(collection)?.set(field.slug, field.type); } catch (e) { // Field might already exist from concurrent creation console.log( @@ -469,19 +591,19 @@ export async function importContent( } } - // Load the collection's field slugs once, so custom meta / ACF + // Load the collection's field types once, so custom meta / ACF // values can land in matching schema fields (created by the // prepare step or already present). - let fieldSlugs = fieldSlugsByCollection.get(collection); - if (!fieldSlugs) { - fieldSlugs = new Set(); + let fieldTypes = fieldTypesByCollection.get(collection); + if (!fieldTypes) { + fieldTypes = new Map(); const collectionDef = await schemaRegistry.getCollection(collection); if (collectionDef) { for (const field of await schemaRegistry.listFields(collectionDef.id)) { - fieldSlugs.add(field.slug); + fieldTypes.set(field.slug, field.type); } } - fieldSlugsByCollection.set(collection, fieldSlugs); + fieldTypesByCollection.set(collection, fieldTypes); } // Generate slug from item slug or title @@ -538,12 +660,18 @@ export async function importContent( // when suggesting the fields, so keys like `event-date` land in // the `event_date` field the prepare step created. Values without // a matching field are dropped -- the user controls the schema, - // we don't invent fields per meta key. + // we don't invent fields per meta key. Values are coerced to the + // field's type (the analysis inferred it from ONE sample; real + // values vary) and dropped when incoercible, so one odd meta + // value can't fail the whole item. const assignMetaValue = (key: string, value: unknown) => { if (value === null || value === "") return; const fieldSlug = sanitizeFieldSlug(key); - if (!(fieldSlug in data) && fieldSlugs.has(fieldSlug)) { - data[fieldSlug] = value; + const fieldType = fieldTypes.get(fieldSlug); + if (!fieldType || fieldSlug in data) return; + const coerced = coerceToFieldType(value, fieldType); + if (coerced !== undefined) { + data[fieldSlug] = coerced; } }; const acf = item.meta?._acf; @@ -612,9 +740,9 @@ export async function importContent( result.imported++; result.byCollection[collection] = (result.byCollection[collection] || 0) + 1; - // eslint-disable-next-line typescript/no-unsafe-type-assertion -- handler success data includes id - const createdData = createResult.data as { id?: string } | undefined; - const createdId = createdData?.id; + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- create handler returns { item, _rev } + const createdData = createResult.data as { item?: { id?: string } } | undefined; + const createdId = createdData?.item?.id; if (createdId) { const wpId = Number(item.sourceId); diff --git a/packages/core/src/import/sources/wordpress-plugin.ts b/packages/core/src/import/sources/wordpress-plugin.ts index e8a0230415..67b6679b59 100644 --- a/packages/core/src/import/sources/wordpress-plugin.ts +++ b/packages/core/src/import/sources/wordpress-plugin.ts @@ -30,6 +30,8 @@ import { mapWpStatus, normalizeUrl, checkSchemaCompatibility, + isPluginBookkeepingMeta, + relativizeContentLinks, sanitizeFieldSlug, } from "../utils.js"; @@ -397,6 +399,7 @@ export const wordpressPluginSource: ImportSource = { // matching schema fields and silently drops the values. const knownSlugs = new Set(requiredFields.map((f) => f.slug)); for (const customField of pt.custom_fields ?? []) { + if (isPluginBookkeepingMeta(customField.key)) continue; const slug = sanitizeFieldSlug(customField.key); if (knownSlugs.has(slug)) continue; knownSlugs.add(slug); @@ -509,7 +512,7 @@ export const wordpressPluginSource: ImportSource = { totalPages = data.pages; for (const post of data.items) { - yield pluginPostToNormalizedItem(post); + yield pluginPostToNormalizedItem(post, siteUrl); yielded++; if (options.limit && yielded >= options.limit) { @@ -628,8 +631,9 @@ function getRequestConfig(input: SourceInput): { /** * Convert plugin post to normalized item */ -function pluginPostToNormalizedItem(post: PluginPost): NormalizedItem { +function pluginPostToNormalizedItem(post: PluginPost, siteUrl: string): NormalizedItem { const content = post.content ? gutenbergToPortableText(post.content) : []; + relativizeContentLinks(content, siteUrl); // Extract categories and tags from taxonomies const categories = @@ -739,7 +743,11 @@ export async function fetchPluginTaxonomies( Array<{ name: string; label: string; + /** Singular label (added in emdash-exporter 1.2.0) */ + label_singular?: string; hierarchical: boolean; + /** WP post types this taxonomy is registered for (added in emdash-exporter 1.2.0) */ + post_types?: string[]; terms: Array<{ id: number; name: string; diff --git a/packages/core/src/import/sources/wxr.ts b/packages/core/src/import/sources/wxr.ts index 63a9d86eca..1e2a04ea2d 100644 --- a/packages/core/src/import/sources/wxr.ts +++ b/packages/core/src/import/sources/wxr.ts @@ -35,6 +35,7 @@ import { getFilenameFromUrl, guessMimeType, checkSchemaCompatibility, + relativizeContentLinks, } from "../utils.js"; export const wxrSource: ImportSource = { @@ -92,7 +93,7 @@ export const wxrSource: ImportSource = { } // Convert to normalized item - yield wxrPostToNormalizedItem(post, attachmentMap); + yield wxrPostToNormalizedItem(post, attachmentMap, wxr.site.link || ""); count++; if (options.limit && count >= options.limit) { @@ -281,8 +282,10 @@ function analyzeWxrData( function wxrPostToNormalizedItem( post: WxrPost, attachmentMap: Map, + siteUrl: string, ): NormalizedItem { const content = post.content ? gutenbergToPortableText(post.content) : []; + if (siteUrl) relativizeContentLinks(content, siteUrl); // Resolve featured image: _thumbnail_id is the attachment ID, look up the URL const thumbnailId = post.meta.get("_thumbnail_id"); diff --git a/packages/core/src/import/types.ts b/packages/core/src/import/types.ts index 8d18839149..d78657bbf1 100644 --- a/packages/core/src/import/types.ts +++ b/packages/core/src/import/types.ts @@ -357,6 +357,8 @@ export interface ImportResult { taxonomyAssignments?: number; /** Source taxonomies skipped because no matching EmDash taxonomy def exists */ missingTaxonomies?: string[]; + /** Custom taxonomy defs auto-created during the import (plugin import) */ + taxonomiesCreated?: string[]; /** Navigation menu import summary (plugin import) */ menus?: { created: number; items: number }; /** Comment import summary (plugin import) */ diff --git a/packages/core/src/import/utils.ts b/packages/core/src/import/utils.ts index 52479c97dc..544d8aa38b 100644 --- a/packages/core/src/import/utils.ts +++ b/packages/core/src/import/utils.ts @@ -4,6 +4,7 @@ * Common constants and functions used across all WordPress import sources. */ +import type { PortableTextBlock } from "@emdash-cms/gutenberg-to-portable-text"; import mime from "mime/lite"; import { RESERVED_FIELD_SLUGS } from "../schema/types.js"; @@ -184,6 +185,72 @@ export function inferMetaType( return "string"; } +// ============================================================================= +// Plugin Bookkeeping Meta +// ============================================================================= + +/** + * Meta prefixes written by well-known WordPress plugins as operational + * bookkeeping (sync state, counters, cache keys) — not content. Without + * this filter, a mature site's analysis suggests dozens of junk fields + * per post type and the real content fields drown in them. + * + * ponytail: curated list of the plugins we've seen in the wild, not a + * taxonomy of the WP ecosystem. Unknown plugins' meta still gets through; + * extend the list as real sites surface new offenders. + */ +const PLUGIN_META_PREFIXES = [ + "aawp_", // AAWP (Amazon affiliate) + "algolia_", // Algolia / WP Search with Algolia + "amazon_polly_", // Amazon Polly + "ampforwp_", // AMP for WP + "classifai_", // ClassifAI + "essb_", // Easy Social Share Buttons + "eg_", // Essential Grid + "gnpub_", // Google News publisher tools + "jetpack_", // Jetpack + "mashsb_", // MashShare + "monsterinsights_", // MonsterInsights + "onesignal_", // OneSignal push + "penci_", // Penci themes + "perfmatters_", // Perfmatters + "pys_", // PixelYourSite + "rank_math_", // Rank Math internals (title/description go through the SEO pass) + "rp4wp_", // Related Posts for WP + "saswp_", // Schema & Structured Data for WP + "sbg_", // Simple Blog Grid + "snap_", // SNAP auto-poster + "spay_", // Simple Pay + "tie_", // TieLabs themes + "wl_", // WordLift + "wpil_", // Link Whisper + "wprm_", // WP Recipe Maker internals + "wpswa_", // WP Search with Algolia + "wpuf_", // WP User Frontend + "yarpp_", // YARPP +]; + +/** Exact meta keys that are plugin/core bookkeeping, not content. */ +const PLUGIN_META_KEYS = new Set([ + "entity_same_as", // WordLift + "exclude_from_search", // search exclusion plugins + "footnotes", // Gutenberg core footnotes store + "inline_featured_image", // inline featured image plugin + "os_meta", // theme option stores + "thirstydata", // ThirstyAffiliates +]); + +/** + * Check whether a meta key is well-known plugin bookkeeping that should + * not become a content field. Hyphens are normalized to underscores + * before matching (e.g. `ampforwp-amp-on-off`). + */ +export function isPluginBookkeepingMeta(key: string): boolean { + const normalized = key.replaceAll("-", "_"); + if (PLUGIN_META_KEYS.has(normalized)) return true; + return PLUGIN_META_PREFIXES.some((prefix) => normalized.startsWith(prefix)); +} + // ============================================================================= // Field Slug Sanitization // ============================================================================= @@ -211,6 +278,112 @@ export function sanitizeFieldSlug(key: string): string { return sanitized; } +// ============================================================================= +// Internal Link Relativization +// ============================================================================= + +const REGEX_SPECIALS = /[.*+?^${}()|[\]\\]/g; +const LEADING_WWW = /^www\./; + +/** + * Turn an absolute URL into a root-relative one when it points at the + * source site (www-insensitive). Returns null when the URL should be + * left alone: external links, non-http(s) schemes, and `/wp-content/` + * media files — those stay absolute so the later media pass can match + * them against its old-URL -> new-URL map. + */ +function relativizeUrl(url: string, sourceHost: string): string | null { + if (!url.startsWith("http://") && !url.startsWith("https://")) return null; + try { + const parsed = new URL(url); + if (parsed.hostname.replace(LEADING_WWW, "") !== sourceHost) return null; + if (parsed.pathname.startsWith("/wp-content/")) return null; + return `${parsed.pathname}${parsed.search}${parsed.hash}` || "/"; + } catch { + return null; + } +} + +function relativizeMarkDefs( + markDefs: Array<{ _type: string; [key: string]: unknown }> | undefined, + sourceHost: string, +): void { + for (const def of markDefs ?? []) { + if (def._type === "link" && typeof def.href === "string") { + def.href = relativizeUrl(def.href, sourceHost) ?? def.href; + } + } +} + +/** + * Rewrite internal links in imported content to root-relative URLs, in + * place. Without this, imported posts keep linking back to the old + * WordPress domain (e.g. `https://oldsite.com/companies/google/`) + * instead of staying on the new site. + * + * ponytail: path structures are kept as-is (WP permalink /2024/05/slug/ + * stays /2024/05/slug/) — mapping old paths onto the new site's routes + * is the planned permalink->redirect-map feature. + */ +export function relativizeContentLinks(blocks: PortableTextBlock[], siteUrl: string): void { + let sourceHost: string; + try { + sourceHost = new URL(siteUrl).hostname.replace(LEADING_WWW, ""); + } catch { + return; + } + const hrefPattern = new RegExp( + `href="https?://(?:www\\.)?${sourceHost.replace(REGEX_SPECIALS, "\\$&")}(/[^"]*)?"`, + "g", + ); + + for (const block of blocks) { + switch (block._type) { + case "block": + relativizeMarkDefs(block.markDefs, sourceHost); + break; + case "image": + // asset.url stays absolute (media pass), only the click-through link + if (block.link) block.link = relativizeUrl(block.link, sourceHost) ?? block.link; + break; + case "table": + for (const row of block.rows) { + for (const cell of row.cells) relativizeMarkDefs(cell.markDefs, sourceHost); + } + break; + case "columns": + for (const column of block.columns) relativizeContentLinks(column.content, siteUrl); + break; + case "cover": + relativizeContentLinks(block.content, siteUrl); + break; + case "button": + if (block.url) block.url = relativizeUrl(block.url, sourceHost) ?? block.url; + break; + case "buttons": + for (const button of block.buttons) { + if (button.url) button.url = relativizeUrl(button.url, sourceHost) ?? button.url; + } + break; + case "htmlBlock": + block.html = block.html.replace(hrefPattern, (_m, path: string | undefined) => { + return `href="${path || "/"}"`; + }); + break; + // URL-less or media-only blocks: media URLs are the media pass's job + case "code": + case "embed": + case "gallery": + case "break": + case "file": + case "pullquote": + break; + default: + block satisfies never; + } + } +} + // ============================================================================= // String Utilities // ============================================================================= diff --git a/packages/core/tests/integration/wordpress-import/plugin-execute-fields.test.ts b/packages/core/tests/integration/wordpress-import/plugin-execute-fields.test.ts index cc8c5a5723..c07cd5f08f 100644 --- a/packages/core/tests/integration/wordpress-import/plugin-execute-fields.test.ts +++ b/packages/core/tests/integration/wordpress-import/plugin-execute-fields.test.ts @@ -11,6 +11,8 @@ import { describe, it, expect, beforeEach, afterEach } from "vitest"; import { handleContentCreate } from "../../../src/api/handlers/content.js"; import { + coerceToFieldType, + ensureCustomTaxonomyDefs, importContent, type WpPluginImportConfig, } from "../../../src/astro/routes/api/import/wordpress-plugin/execute.js"; @@ -70,11 +72,28 @@ describe("WordPress plugin import — field auto-creation", () => { }), ]; - const { result } = await importContent(generate(items), config, emdash, manifest, undefined); + const { result, contentIdMap, collectionByWpId } = await importContent( + generate(items), + config, + emdash, + manifest, + undefined, + ); expect(result.errors).toEqual([]); expect(result.imported).toBe(2); + // The WP-ID maps must be populated for created items -- menus, + // comments, and taxonomy attachment all resolve through them. + // Regression: the route read `data.id` but the create handler + // returns `{ item, _rev }`, so the maps stayed empty (309 comments + // skipped on a live migration). + expect([...contentIdMap.keys()]).toEqual([1, 2]); + expect(collectionByWpId.get(1)).toBe("post"); + for (const id of contentIdMap.values()) { + expect(id).toMatch(/^[0-9A-Z]{26}$/); // ULID + } + const row = await db // eslint-disable-next-line typescript/no-explicit-any -- dynamic ec_ table not in the static schema .selectFrom("ec_post" as any) @@ -87,3 +106,96 @@ describe("WordPress plugin import — field auto-creation", () => { }); }); }); + +describe("WordPress plugin import — custom taxonomy def auto-creation", () => { + let db: Kysely; + + beforeEach(async () => { + db = await setupTestDatabaseWithCollections(); + }); + + afterEach(async () => { + await teardownTestDatabase(db); + }); + + const term = { id: 1, name: "Term", slug: "term", description: "", parent: null, count: 1 }; + const config: WpPluginImportConfig = { + postTypeMappings: { + post: { collection: "post", enabled: true }, + company: { collection: "post", enabled: true }, + skipped_cpt: { collection: "old", enabled: false }, + }, + skipExisting: false, + }; + + it("creates defs for custom CPT taxonomies, scoped to mapped collections", async () => { + const created = await ensureCustomTaxonomyDefs( + db, + [ + { + name: "company", + label: "Companies", + label_singular: "Company", + hierarchical: true, + post_types: ["company", "skipped_cpt"], + terms: [term], + }, + // builtins and empty taxonomies must be skipped + { name: "category", label: "Categories", hierarchical: true, terms: [term] }, + { name: "post_format", label: "Formats", hierarchical: false, terms: [term] }, + { name: "empty_tax", label: "Empty", hierarchical: false, terms: [] }, + // names outside /^[a-z][a-z0-9_]*$/ stay in missingTaxonomies + { name: "bad-name", label: "Bad", hierarchical: false, terms: [term] }, + ], + config, + ); + + expect(created).toEqual(["company"]); + + const row = await db + .selectFrom("_emdash_taxonomy_defs") + .selectAll() + .where("name", "=", "company") + .executeTakeFirstOrThrow(); + expect(row.label).toBe("Companies"); + expect(row.label_singular).toBe("Company"); + expect(row.hierarchical).toBe(1); + // disabled mapping (skipped_cpt -> old) must not leak into collections + expect(JSON.parse(row.collections ?? "[]")).toEqual(["post"]); + }); + + it("is idempotent and tolerates old plugins without post_types", async () => { + const taxonomies = [ + { name: "plattform", label: "Plattformen", hierarchical: false, terms: [term] }, + ]; + expect(await ensureCustomTaxonomyDefs(db, taxonomies, config)).toEqual(["plattform"]); + expect(await ensureCustomTaxonomyDefs(db, taxonomies, config)).toEqual([]); + + const row = await db + .selectFrom("_emdash_taxonomy_defs") + .selectAll() + .where("name", "=", "plattform") + .executeTakeFirstOrThrow(); + // no post_types -> no collection filter ("any collection") + expect(JSON.parse(row.collections ?? "[]")).toEqual([]); + }); +}); + +describe("coerceToFieldType", () => { + it("coerces stringly-typed WP meta to the field type, drops incoercible values", () => { + // Real-world failures from a live migration: a field inferred as + // integer from one sample, but other posts hold strings/booleans. + expect(coerceToFieldType("5", "integer")).toBe(5); + expect(coerceToFieldType("abc", "integer")).toBeUndefined(); + expect(coerceToFieldType(false, "string")).toBeUndefined(); + expect(coerceToFieldType(42, "string")).toBe("42"); + expect(coerceToFieldType("2.5", "number")).toBe(2.5); + expect(coerceToFieldType("yes", "boolean")).toBe(true); + expect(coerceToFieldType("0", "boolean")).toBe(false); + expect(coerceToFieldType("maybe", "boolean")).toBeUndefined(); + expect(coerceToFieldType("2026-01-02", "datetime")).toBe("2026-01-02T00:00:00.000Z"); + expect(coerceToFieldType("not a date", "datetime")).toBeUndefined(); + expect(coerceToFieldType({ a: 1 }, "json")).toEqual({ a: 1 }); + expect(coerceToFieldType("123", "image")).toBeUndefined(); + }); +}); diff --git a/packages/core/tests/unit/import/relativize-links.test.ts b/packages/core/tests/unit/import/relativize-links.test.ts new file mode 100644 index 0000000000..9e91a6f71b --- /dev/null +++ b/packages/core/tests/unit/import/relativize-links.test.ts @@ -0,0 +1,125 @@ +/** + * Internal links in imported content must be rewritten to root-relative + * URLs — otherwise migrated posts keep sending readers back to the old + * WordPress domain (live-migration finding: an imported post linked to + * `https://techgarage.blog/companies/google/` instead of the new site). + */ + +import type { PortableTextBlock } from "@emdash-cms/gutenberg-to-portable-text"; +import { describe, it, expect } from "vitest"; + +import { relativizeContentLinks } from "../../../src/import/utils.js"; + +const SITE = "https://techgarage.blog"; + +function textBlock(markDefs: Array>): PortableTextBlock { + return { + _type: "block", + _key: "b1", + children: [{ _type: "span", _key: "s1", text: "x", marks: ["l1"] }], + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- test fixture + markDefs: markDefs as never, + }; +} + +describe("relativizeContentLinks", () => { + it("rewrites internal link markDefs, leaves external and media links alone", () => { + const blocks = [ + textBlock([ + { _type: "link", _key: "l1", href: `${SITE}/companies/google/` }, + { _type: "link", _key: "l2", href: "https://www.techgarage.blog/posts/a?x=1#frag" }, + { _type: "link", _key: "l3", href: "https://example.com/other" }, + { _type: "link", _key: "l4", href: `${SITE}/wp-content/uploads/img.jpg` }, + { _type: "link", _key: "l5", href: "mailto:hi@techgarage.blog" }, + ]), + ]; + + relativizeContentLinks(blocks, SITE); + + const defs = (blocks[0] as { markDefs: Array<{ href: string }> }).markDefs; + expect(defs[0]?.href).toBe("/companies/google/"); + // www-insensitive, query + hash preserved + expect(defs[1]?.href).toBe("/posts/a?x=1#frag"); + expect(defs[2]?.href).toBe("https://example.com/other"); + // media stays absolute for the media pass's URL map + expect(defs[3]?.href).toBe(`${SITE}/wp-content/uploads/img.jpg`); + expect(defs[4]?.href).toBe("mailto:hi@techgarage.blog"); + }); + + it("rewrites image click-through links, buttons, and raw html blocks", () => { + const blocks: PortableTextBlock[] = [ + { + _type: "image", + _key: "i1", + asset: { _type: "reference", _ref: `${SITE}/wp-content/uploads/a.jpg` }, + link: `${SITE}/gallery/`, + }, + { _type: "button", _key: "bt1", text: "Go", url: `${SITE}/pricing/` }, + { + _type: "htmlBlock", + _key: "h1", + html: `About ext`, + }, + ]; + + relativizeContentLinks(blocks, SITE); + + expect(blocks[0]).toMatchObject({ + asset: { _ref: `${SITE}/wp-content/uploads/a.jpg` }, // untouched + link: "/gallery/", + }); + expect(blocks[1]).toMatchObject({ url: "/pricing/" }); + expect(blocks[2]).toMatchObject({ + html: 'About ext', + }); + }); + + it("recurses into columns and rewrites table cell links", () => { + const blocks: PortableTextBlock[] = [ + { + _type: "columns", + _key: "c1", + columns: [ + { + _type: "column", + _key: "co1", + content: [textBlock([{ _type: "link", _key: "l1", href: `${SITE}/nested/` }])], + }, + ], + }, + { + _type: "table", + _key: "t1", + rows: [ + { + _type: "tableRow", + _key: "r1", + cells: [ + { + _type: "tableCell", + _key: "ce1", + content: [], + // eslint-disable-next-line typescript/no-unsafe-type-assertion -- test fixture + markDefs: [{ _type: "link", _key: "l2", href: `${SITE}/cell/` }] as never, + }, + ], + }, + ], + }, + ]; + + relativizeContentLinks(blocks, SITE); + + expect(JSON.stringify(blocks)).toContain('"/nested/"'); + expect(JSON.stringify(blocks)).toContain('"/cell/"'); + expect(JSON.stringify(blocks)).not.toContain("techgarage.blog"); + }); + + it("is a no-op for an unparseable site url", () => { + const blocks = [textBlock([{ _type: "link", _key: "l1", href: `${SITE}/a/` }])]; + relativizeContentLinks(blocks, "not a url"); + expect((blocks[0] as { markDefs: Array<{ href: string }> }).markDefs[0]?.href).toBe( + `${SITE}/a/`, + ); + }); +}); diff --git a/packages/core/tests/unit/import/wordpress-plugin-source.test.ts b/packages/core/tests/unit/import/wordpress-plugin-source.test.ts index 1e684b8e92..79691e5860 100644 --- a/packages/core/tests/unit/import/wordpress-plugin-source.test.ts +++ b/packages/core/tests/unit/import/wordpress-plugin-source.test.ts @@ -146,6 +146,12 @@ describe("WordPress Plugin Source — fetch behaviour", () => { { key: "venue", count: 2, inferred_type: "weird_type", sample: "Hall A" }, // Collides with a base field -- must not be duplicated { key: "title", count: 3, inferred_type: "string", sample: "x" }, + // Plugin bookkeeping -- must not be suggested as content fields + { key: "wpil_sync_report3", count: 3, inferred_type: "integer", sample: "1" }, + { key: "rank_math_seo_score", count: 3, inferred_type: "integer", sample: "80" }, + { key: "entity_same_as", count: 2, inferred_type: "string", sample: "https://x" }, + // Hyphenated variant must be caught too + { key: "ampforwp-amp-on-off", count: 3, inferred_type: "string", sample: "default" }, ], hierarchical: false, has_archive: true, @@ -167,6 +173,11 @@ describe("WordPress Plugin Source — fetch behaviour", () => { expect(bySlug.get("venue")).toMatchObject({ type: "string" }); // Base fields are not duplicated expect(fields.filter((f) => f.slug === "title")).toHaveLength(1); + // Plugin bookkeeping meta is filtered out + expect(bySlug.has("wpil_sync_report3")).toBe(false); + expect(bySlug.has("rank_math_seo_score")).toBe(false); + expect(bySlug.has("entity_same_as")).toBe(false); + expect(bySlug.has("ampforwp_amp_on_off")).toBe(false); }); it("fetches every media page during analyze, not just the first", async () => { From 54643314954d061e4ecfe0d8f7bd77c385cc5848 Mon Sep 17 00:00:00 2001 From: swissky <30409887+swissky@users.noreply.github.com> Date: Mon, 6 Jul 2026 09:15:15 +0200 Subject: [PATCH 4/4] =?UTF-8?q?fix(import):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20pad=20migration=20key=20base64,=20handle=20all=20href=20quot?= =?UTF-8?q?e=20styles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Restore base64 padding in decodeMigrationKey before atob. Browsers' forgiving-base64 accepts unpadded input (verified: remainder-1 lengths are impossible for valid base64), but strict polyfills may not. - relativizeContentLinks htmlBlock handler now matches single-quoted, unquoted, and uppercase href attributes, not just href="...". --- .../admin/src/components/WordPressImport.tsx | 5 ++++- packages/core/src/import/utils.ts | 16 +++++++++++----- .../tests/unit/import/relativize-links.test.ts | 16 ++++++++++++++++ 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/packages/admin/src/components/WordPressImport.tsx b/packages/admin/src/components/WordPressImport.tsx index bac12f5144..b4275eb443 100644 --- a/packages/admin/src/components/WordPressImport.tsx +++ b/packages/admin/src/components/WordPressImport.tsx @@ -77,7 +77,10 @@ const MIGRATION_KEY_PREFIX = "em1."; function decodeMigrationKey(key: string): { url: string; user: string; pass: string } | null { if (!key.startsWith(MIGRATION_KEY_PREFIX)) return null; try { - const base64 = key.slice(MIGRATION_KEY_PREFIX.length).replace(/-/g, "+").replace(/_/g, "/"); + // atob's forgiving-base64 accepts unpadded input, but restore the + // padding anyway so strict polyfills don't reject valid keys. + const base64url = key.slice(MIGRATION_KEY_PREFIX.length).replace(/-/g, "+").replace(/_/g, "/"); + const base64 = base64url.padEnd(base64url.length + ((4 - (base64url.length % 4)) % 4), "="); const parsed: unknown = JSON.parse(atob(base64)); if ( typeof parsed === "object" && diff --git a/packages/core/src/import/utils.ts b/packages/core/src/import/utils.ts index 544d8aa38b..f18ba96ddc 100644 --- a/packages/core/src/import/utils.ts +++ b/packages/core/src/import/utils.ts @@ -332,9 +332,12 @@ export function relativizeContentLinks(blocks: PortableTextBlock[], siteUrl: str } catch { return; } + // Raw HTML in the wild uses double-quoted, single-quoted, and unquoted + // href values; the backreference \1 matches the closing quote (or + // nothing, for unquoted). Rewritten links are normalized to href="...". const hrefPattern = new RegExp( - `href="https?://(?:www\\.)?${sourceHost.replace(REGEX_SPECIALS, "\\$&")}(/[^"]*)?"`, - "g", + `href=(["']?)https?://(?:www\\.)?${sourceHost.replace(REGEX_SPECIALS, "\\$&")}(/[^"'\\s>]*)?\\1`, + "gi", ); for (const block of blocks) { @@ -366,9 +369,12 @@ export function relativizeContentLinks(blocks: PortableTextBlock[], siteUrl: str } break; case "htmlBlock": - block.html = block.html.replace(hrefPattern, (_m, path: string | undefined) => { - return `href="${path || "/"}"`; - }); + block.html = block.html.replace( + hrefPattern, + (_m, _quote: string, path: string | undefined) => { + return `href="${path || "/"}"`; + }, + ); break; // URL-less or media-only blocks: media URLs are the media pass's job case "code": diff --git a/packages/core/tests/unit/import/relativize-links.test.ts b/packages/core/tests/unit/import/relativize-links.test.ts index 9e91a6f71b..2014b21d12 100644 --- a/packages/core/tests/unit/import/relativize-links.test.ts +++ b/packages/core/tests/unit/import/relativize-links.test.ts @@ -74,6 +74,22 @@ describe("relativizeContentLinks", () => { }); }); + it("handles single-quoted, unquoted, and uppercase hrefs in raw html", () => { + const blocks: PortableTextBlock[] = [ + { + _type: "htmlBlock", + _key: "h1", + html: `s b u ext`, + }, + ]; + + relativizeContentLinks(blocks, SITE); + + expect(blocks[0]).toMatchObject({ + html: `s b u ext`, + }); + }); + it("recurses into columns and rewrites table cell links", () => { const blocks: PortableTextBlock[] = [ {