fix: mask empty and unclosed fences, keep require() images, rewrite @site images

package.json

@@ -39,7 +39,7 @@
     "format:check": "oxfmt --check",
     "cleanup": "node cleanup.js",
     "prepublishOnly": "npm run build && npm run cleanup",
-    "test:unit": "node tests/test-plugin-options-validation.js && node tests/test-plugin-validation-integration.js && node tests/test-regex-escaping.js && node tests/test-baseurl-handling.js && node tests/test-baseurl-url-construction.js && node tests/test-baseurl-individual-markdown.js && node tests/test-compound-number-prefixes.js && node tests/test-relative-urls.js && node tests/test-content-cleaning-fences.js && node tests/test-jsx-attr-stripping.js && node tests/test-preserve-components.js && node tests/test-tabitem-attr-edge-cases.js && node tests/test-partial-dollar-replacement.js && node tests/test-partial-fence-masking.js && node tests/test-content-loss.js && node tests/test-output-assembly.js && node tests/test-crlf-handling.js && node tests/test-path-transforms.js && node tests/test-header-deduplication.js && node tests/test-import-removal.js && node tests/test-partials.js && node tests/test-site-alias-partials.js && node tests/test-partial-code-fence-imports.js && node tests/test-mdx-page-body-imports.js && node tests/test-missing-partials.js && node tests/test-circular-imports.js && node tests/test-root-content.js && node tests/test-rewrite-image-urls.js && node tests/test-filenames.js && node tests/test-url-encoding.js && node tests/test-nested-paths.js && node tests/test-filename-sanitization.js && node tests/test-yaml-encoding.js && node tests/test-url-error-handling.js && node tests/test-regex-lastindex.js && node tests/test-whitespace-paths.js && node tests/test-unique-identifier-iteration-limit.js && node tests/test-error-handling.js && node tests/test-file-io-error-handling.js && node tests/test-parallel-processing.js && node tests/test-path-length-validation.js && node tests/test-bom-handling.js && node tests/test-batch-processing.js && node tests/test-input-validation.js && node tests/test-add-md-extension.js && node tests/test-import-description-skip.js && node tests/test-description-non-prose.js && node tests/test-filename-sanitization-improved.js",
+    "test:unit": "node tests/test-plugin-options-validation.js && node tests/test-plugin-validation-integration.js && node tests/test-regex-escaping.js && node tests/test-baseurl-handling.js && node tests/test-baseurl-url-construction.js && node tests/test-baseurl-individual-markdown.js && node tests/test-compound-number-prefixes.js && node tests/test-relative-urls.js && node tests/test-content-cleaning-fences.js && node tests/test-jsx-attr-stripping.js && node tests/test-preserve-components.js && node tests/test-tabitem-attr-edge-cases.js && node tests/test-partial-dollar-replacement.js && node tests/test-partial-fence-masking.js && node tests/test-content-loss.js && node tests/test-output-assembly.js && node tests/test-crlf-handling.js && node tests/test-path-transforms.js && node tests/test-header-deduplication.js && node tests/test-import-removal.js && node tests/test-partials.js && node tests/test-site-alias-partials.js && node tests/test-partial-code-fence-imports.js && node tests/test-mdx-page-body-imports.js && node tests/test-missing-partials.js && node tests/test-circular-imports.js && node tests/test-root-content.js && node tests/test-rewrite-image-urls.js && node tests/test-filenames.js && node tests/test-url-encoding.js && node tests/test-nested-paths.js && node tests/test-filename-sanitization.js && node tests/test-yaml-encoding.js && node tests/test-url-error-handling.js && node tests/test-regex-lastindex.js && node tests/test-whitespace-paths.js && node tests/test-unique-identifier-iteration-limit.js && node tests/test-error-handling.js && node tests/test-file-io-error-handling.js && node tests/test-parallel-processing.js && node tests/test-path-length-validation.js && node tests/test-bom-handling.js && node tests/test-batch-processing.js && node tests/test-input-validation.js && node tests/test-add-md-extension.js && node tests/test-import-description-skip.js && node tests/test-description-non-prose.js && node tests/test-filename-sanitization-improved.js && node tests/test-content-edge-cases.js",
     "test:integration": "node tests/test-path-transformation.js && node tests/test-multi-doc.js && node tests/test-draft-integration.js && node tests/test-md-extension-generation.js && node tests/test-regression-fixes.js && node tests/test-multi-version.js && node tests/test-versions-i18n.js && node tests/test-review-followups.js && node tests/test-frontmatter-slug-priority.js && node tests/test-numeric-frontmatter-slug.js && node tests/test-index-filename-case.js && node tests/test-log-level-and-batch-size.js && node tests/test-multi-section-route-scoping.js && node tests/test-path-safety.js && node tests/test-url-resolution.js && node tests/test-small-fixes.js",
     "test": "npm run build && npm run test:unit && npm run test:integration"
   },

src/content.ts

@@ -32,29 +32,58 @@ export function maskCodeSegments(content: string): {
     segments.push(code);
     return token;
   };
+  const restore = (s: string): string =>
+    s.replace(new RegExp(`CODE${id}:(\\d+)`, 'g'), (_t, i) => segments[Number(i)] ?? '');
 
   // Fenced code blocks first: opening fence (>=3 backticks or tildes) through a
   // closing fence of the same character that is at least as long as the
   // opener (CommonMark), so a ```` fence can show a ``` fence inside it. The
+  // block may be empty (the closer on the line after the opener). The
   // leading newline (if any) stays outside the token so line structure is
   // unchanged.
   let masked = content.replace(
-    /(^|\n)([ \t]*(?:(`{3,})[^\n]*\n[\s\S]*?\n[ \t]*\3`*|(~{3,})[^\n]*\n[\s\S]*?\n[ \t]*\4~*)[ \t]*)(?=\n|$)/g,
+    /(^|\n)([ \t]*(?:(`{3,})[^\n]*(?:\n[\s\S]*?)??\n[ \t]*\3`*|(~{3,})[^\n]*(?:\n[\s\S]*?)??\n[ \t]*\4~*)[ \t]*)(?=\n|$)/g,
     (_match, lead, block) => `${lead}${store(block)}`,
   );
 
+  // An opening fence left without a closer runs to the end of its container
+  // (CommonMark): the end of the document, or for an indented opener (inside
+  // a list item) the first non-blank line indented less than the opener. A
+  // backtick opener's info string has no backticks, so a line of inline code
+  // is not an opener. Blocks masked above inside the region are restored into
+  // it, so its token holds the region as written.
+  const unclosedFence = /(^|\n)([ \t]*)(?:`{3,}[^`\n]*|~{3,}[^\n]*)(?=\n|$)/;
+  let open: RegExpExecArray | null;
+  let from = 0;
+  while ((open = unclosedFence.exec(masked.slice(from))) !== null) {
+    const start = from + open.index + open[1].length;
+    const indent = open[2].length;
+    let end = masked.length;
+    if (indent > 0) {
+      const dedent = new RegExp(`\\n(?![ \\t]{${indent}}|[ \\t]*(?:\\n|$))`, 'g');
+      dedent.lastIndex = start;
+      const next = dedent.exec(masked);
+      if (next) end = next.index;
+    }
+    const token = store(restore(masked.slice(start, end)));
+    masked = masked.slice(0, start) + token + masked.slice(end);
+    from = start + token.length;
+  }
+
   // Fences inside blockquotes: every line of the block carries a `>` prefix.
+  // An opener without a closer runs to the end of the blockquote.
   masked = masked.replace(
     /(^|\n)([ \t]*>(?:[ \t]*>)*[ \t]*(?:(`{3,})[^\n]*(?:\n[ \t]*>[^\n]*?)*?\n[ \t]*>(?:[ \t]*>)*[ \t]*\3`*|(~{3,})[^\n]*(?:\n[ \t]*>[^\n]*?)*?\n[ \t]*>(?:[ \t]*>)*[ \t]*\4~*)[ \t]*)(?=\n|$)/g,
     (_match, lead, block) => `${lead}${store(block)}`,
   );
+  masked = masked.replace(
+    /(^|\n)([ \t]*>(?:[ \t]*>)*[ \t]*(?:`{3,}[^`\n]*|~{3,}[^\n]*)(?:\n[ \t]*>[^\n]*)*)(?=\n|$)/g,
+    (_match, lead, block) => `${lead}${store(restore(block))}`,
+  );
 
   // Then inline code spans (`code`, ``co`de``) — a span never crosses a line.
   masked = masked.replace(/(`+)(?:(?!\1)[^\n])+?\1/g, (m) => store(m));
 
-  const restore = (s: string): string =>
-    s.replace(new RegExp(`CODE${id}:(\\d+)`, 'g'), (_t, i) => segments[Number(i)] ?? '');
-
   return { masked, restore };
 }
 
@@ -372,6 +401,24 @@ function quotedTagAttr(tag: string, name: string): string | undefined {
   return isNonEmptyString(raw) ? raw : undefined;
 }
 
+/**
+ * Extract the path from a JSX `src` brace expression on an `<img>` tag: a
+ * string literal (`{"./x.png"}`, `{'./x.png'}`, or a template literal without
+ * `${}`), or a `require()` of one (`{require('./x.png').default}`), which the
+ * bundler resolves like a relative import. Any other expression yields
+ * undefined.
+ */
+function imgSrcExpression(tag: string): string | undefined {
+  const m = tag.match(/\ssrc\s*=\s*\{([^{}]*)\}/i);
+  if (!m) return undefined;
+  const literal = /(?:"([^"]*)"|'([^']*)'|`([^`]*)`)/.source;
+  const expression = m[1]
+    .trim()
+    .match(new RegExp(`^(?:${literal}|require\\(\\s*${literal}\\s*\\)(?:\\.default)?)$`));
+  const text = expression?.slice(1).find(isDefined);
+  return isNonEmptyString(text) && !text.includes('${') ? text : undefined;
+}
+
 /**
  * Clean markdown content for LLM consumption
  * @param content - Raw markdown content
@@ -416,8 +463,11 @@ export function cleanMarkdownContent(
   // capitalized name (<Img>, <Table>) is a component, which the JSX pass
   // further down strips or keeps per preserveComponents.
   cleaned = cleaned.replace(new RegExp(`<img\\b${TAG_ATTRS}/?>`, 'g'), (tag) => {
-    const src = quotedTagAttr(tag, 'src');
-    return src ? `![${quotedTagAttr(tag, 'alt') ?? ''}](${src})` : '';
+    // Restore first: a template-literal value (src={`x`}) was masked as
+    // inline code along with the rest of the content.
+    const source = restore(tag);
+    const src = quotedTagAttr(source, 'src') ?? imgSrcExpression(source);
+    return src ? `![${quotedTagAttr(source, 'alt') ?? ''}](${src})` : '';
   });
 
   // Remove common HTML tags (code blocks are already masked out above),

src/generator.ts

@@ -79,19 +79,33 @@ function cleanDescriptionForToc(description: string, fromFrontMatter: boolean =
     cleaned = cleaned.replace(/!?\[([^\]]*)\]\([^)]*\)/g, '$1');
   }
 
-  // Truncate if too long (150 characters max with ellipsis)
-  if (cleaned.length <= 150) return cleaned;
-  let truncated = cleaned.substring(0, 147);
-  // A cut inside a code span or link would leave it open and break the TOC
-  // line's markdown, so cut before the unclosed opener instead.
-  if ((truncated.match(/`/g) || []).length % 2 === 1) {
-    truncated = truncated.substring(0, truncated.lastIndexOf('`')).trimEnd();
+  // Truncate if too long (150 characters max with ellipsis). A front matter
+  // description cut to its first line can also leave a code span or link
+  // open (it closes on a later line); that line is balanced and marked as cut
+  // too, and a first line with nothing open is kept as written.
+  if (cleaned.length <= 150) {
+    if (!fromFrontMatter || lines.length < 2) return cleaned;
+    const balanced = cutBeforeOpenSpan(cleaned);
+    if (balanced === cleaned) return cleaned;
+    return balanced ? `${balanced}...` : '';
   }
-  const openLink = truncated.search(/!?\[[^\]]*$|!?\[[^\]]*\]\([^)]*$/);
+  return `${cutBeforeOpenSpan(cleaned.substring(0, 147))}...`;
+}
+
+/**
+ * Cut text before an unclosed code span or link opener: a cut inside one
+ * would leave it open and break the TOC line's markdown.
+ */
+function cutBeforeOpenSpan(text: string): string {
+  let cut = text;
+  if ((cut.match(/`/g) || []).length % 2 === 1) {
+    cut = cut.substring(0, cut.lastIndexOf('`')).trimEnd();
+  }
+  const openLink = cut.search(/!?\[[^\]]*$|!?\[[^\]]*\]\([^)]*$/);
   if (openLink !== -1) {
-    truncated = truncated.substring(0, openLink).trimEnd();
+    cut = cut.substring(0, openLink).trimEnd();
   }
-  return `${truncated}...`;
+  return cut;
 }
 
 /**

src/images.ts

@@ -6,6 +6,7 @@ import * as fs from 'fs/promises';
 import * as path from 'path';
 import { Dirent } from 'fs';
 import { maskCodeSegments } from './content';
+import { isDefined } from './guards';
 
 /** Image extensions recognised by Docusaurus / browsers (regex alternation). */
 const IMAGE_EXTENSIONS = 'png|jpe?g|gif|svg|webp|bmp|ico|avif|tiff?';
@@ -57,6 +58,38 @@ export async function buildImageAssetMap(outDir: string): Promise<Map<string, st
   return map;
 }
 
+/**
+ * Find the build-output copy of a site file under a static directory: for
+ * `<siteDir>/<dir>/<p>`, the file `<outDir>/<p>` with the same bytes (each
+ * leading directory is tried as the static directory, shortest first).
+ * Returns its site-root-relative URL path, or null when there is none.
+ */
+async function findStaticCopy(
+  sourceFile: string,
+  siteDir: string,
+  outDir: string,
+): Promise<string | null> {
+  const segments = path.relative(siteDir, sourceFile).split(path.sep);
+  if (segments[0] === '..' || path.isAbsolute(segments[0])) return null;
+  let srcBytes: Buffer;
+  try {
+    srcBytes = await fs.readFile(sourceFile);
+  } catch {
+    return null;
+  }
+  for (let i = 1; i < segments.length; i++) {
+    const rel = segments.slice(i);
+    try {
+      if (srcBytes.equals(await fs.readFile(path.join(outDir, ...rel)))) {
+        return `/${rel.join('/')}`;
+      }
+    } catch {
+      /* no copy at this depth */
+    }
+  }
+  return null;
+}
+
 /**
  * Rewrite relative image references in markdown content to absolute URLs
  * pointing to the hashed build output in `assets/images/`.
@@ -67,8 +100,13 @@ export async function buildImageAssetMap(outDir: string): Promise<Map<string, st
  *   - HTML img tags:    `<img src="./img/foo.png" />`  (both quote styles)
  *
  * As in Docusaurus, a markdown image path that is not absolute, root-relative,
- * or an alias (`@site/...`) resolves against the source file's directory, and
- * is percent-decoded before the lookup.
+ * or an alias resolves against the source file's directory, and is
+ * percent-decoded before the lookup. When `siteDir` is given, a `@site/<p>`
+ * path resolves to `<siteDir>/<p>`: Docusaurus bundles it like a relative
+ * image, so it takes the same lookup. A `@site/` image with no bundled asset
+ * (one small enough to be inlined as a data URI) that the build serves from a
+ * static directory (`@site/static/<p>` copied to `<outDir>/<p>`, confirmed by
+ * comparing bytes) is rewritten to `<siteUrl><p>`.
  *
  * Resolution strategy:
  *   1. Extract the basename from the relative path (e.g., `foo.png`).
@@ -85,6 +123,7 @@ export async function buildImageAssetMap(outDir: string): Promise<Map<string, st
  * @param imageAssetMap  - Lookup map built by `buildImageAssetMap`
  * @param siteUrl        - Site base URL used to build absolute image URLs
  * @param outDir         - Build output directory (needed for byte comparison)
+ * @param siteDir        - Site directory, for resolving `@site/` image paths
  * @returns Content with rewritten image URLs
  */
 export async function rewriteRelativeImageUrls(
@@ -93,6 +132,7 @@ export async function rewriteRelativeImageUrls(
   imageAssetMap: Map<string, string[]>,
   siteUrl: string,
   outDir: string,
+  siteDir?: string,
 ): Promise<string> {
   const baseUrl = siteUrl.endsWith('/') ? siteUrl.slice(0, -1) : siteUrl;
   const sourceDir = path.dirname(sourceFilePath);
@@ -112,7 +152,9 @@ export async function rewriteRelativeImageUrls(
   // filtering is done separately with imgExtRe so we don't accidentally miss
   // legitimate multi-level paths.
   const imageRefRe = /(!\[[^\]]*\]\()(?:<([^<>\n]+)>|([^)"'\s<>]+))|(src=["'])(\.[^"'\s]+)/gi;
-  const isRelative = (p: string): boolean => !/^(?:[a-z][a-z0-9+.-]*:|[/#@~])/i.test(p);
+  const isSiteAlias = (p: string): boolean => isDefined(siteDir) && p.startsWith('@site/');
+  const isRelative = (p: string): boolean =>
+    isSiteAlias(p) || !/^(?:[a-z][a-z0-9+.-]*:|[/#@~])/i.test(p);
   // The file-system path a reference names: no query/fragment, percent-decoded.
   const toFsPath = (ref: string): string => {
     const bare = ref.split('?')[0].split('#')[0];
@@ -123,6 +165,12 @@ export async function rewriteRelativeImageUrls(
     }
   };
 
+  // The source file an image reference names.
+  const toSourceFile = (ref: string): string =>
+    isSiteAlias(ref)
+      ? path.join(siteDir!, toFsPath(ref).slice('@site/'.length))
+      : path.resolve(sourceDir, toFsPath(ref));
+
   // Collect unique relative paths that point to image files
   const uniquePaths = new Set<string>();
   let m: RegExpExecArray | null;
@@ -148,7 +196,7 @@ export async function rewriteRelativeImageUrls(
       assetPath = candidates[0];
     } else if (candidates.length > 1) {
       // Multiple files with same basename — byte-compare to find the right one
-      const absSource = path.resolve(sourceDir, toFsPath(relPath));
+      const absSource = toSourceFile(relPath);
       try {
         const srcBytes = await fs.readFile(absSource);
         for (const candidate of candidates) {
@@ -167,6 +215,13 @@ export async function rewriteRelativeImageUrls(
       }
     }
 
+    // A `@site/<dir>/<p>` image with no bundled asset: the build output
+    // serves it at `/<p>` when `<dir>` is a static directory, which copies
+    // its files to the output root. Matching bytes confirm the copy.
+    if (!assetPath && isSiteAlias(relPath)) {
+      assetPath = await findStaticCopy(toSourceFile(relPath), siteDir!, outDir);
+    }
+
     // Preserve any query string / fragment (e.g. "?raw=1", "#anchor") so we
     // don't change semantics for downstream tooling that relies on them.
     const suffixMatch = relPath.match(/[?#].*$/);

src/processor.ts

@@ -401,6 +401,7 @@ export async function processMarkdownFile(
           imageAssetMap,
           siteUrl,
           outDir,
+          siteDir,
         )
       : cleanedContent;