From 259fd913bed9a4fe1dac7a9d3d042ce898fead08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E4=B8=96=E8=80=80?= Date: Tue, 8 Sep 2026 11:52:21 +0800 Subject: [PATCH 1/3] fix(release): sanitize public release notes --- .github/workflows/github-draft-release-v2.yml | 18 +- scripts/sanitize-release-notes.mjs | 166 ++++++++++++++++++ tests/release-workflow.test.ts | 110 +++++++++++- 3 files changed, 280 insertions(+), 14 deletions(-) create mode 100644 scripts/sanitize-release-notes.mjs diff --git a/.github/workflows/github-draft-release-v2.yml b/.github/workflows/github-draft-release-v2.yml index badaa064e..ef10261c9 100644 --- a/.github/workflows/github-draft-release-v2.yml +++ b/.github/workflows/github-draft-release-v2.yml @@ -692,6 +692,13 @@ jobs: fi fi + sanitized_notes="${notes}.sanitized" + if ! node scripts/sanitize-release-notes.mjs "$notes" "$sanitized_notes"; then + echo "::error title=Release notes sanitization failed::Reserved Doc Agent metadata was malformed or could not be removed safely. Manual recovery: inspect the selected release notes source and keep internal audit data in the structured Release assets." >&2 + exit 1 + fi + mv "$sanitized_notes" "$notes" + cat >> "$notes" < - EOF - name: Build auditable release evidence diff --git a/scripts/sanitize-release-notes.mjs b/scripts/sanitize-release-notes.mjs new file mode 100644 index 000000000..8ecdb72b3 --- /dev/null +++ b/scripts/sanitize-release-notes.mjs @@ -0,0 +1,166 @@ +#!/usr/bin/env node + +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const RESERVED_MARKERS = [ + /^|$)/, + /^|$)/, +]; + +function isReservedMarker(value) { + return RESERVED_MARKERS.some((pattern) => pattern.test(value)); +} + +function reservedMarkerOffset(line) { + let offset = line.indexOf(""); + if (closeOffset === -1) continue; + + if (line.slice(closeOffset + 3).trim() !== "") { + throw new Error( + `Reserved release metadata must occupy complete lines (line ${lineNumber})`, + ); + } + + removingReservedComment = false; + continue; + } + + if (fence) { + publicLines.push(line); + if (isFenceClosing(line, fence)) fence = null; + continue; + } + + const openingFence = fenceOpening(line); + if (openingFence) { + fence = openingFence; + publicLines.push(line); + continue; + } + + const markerOffset = reservedMarkerOffset(line); + if (markerOffset === -1) { + publicLines.push(line); + continue; + } + + assertCommentOccupiesCompleteLines(line, markerOffset, lineNumber); + const closeOffset = line.indexOf("-->", markerOffset + 4); + if (closeOffset === -1) { + removingReservedComment = true; + reservedCommentStart = lineNumber; + continue; + } + + if (line.slice(closeOffset + 3).trim() !== "") { + throw new Error( + `Reserved release metadata must occupy complete lines (line ${lineNumber})`, + ); + } + } + + if (removingReservedComment) { + throw new Error( + `Unterminated reserved release metadata starting at line ${reservedCommentStart}`, + ); + } + + const publicMarkdown = publicLines.join("\n").trimEnd(); + if (publicMarkdown.trim() === "") { + throw new Error("Release notes contain no public content after sanitization"); + } + + const sanitized = `${publicMarkdown}\n`; + assertNoReservedMetadata(sanitized); + return sanitized; +} + +function assertNoReservedMetadata(markdown) { + const lines = markdown.split(/\r?\n/); + let fence = null; + + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]; + if (fence) { + if (isFenceClosing(line, fence)) fence = null; + continue; + } + + const openingFence = fenceOpening(line); + if (openingFence) { + fence = openingFence; + continue; + } + + if (reservedMarkerOffset(line) !== -1) { + throw new Error( + `Reserved release metadata remains after sanitization (line ${index + 1})`, + ); + } + } +} + +function main() { + const [inputArg, outputArg, ...extraArgs] = process.argv.slice(2); + if (!inputArg || !outputArg || extraArgs.length > 0) { + throw new Error("Usage: node scripts/sanitize-release-notes.mjs "); + } + + const inputPath = resolve(inputArg); + const outputPath = resolve(outputArg); + if (inputPath === outputPath) { + throw new Error("Input and output paths must be different"); + } + + const markdown = readFileSync(inputPath, "utf8"); + const sanitized = sanitizeReleaseNotes(markdown); + writeFileSync(outputPath, sanitized, "utf8"); +} + +try { + main(); +} catch (error) { + console.error(`sanitize-release-notes: ${error instanceof Error ? error.message : error}`); + process.exitCode = 1; +} diff --git a/tests/release-workflow.test.ts b/tests/release-workflow.test.ts index b28aecd3f..12b0b0d15 100644 --- a/tests/release-workflow.test.ts +++ b/tests/release-workflow.test.ts @@ -9,6 +9,7 @@ const repoRoot = resolve(import.meta.dirname, ".."); const legacyWorkflowPath = resolve(repoRoot, ".github/workflows/github-release.yml"); const draftWorkflowPath = resolve(repoRoot, ".github/workflows/github-draft-release-v2.yml"); const releaseCompareScriptPath = resolve(repoRoot, "scripts/build-release-compare.mjs"); +const releaseNotesSanitizerPath = resolve(repoRoot, "scripts/sanitize-release-notes.mjs"); const ossIntegrityScriptPath = resolve(repoRoot, "scripts/internal/shared/oss-object-integrity.mjs"); const draftSource = readFileSync(draftWorkflowPath, "utf8"); const releaseCompareSource = readFileSync(releaseCompareScriptPath, "utf8"); @@ -57,6 +58,103 @@ function readJson(relativePath: string): { return JSON.parse(readFileSync(resolve(repoRoot, relativePath), "utf8")); } +function runReleaseNotesSanitizer(markdown: string) { + const tempDir = mkdtempSync(resolve(tmpdir(), "memmy-release-notes-sanitizer-")); + const inputPath = resolve(tempDir, "input.md"); + const outputPath = resolve(tempDir, "output.md"); + writeFileSync(inputPath, markdown); + + const result = spawnSync( + "node", + [releaseNotesSanitizerPath, inputPath, outputPath], + { cwd: repoRoot, encoding: "utf8" }, + ); + + return { + result, + output: existsSync(outputPath) ? readFileSync(outputPath, "utf8") : "", + }; +} + +describe("public release notes sanitizer", () => { + it("removes reserved audit comments but preserves public and fenced content", () => { + const { result, output } = runReleaseNotesSanitizer(` +# Memmy v1.1.3 + +Public release notes. + + + + + + + +\`\`\`markdown + + +\`\`\` + +~~~text + +~~~ +`); + + expect(result.status, result.stderr).toBe(0); + expect(output).toContain("# Memmy v1.1.3"); + expect(output).toContain("Public release notes."); + expect(output).toContain(""); + expect(output).toContain(""); + expect(output).toContain("inside: backtick-fence"); + expect(output).toContain("inside: tilde-fence"); + expect(output).not.toContain("memmy-official-changelog-v2"); + expect(output).not.toContain("target_sha: abc123"); + expect(output.endsWith("\n")).toBe(true); + }); + + it("fails closed for unterminated or inline reserved metadata", () => { + const unterminated = runReleaseNotesSanitizer(` +# Memmy + +\n", + ); + expect(inline.result.status).not.toBe(0); + expect(inline.result.stderr).toContain("must occupy complete lines"); + expect(inline.output).toBe(""); + + const afterOrdinaryComment = runReleaseNotesSanitizer( + " \n", + ); + expect(afterOrdinaryComment.result.status).not.toBe(0); + expect(afterOrdinaryComment.result.stderr).toContain("must occupy complete lines"); + expect(afterOrdinaryComment.output).toBe(""); + + const metadataOnly = runReleaseNotesSanitizer( + "\n", + ); + expect(metadataOnly.result.status).not.toBe(0); + expect(metadataOnly.result.stderr).toContain("no public content"); + expect(metadataOnly.output).toBe(""); + }); +}); + describe("Memmy release workflow metadata", () => { it("keeps Memmy metadata aligned while preserving the independent Memory version", () => { const version = readJson("package.json").version; @@ -451,6 +549,15 @@ describe("GitHub Draft Release v2 workflow", () => { expect(releaseNotes).toContain("Release notes generation produced an empty body"); expect(releaseNotes).toContain("RELEASE_NOTES_SOURCE.json"); expect(releaseNotes).toContain("QUALITY_REPORT.json"); + expect(existsSync(releaseNotesSanitizerPath)).toBe(true); + expect(releaseNotes).toContain( + 'node scripts/sanitize-release-notes.mjs "$notes" "$sanitized_notes"', + ); + expect(releaseNotes).toContain('mv "$sanitized_notes" "$notes"'); + expect(releaseNotes).toContain("Release notes sanitization failed"); + expect(releaseNotes).not.toContain("|$)/, /^|$)/, ]; +const CJK_RE = /[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff]/; function isReservedMarker(value) { return RESERVED_MARKERS.some((pattern) => pattern.test(value)); @@ -41,7 +42,130 @@ function assertCommentOccupiesCompleteLines(line, markerOffset, lineNumber) { } } -export function sanitizeReleaseNotes(markdown) { +function removeCjkHeadingSections(markdown) { + const lines = markdown.split(/\r?\n/); + const output = []; + let fence = null; + let htmlComment = false; + let skippedHeadingDepth = null; + + for (const line of lines) { + if (fence) { + if (skippedHeadingDepth === null) output.push(line); + if (isFenceClosing(line, fence)) fence = null; + continue; + } + + const openingFence = fenceOpening(line); + if (openingFence) { + fence = openingFence; + if (skippedHeadingDepth === null) output.push(line); + continue; + } + + if (htmlComment) { + if (skippedHeadingDepth === null) output.push(line); + if (line.includes("-->")) htmlComment = false; + continue; + } + if (line.includes("", line.indexOf("")) htmlComment = false; + continue; + } + const commentOffset = line.indexOf("", commentOffset + 4)) htmlComment = true; + const visiblePrefix = visibleLineForLanguageValidation(line.slice(0, commentOffset)); + if (CJK_RE.test(visiblePrefix)) { + throw new Error( + `English public release notes contain visible CJK text (line ${index + 1})`, + ); + } + continue; + } + + const visible = visibleLineForLanguageValidation(line); + if (CJK_RE.test(visible)) { + throw new Error( + `English public release notes contain visible CJK text (line ${index + 1})`, + ); + } + if ( + /[A-Za-z]/.test(visible) && + !/^[ \t]{0,3}#{1,6}[ \t]/.test(visible) && + !/^[ \t]*(?:[-*_][ \t]*){3,}$/.test(visible) + ) { + hasBodyContent = true; + } + } + + if (!hasBodyContent) { + throw new Error("English public release notes contain no English body content"); + } +} + +function normalizePublicLanguage(markdown, publicLanguage) { + if (!publicLanguage) return markdown; + if (publicLanguage !== "en") { + throw new Error(`Unsupported public release language: ${publicLanguage}`); + } + const normalized = removeCjkHeadingSections(markdown).trimEnd(); + assertEnglishPublicBody(normalized); + return normalized; +} + +export function sanitizeReleaseNotes(markdown, { publicLanguage = "" } = {}) { const lines = markdown.split(/\r?\n/); const publicLines = []; let fence = null; @@ -106,7 +230,10 @@ export function sanitizeReleaseNotes(markdown) { ); } - const publicMarkdown = publicLines.join("\n").trimEnd(); + const publicMarkdown = normalizePublicLanguage( + publicLines.join("\n").trimEnd(), + publicLanguage, + ); if (publicMarkdown.trim() === "") { throw new Error("Release notes contain no public content after sanitization"); } @@ -142,9 +269,17 @@ function assertNoReservedMetadata(markdown) { } function main() { - const [inputArg, outputArg, ...extraArgs] = process.argv.slice(2); - if (!inputArg || !outputArg || extraArgs.length > 0) { - throw new Error("Usage: node scripts/sanitize-release-notes.mjs "); + const [inputArg, outputArg, languageFlag, languageArg, ...extraArgs] = process.argv.slice(2); + const hasLanguage = languageFlag !== undefined || languageArg !== undefined; + if ( + !inputArg || + !outputArg || + extraArgs.length > 0 || + (hasLanguage && (languageFlag !== "--language" || !languageArg)) + ) { + throw new Error( + "Usage: node scripts/sanitize-release-notes.mjs [--language en]", + ); } const inputPath = resolve(inputArg); @@ -154,7 +289,9 @@ function main() { } const markdown = readFileSync(inputPath, "utf8"); - const sanitized = sanitizeReleaseNotes(markdown); + const sanitized = sanitizeReleaseNotes(markdown, { + publicLanguage: languageArg || "", + }); writeFileSync(outputPath, sanitized, "utf8"); } diff --git a/tests/release-workflow.test.ts b/tests/release-workflow.test.ts index 12b0b0d15..ada6a14c1 100644 --- a/tests/release-workflow.test.ts +++ b/tests/release-workflow.test.ts @@ -58,7 +58,7 @@ function readJson(relativePath: string): { return JSON.parse(readFileSync(resolve(repoRoot, relativePath), "utf8")); } -function runReleaseNotesSanitizer(markdown: string) { +function runReleaseNotesSanitizer(markdown: string, publicLanguage?: "en") { const tempDir = mkdtempSync(resolve(tmpdir(), "memmy-release-notes-sanitizer-")); const inputPath = resolve(tempDir, "input.md"); const outputPath = resolve(tempDir, "output.md"); @@ -66,7 +66,12 @@ function runReleaseNotesSanitizer(markdown: string) { const result = spawnSync( "node", - [releaseNotesSanitizerPath, inputPath, outputPath], + [ + releaseNotesSanitizerPath, + inputPath, + outputPath, + ...(publicLanguage ? ["--language", publicLanguage] : []), + ], { cwd: repoRoot, encoding: "utf8" }, ); @@ -153,6 +158,84 @@ schema_version: 2 expect(metadataOnly.result.stderr).toContain("no public content"); expect(metadataOnly.output).toBe(""); }); + + it("keeps only English sections when a reviewed source contains parallel Chinese sections", () => { + const { result, output } = runReleaseNotesSanitizer( + `# Memmy v1.1.3 + +## Fixes + +- Fixed packaged Memory startup. + +## 修复 + +- 修复随包 Memory 的启动问题。 + +## Upgrade notes + +- Memory remains independently versioned. + +## 升级说明 + +- Memory 继续独立发版。 +`, + "en", + ); + + expect(result.status, result.stderr).toBe(0); + expect(output).toContain("## Fixes"); + expect(output).toContain("Fixed packaged Memory startup."); + expect(output).toContain("## Upgrade notes"); + expect(output).not.toMatch(/[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff]/); + expect(output.match(/^## Fixes$/gm)).toHaveLength(1); + }); + + it("rejects Chinese prose that remains inside an English section", () => { + const { result, output } = runReleaseNotesSanitizer( + `# Memmy v1.1.3 + +## Fixes + +- 修复随包 Memory 的启动问题。 +`, + "en", + ); + + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("English public release notes contain visible CJK text"); + expect(output).toBe(""); + }); + + it("allows CJK characters in code spans while validating English prose", () => { + const { result, output } = runReleaseNotesSanitizer( + `# Memmy v1.1.3 + +## Fixes + +- Fixed startup when the configured path is \`C:\\\\用户\\\\Memmy\`. +`, + "en", + ); + + expect(result.status, result.stderr).toBe(0); + expect(output).toContain("`C:\\\\用户\\\\Memmy`"); + }); + + it("normalizes the real v1.1.3 reviewed notes to a single English public body", () => { + const notes = readFileSync( + resolve(repoRoot, ".github/release-notes/v1.1.3.md"), + "utf8", + ); + const { result, output } = runReleaseNotesSanitizer(notes, "en"); + + expect(result.status, result.stderr).toBe(0); + expect(output).toContain("# Memmy v1.1.3"); + expect(output).toContain("## Fixes"); + expect(output).toContain("## Upgrade notes"); + expect(output).not.toMatch(/[\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff]/); + expect(output.match(/^## Fixes$/gm)).toHaveLength(1); + expect(output.match(/^## Upgrade notes$/gm)).toHaveLength(1); + }); }); describe("Memmy release workflow metadata", () => { @@ -537,6 +620,7 @@ describe("GitHub Draft Release v2 workflow", () => { expect(releaseNotes).toContain("DOC_AGENT_RELEASE_NOTES_REQUEST.json"); expect(releaseNotes).toContain("MEMMY_RELEASE_STYLE_EXAMPLES.json"); expect(releaseNotes).toContain("candidate_count: 3"); + expect(releaseNotes).toContain('public_release_language: "en"'); expect(releaseNotes).toContain(".release_notes_md // .release_notes_markdown"); expect(releaseNotes).toContain("Doc Agent draft configuration missing"); expect(releaseNotes).toContain("Doc Agent draft generation failed"); @@ -551,10 +635,12 @@ describe("GitHub Draft Release v2 workflow", () => { expect(releaseNotes).toContain("QUALITY_REPORT.json"); expect(existsSync(releaseNotesSanitizerPath)).toBe(true); expect(releaseNotes).toContain( - 'node scripts/sanitize-release-notes.mjs "$notes" "$sanitized_notes"', + 'node scripts/sanitize-release-notes.mjs "$notes" "$sanitized_notes" --language en', ); expect(releaseNotes).toContain('mv "$sanitized_notes" "$notes"'); expect(releaseNotes).toContain("Release notes sanitization failed"); + expect(releaseNotes).toContain('public_release_language: "en"'); + expect(releaseNotes).toContain("language_validation"); expect(releaseNotes).not.toContain("