From ca48d7c4d121685f7a509240b45b495b978a258c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Wed, 29 Jul 2026 00:41:24 +0000 Subject: [PATCH 1/2] fix(render): preserve transparency in GIF output --- .../producer/src/services/distributed/plan.ts | 6 +- .../src/services/render/renderFormat.test.ts | 15 +++++ .../src/services/render/renderFormat.ts | 5 ++ .../render/stages/encodeStage.test.ts | 60 +++++++++++++++++++ .../src/services/render/stages/encodeStage.ts | 6 +- .../services/render/stages/gifEncodeArgs.ts | 7 ++- .../src/services/renderOrchestrator.ts | 13 ++-- packages/producer/src/transparency-test.ts | 52 ++++++++++++++-- .../tests/transparency-regression/meta.json | 2 +- .../transparency-regression/src/index.html | 1 + 10 files changed, 148 insertions(+), 19 deletions(-) create mode 100644 packages/producer/src/services/render/renderFormat.test.ts create mode 100644 packages/producer/src/services/render/renderFormat.ts diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts index 63121e2890..ef989c3676 100644 --- a/packages/producer/src/services/distributed/plan.ts +++ b/packages/producer/src/services/distributed/plan.ts @@ -66,6 +66,7 @@ import { validateNoSystemFonts, } from "../render/planValidation.js"; import { snapshotRuntimeEnv } from "../render/runtimeEnvSnapshot.js"; +import { outputNeedsAlpha } from "../render/renderFormat.js"; import { buildSyntheticRenderJob, buildPlanVideosJson, @@ -888,7 +889,7 @@ export async function plan( // move the contents over once the staged work completes. const finalCompiledDir = join(planDir, "compiled"); - // webm + mov + png-sequence carry alpha — flip force-screenshot so + // Alpha-capable distributed formats flip force-screenshot so // compileStage takes the alpha-aware capture path (BeginFrame doesn't // preserve alpha on Linux headless-shell). Must match the in-process // renderer's needsAlpha logic in `renderOrchestrator.ts` so chunked @@ -897,8 +898,7 @@ export async function plan( // into the planDir and every chunk worker captures opaque RGB — the // libvpx-vp9 alpha sub-stream then encodes either uniform alpha or // gets downgraded by the encoder, producing un-keyable webm output. - const needsAlpha = - config.format === "png-sequence" || config.format === "mov" || config.format === "webm"; + const needsAlpha = outputNeedsAlpha(config.format); // ── Compile ── const compileResult = await runCompileStage({ diff --git a/packages/producer/src/services/render/renderFormat.test.ts b/packages/producer/src/services/render/renderFormat.test.ts new file mode 100644 index 0000000000..7b465993c8 --- /dev/null +++ b/packages/producer/src/services/render/renderFormat.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from "bun:test"; +import { outputNeedsAlpha } from "./renderFormat.js"; + +describe("outputNeedsAlpha", () => { + it("uses alpha-aware capture for transparent-capable formats", () => { + expect(outputNeedsAlpha("gif")).toBe(true); + expect(outputNeedsAlpha("webm")).toBe(true); + expect(outputNeedsAlpha("mov")).toBe(true); + expect(outputNeedsAlpha("png-sequence")).toBe(true); + }); + + it("preserves opaque capture for mp4", () => { + expect(outputNeedsAlpha("mp4")).toBe(false); + }); +}); diff --git a/packages/producer/src/services/render/renderFormat.ts b/packages/producer/src/services/render/renderFormat.ts new file mode 100644 index 0000000000..e07118efe7 --- /dev/null +++ b/packages/producer/src/services/render/renderFormat.ts @@ -0,0 +1,5 @@ +export type RenderOutputFormat = "mp4" | "webm" | "mov" | "png-sequence" | "gif"; + +export function outputNeedsAlpha(format: RenderOutputFormat): boolean { + return format !== "mp4"; +} diff --git a/packages/producer/src/services/render/stages/encodeStage.test.ts b/packages/producer/src/services/render/stages/encodeStage.test.ts index 58df97ae65..537fc401e6 100644 --- a/packages/producer/src/services/render/stages/encodeStage.test.ts +++ b/packages/producer/src/services/render/stages/encodeStage.test.ts @@ -120,6 +120,7 @@ describe("gif encode args", () => { outputPath: "/tmp/hf/demo.gif", fps: { num: 15, den: 1 }, loop: 0, + preserveAlpha: false, }; it("builds the palettegen pass with diff statistics", () => { @@ -151,6 +152,21 @@ describe("gif encode args", () => { "/tmp/hf/demo.gif", ]); }); + + it("reserves transparency and applies the GIF alpha threshold for RGBA frames", () => { + const transparentInput = { + ...input, + framePattern: "frame_%06d.png", + preserveAlpha: true, + }; + + expect(buildGifPalettegenArgs(transparentInput)).toContain( + "fps=15,palettegen=stats_mode=diff:reserve_transparent=1", + ); + expect(buildGifPaletteuseArgs(transparentInput)).toContain( + "fps=15 [x]; [x][1:v] paletteuse=dither=sierra2_4a:alpha_threshold=128", + ); + }); }); describe("runEncodeStage config plumbing", () => { @@ -221,4 +237,48 @@ describe("runEncodeStage config plumbing", () => { resolvedEngineConfig.ffmpegEncodeTimeout, ); }); + + it("encodes alpha GIFs from PNG frames with explicit transparency filters", async () => { + const { runEncodeStage } = await import("./encodeStage.js"); + const paths = createFramesDir("png"); + + await runEncodeStage( + makeInput({ + framesDir: paths.framesDir, + outputPath: join(paths.root, "out.gif"), + videoOnlyPath: join(paths.root, "video-only.mp4"), + isGif: true, + needsAlpha: true, + }), + ); + + expect(runFfmpegMock).toHaveBeenCalledTimes(2); + expect(runFfmpegMock.mock.calls[0]?.[0]).toContain(join(paths.framesDir, "frame_%06d.png")); + expect(runFfmpegMock.mock.calls[0]?.[0]).toContain( + "fps=30,palettegen=stats_mode=diff:reserve_transparent=1", + ); + expect(runFfmpegMock.mock.calls[1]?.[0]).toContain(join(paths.framesDir, "frame_%06d.png")); + expect(runFfmpegMock.mock.calls[1]?.[0]).toContain( + "fps=30 [x]; [x][1:v] paletteuse=dither=sierra2_4a:alpha_threshold=128", + ); + }); + + it("keeps opaque GIF encoding on JPEG frames without alpha-only filters", async () => { + const { runEncodeStage } = await import("./encodeStage.js"); + const paths = createFramesDir("jpg"); + + await runEncodeStage( + makeInput({ + framesDir: paths.framesDir, + outputPath: join(paths.root, "out.gif"), + videoOnlyPath: join(paths.root, "video-only.mp4"), + isGif: true, + needsAlpha: false, + }), + ); + + expect(runFfmpegMock.mock.calls[0]?.[0]).toContain(join(paths.framesDir, "frame_%06d.jpg")); + expect(runFfmpegMock.mock.calls[0]?.[0]).not.toContain("reserve_transparent"); + expect(runFfmpegMock.mock.calls[1]?.[0]).not.toContain("alpha_threshold"); + }); }); diff --git a/packages/producer/src/services/render/stages/encodeStage.ts b/packages/producer/src/services/render/stages/encodeStage.ts index c66e4bedf7..dcebb4a70d 100644 --- a/packages/producer/src/services/render/stages/encodeStage.ts +++ b/packages/producer/src/services/render/stages/encodeStage.ts @@ -123,6 +123,7 @@ async function encodeGifFromDir( fps: Fps; loop: number; palettePath: string; + preserveAlpha: boolean; signal?: AbortSignal; timeout: number; }, @@ -148,6 +149,7 @@ async function encodeGifFromDir( outputPath, fps: input.fps, loop: input.loop, + preserveAlpha: input.preserveAlpha, }; try { const paletteResult = await runFfmpeg(buildGifPalettegenArgs(argsInput), { @@ -258,12 +260,14 @@ export async function runEncodeStage(input: EncodeStageInput): Promise { + const result = await runFfmpeg( + ["-y", "-i", gifPath, "-frames:v", "1", "-pix_fmt", "rgba", "-update", "1", outPng], + { timeout: 60_000 }, + ); + if (!result.success) { + throw new Error( + `ffmpeg failed extracting frame 0 from ${gifPath}: ${result.stderr.slice(-400)}`, + ); + } +} + async function runWebmCheck(workRoot: string): Promise { console.log("\n[webm] rendering transparency-regression …"); const outDir = join(workRoot, "webm"); @@ -141,6 +154,34 @@ async function runWebmCheck(workRoot: string): Promise { console.log("[webm] PASS — transparent + opaque-red pixels verified"); } +async function runGifCheck(workRoot: string): Promise { + console.log("\n[gif] rendering transparency-regression …"); + const outDir = join(workRoot, "gif"); + mkdirSync(outDir, { recursive: true }); + const outPath = join(outDir, "out.gif"); + + const job = createRenderJob({ + fps: { num: 15, den: 1 }, + quality: "draft", + format: "gif", + gifLoop: 0, + }); + + await executeRenderJob(job, FIXTURE_SRC, outPath); + assert.equal(job.status, "complete", `gif render did not complete: status=${job.status}`); + assert.ok(existsSync(outPath), `gif output not written to ${outPath}`); + const size = (await import("node:fs")).statSync(outPath).size; + assert.ok(size > 0, `gif output ${outPath} is empty`); + console.log(`[gif] rendered ${outPath} (${size} bytes)`); + + const framePng = join(outDir, "frame-0.png"); + await extractFirstFrameFromGif(outPath, framePng); + const decoded = decodePng(readFileSync(framePng)); + assertAlphaPixel(decoded, TRANSPARENT_X, TRANSPARENT_Y, "transparent", "gif"); + assertAlphaPixel(decoded, OPAQUE_X, OPAQUE_Y, "opaque-red", "gif"); + console.log("[gif] PASS — transparent + opaque-red pixels verified"); +} + async function runPngSequenceCheck(workRoot: string): Promise { console.log("\n[png-sequence] rendering transparency-regression …"); const outDir = join(workRoot, "pngs"); @@ -165,14 +206,14 @@ async function runPngSequenceCheck(workRoot: string): Promise { .sort(); assert.equal( frames.length, - FPS, // 1 second at 30fps = 30 frames - `png-sequence expected ${FPS} frames, got ${frames.length}: ${frames.join(",")}`, + PNG_SEQUENCE_FRAME_COUNT, // 1 second at 30fps = 30 frames + `png-sequence expected ${PNG_SEQUENCE_FRAME_COUNT} frames, got ${frames.length}: ${frames.join(",")}`, ); assert.equal(frames[0], "frame_000001.png", "first frame should be frame_000001.png"); assert.equal( frames[frames.length - 1], - `frame_${String(FPS).padStart(6, "0")}.png`, - `last frame should be frame_${String(FPS).padStart(6, "0")}.png`, + `frame_${String(PNG_SEQUENCE_FRAME_COUNT).padStart(6, "0")}.png`, + `last frame should be frame_${String(PNG_SEQUENCE_FRAME_COUNT).padStart(6, "0")}.png`, ); console.log(`[png-sequence] wrote ${frames.length} frames to ${outDir}`); @@ -195,6 +236,7 @@ async function main(): Promise { try { await runWebmCheck(workRoot); + await runGifCheck(workRoot); await runPngSequenceCheck(workRoot); console.log("\nAll transparency assertions passed."); } finally { diff --git a/packages/producer/tests/transparency-regression/meta.json b/packages/producer/tests/transparency-regression/meta.json index 31d4eea61d..48ecf3ce02 100644 --- a/packages/producer/tests/transparency-regression/meta.json +++ b/packages/producer/tests/transparency-regression/meta.json @@ -1,6 +1,6 @@ { "name": "Transparency Regression", - "description": "Asserts that webm + png-sequence outputs preserve a real alpha channel end-to-end. Exercised by `tsx src/transparency-test.ts`, NOT by the standard regression harness (which compares against a golden MP4).", + "description": "Asserts that webm + gif + png-sequence outputs preserve transparent pixels end-to-end. Exercised by `tsx src/transparency-test.ts`, NOT by the standard regression harness (which compares against a golden MP4).", "tags": ["transparency", "alpha", "smoke"], "renderConfig": { "fps": 30 diff --git a/packages/producer/tests/transparency-regression/src/index.html b/packages/producer/tests/transparency-regression/src/index.html index 526c671b6c..24e5275546 100644 --- a/packages/producer/tests/transparency-regression/src/index.html +++ b/packages/producer/tests/transparency-regression/src/index.html @@ -34,6 +34,7 @@ Date: Wed, 29 Jul 2026 01:52:28 +0000 Subject: [PATCH 2/2] fix(render): preserve GIF shader transitions --- .../src/services/render/renderFormat.test.ts | 15 +++- .../src/services/render/renderFormat.ts | 4 + .../src/services/renderOrchestrator.ts | 24 +++--- packages/producer/src/transparency-test.ts | 85 ++++++++++++++++++- 4 files changed, 114 insertions(+), 14 deletions(-) diff --git a/packages/producer/src/services/render/renderFormat.test.ts b/packages/producer/src/services/render/renderFormat.test.ts index 7b465993c8..1435d42cfc 100644 --- a/packages/producer/src/services/render/renderFormat.test.ts +++ b/packages/producer/src/services/render/renderFormat.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "bun:test"; -import { outputNeedsAlpha } from "./renderFormat.js"; +import { outputNeedsAlpha, outputSupportsPageSideShaderCompositing } from "./renderFormat.js"; describe("outputNeedsAlpha", () => { it("uses alpha-aware capture for transparent-capable formats", () => { @@ -13,3 +13,16 @@ describe("outputNeedsAlpha", () => { expect(outputNeedsAlpha("mp4")).toBe(false); }); }); + +describe("outputSupportsPageSideShaderCompositing", () => { + it("supports opaque MP4 capture and RGBA GIF disk frames", () => { + expect(outputSupportsPageSideShaderCompositing("mp4")).toBe(true); + expect(outputSupportsPageSideShaderCompositing("gif")).toBe(true); + }); + + it("keeps the alpha video and PNG sequence formats on their existing paths", () => { + expect(outputSupportsPageSideShaderCompositing("webm")).toBe(false); + expect(outputSupportsPageSideShaderCompositing("mov")).toBe(false); + expect(outputSupportsPageSideShaderCompositing("png-sequence")).toBe(false); + }); +}); diff --git a/packages/producer/src/services/render/renderFormat.ts b/packages/producer/src/services/render/renderFormat.ts index e07118efe7..fd654e7386 100644 --- a/packages/producer/src/services/render/renderFormat.ts +++ b/packages/producer/src/services/render/renderFormat.ts @@ -3,3 +3,7 @@ export type RenderOutputFormat = "mp4" | "webm" | "mov" | "png-sequence" | "gif" export function outputNeedsAlpha(format: RenderOutputFormat): boolean { return format !== "mp4"; } + +export function outputSupportsPageSideShaderCompositing(format: RenderOutputFormat): boolean { + return format === "mp4" || format === "gif"; +} diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index 8c4144debf..99569d4375 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -101,7 +101,11 @@ import { VIRTUAL_TIME_SHIM, } from "./fileServer.js"; import { defaultLogger, type ProducerLogger } from "../logger.js"; -import { outputNeedsAlpha, type RenderOutputFormat } from "./render/renderFormat.js"; +import { + outputNeedsAlpha, + outputSupportsPageSideShaderCompositing, + type RenderOutputFormat, +} from "./render/renderFormat.js"; import { createMemorySampler, type MemorySampler, updateJobStatus } from "./render/shared.js"; import { buildRenderErrorDetails } from "./render/cleanup.js"; import { publishRenderFailure } from "./render/renderEventPublisher.js"; @@ -2811,19 +2815,16 @@ async function executeRenderPipeline(input: { // Page-side compositing opt-in: when the engine is configured to run the // shader blend inside Chrome via a page-side WebGL canvas, the layered // Node-side composite path is unnecessary for SDR shader transitions. - // The streaming path takes ONE opaque RGB screenshot per output frame — - // exactly the single capture the page-side compositor produces. HDR - // content still forces the layered path (HDR layers need per-layer - // alpha + native HDR raw frame compositing in Node; that's out of scope - // for this opt-in). GIF also uses this path for shader transitions - // because its two-pass palette encoder needs disk frames, not the - // layered path's streaming raw-video encoder. + // MP4's streaming path takes one opaque RGB screenshot per output frame. + // GIF takes the same page-side composite through its RGBA PNG disk-frame + // path so the palette encoder can preserve transparency. HDR content still + // forces the layered path (HDR layers need per-layer alpha + native HDR raw + // frame compositing in Node; that's out of scope for this opt-in). const usePageSideCompositingForTransitions = (cfg.enablePageSideCompositing || isGif) && compiled.hasShaderTransitions && !hasHdrContent && - !isPngSequence && - !needsAlpha; + outputSupportsPageSideShaderCompositing(outputFormat); if (usePageSideCompositingForTransitions) { activeFileServer.addPreHeadScript(HF_PAGE_SIDE_COMPOSITING_STUB); if ( @@ -2847,7 +2848,8 @@ async function executeRenderPipeline(input: { updateCaptureObservability({ forceScreenshot: captureForceScreenshot }); log.info( "[Render] Page-side compositing enabled — bypassing Node-side layered " + - "shader-blend path. Engine will capture one opaque RGB screenshot per output frame.", + `shader-blend path. Engine will capture one ${needsAlpha ? "RGBA PNG" : "opaque RGB"} ` + + "screenshot per output frame.", ); } const useLayeredComposite = diff --git a/packages/producer/src/transparency-test.ts b/packages/producer/src/transparency-test.ts index 9219e59d4d..7cb87058c2 100644 --- a/packages/producer/src/transparency-test.ts +++ b/packages/producer/src/transparency-test.ts @@ -2,12 +2,15 @@ * Transparency Regression Test * * Exercises the alpha-output pipelines (webm + gif + png-sequence) end-to-end - * against `tests/transparency-regression/`. Asserts that: + * against `tests/transparency-regression/`, then renders the real page-side + * shader fixture to GIF. Asserts that: * * 1. Pixels that were transparent in the browser stay transparent in the * output (alpha = 0). * 2. Pixels covered by the opaque red `.card` element stay fully opaque * (alpha = 255) and keep their red color. + * 3. GIF's RGBA disk-frame path still captures the authored WebGL shader + * transition instead of the virtual-time DOM fallback. * * This is intentionally NOT wired into `regression-harness.ts` — the harness * compares each fixture against a golden MP4, but transparency requires a @@ -22,12 +25,15 @@ import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { decodePng, runFfmpeg } from "@hyperframes/engine"; +import { decodePng, psnrDb, runFfmpeg } from "@hyperframes/engine"; import { createRenderJob, executeRenderJob } from "./services/renderOrchestrator.js"; const moduleDir = dirname(fileURLToPath(import.meta.url)); const FIXTURE_DIR = resolve(moduleDir, "../tests/transparency-regression"); const FIXTURE_SRC = join(FIXTURE_DIR, "src"); +const SHADER_FIXTURE_DIR = resolve(moduleDir, "../tests/page-side-shader-compositor-render-compat"); +const SHADER_FIXTURE_SRC = join(SHADER_FIXTURE_DIR, "src"); +const SHADER_GOLDEN = join(SHADER_FIXTURE_DIR, "output", "output.mp4"); const WIDTH = 200; const HEIGHT = 200; @@ -127,6 +133,33 @@ async function extractFirstFrameFromGif(gifPath: string, outPng: string): Promis } } +async function extractFrameAtIndex( + inputPath: string, + frameIndex: number, + outPng: string, +): Promise { + const result = await runFfmpeg( + [ + "-y", + "-i", + inputPath, + "-vf", + `select=eq(n\\,${frameIndex})`, + "-frames:v", + "1", + "-update", + "1", + outPng, + ], + { timeout: 60_000 }, + ); + if (!result.success) { + throw new Error( + `ffmpeg failed extracting frame ${frameIndex} from ${inputPath}: ${result.stderr.slice(-400)}`, + ); + } +} + async function runWebmCheck(workRoot: string): Promise { console.log("\n[webm] rendering transparency-regression …"); const outDir = join(workRoot, "webm"); @@ -182,6 +215,50 @@ async function runGifCheck(workRoot: string): Promise { console.log("[gif] PASS — transparent + opaque-red pixels verified"); } +async function runGifShaderTransitionCheck(workRoot: string): Promise { + console.log("\n[gif-shader] rendering page-side shader transition …"); + const outDir = join(workRoot, "gif-shader"); + mkdirSync(outDir, { recursive: true }); + const outPath = join(outDir, "out.gif"); + + const job = createRenderJob({ + fps: { num: 15, den: 1 }, + quality: "draft", + format: "gif", + gifLoop: 0, + workers: 1, + }); + + await executeRenderJob(job, SHADER_FIXTURE_SRC, outPath); + assert.equal(job.status, "complete", `gif shader render did not complete: status=${job.status}`); + assert.ok(existsSync(outPath), `gif shader output not written to ${outPath}`); + + const gifBefore = join(outDir, "gif-before.png"); + const gifTransition = join(outDir, "gif-transition.png"); + const goldenBefore = join(outDir, "golden-before.png"); + const goldenTransition = join(outDir, "golden-transition.png"); + await Promise.all([ + extractFrameAtIndex(outPath, 7, gifBefore), + extractFrameAtIndex(outPath, 17, gifTransition), + extractFrameAtIndex(SHADER_GOLDEN, 14, goldenBefore), + extractFrameAtIndex(SHADER_GOLDEN, 34, goldenTransition), + ]); + + const beforePsnr = await psnrDb(readFileSync(gifBefore), readFileSync(goldenBefore)); + const transitionPsnr = await psnrDb(readFileSync(gifTransition), readFileSync(goldenTransition)); + assert.ok( + beforePsnr >= 25, + `gif shader control frame expected >=25 dB against the golden, got ${beforePsnr.toFixed(2)} dB`, + ); + assert.ok( + transitionPsnr >= 20, + `gif shader transition expected >=20 dB against the golden, got ${transitionPsnr.toFixed(2)} dB`, + ); + console.log( + `[gif-shader] PASS — control ${beforePsnr.toFixed(2)} dB, transition ${transitionPsnr.toFixed(2)} dB`, + ); +} + async function runPngSequenceCheck(workRoot: string): Promise { console.log("\n[png-sequence] rendering transparency-regression …"); const outDir = join(workRoot, "pngs"); @@ -229,6 +306,9 @@ async function main(): Promise { if (!existsSync(FIXTURE_SRC)) { throw new Error(`Fixture missing: ${FIXTURE_SRC}`); } + if (!existsSync(SHADER_FIXTURE_SRC) || !existsSync(SHADER_GOLDEN)) { + throw new Error(`Shader fixture or golden missing: ${SHADER_FIXTURE_DIR}`); + } const workRoot = join(tmpdir(), `hf-transparency-${process.pid}-${Date.now()}`); mkdirSync(workRoot, { recursive: true }); const keepWork = process.env.KEEP_TEMP === "1"; @@ -237,6 +317,7 @@ async function main(): Promise { try { await runWebmCheck(workRoot); await runGifCheck(workRoot); + await runGifShaderTransitionCheck(workRoot); await runPngSequenceCheck(workRoot); console.log("\nAll transparency assertions passed."); } finally {