From 53e3175a41355b641a4ec3a6d68314c404f3f4c1 Mon Sep 17 00:00:00 2001 From: James Xiao Date: Tue, 26 May 2026 15:52:21 +0800 Subject: [PATCH 1/2] fix(core): use raw data-start for media elements in preview visibility loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For video and audio elements, data-start is authored in global (composition-root) time — the same contract used by the render pipeline's discoverMediaFromBrowser, which reads the raw attribute directly. Previously, the visibility loop called resolveStartForElement which adds the nearest ancestor composition's global start on top, causing a double-offset that kept pip-wired media permanently hidden when the host composition did not start at t=0. Example: a pip video with data-start="45.40" inside a host composition that also starts at data-start="45.40" resolved to 90.80, so the video was always hidden during its actual [45.40, 52.46] window. Non-media elements (divs, sections, etc.) continue to use the accumulating resolver because their data-start values are local to their composition. Co-Authored-By: Claude Sonnet 4.6 --- packages/core/src/runtime/init.test.ts | 62 ++++++++++++++++++++++++++ packages/core/src/runtime/init.ts | 10 ++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index 3a3ffa780..4122095a9 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -477,6 +477,68 @@ describe("initSandboxRuntimeModular", () => { expect(hookHost.style.visibility).toBe("visible"); }); + it("shows pip video at global start time even when host composition starts late", () => { + // Regression: resolveStartForElement used to add the host composition's start on top of + // the video's own data-start, causing double-offset. A pip video with data-start="45.40" + // inside a host at data-start="45.40" would resolve to 90.80 and stay permanently hidden. + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-start", "0"); + root.setAttribute("data-width", "1920"); + root.setAttribute("data-height", "1080"); + document.body.appendChild(root); + + const host = document.createElement("div"); + host.setAttribute("data-composition-id", "scene-pip"); + host.setAttribute("data-start", "45.40"); + host.setAttribute("data-duration", "7.06"); + root.appendChild(host); + + const innerRoot = document.createElement("div"); + innerRoot.setAttribute("data-composition-id", "scene-pip"); + host.appendChild(innerRoot); + + // pip-wired video: data-start is authored in global time (same value as host) + const pipVideo = document.createElement("video"); + pipVideo.setAttribute("data-start", "45.40"); + pipVideo.setAttribute("data-duration", "7.06"); + Object.defineProperty(pipVideo, "paused", { value: true, configurable: true }); + Object.defineProperty(pipVideo, "readyState", { value: 0, configurable: true }); + Object.defineProperty(pipVideo, "currentTime", { + value: 0, + writable: true, + configurable: true, + }); + pipVideo.load = () => {}; + innerRoot.appendChild(pipVideo); + + (window as Window & { __timelines?: Record }).__timelines = { + main: createMockTimeline(60), + "scene-pip": createMockTimeline(7.06), + }; + + initSandboxRuntimeModular(); + + const player = ( + window as Window & { + __player?: { seek: (timeSeconds: number) => void }; + } + ).__player; + expect(player).toBeDefined(); + + // Before the fix: resolveStartForElement(pipVideo) = 45.40 + 45.40 = 90.80, so the + // video would be hidden at t=46 (90.80 > 46). After the fix: start = 45.40, visible. + player?.seek(46); + expect(pipVideo.style.visibility).toBe("visible"); + + player?.seek(53); + expect(pipVideo.style.visibility).toBe("hidden"); + + player?.seek(44); + expect(pipVideo.style.visibility).toBe("hidden"); + }); + it("plays scheduled child timelines without a captured root timeline when audio has failed", () => { const raf = createManualRaf(); vi.spyOn(performance, "now").mockImplementation(() => raf.now()); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index 2357cb24f..ed7e65570 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -1329,7 +1329,15 @@ export function initSandboxRuntimeModular(): void { const tag = rawNode.tagName.toLowerCase(); if (tag === "script" || tag === "style" || tag === "link" || tag === "meta") continue; - const start = resolveStartForElement(rawNode, 0); + // For media elements (video/audio) data-start is authored in global (composition-root) + // time — the same contract used by the render pipeline's discoverMediaFromBrowser which + // reads the raw attribute directly. Calling resolveStartForElement would add the nearest + // ancestor composition's start a second time, creating a double-offset that keeps the + // element permanently hidden when its host composition does not start at t=0. + const isMediaElement = tag === "video" || tag === "audio"; + const start = isMediaElement + ? Math.max(0, Number(rawNode.getAttribute("data-start") ?? 0) || 0) + : resolveStartForElement(rawNode, 0); let duration = resolveDurationForElement(rawNode); const compId = rawNode.getAttribute("data-composition-id"); if (compId) { From 5272d62297bfe86e5abb1454617086287bba52e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20=C3=81ngel?= Date: Tue, 26 May 2026 16:53:11 +0000 Subject: [PATCH 2/2] fix(core): extend media start fix to all consumers, guard auto-start Narrow the raw data-start read to media elements without data-hf-auto-start (explicitly authored global coordinates). Elements with auto-injected data-start="0" remain composition-local via the resolver. Apply consistently across all three consumers: - visibility loop (init.ts) - refreshRuntimeMediaCache start/duration (init.ts) - resolveMediaWindowEndSeconds (timeline.ts) Add regression test for auto-injected data-start="0" inside a late-starting host to prove it doesn't regress. --- packages/core/src/runtime/init.test.ts | 57 ++++++++++++++++++++++++++ packages/core/src/runtime/init.ts | 23 +++++++---- packages/core/src/runtime/timeline.ts | 4 +- 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/packages/core/src/runtime/init.test.ts b/packages/core/src/runtime/init.test.ts index 4122095a9..12d712a19 100644 --- a/packages/core/src/runtime/init.test.ts +++ b/packages/core/src/runtime/init.test.ts @@ -539,6 +539,63 @@ describe("initSandboxRuntimeModular", () => { expect(pipVideo.style.visibility).toBe("hidden"); }); + it("shows auto-injected video at host time, not at t=0", () => { + const root = document.createElement("div"); + root.setAttribute("data-composition-id", "main"); + root.setAttribute("data-root", "true"); + root.setAttribute("data-start", "0"); + root.setAttribute("data-width", "1920"); + root.setAttribute("data-height", "1080"); + document.body.appendChild(root); + + const host = document.createElement("div"); + host.setAttribute("data-composition-id", "intro"); + host.setAttribute("data-start", "10"); + host.setAttribute("data-duration", "5"); + root.appendChild(host); + + const innerRoot = document.createElement("div"); + innerRoot.setAttribute("data-composition-id", "intro"); + host.appendChild(innerRoot); + + const video = document.createElement("video"); + video.setAttribute("data-start", "0"); + video.setAttribute("data-hf-auto-start", ""); + video.setAttribute("data-duration", "5"); + Object.defineProperty(video, "paused", { value: true, configurable: true }); + Object.defineProperty(video, "readyState", { value: 0, configurable: true }); + Object.defineProperty(video, "currentTime", { + value: 0, + writable: true, + configurable: true, + }); + video.load = () => {}; + innerRoot.appendChild(video); + + (window as Window & { __timelines?: Record }).__timelines = { + main: createMockTimeline(30), + intro: createMockTimeline(5), + }; + + initSandboxRuntimeModular(); + + const player = ( + window as Window & { + __player?: { seek: (timeSeconds: number) => void }; + } + ).__player; + expect(player).toBeDefined(); + + player?.seek(12); + expect(video.style.visibility).toBe("visible"); + + player?.seek(5); + expect(video.style.visibility).toBe("hidden"); + + player?.seek(16); + expect(video.style.visibility).toBe("hidden"); + }); + it("plays scheduled child timelines without a captured root timeline when audio has failed", () => { const raf = createManualRaf(); vi.spyOn(performance, "now").mockImplementation(() => raf.now()); diff --git a/packages/core/src/runtime/init.ts b/packages/core/src/runtime/init.ts index ed7e65570..1e32a71e9 100644 --- a/packages/core/src/runtime/init.ts +++ b/packages/core/src/runtime/init.ts @@ -1278,6 +1278,9 @@ export function initSandboxRuntimeModular(): void { element.hasAttribute("data-start") || Boolean(resolveMediaCompositionContext(element).compositionRoot), resolveStartSeconds: (element) => { + if (!element.hasAttribute("data-hf-auto-start") && element.hasAttribute("data-start")) { + return Math.max(0, Number(element.getAttribute("data-start") ?? 0) || 0); + } const context = resolveMediaCompositionContext( element as HTMLVideoElement | HTMLAudioElement, ); @@ -1285,7 +1288,10 @@ export function initSandboxRuntimeModular(): void { }, resolveDurationSeconds: (element) => { const context = resolveMediaCompositionContext(element); - const start = resolveStartForElement(element, context.inheritedStart ?? 0); + const start = + !element.hasAttribute("data-hf-auto-start") && element.hasAttribute("data-start") + ? Math.max(0, Number(element.getAttribute("data-start") ?? 0) || 0) + : resolveStartForElement(element, context.inheritedStart ?? 0); const mediaStart = Number.parseFloat(element.dataset.playbackStart ?? element.dataset.mediaStart ?? "0") || 0; @@ -1329,13 +1335,14 @@ export function initSandboxRuntimeModular(): void { const tag = rawNode.tagName.toLowerCase(); if (tag === "script" || tag === "style" || tag === "link" || tag === "meta") continue; - // For media elements (video/audio) data-start is authored in global (composition-root) - // time — the same contract used by the render pipeline's discoverMediaFromBrowser which - // reads the raw attribute directly. Calling resolveStartForElement would add the nearest - // ancestor composition's start a second time, creating a double-offset that keeps the - // element permanently hidden when its host composition does not start at t=0. - const isMediaElement = tag === "video" || tag === "audio"; - const start = isMediaElement + // Media elements with explicitly authored data-start (no data-hf-auto-start + // marker) use global coordinates — matching the render pipeline's + // discoverMediaFromBrowser. resolveStartForElement would add the host + // composition's offset a second time. Auto-injected data-start="0" + // (data-hf-auto-start present) is composition-local and needs the resolver. + const isGlobalMediaStart = + (tag === "video" || tag === "audio") && !rawNode.hasAttribute("data-hf-auto-start"); + const start = isGlobalMediaStart ? Math.max(0, Number(rawNode.getAttribute("data-start") ?? 0) || 0) : resolveStartForElement(rawNode, 0); let duration = resolveDurationForElement(rawNode); diff --git a/packages/core/src/runtime/timeline.ts b/packages/core/src/runtime/timeline.ts index 4d598fb04..1aa234ee3 100644 --- a/packages/core/src/runtime/timeline.ts +++ b/packages/core/src/runtime/timeline.ts @@ -220,7 +220,9 @@ export function collectRuntimeTimelinePayload(params: { if (mediaNodes.length === 0) return null; let maxWindowEndSeconds = 0; for (const mediaNode of mediaNodes) { - const start = startResolver.resolveStartForElement(mediaNode, 0); + const start = !mediaNode.hasAttribute("data-hf-auto-start") + ? Math.max(0, Number(mediaNode.getAttribute("data-start") ?? 0) || 0) + : startResolver.resolveStartForElement(mediaNode, 0); if (!Number.isFinite(start)) continue; const duration = resolveMediaElementDurationSeconds(mediaNode); if (duration == null || duration <= 0) continue;