fix(core): mute preview audio per-element so a slow-decoding track isn't silenced (#1602)

The runtime plays audio two ways — a Web Audio transport (sample-accurate) and
the HTMLMediaElement as a fallback — and mutes the elements when Web Audio takes
over so they don't double-play. That mute gate was global: it muted every element
the moment ANY Web Audio source was active (webAudio.isActive()). A track Web
Audio had not claimed yet (its larger buffer decodes slower) was muted on the
fallback AND not playing on Web Audio = silent, while the other tracks played.
With TTS narration + BGM + SFX, the narration (largest buffer) lost the decode
race and dropped out intermittently, with every file fully loaded.

Make the mute per-element: an element is muted only when its own Web Audio source
is live, or the user / parent-proxy force-mute is set. A track Web Audio has not
claimed stays audible on the HTMLMedia fallback until the transport takes it over
— which also lets narration start immediately on cold play instead of waiting for
its buffer to decode.

Also in this change:
- Don't permanently blacklist a transient fetch failure in the Web Audio decoder
  (_failedSrcs was never cleared); only blacklist genuinely undecodable bytes, so
  a late-arriving asset (404 then available) self-heals on the next play.
- Stop re-issuing play() every tick on an errored / no-source element.

Author: miguel-heygen

packages/core/src/runtime/init.ts

@@ -1506,11 +1506,12 @@ export function initSandboxRuntimeModular(): void {
       timeSeconds: state.currentTime,
       playing: state.isPlaying,
       playbackRate: state.playbackRate,
-      outputMuted: state.mediaOutputMuted || webAudio.isActive(),
+      outputMuted: state.mediaOutputMuted,
       userMuted: state.bridgeMuted,
       userVolume: state.bridgeVolume,
       forceSync,
       onElementVolume: (el, volume) => webAudio.setElementVolume(el, volume),
+      isWebAudioOwned: (el) => webAudio.ownsElement(el),
       onAutoplayBlocked: () => {
         if (state.mediaAutoplayBlockedPosted) return;
         state.mediaAutoplayBlockedPosted = true;

packages/core/src/runtime/media.test.ts

@@ -262,6 +262,41 @@ describe("syncRuntimeMedia", () => {
     expect(clip.el.play).toHaveBeenCalled();
   });
 
+  describe("play() storm guard (unplayable elements)", () => {
+    it("does not play() an element with a media error", () => {
+      const clip = createMockClip({ start: 0, end: 10 });
+      Object.defineProperty(clip.el, "error", { value: { code: 4 }, configurable: true });
+      syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
+      expect(clip.el.play).not.toHaveBeenCalled();
+    });
+
+    it("does not play() an element whose networkState is NO_SOURCE", () => {
+      const clip = createMockClip({ start: 0, end: 10 });
+      Object.defineProperty(clip.el, "networkState", { value: 3, configurable: true });
+      syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
+      expect(clip.el.play).not.toHaveBeenCalled();
+    });
+
+    it("does not re-play() across ticks while the element stays errored", () => {
+      const clip = createMockClip({ start: 0, end: 10 });
+      Object.defineProperty(clip.el, "error", { value: { code: 4 }, configurable: true });
+      for (const t of [5, 5.1, 5.2, 5.3]) {
+        syncRuntimeMedia({ clips: [clip], timeSeconds: t, playing: true, playbackRate: 1 });
+      }
+      expect(clip.el.play).not.toHaveBeenCalled();
+    });
+
+    it("plays again once the element recovers (error clears)", () => {
+      const clip = createMockClip({ start: 0, end: 10 });
+      Object.defineProperty(clip.el, "error", { value: { code: 4 }, configurable: true });
+      syncRuntimeMedia({ clips: [clip], timeSeconds: 5, playing: true, playbackRate: 1 });
+      expect(clip.el.play).not.toHaveBeenCalled();
+      Object.defineProperty(clip.el, "error", { value: null, configurable: true });
+      syncRuntimeMedia({ clips: [clip], timeSeconds: 5.1, playing: true, playbackRate: 1 });
+      expect(clip.el.play).toHaveBeenCalled();
+    });
+  });
+
   it("forces preload=auto on every active element, not just during play", () => {
     // Streaming formats (MP3) may arrive with preload="metadata", which only
     // buffers the first few seconds. Setting preload="auto" on every active
@@ -469,6 +504,84 @@ describe("syncRuntimeMedia", () => {
     expect(onElementVolume).toHaveBeenLastCalledWith(clip.el, 0.375);
   });
 
+  describe("per-element mute (Web Audio ownership)", () => {
+    it("mutes a clip whose element the transport owns", () => {
+      const clip = createMockClip({ start: 0, end: 10 });
+      syncRuntimeMedia({
+        clips: [clip],
+        timeSeconds: 5,
+        playing: true,
+        playbackRate: 1,
+        isWebAudioOwned: (el) => el === clip.el,
+      });
+      expect(clip.el.muted).toBe(true);
+    });
+
+    it("leaves an un-owned clip audible while another track is on Web Audio", () => {
+      // Regression: the un-owned track used to be muted by the global gate the
+      // moment any source was active → silent while the owned track played.
+      const owned = createMockClip({ start: 0, end: 10 });
+      const unowned = createMockClip({ start: 0, end: 10 });
+      syncRuntimeMedia({
+        clips: [owned, unowned],
+        timeSeconds: 5,
+        playing: true,
+        playbackRate: 1,
+        isWebAudioOwned: (el) => el === owned.el,
+      });
+      expect(owned.el.muted).toBe(true);
+      expect(unowned.el.muted).toBe(false);
+    });
+
+    it("force-mutes every element when outputMuted (parent proxy owns all audio)", () => {
+      const clip = createMockClip({ start: 0, end: 10 });
+      syncRuntimeMedia({
+        clips: [clip],
+        timeSeconds: 5,
+        playing: true,
+        playbackRate: 1,
+        outputMuted: true,
+        isWebAudioOwned: () => false,
+      });
+      expect(clip.el.muted).toBe(true);
+    });
+
+    it("force-mutes every element when userMuted", () => {
+      const clip = createMockClip({ start: 0, end: 10 });
+      syncRuntimeMedia({
+        clips: [clip],
+        timeSeconds: 5,
+        playing: true,
+        playbackRate: 1,
+        userMuted: true,
+        isWebAudioOwned: () => false,
+      });
+      expect(clip.el.muted).toBe(true);
+    });
+
+    it("mutes only once the transport takes the element over", () => {
+      const clip = createMockClip({ start: 0, end: 10 });
+      let owned = false;
+      syncRuntimeMedia({
+        clips: [clip],
+        timeSeconds: 5,
+        playing: true,
+        playbackRate: 1,
+        isWebAudioOwned: () => owned,
+      });
+      expect(clip.el.muted).toBe(false); // decoding — audible via HTMLMedia fallback
+      owned = true;
+      syncRuntimeMedia({
+        clips: [clip],
+        timeSeconds: 5.1,
+        playing: true,
+        playbackRate: 1,
+        isWebAudioOwned: () => owned,
+      });
+      expect(clip.el.muted).toBe(true);
+    });
+  });
+
   it("hard-syncs on the first active tick (sub-composition activation, mediaStart offsets)", () => {
     const clip = createMockClip({ start: 0, end: 10, mediaStart: 0 });
     Object.defineProperty(clip.el, "currentTime", { value: 0, writable: true });

packages/core/src/runtime/media.ts

@@ -113,6 +113,14 @@ function markPlayRequested(el: HTMLMediaElement): void {
   el.addEventListener("error", clear, { once: true });
 }
 
+// HTMLMediaElement.NETWORK_NO_SOURCE — no usable source (404 / unsupported).
+const MEDIA_NETWORK_NO_SOURCE = 3;
+// An element that errored or has no source can't play; re-issuing play() every
+// tick just floods rejections. Skip it until its state changes (src reload).
+function isUnplayable(el: HTMLMediaElement): boolean {
+  return el.error != null || el.networkState === MEDIA_NETWORK_NO_SOURCE;
+}
+
 const lastRuntimeAppliedVolume = new WeakMap<HTMLMediaElement, number>();
 
 function clampVolume(volume: number): number {
@@ -126,11 +134,8 @@ export function syncRuntimeMedia(params: {
   timeSeconds: number;
   playing: boolean;
   playbackRate: number;
-  /**
-   * Parent-frame audio-owner has taken over audible playback. Assert
-   * `el.muted = true` on every active media element per tick so that any
-   * sub-composition media inserted mid-playback inherits the silence.
-   */
+  /** Force-mute every element (parent-frame proxy owns all audio). Asserted per
+   *  tick so sub-composition media added mid-playback inherits the silence. */
   outputMuted?: boolean;
   /**
    * User's explicit mute preference (set via `onSetMuted`). Symmetric to
@@ -151,11 +156,13 @@ export function syncRuntimeMedia(params: {
    */
   onAutoplayBlocked?: () => void;
   onElementVolume?: (el: HTMLMediaElement, volume: number) => void;
+  /** Is THIS element owned by the Web Audio transport? Owned → mute it (transport
+   *  plays it); not owned → leave audible (HTMLMedia fallback). Per-element, not a
+   *  global flag, so a not-yet-claimed track isn't muted by other tracks. */
+  isWebAudioOwned?: (el: HTMLMediaElement) => boolean;
   forceSync?: boolean;
 }): void {
-  // Either flag silences output. Combined up front so the per-clip loop is
-  // a single branch instead of two.
-  const shouldMute = !!(params.outputMuted || params.userMuted);
+  const forceMuteAll = !!(params.outputMuted || params.userMuted);
   for (const clip of params.clips) {
     const { el } = clip;
     if (!el.isConnected) continue;
@@ -205,7 +212,9 @@ export function syncRuntimeMedia(params: {
       el.volume = effectiveVolume;
       lastRuntimeAppliedVolume.set(el, effectiveVolume);
       params.onElementVolume?.(el, effectiveVolume);
-      if (shouldMute) el.muted = true;
+      // Mute only when force-muted or the transport owns this element; an unclaimed
+      // track stays audible via the HTMLMedia fallback.
+      if (forceMuteAll || params.isWebAudioOwned?.(el)) el.muted = true;
       // Ensure full preload for every active media element. Streaming
       // formats (MP3) may arrive with preload="metadata", which only
       // buffers the first few seconds and causes seeks to silently fail
@@ -297,7 +306,7 @@ export function syncRuntimeMedia(params: {
         }
         playRequested.delete(el);
       }
-      if (params.playing && el.paused && !playRequested.has(el)) {
+      if (params.playing && el.paused && !playRequested.has(el) && !isUnplayable(el)) {
         // `HTMLMediaElement.play()` is spec'd to queue playback and resolve
         // once enough data is buffered, so we can unconditionally call it —
         // no need to gate on `readyState` or defer to a `canplay` listener.

packages/core/src/runtime/webAudioTransport.test.ts

@@ -129,6 +129,42 @@ describe("WebAudioTransport", () => {
     expect(transport.isActive()).toBe(false);
   });
 
+  describe("ownsElement (per-element mute gate)", () => {
+    function withSource(el: HTMLMediaElement) {
+      const transport = new WebAudioTransport();
+      const source = {
+        el,
+        sourceNode: { stop: vi.fn(), disconnect: vi.fn() } as unknown as AudioBufferSourceNode,
+        gainNode: { disconnect: vi.fn() } as unknown as GainNode,
+        compositionStart: 0,
+        mediaStart: 0,
+        scheduledAt: 0,
+        priorMuted: false,
+      };
+      (transport as unknown as { _activeSources: (typeof source)[] })._activeSources = [source];
+      (transport as unknown as { _paused: boolean })._paused = false;
+      return transport;
+    }
+
+    it("returns true for an element the transport plays", () => {
+      const el = { muted: false } as HTMLMediaElement;
+      expect(withSource(el).ownsElement(el)).toBe(true);
+    });
+
+    it("returns false for an element the transport does not play", () => {
+      const el = { muted: false } as HTMLMediaElement;
+      const other = { muted: false } as HTMLMediaElement;
+      expect(withSource(el).ownsElement(other)).toBe(false);
+    });
+
+    it("returns false after stopAll releases the element", () => {
+      const el = { muted: false } as HTMLMediaElement;
+      const transport = withSource(el);
+      transport.stopAll();
+      expect(transport.ownsElement(el)).toBe(false);
+    });
+  });
+
   describe("schedulePlayback timing", () => {
     it("starts in-progress clips immediately with correct buffer offset", async () => {
       const { transport, mock, gen } = setupTransport(100);
@@ -379,4 +415,54 @@ describe("WebAudioTransport", () => {
       expect(transport.isActive()).toBe(false);
     });
   });
+
+  describe("decodeAudioElement retry policy (late-asset self-heal)", () => {
+    function transportWithDecode(decodeImpl: () => Promise<AudioBuffer>) {
+      const transport = new WebAudioTransport();
+      const ctx = { state: "running", decodeAudioData: vi.fn(decodeImpl) };
+      (transport as unknown as { _ctx: unknown })._ctx = ctx;
+      return transport;
+    }
+    const el = (src: string) =>
+      ({ getAttribute: () => src, currentSrc: "" }) as unknown as HTMLMediaElement;
+    const failedSrcs = (t: WebAudioTransport) =>
+      (t as unknown as { _failedSrcs: Set<string> })._failedSrcs;
+
+    it("does NOT blacklist a transient fetch failure — a later play retries and succeeds", async () => {
+      const transport = transportWithDecode(async () => ({}) as AudioBuffer);
+      const fetchMock = vi
+        .fn()
+        .mockResolvedValueOnce({ ok: false, status: 404 }) // asset not uploaded yet
+        .mockResolvedValueOnce({ ok: true, arrayBuffer: async () => new ArrayBuffer(8) });
+      vi.stubGlobal("fetch", fetchMock);
+
+      const first = await transport.decodeAudioElement(el("tts.wav"));
+      expect(first).toBeNull();
+      expect(failedSrcs(transport).has("tts.wav")).toBe(false); // not permanently silenced
+
+      const second = await transport.decodeAudioElement(el("tts.wav"));
+      expect(second).not.toBeNull(); // self-heals once the asset is available
+      expect(fetchMock).toHaveBeenCalledTimes(2);
+      vi.unstubAllGlobals();
+    });
+
+    it("DOES blacklist genuinely undecodable bytes — not retried", async () => {
+      const transport = transportWithDecode(async () => {
+        throw new Error("unsupported codec");
+      });
+      const fetchMock = vi
+        .fn()
+        .mockResolvedValue({ ok: true, arrayBuffer: async () => new ArrayBuffer(8) });
+      vi.stubGlobal("fetch", fetchMock);
+
+      const first = await transport.decodeAudioElement(el("corrupt.wav"));
+      expect(first).toBeNull();
+      expect(failedSrcs(transport).has("corrupt.wav")).toBe(true); // bad data is permanent
+
+      const second = await transport.decodeAudioElement(el("corrupt.wav"));
+      expect(second).toBeNull();
+      expect(fetchMock).toHaveBeenCalledTimes(1); // short-circuited, no re-fetch
+      vi.unstubAllGlobals();
+    });
+  });
 });

packages/core/src/runtime/webAudioTransport.ts

@@ -1,10 +1,25 @@
 import { swallow } from "./diagnostics";
+import { getDebugSurface } from "./globals.js";
 
 function normalizeRate(rate: number): number {
   if (!Number.isFinite(rate) || rate <= 0) return 1;
   return rate;
 }
 
+/**
+ * Breadcrumb for the per-element-mute handoff: the transport just claimed a track
+ * that was audibly playing through the HTMLMedia fallback. Quiet unless
+ * `__hfDebug` — a hook for diagnosing the race if it ever regresses.
+ */
+function logFallbackHandoff(el: HTMLMediaElement, priorMuted: boolean): void {
+  if (priorMuted || el.paused || !getDebugSurface().__hfDebug) return;
+  // eslint-disable-next-line no-console -- intentional debug surface
+  console.debug(
+    "[hyperframes] webAudioTransport claimed fallback-playing element:",
+    el.currentSrc || el.getAttribute("src") || "",
+  );
+}
+
 /**
  * Start a buffer source, bounding it to the clip's authored window
  * (`data-duration`) so a trimmed clip stops at its edge instead of running the
@@ -96,14 +111,32 @@ export class WebAudioTransport {
     if (this._bufferCache.has(src)) return this._bufferCache.get(src)!;
     if (this._failedSrcs.has(src)) return null;
     if (!this._ctx) return null;
+
+    // Fetch the bytes. A network error or non-OK status (e.g. a 404 for an
+    // asset that simply has not been uploaded yet) is TRANSIENT — return null
+    // WITHOUT blacklisting, so the next play/seek generation retries once the
+    // asset becomes available. (Previously these were added to `_failedSrcs`,
+    // which is never cleared, permanently silencing a merely-late track.)
+    let arrayBuffer: ArrayBuffer;
     try {
-      const response = await fetch(src);
+      // `no-store`: a retry must actually re-request the asset — not replay a
+      // cached 404/stale response from the failed attempt that we chose not to
+      // blacklist.
+      const response = await fetch(src, { cache: "no-store" });
       if (!response.ok) {
-        this._failedSrcs.add(src);
         swallow("webAudioTransport.fetch", new Error(`${response.status} ${src}`));
         return null;
       }
-      const arrayBuffer = await response.arrayBuffer();
+      arrayBuffer = await response.arrayBuffer();
+    } catch (err) {
+      swallow("webAudioTransport.fetch", err);
+      return null;
+    }
+
+    // A decode failure means the bytes themselves are unusable (corrupt or an
+    // unsupported codec) — that IS permanent, so blacklist to avoid re-decoding
+    // the same bad payload on every generation.
+    try {
       const audioBuffer = await this._ctx.decodeAudioData(arrayBuffer);
       this._bufferCache.set(src, audioBuffer);
       return audioBuffer;
@@ -177,6 +210,7 @@ export class WebAudioTransport {
 
       const priorMuted = el.muted;
       el.muted = true;
+      logFallbackHandoff(el, priorMuted);
 
       const scheduled: ScheduledSource = {
         el,
@@ -281,9 +315,16 @@ export class WebAudioTransport {
     return this._activeSources.length > 0 && !this._paused;
   }
 
+  /** Whether the transport currently plays THIS element (the runtime mutes it to
+   *  avoid double audio; an unclaimed track stays audible). */
+  ownsElement(el: HTMLMediaElement): boolean {
+    return !this._paused && this._activeSources.some((s) => s.el === el);
+  }
+
   destroy(): void {
     this.stopAll();
     this._bufferCache.clear();
+    this._failedSrcs.clear();
     if (this._ctx) {
       try {
         void this._ctx.close();