diff --git a/.fallowrc.jsonc b/.fallowrc.jsonc index 3a9511683b..e27b71cfa4 100644 --- a/.fallowrc.jsonc +++ b/.fallowrc.jsonc @@ -149,13 +149,6 @@ "file": "packages/studio/src/utils/studioHelpers.ts", "exports": ["resolveDroppedAssetDimensions"], }, - // Audio FX worklets sit near the bottom of the audio stack: the worklet - // source and its test reset are consumed by the runtime and engine PRs - // upstack, so a per-PR audit against the merge base sees them as unused. - { - "file": "packages/core/src/audio/audioFxWorklets.ts", - "exports": ["AUDIO_FX_WORKLET_SOURCE", "__resetAudioFxWorkletsForTests"], - }, { "file": "packages/core/src/audio/audioFxGraph.ts", "exports": ["ensureAudioFxWorklets"], diff --git a/packages/core/package.json b/packages/core/package.json index 06ac920aa8..29f6a2a87d 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -549,7 +549,7 @@ }, "scripts": { "build": "bun run build:hyperframes-runtime && bun run build:position-edits-render && bun run build:audio-fx-runtime && tsc && tsx scripts/rewrite-esm-extensions.ts", - "test": "bun run check:position-edits-render && vitest run", + "test": "bun run check:position-edits-render && bun run build:audio-fx-runtime && vitest run", "test:watch": "vitest", "test:coverage": "vitest run --coverage", "test:runtime-coverage": "vitest run --coverage src/runtime", diff --git a/packages/core/src/audio/audioFxGraph.test.ts b/packages/core/src/audio/audioFxGraph.test.ts index 9c54f12bc9..f1eca2b743 100644 --- a/packages/core/src/audio/audioFxGraph.test.ts +++ b/packages/core/src/audio/audioFxGraph.test.ts @@ -155,6 +155,16 @@ describe("buildFxNode", () => { expect(workletNodes[0]!.messages[0]).toMatchObject({ threshold: -30 }); }); + it("tells a worklet processor to retire on dispose, not just disconnect it", () => { + // Disconnecting leaves the processor alive — it lives until `process()` + // returns false — so every rebuild that dropped a worklet effect left one + // running on the audio thread for the rest of the session. + workletNodes.length = 0; + const h = buildFxNode(asCtx(ctx()), "compressor", defaultAudioFxParams("compressor")); + h.dispose(); + expect(workletNodes[0]!.messages).toEqual([{ __hfDispose: true }]); + }); + it("rebuilds the saturation curve for the selected shape", () => { const c = ctx(); buildFxNode(asCtx(c), "saturate", { ...defaultAudioFxParams("saturate"), type: "hard" }); @@ -206,6 +216,39 @@ describe("buildFxChain", () => { expect(c.created.find((x) => x.kind === "biquad")!.frequency.value).toBe(2000); }); + it("lines new values up against the built nodes, skipping a bypassed one", () => { + // A bypassed node is not in the graph, so the update has to walk the + // ENABLED nodes to stay aligned with what was built. Walking `next.nodes` + // instead shifts everything after the bypass by one and pushes each node's + // parameters into its neighbour — and the shape is unchanged either way, + // so nothing forces a rebuild that would hide it. + const c = ctx(); + const withBypass = (frequency: number): HfAudioFxChain => ({ + version: 1, + nodes: [ + { + type: "highpass", + enabled: true, + params: { ...defaultAudioFxParams("highpass"), frequency: 100 }, + }, + { type: "peaking", enabled: false, params: defaultAudioFxParams("peaking") }, + { + type: "lowpass", + enabled: true, + params: { ...defaultAudioFxParams("lowpass"), frequency }, + }, + ], + }); + const h = buildFxChain(asCtx(c), withBypass(8000)); + const biquads = c.created.filter((n) => n.kind === "biquad"); + expect(biquads).toHaveLength(2); + + expect(h.update(withBypass(3000))).toBe(true); + // The lowpass took the new cutoff; the highpass was left where it was. + expect(biquads[0]!.frequency.value).toBe(100); + expect(biquads[1]!.frequency.value).toBe(3000); + }); + it("reports that a rebuild is needed when the chain shape changes", () => { const c = ctx(); const h = buildFxChain(asCtx(c), chain("peaking")); @@ -438,3 +481,54 @@ describe("automatable parameters", () => { expect(onePole.automation?.frequency).toBeUndefined(); }); }); + +describe("phaser automation targets", () => { + /** + * `in_gain` and `out_gain` trim the signal entering and leaving the effect, + * which the builder drives through inTrim/outTrim while pinning wet and dry + * to 1. The automation map used to aim both lanes at wet/dry — so an envelope + * modulated a constant, the trim it was supposed to move stayed frozen, and + * "fade the phaser out" left the dry leg playing at full level. + * + * Asserted by VALUE rather than by node identity: the trims are internal, and + * the only honest question is whether the param a lane would drive is the one + * the knob sets. + */ + it("drives the trims a lane is named for, not the pinned wet/dry pair", () => { + const handle = buildFxNode(ctx() as unknown as BaseAudioContext, "phaser", { + ...defaultAudioFxParams("phaser"), + in_gain: 0.25, + out_gain: 0.5, + }); + expect(handle.automation?.in_gain?.[0]?.param.value).toBeCloseTo(0.25, 6); + expect(handle.automation?.out_gain?.[0]?.param.value).toBeCloseTo(0.5, 6); + }); +}); + +describe("chain update keeps ids with their effects", () => { + const band = (id: string, frequency: number) => ({ + type: "peaking", + id, + enabled: true, + params: { ...defaultAudioFxParams("peaking"), frequency }, + }); + + /** + * Reordering two effects of the same type leaves the shape string identical, + * so the chain updates in place rather than rebuilding — correct for the + * audio, since the params move with the position. The ids have to move too: + * a lane addresses its effect by id, and an id captured at build time names + * whichever effect used to occupy that slot. The scheduler would then drive + * `fx.n2.frequency` into the band that is now n1 — the exact swap that + * HfAudioFxNode.id documents itself as preventing, and the one the voiceover + * carve's all-peaking chains make easy to hit. + */ + it("moves an id with its slot when same-type effects are reordered", () => { + const chain: HfAudioFxChain = { version: 1, nodes: [band("n1", 200), band("n2", 4000)] }; + const handle = buildFxChain(ctx() as unknown as BaseAudioContext, chain); + + const swapped: HfAudioFxChain = { version: 1, nodes: [band("n2", 4000), band("n1", 200)] }; + expect(handle.update(swapped)).toBe(true); + expect(handle.nodes.map((n) => n.id)).toEqual(["n2", "n1"]); + }); +}); diff --git a/packages/core/src/audio/audioFxGraph.ts b/packages/core/src/audio/audioFxGraph.ts index a366b823e7..f37f394214 100644 --- a/packages/core/src/audio/audioFxGraph.ts +++ b/packages/core/src/audio/audioFxGraph.ts @@ -9,6 +9,7 @@ */ import { + enabledAudioFxNodes, getAudioFxDef, normalizeAudioFxParams, type HfAudioFxChain, @@ -176,7 +177,17 @@ function workletBuilder(processor: string): Builder { input: node, output: node, update: (v) => node.port.postMessage({ ...v }), - dispose: () => node.disconnect(), + dispose: () => { + // Disconnecting is not enough to retire an AudioWorkletProcessor: it + // lives until its `process()` returns false, and these all returned + // true unconditionally. So every chain rebuild that dropped a limiter, + // compressor, gate or bitcrush left it running on the audio thread for + // the rest of the session, and a few edits to a carved bed accumulated + // a stack of them. The processors treat this message as their cue to + // stop. + node.port.postMessage({ __hfDispose: true }); + node.disconnect(); + }, }; }; } @@ -360,8 +371,13 @@ const allpassPhaser: Builder = (ctx, p) => { // frequency at once — not one knob, one param — so they stay unautomated. automation: { speed: [{ param: lfo.frequency }], - in_gain: [{ param: dry.gain }], - out_gain: [{ param: wet.gain }], + // The trims, not wet/dry. apply() drives inTrim/outTrim from these knobs + // and pins wet and dry to 1 — so a lane aimed at wet/dry modulated a + // constant and left the trim frozen, and the next values-only edit slammed + // it back over the running envelope. The comment above records that this + // wiring was already moved once; the automation map was missed. + in_gain: [{ param: inTrim.gain }], + out_gain: [{ param: outTrim.gain }], }, dispose: () => { try { @@ -465,8 +481,7 @@ export interface FxChainHandle { * into the running nodes; when it changes, the caller rebuilds. */ function shapeOf(chain: HfAudioFxChain): string { - return chain.nodes - .filter((node) => node.enabled !== false) + return enabledAudioFxNodes(chain) .map((node) => { const p = normalizeAudioFxParams(node.type, node.params); const poles = p.poles !== undefined ? `:${p.poles}` : ""; @@ -491,8 +506,7 @@ export function buildFxChain(ctx: BaseAudioContext, chain: HfAudioFxChain): FxCh const handles: { id?: string; type: string; handle: FxNodeHandle }[] = []; let tail: AudioNode = input; - for (const node of chain.nodes) { - if (node.enabled === false) continue; + for (const node of enabledAudioFxNodes(chain)) { const handle = buildFxNode(ctx, node.type, node.params ?? {}); tail.connect(handle.input); tail = handle.output; @@ -500,7 +514,7 @@ export function buildFxChain(ctx: BaseAudioContext, chain: HfAudioFxChain): FxCh } tail.connect(output); - let shape = shapeOf(chain); + const shape = shapeOf(chain); return { input, @@ -508,11 +522,22 @@ export function buildFxChain(ctx: BaseAudioContext, chain: HfAudioFxChain): FxCh nodes: handles, update(next) { if (shapeOf(next) !== shape) return false; - const active = next.nodes.filter((node) => node.enabled !== false); - active.forEach((node, i) => { - handles[i]?.handle.update(normalizeAudioFxParams(node.type, node.params)); + enabledAudioFxNodes(next).forEach((node, i) => { + const held = handles[i]; + if (!held) return; + held.handle.update(normalizeAudioFxParams(node.type, node.params)); + // The id follows the position, because the params just did. Reordering + // two effects of the same type leaves the shape identical, so the graph + // is updated in place — but a lane addresses its effect BY id, and an id + // captured at build time then names whichever effect used to be here. + // The scheduler would drive `fx.n2.frequency` into the band that is now + // n1: exactly what HfAudioFxNode.id documents itself as preventing. + if (node.id === undefined) delete held.id; + else held.id = node.id; }); - shape = shapeOf(next); + // `shape` is not reassigned: the early return above already established + // that `shapeOf(next)` equals it, so recomputing was a whole normalise + + // join per observer tick to write back the string that was already there. return true; }, dispose() { diff --git a/packages/core/src/audio/audioFxWorklets.test.ts b/packages/core/src/audio/audioFxWorklets.test.ts new file mode 100644 index 0000000000..1e9e06ca66 --- /dev/null +++ b/packages/core/src/audio/audioFxWorklets.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it, vi } from "vitest"; +import { audioFxWorkletsReady, ensureAudioFxWorklets } from "./audioFxWorklets.js"; + +/** Just enough of a BaseAudioContext for the registration cache to key on. */ +const contextWith = (addModule: (url: string) => Promise): BaseAudioContext => + ({ audioWorklet: { addModule } }) as unknown as BaseAudioContext; + +describe("ensureAudioFxWorklets", () => { + it("registers once per context and reuses the result", async () => { + const addModule = vi.fn(async () => undefined); + const ctx = contextWith(addModule); + + await ensureAudioFxWorklets(ctx); + await ensureAudioFxWorklets(ctx); + + expect(addModule).toHaveBeenCalledTimes(1); + expect(audioFxWorkletsReady(ctx)).toBe(true); + }); + + it("retries after a failure instead of replaying it forever", async () => { + // The rejected promise used to stay in the cache, so every later attempt + // got the same rejection back — the limiter, compressor, gate and bitcrush + // were silent for the life of the context after one transient failure. + const addModule = vi + .fn<(url: string) => Promise>() + .mockRejectedValueOnce(new Error("module load failed")) + .mockResolvedValue(undefined); + const ctx = contextWith(addModule); + + await expect(ensureAudioFxWorklets(ctx)).rejects.toThrow("module load failed"); + expect(audioFxWorkletsReady(ctx)).toBe(false); + + await expect(ensureAudioFxWorklets(ctx)).resolves.toBeUndefined(); + expect(addModule).toHaveBeenCalledTimes(2); + expect(audioFxWorkletsReady(ctx)).toBe(true); + }); + + it("refuses a context with no AudioWorklet rather than hanging", async () => { + const ctx = {} as BaseAudioContext; + await expect(ensureAudioFxWorklets(ctx)).rejects.toThrow(/secure context/); + }); +}); + +/** + * A processor lives until its `process()` returns false — disconnecting the + * node does not retire it. These all returned true unconditionally, so every + * chain rebuild that dropped a worklet effect left it running on the audio + * thread for the rest of the session. + * + * The source is taken from the data: URL registration actually hands to + * `addModule`, so this also proves the URL carries what it claims to. + */ +describe("the worklet processors themselves", () => { + /** Evaluate the registered module and hand back the processor classes by name. */ + async function loadProcessors(): Promise Processor>> { + let moduleSource = ""; + await ensureAudioFxWorklets( + contextWith(async (url: string) => { + moduleSource = atob(url.replace("data:text/javascript;base64,", "")); + }), + ); + const made = new Map Processor>(); + class Base { + port = { + onmessage: null as ((e: { data: unknown }) => void) | null, + postMessage: (data: unknown) => this.port.onmessage?.({ data }), + }; + } + new Function("AudioWorkletProcessor", "registerProcessor", "sampleRate", moduleSource)( + Base, + (name: string, cls: new (o: unknown) => Processor) => made.set(name, cls), + 48000, + ); + return made; + } + + interface Processor { + port: { postMessage(data: unknown): void }; + process(inputs: Float32Array[][], outputs: Float32Array[][]): boolean; + } + + const block = (): Float32Array[][] => [[new Float32Array(128)]]; + + it("every processor keeps running until it is told to stop, then retires", async () => { + const processors = await loadProcessors(); + expect([...processors.keys()]).toEqual([ + "hf-compressor", + "hf-limiter", + "hf-gate", + "hf-bitcrush", + ]); + + for (const [name, Cls] of processors) { + const p = new Cls({ processorOptions: {} }); + expect(p.process(block(), block()), `${name} retired before it was disposed`).toBe(true); + p.port.postMessage({ __hfDispose: true }); + expect(p.process(block(), block()), `${name} kept running after dispose`).toBe(false); + // And it stays retired — a later parameter update must not revive it. + p.port.postMessage({ mix: 0.5 }); + expect(p.process(block(), block()), `${name} came back to life`).toBe(false); + } + }); +}); diff --git a/packages/core/src/audio/audioFxWorklets.ts b/packages/core/src/audio/audioFxWorklets.ts index e421d34099..f2261ea8a1 100644 --- a/packages/core/src/audio/audioFxWorklets.ts +++ b/packages/core/src/audio/audioFxWorklets.ts @@ -60,11 +60,13 @@ class HfCompressor extends AudioWorkletProcessor { this.p = o.processorOptions || {}; this.env = new EnvBank(this.p.attack ?? 20, this.p.release ?? 250); this.port.onmessage = (e) => { + if (e.data && e.data.__hfDispose) { this.dead = true; return; } this.p = { ...this.p, ...e.data }; this.env.set(this.p.attack ?? 20, this.p.release ?? 250); }; } process(inputs, outputs) { + if (this.dead) return false; const i = inputs[0], o = outputs[0]; if (!i || !i.length) return true; const p = this.p; @@ -105,11 +107,13 @@ class HfLimiter extends AudioWorkletProcessor { this.p = o.processorOptions || {}; this.env = new EnvBank(this.p.attack ?? 5, this.p.release ?? 50); this.port.onmessage = (e) => { + if (e.data && e.data.__hfDispose) { this.dead = true; return; } this.p = { ...this.p, ...e.data }; this.env.set(this.p.attack ?? 5, this.p.release ?? 50); }; } process(inputs, outputs) { + if (this.dead) return false; const i = inputs[0], o = outputs[0]; if (!i || !i.length) return true; const ceiling = dbToLin(this.p.limit ?? -1); @@ -136,11 +140,13 @@ class HfGate extends AudioWorkletProcessor { this.env = new EnvBank(this.p.attack ?? 1, this.p.release ?? 100); this.gains = []; this.port.onmessage = (e) => { + if (e.data && e.data.__hfDispose) { this.dead = true; return; } this.p = { ...this.p, ...e.data }; this.env.set(this.p.attack ?? 1, this.p.release ?? 100); }; } process(inputs, outputs) { + if (this.dead) return false; const i = inputs[0], o = outputs[0]; if (!i || !i.length) return true; const p = this.p; @@ -183,9 +189,13 @@ class HfBitcrush extends AudioWorkletProcessor { this.p = o.processorOptions || {}; this.holds = []; this.held = []; - this.port.onmessage = (e) => { this.p = { ...this.p, ...e.data }; }; + this.port.onmessage = (e) => { + if (e.data && e.data.__hfDispose) { this.dead = true; return; } + this.p = { ...this.p, ...e.data }; + }; } process(inputs, outputs) { + if (this.dead) return false; const i = inputs[0], o = outputs[0]; if (!i || !i.length) return true; const p = this.p; @@ -243,7 +253,16 @@ export function ensureAudioFxWorklets(ctx: BaseAudioContext): Promise { )}`; await ctx.audioWorklet.addModule(url); readyContexts.add(ctx); - })(); + })().catch((err: unknown) => { + // A failed registration must not be remembered. `readyContexts` is only + // written on success, so callers correctly keep asking — and every ask + // replayed this same rejected promise, leaving the limiter, compressor, + // gate and bitcrush silent for the life of the context with no way back. + // One transient failure (a slow module load, a context still warming up) + // permanently disabled half the rack. + registered.delete(ctx); + throw err; + }); registered.set(ctx, modulePromise); } return modulePromise; diff --git a/packages/core/src/audioAutomation.test.ts b/packages/core/src/audioAutomation.test.ts index 7b9af2c747..61fefbf313 100644 --- a/packages/core/src/audioAutomation.test.ts +++ b/packages/core/src/audioAutomation.test.ts @@ -3,6 +3,8 @@ import { applyCurve, shapeProgress, fxAutomationTarget, + HF_AUDIO_AUTOMATION_ATTR, + HF_AUDIO_AUTOMATION_DATA_KEY, isConstantLane, parseAutomation, parseAutomationTarget, @@ -30,6 +32,14 @@ const lane = (points: HfAutomationLane["points"], target = "volume"): HfAutomati points, }); +describe("the automation attribute's two spellings", () => { + it("names the same attribute either way", () => { + // Same split as the FX chain: written as an attribute, read as a dataset + // key. Derived, so a rename cannot half-land. + expect(HF_AUDIO_AUTOMATION_ATTR).toBe(`data-${HF_AUDIO_AUTOMATION_DATA_KEY}`); + }); +}); + describe("targets", () => { it("reads volume and fx targets, and rejects anything else", () => { expect(parseAutomationTarget("volume")).toEqual({ kind: "volume" }); diff --git a/packages/core/src/audioAutomation.ts b/packages/core/src/audioAutomation.ts index a2cdef1c92..fe6909b719 100644 --- a/packages/core/src/audioAutomation.ts +++ b/packages/core/src/audioAutomation.ts @@ -11,10 +11,13 @@ * curve, three consumers, or the picture and the sound disagree. */ -import { getAudioFxDef, type HfAudioFxChain, type HfAudioFxNumberParam } from "./audioFx.js"; +import { getAudioFxDef, type HfAudioFxChain } from "./audioFx.js"; export const HF_AUDIO_AUTOMATION_ATTR = "data-automation"; +/** The same attribute as a `dataset` / `dataAttributes` key. See `HF_AUDIO_FX_DATA_KEY`. */ +export const HF_AUDIO_AUTOMATION_DATA_KEY = HF_AUDIO_AUTOMATION_ATTR.slice("data-".length); + /** Automation files are versioned; a reader must refuse a version it doesn't know. */ export const HF_AUDIO_AUTOMATION_VERSION = 1; @@ -138,15 +141,14 @@ export function resolveAutomationRange( const def = getAudioFxDef(node.type); const param = def?.params.find((p) => p.key === parsed.param); if (!param || param.kind !== "number") return null; - const p = param as HfAudioFxNumberParam; return { - min: p.min, - max: p.max, - step: p.step, - unit: p.unit, - label: `${def?.label ?? node.type} · ${p.label}`, - scale: p.scale === "log" && p.min > 0 ? "log" : "linear", - default: p.default, + min: param.min, + max: param.max, + step: param.step, + unit: param.unit, + label: `${def?.label ?? node.type} · ${param.label}`, + scale: param.scale === "log" && param.min > 0 ? "log" : "linear", + default: param.default, }; } diff --git a/packages/core/src/audioCarve.test.ts b/packages/core/src/audioCarve.test.ts index 519afc875e..468ee27b4a 100644 --- a/packages/core/src/audioCarve.test.ts +++ b/packages/core/src/audioCarve.test.ts @@ -249,6 +249,48 @@ describe("analyseCarveBands", () => { }); }); +/** + * Both analysis loops reuse one pair of FFT scratch arrays across every window + * rather than allocating a pair per hop — a 5-minute 48 kHz voiceover is ~7000 + * hops, so ~460 MB of transient Float64Array used to churn through the main + * thread for one carve. `re` is fully overwritten each window, but `im` is only + * ever added to, so it has to be cleared; missing that, the imaginary part + * accumulates across windows and every spectrum after the first is wrong by a + * growing amount. + */ +describe("reused FFT scratch across windows", () => { + /** A steady tone on an exact bin centre (48000/4096 x 128), so every window is identical. */ + const steady = (seconds: number): Float32Array => { + const n = Math.floor(SR * seconds); + const out = new Float32Array(n); + for (let i = 0; i < n; i++) out[i] = 0.5 * Math.sin((2 * Math.PI * 1500 * i) / SR); + return out; + }; + + it("measures the same bands however many windows the clip has", () => { + // Every window carries the same spectrum, so the Welch average cannot + // depend on how many were averaged — unless one window is contaminating + // the next. + const short = analyseCarveBands(steady(0.5), SR, PROFILE); + const long = analyseCarveBands(steady(12), SR, PROFILE); + expect(short.length).toBeGreaterThan(0); + expect(long).toEqual(short); + }); + + it("keeps a steady tone's dynamics envelope flat instead of drifting", () => { + const [lane] = analyseCarveDynamics(steady(12), SR, [{ freq: 1600, gainDb: -8, q: 1.4 }]); + const value = (t: number): number => + sampleAutomationLane({ target: "fx.n1.gain", points: lane!.points }, t); + // Past the attack the cut has to sit still, because the signal does. A + // window contaminated by the one before it grows the measured power over + // the clip, and the envelope — which is relative to the band's own peak — + // slides with it. + expect(value(4)).toBeLessThan(-1); + expect(value(8)).toBeCloseTo(value(4), 0); + expect(value(11)).toBeCloseTo(value(4), 0); + }); +}); + describe("carveBandsToChain", () => { it("turns bands into peaking nodes carrying the analysed values", () => { const chain = carveBandsToChain([{ freq: 1000, gainDb: -6, q: 1.4 }]); diff --git a/packages/core/src/audioCarve.ts b/packages/core/src/audioCarve.ts index 1a0fac18a0..4aff6ee6dd 100644 --- a/packages/core/src/audioCarve.ts +++ b/packages/core/src/audioCarve.ts @@ -313,12 +313,23 @@ function powerSpectrum( const bins = FRAME / 2 + 1; const acc = new Float64Array(bins); + // Reused across hops. These used to be allocated inside the loop: a 5-minute + // 48 kHz voiceover is ~7000 hops, so ~460 MB of transient Float64Array + // churned through the main thread for a single carve. `re` is fully + // overwritten below; only `im` has to be cleared. + const re = new Float64Array(FRAME); + const im = new Float64Array(FRAME); + // + // Every hop is still read. Striding them — Welch's average is supposed to + // settle long before 7000 windows — was measured on a 5-minute voiceover and + // moves the result: at strength 0.9 the chosen band set changed (630 Hz for + // 160 Hz), and it did not converge back to the full read even at 2048 + // windows. 27x faster is not worth silently redrawing the author's carve. let frames = 0; for (let start = 0; start + FRAME <= n; start += HOP) { // Goertzel-free naive DFT would be O(n^2); use a real FFT via recursion on // a copied frame. FRAME is a power of two so the radix-2 split is exact. - const re = new Float64Array(FRAME); - const im = new Float64Array(FRAME); + im.fill(0); for (let i = 0; i < FRAME; i++) re[i] = (padded[start + i] ?? 0) * window[i]!; fft(re, im); for (let k = 0; k < bins; k++) acc[k]! += re[k]! * re[k]! + im[k]! * im[k]!; @@ -554,9 +565,13 @@ export function analyseCarveDynamics( const times: number[] = []; const perBand = bands.map(() => [] as number[]); + // Reused across windows, as in powerSpectrum. `re` is fully overwritten + // below; only `im` has to be cleared. The hop here is already bounded by + // POINT_BUDGET, so there is nothing to stride. + const re = new Float64Array(FRAME); + const im = new Float64Array(FRAME); for (let start = 0; start < voice.length; start += hop) { - const re = new Float64Array(FRAME); - const im = new Float64Array(FRAME); + im.fill(0); for (let i = 0; i < FRAME; i++) re[i] = (voice[start + i] ?? 0) * window[i]!; fft(re, im); const power: number[] = []; diff --git a/packages/core/src/audioFx.test.ts b/packages/core/src/audioFx.test.ts index e59819b952..09b60ae660 100644 --- a/packages/core/src/audioFx.test.ts +++ b/packages/core/src/audioFx.test.ts @@ -5,7 +5,9 @@ import { enabledAudioFxNodes, getAudioFxDef, HF_AUDIO_FX, + HF_AUDIO_FX_ATTR, HF_AUDIO_FX_CHAIN_VERSION, + HF_AUDIO_FX_DATA_KEY, HF_AUDIO_FX_IDS, normalizeAudioFxParams, parseAudioFxChain, @@ -47,6 +49,15 @@ describe("effect registry", () => { }); }); +describe("the chain attribute's two spellings", () => { + it("names the same attribute either way", () => { + // The studio WRITES through the attribute and READS through the dataset + // key, so the read side used to carry its own `"fx-chain"` literal and a + // rename would only half-land. Derived now; this is what keeps it derived. + expect(HF_AUDIO_FX_ATTR).toBe(`data-${HF_AUDIO_FX_DATA_KEY}`); + }); +}); + describe("normalizeAudioFxParams", () => { it("fills missing keys with defaults", () => { expect(normalizeAudioFxParams("peaking", {})).toEqual(defaultAudioFxParams("peaking")); @@ -69,6 +80,27 @@ describe("normalizeAudioFxParams", () => { expect(v.gain).toBe(0); }); + it("treats a blank or missing value as absent rather than as zero", () => { + // `Number(null)`, `Number("")`, `Number(false)` and `Number([])` are all 0 + // and all finite, so these used to clamp to 0 instead of falling back. Zero + // is a legal setting for most of these knobs, so nothing downstream could + // tell: a compressor whose threshold arrived as null sat at 0 dB and never + // engaged, silently, rather than at its declared -24 dB. + const def = defaultAudioFxParams("compressor").threshold; + expect(def).not.toBe(0); + for (const blank of [null, undefined, "", " ", false, [], {}]) { + expect( + normalizeAudioFxParams("compressor", { threshold: blank as unknown as number }).threshold, + `${JSON.stringify(blank)} was read as a number`, + ).toBe(def); + } + // A string that really does spell a number still counts — that is how the + // panel's inputs arrive. + expect( + normalizeAudioFxParams("compressor", { threshold: "-30" as unknown as number }).threshold, + ).toBe(-30); + }); + it("falls back to the default for an unrecognised enum value", () => { expect(normalizeAudioFxParams("saturate", { type: "sawtooth" }).type).toBe("tanh"); expect(normalizeAudioFxParams("saturate", { type: "atan" }).type).toBe("atan"); diff --git a/packages/core/src/audioFx.ts b/packages/core/src/audioFx.ts index 1e38c4f298..8ab6c60cc0 100644 --- a/packages/core/src/audioFx.ts +++ b/packages/core/src/audioFx.ts @@ -16,6 +16,16 @@ export const HF_AUDIO_FX_ATTR = "data-fx-chain"; +/** + * The same attribute as a `dataset` / `dataAttributes` key — the `data-` prefix + * is not part of that spelling. + * + * Derived rather than restated: the studio writes through + * `HF_AUDIO_FX_ATTR` and reads through the key, so a hardcoded `"fx-chain"` + * on the read side is a rename waiting to half-land. + */ +export const HF_AUDIO_FX_DATA_KEY = HF_AUDIO_FX_ATTR.slice("data-".length); + /** Chain files are versioned; a reader must refuse a version it doesn't know. */ export const HF_AUDIO_FX_CHAIN_VERSION = 1; @@ -761,7 +771,20 @@ export function normalizeAudioFxParams( out[p.key] = ok ? (raw as string) : p.default; continue; } - const n = typeof raw === "number" ? raw : Number(raw); + // Only a number, or a string that actually spells one. `Number(null)`, + // `Number("")`, `Number(false)` and `Number([])` are all 0 and all pass + // Number.isFinite, so a missing or blanked value used to clamp to 0 rather + // than fall back to the declared default — and 0 is a legal value for most + // of these knobs, so nothing downstream could tell. A compressor whose + // threshold arrived as null sat at 0 dB and never engaged, silently, + // instead of at its -24 dB default. `numberOrNull` in audioAutomation.ts + // already guards exactly this. + const n = + typeof raw === "number" + ? raw + : typeof raw === "string" && raw.trim() !== "" + ? Number(raw) + : Number.NaN; out[p.key] = Number.isFinite(n) ? Math.min(p.max, Math.max(p.min, n)) : p.default; } return out; diff --git a/packages/core/src/runtime/audioFx.test.ts b/packages/core/src/runtime/audioFx.test.ts index 9774515810..09374e12e4 100644 --- a/packages/core/src/runtime/audioFx.test.ts +++ b/packages/core/src/runtime/audioFx.test.ts @@ -315,6 +315,124 @@ describe("attachElementFxChain", () => { }); }); + /** + * Lanes are committed to absolute context times, so the schedule is only + * right for the rate it was booked at. Bumping `playbackRate` alone left a + * lowpass sweeping over its original 10 wall-clock seconds while the audio + * underneath ran through 20 clip-seconds of material — and the runtime's + * stopAll()+reschedule recovery never fired for an unbounded source. + */ + describe("a rate change mid-playback", () => { + /** Records what was booked and when, without the browser's overlap rules. */ + class TimedParam { + curves: { time: number; duration: number }[] = []; + ramps: number[] = []; + value = 0; + setValueAtTime(v: number): void { + this.value = v; + } + linearRampToValueAtTime(v: number, t: number): void { + this.ramps.push(t); + this.value = v; + } + setValueCurveAtTime(_v: Float32Array, time: number, duration: number): void { + this.curves.push({ time, duration }); + } + cancelScheduledValues(): void {} + cancelAndHoldAtTime(): void {} + /** The last span booked, however the scheduler chose to express it. */ + last(): { time: number; duration: number } | undefined { + return this.curves.at(-1); + } + } + + const sweep = { + version: 1, + nodes: [{ type: "lowpass", id: "n1", params: { frequency: 300, q: 0.707 } }], + }; + const lane = JSON.stringify({ + version: 1, + lanes: [ + { + target: "fx.n1.frequency", + points: [ + { t: 0, v: 300 }, + { t: 8, v: 3000 }, + ], + }, + ], + }); + + const build = () => { + const clock = { currentTime: 0 }; + const made: { frequency: TimedParam }[] = []; + class TimedNode extends Node { + override frequency = new TimedParam() as unknown as { value: number }; + } + class TimedCtx extends Ctx { + get currentTime(): number { + return clock.currentTime; + } + override createBiquadFilter(): Node { + const n = new TimedNode(); + made.push(n as unknown as { frequency: TimedParam }); + return n; + } + } + const node = document.createElement("audio"); + node.setAttribute("data-fx-chain", JSON.stringify(sweep)); + node.setAttribute("data-automation", lane); + document.body.append(node); + const handle = attachElementFxChain( + new TimedCtx() as unknown as BaseAudioContext, + node, + new Node() as never, + new Node() as never, + { scheduledAt: 0, elapsed: 0, rate: 1 }, + ); + return { clock, node, handle, param: () => made[0]?.frequency as unknown as TimedParam }; + }; + + it("re-aims the envelope so the sweep still ends with the material", () => { + const { clock, handle, param } = build(); + // Booked at 1x: the whole 8 s lane spans 8 s of context time. + expect(param().last()).toEqual({ time: 0, duration: 8 }); + + clock.currentTime = 2; + handle?.setRate(2); + + // 6 clip-seconds are left, and at 2x they take 3 wall-clock seconds. + // Without this the sweep kept its original plan to t=8 while the audio + // ran out at t=5. + expect(param().last()).toEqual({ time: 2, duration: 3 }); + }); + + it("measures later edits from the new rate, not the one it started at", async () => { + // `elapsed` advances at whatever rate the reference frame holds, so a + // frame left at 1x re-aims every subsequent edit at the wrong clip + // position for as long as the track plays. + const { clock, node, handle, param } = build(); + clock.currentTime = 2; + handle?.setRate(2); + + clock.currentTime = 4; + // 2 wall-clock seconds at 2x is 4 clip-seconds, so the playhead is at 6 + // and 2 clip-seconds remain: 1 second of wall clock. + node.setAttribute("data-automation", lane); + await new Promise((r) => setTimeout(r, 0)); + expect(param().last()).toEqual({ time: 4, duration: 1 }); + }); + + it("ignores a rate that is not a rate", () => { + const { clock, handle, param } = build(); + clock.currentTime = 2; + handle?.setRate(0); + handle?.setRate(Number.NaN); + handle?.setRate(1); + expect(param().last()).toEqual({ time: 0, duration: 8 }); + }); + }); + it("tears the chain down on dispose", () => { const src = new Node(); const dst = new Node(); diff --git a/packages/core/src/runtime/audioFx.ts b/packages/core/src/runtime/audioFx.ts index 8bd4c4e3ad..cec709e7a8 100644 --- a/packages/core/src/runtime/audioFx.ts +++ b/packages/core/src/runtime/audioFx.ts @@ -81,15 +81,29 @@ function readChain(el: { getAttribute?(name: string): string | null }): { * first effect is then heard without rescheduling the source. * * With `timing`, the element's automation lanes are scheduled onto the built - * effects as AudioParam ramps, and rescheduled when the attribute is edited. + * effects as AudioParam ramps, and rescheduled when the attribute is edited or + * `setRate` reports the transport changed speed. */ +export interface ElementFxHandle { + dispose(): void; + /** + * Re-aim every booked envelope at a new playback rate. + * + * Lanes are committed to absolute context times, so a param scheduled at 1× + * keeps its original wall-clock plan while the audio underneath runs at the + * new speed: a lowpass sweeping over 10 clip-seconds, switched to 2×, eats + * 20 s of material in 10 s of wall clock with the sweep unchanged. + */ + setRate(rate: number): void; +} + export function attachElementFxChain( ctx: BaseAudioContext, el: { getAttribute?(name: string): string | null }, source: AudioNode, destination: AudioNode, timing?: AutomationTiming, -): { dispose(): void } | null { +): ElementFxHandle | null { const { chain } = readChain(el); // Null means the source runs straight into its gain: an empty chain, or one @@ -166,20 +180,26 @@ export function attachElementFxChain( at && handle ? scheduleChainAutomation(readAutomation(el, next), next, handle.nodes, at) : []; }; + // The reference frame every later reschedule measures from. Mutable because a + // rate change rebases it: `elapsed` has to stop advancing at the old rate the + // instant the new one takes effect, or every subsequent edit re-aims the + // envelope at the wrong clip position. + let frame: AutomationTiming | null = timing ? { ...timing } : null; + attach(chain); - scheduleFor(chain, timing ?? null); + scheduleFor(chain, frame); /** * Re-aim the envelope at the live playhead. An edit lands mid-playback, so * the clip has advanced past the offset the source was scheduled with. */ const timingNow = (): AutomationTiming | null => { - if (!timing) return null; - const now = typeof ctx.currentTime === "number" ? ctx.currentTime : timing.scheduledAt; + if (!frame) return null; + const now = typeof ctx.currentTime === "number" ? ctx.currentTime : frame.scheduledAt; return { scheduledAt: now, - elapsed: timing.elapsed + (now - timing.scheduledAt) * timing.rate, - rate: timing.rate, + elapsed: frame.elapsed + (now - frame.scheduledAt) * frame.rate, + rate: frame.rate, }; }; @@ -239,6 +259,15 @@ export function attachElementFxChain( } return { + setRate: (rate: number) => { + const at = timingNow(); + if (disposed || !at || !Number.isFinite(rate) || rate <= 0 || rate === at.rate) return; + // Rebased at the playhead the OLD rate carried us to, then replayed from + // there at the new one. + frame = { ...at, rate }; + cancelParamLane(automated, at.scheduledAt); + scheduleFor(readChain(el).chain, frame); + }, dispose: () => { disposed = true; observer?.disconnect(); diff --git a/packages/core/src/runtime/media.test.ts b/packages/core/src/runtime/media.test.ts index c6fa207ed8..bf697161b8 100644 --- a/packages/core/src/runtime/media.test.ts +++ b/packages/core/src/runtime/media.test.ts @@ -365,6 +365,37 @@ describe("syncRuntimeMedia", () => { expect(only).toBeCloseTo(0.55, 5); }); + /** + * The render bakes the lane at CLIP-LOCAL time: prepareAudioTrack already + * cut the wav with `-ss mediaStart`, so its t=0 is the clip's start, and + * normaliseEnvelope subtracts trackStart. Preview used to sample at MEDIA + * time — mediaStart included, scaled by playbackRate, wrapped on a loop — so + * the same envelope played somewhere else than it rendered. + */ + it("samples the lane at clip-local time, the way the render bakes it", () => { + const trimmed = (t: number) => { + const clip = createMockClip({ start: 0, end: 10, volume: 0.55, mediaStart: 30 }); + Object.defineProperty(clip.el, "readyState", { value: 4, writable: true }); + clip.el.setAttribute("data-automation", DUCK); + let seen = -1; + syncRuntimeMedia({ + clips: [clip], + timeSeconds: t, + playing: true, + playbackRate: 1, + onElementVolume: (_el, v) => { + seen = v; + }, + }); + return seen; + }; + // `data-media-start="30"` on a clip whose lane holds 0.8 until t=2 then + // ducks to 0.1 by t=3. At media time the playhead is already 30 s past the + // last point, so preview held 0.1 from the first frame and never ducked. + expect(trimmed(1)).toBeCloseTo(0.8, 5); + expect(trimmed(5)).toBeCloseTo(0.1, 5); + }); + it("supersedes keyframes probed from the timeline", () => { // Both present: the lane is the explicit one, and `lint` warns about it. const clip = createMockClip({ start: 0, end: 10, volume: 0.55 }); diff --git a/packages/core/src/runtime/media.ts b/packages/core/src/runtime/media.ts index 8dabe8c37b..a6aa4c2183 100644 --- a/packages/core/src/runtime/media.ts +++ b/packages/core/src/runtime/media.ts @@ -274,7 +274,15 @@ export function syncRuntimeMedia(params: { // An explicit volume lane owns the fader. It is checked before the probed // keyframes because the two would otherwise fight, and it is the one the // author drew — `lint` warns when a track carries both. - const laneGain = elementVolumeLaneGain(el, relTime); + // Clip-local, NOT `relTime`. A lane's `t` is "seconds from the start of + // the clip" (see HfAutomationPoint), and the render honours that: the wav + // is already cut with `-ss mediaStart`, so its t=0 IS the clip's start. + // `relTime` is MEDIA time — it carries mediaStart, scales by playbackRate + // and wraps on a loop — so feeding it here played the envelope at a + // different position than it renders, or ran it off the end entirely on a + // trimmed clip. The FX lanes on this same feature use clip-local elapsed; + // there is one time base, and this is it. + const laneGain = elementVolumeLaneGain(el, params.timeSeconds - clip.start); if (laneGain !== null) { authorVolume = clampVolume(laneGain); } else if (clip.volumeKeyframes && clip.volumeKeyframes.length > 0) { diff --git a/packages/core/src/runtime/webAudioTransport.test.ts b/packages/core/src/runtime/webAudioTransport.test.ts index 15f1825a13..35f935bc5a 100644 --- a/packages/core/src/runtime/webAudioTransport.test.ts +++ b/packages/core/src/runtime/webAudioTransport.test.ts @@ -289,6 +289,23 @@ describe("WebAudioTransport", () => { expect(mock.sourceNode.playbackRate.value).toBe(2); }); + it("setRate re-aims each source's FX automation, not just its playback rate", async () => { + // The lanes are committed to absolute context times when the source is + // scheduled, so bumping playbackRate alone left every automated parameter + // running its original plan over audio moving at a different speed. + const { transport, mock, gen } = setupTransport(100); + await transport.schedulePlayback(mockEl, mockBuffer, 5, 0, 8, 1, gen, 1); + const active = (transport as unknown as { _activeSources: { fx?: unknown }[] }) + ._activeSources; + const setRate = vi.fn(); + active[0]!.fx = { dispose: vi.fn(), setRate }; + + transport.setRate(2); + + expect(setRate).toHaveBeenCalledWith(2); + expect(mock.sourceNode.playbackRate.value).toBe(2); + }); + it("setRate before any sources are scheduled does not throw", () => { const transport = new WebAudioTransport(); expect(() => transport.setRate(2)).not.toThrow(); @@ -388,6 +405,19 @@ describe("WebAudioTransport", () => { expect(transport.isActive()).toBe(false); }); + it("disposes the FX graph when a clip ends naturally", async () => { + // stopAll() disposes by walking _activeSources, and the splice above had + // already removed this entry — so the handle, its MutationObserver and any + // running LFO survived the clip for the rest of the session. + const { transport, mock, gen } = setupTransport(100); + await transport.schedulePlayback(mockEl, mockBuffer, 0, 0, 0, 1, gen); + + mock.sourceNode._fireEnded(); + + expect(mock.sourceNode.disconnect).toHaveBeenCalled(); + expect(mock.gainNode.disconnect).toHaveBeenCalled(); + }); + it("registers onended listener on the sourceNode", async () => { const { transport, mock, gen } = setupTransport(100); diff --git a/packages/core/src/runtime/webAudioTransport.ts b/packages/core/src/runtime/webAudioTransport.ts index fedcdca0bf..d77aa7152a 100644 --- a/packages/core/src/runtime/webAudioTransport.ts +++ b/packages/core/src/runtime/webAudioTransport.ts @@ -1,4 +1,4 @@ -import { attachElementFxChain } from "./audioFx.js"; +import { attachElementFxChain, type ElementFxHandle } from "./audioFx.js"; import type { AutomationTiming } from "../audio/audioFxAutomation.js"; import { swallow } from "./diagnostics"; import { getDebugSurface } from "./globals.js"; @@ -63,7 +63,7 @@ export type ScheduledSource = { sourceNode: AudioBufferSourceNode; gainNode: GainNode; /** FX chain spliced between source and gain, when the element carries one. */ - fx?: { dispose(): void } | null; + fx?: ElementFxHandle | null; compositionStart: number; mediaStart: number; scheduledAt: number; @@ -243,6 +243,21 @@ export class WebAudioTransport { if (idx !== -1) { this._activeSources.splice(idx, 1); el.muted = priorMuted; + // The graph goes with it. Splicing alone left the FX handle alive and + // then UNREACHABLE — stopAll() disposes by walking this array, which + // the splice just emptied of this entry. Every clip that finished + // naturally leaked its MutationObserver for the session, and each one + // still answered later `data-fx-chain` edits by rebuilding a whole + // graph (impulse response, chorus/phaser oscillators started and never + // stopped) around a dead source. Not disposed when idx is -1: stopAll() + // has already done it, and `stop()` is what fired this event. + try { + sourceNode.disconnect(); + fx?.dispose(); + gainNode.disconnect(); + } catch { + // Already torn down. + } if (this._activeSources.length === 0) this._paused = true; } }); @@ -259,6 +274,14 @@ export class WebAudioTransport { * `getTime()` stays continuous across the change. Sources scheduled to * start in the future keep their original wallclock start time — callers * that need rate-correct future starts should `stopAll()` and reschedule. + * + * Each source's FX automation is re-aimed too. Lanes are committed to + * absolute context times when the source is scheduled, so bumping only + * `playbackRate` left every automated parameter running its original plan + * over audio moving at a different speed. The `stopAll()`+reschedule recovery + * in the runtime is no help here: it only fires for bounded sources, and a + * project-level music bed with no `data-duration` is unbounded, so it never + * recovered at all. */ setRate(rate: number): boolean { const safeRate = normalizeRate(rate); @@ -271,6 +294,7 @@ export class WebAudioTransport { for (const source of this._activeSources) { try { source.sourceNode.playbackRate.value = safeRate; + source.fx?.setRate(safeRate); } catch (err) { swallow("webAudioTransport.setRate", err); } diff --git a/packages/core/stubs/audio-fx-runtime-entry.ts b/packages/core/stubs/audio-fx-runtime-entry.ts index 81f8e5f035..d6a78b9439 100644 --- a/packages/core/stubs/audio-fx-runtime-entry.ts +++ b/packages/core/stubs/audio-fx-runtime-entry.ts @@ -81,6 +81,11 @@ async function render( const chain: HfAudioFxChain = parseAudioFxChain(chainJson); const channels = Math.max(1, planes.length); const frames = planes[0]?.length ?? 0; + // Nothing to process, and `new OfflineAudioContext(ch, 0, rate)` throws — an + // error the render treats as fatal. applyAudioFxChain screens empty tracks + // out before they reach the browser; this is the same guard at the point the + // constructor would actually blow up. + if (frames === 0) return planes; const parsedAutomation = automationJson ? resolveAutomation(parseAutomation(automationJson), chain) : null; diff --git a/packages/engine/src/services/audioFxRender.test.ts b/packages/engine/src/services/audioFxRender.test.ts index 6b07f35c3a..bd78710c83 100644 --- a/packages/engine/src/services/audioFxRender.test.ts +++ b/packages/engine/src/services/audioFxRender.test.ts @@ -140,7 +140,7 @@ describe("applyAudioFxChain", () => { join(dir, "out.wav"), { trackId: "t" }, ); - expect(out).toBe(input); + expect(out).toEqual({ path: input, envelopeBaked: false }); expect(existsSync(join(dir, "out.wav"))).toBe(false); }); @@ -173,7 +173,7 @@ describe("browser render", () => { outPath, { trackId: "t" }, ); - expect(result).toBe(outPath); + expect(result).toEqual({ path: outPath, envelopeBaked: false }); const before = readWav(input).samples; const after = readWav(outPath).samples; expect(after.length).toBe(before.length); @@ -243,6 +243,139 @@ describe("browser render", () => { expect(tail).toBeGreaterThan(head + 15); }, 180_000); + it("ducks a hot chain before quantising it, not after", async () => { + // A +6 dB peaking band on a 0.9 tone leaves the chain output around 1.8 — + // well past full scale — and the volume lane immediately halves it. Baking + // the envelope into the float samples lands that at ~0.9 intact. Letting + // writeWav clamp first and ducking the file afterwards lands at ~0.5 with + // the tops sheared off: distortion the render bakes in and preview, which + // is float all the way through, never has. + // + // Stereo with a step partway through, so the same run also proves the + // envelope is walked per frame across all channels rather than per channel: + // one walker restarted for a second plane would hand it the tail gain for + // its whole length. + const input = join(dir, "hot-in.wav"); + const frames = Math.floor(SR * 0.3); + const s = new Float32Array(frames * 2); + for (let i = 0; i < frames; i++) { + const v = 0.9 * Math.sin((2 * Math.PI * 440 * i) / SR); + s[i * 2] = v; + s[i * 2 + 1] = v; + } + writeWav(input, s, SR, 2); + + const outPath = join(dir, "hot-out.wav"); + const result = await applyAudioFxChain( + input, + { + version: 1, + nodes: [{ type: "peaking", enabled: true, params: { frequency: 440, gain: 6, q: 1 } }], + }, + outPath, + { + trackId: "t", + envelope: { + // 0.5 for the first 150 ms, then 0.25 — the step is a segment + // advance, which is what the walker's cursor exists for. + keyframes: [ + { time: 0, volume: 0.5 }, + { time: 0.15, volume: 0.5 }, + { time: 0.1501, volume: 0.25 }, + { time: 0.3, volume: 0.25 }, + ], + trackStart: 0, + baseVolume: 1, + }, + }, + ); + expect(result.envelopeBaked).toBe(true); + + const out = readWav(outPath); + expect(out.channels).toBe(2); + const channel = (c: number): Float32Array => + Float32Array.from({ length: frames }, (_, i) => out.samples[i * 2 + c] ?? 0); + const window = (s: Float32Array, from: number, to: number): Float32Array => + s.slice(Math.floor(from * SR), Math.floor(to * SR)); + + for (const c of [0, 1]) { + const plane = channel(c); + // Past the filter's settling transient, before the step. + const loud = window(plane, 0.1, 0.14); + const peak = Math.max(...Array.from(loud, Math.abs)); + // ~0.9. Clamped-then-ducked gives 0.5; a dropped envelope gives 1.0; a + // walker restarted per plane gives channel 1 the 0.25 tail gain. + expect(peak).toBeGreaterThan(0.8); + expect(peak).toBeLessThan(0.95); + // And still a sine, not a squared-off one: clipping 1.8 down to 1.0 pulls + // the crest factor from 1.41 towards 1.15. + expect(peak / rms(loud)).toBeGreaterThan(1.35); + // The step landed, so the envelope was sampled over time, not once. + const quiet = window(plane, 0.2, 0.29); + expect(Math.max(...Array.from(quiet, Math.abs))).toBeCloseTo(peak / 2, 1); + } + }, 180_000); + + it("round-trips a track larger than one transfer chunk", async () => { + // The PCM used to cross in a single evaluate pair, so a stereo track past + // ~8.7 minutes blew puppeteer's 256 MB frame cap and failed the render. + // It now goes a chunk at a time; this clip is 45 s stereo, so each plane + // spans two chunks and the seam is inside the audio rather than at its end. + // + // A ramp, not a tone: every sample is a unique position marker, so a chunk + // dropped, reordered or over-read shows up as a value at the wrong place + // instead of hiding inside a periodic signal. + const input = join(dir, "long-in.wav"); + const frames = SR * 45; + const at = (i: number): number => -0.9 + (1.8 * i) / (frames - 1); + const s = new Float32Array(frames * 2); + for (let i = 0; i < frames; i++) { + s[i * 2] = at(i); + s[i * 2 + 1] = -at(i); + } + writeWav(input, s, SR, 2); + + const outPath = join(dir, "long-out.wav"); + await applyAudioFxChain( + input, + // Transparent: a peaking band at 0 dB is unity, so the output is the + // input and any difference is the transfer's doing. + { + version: 1, + nodes: [{ type: "peaking", enabled: true, params: { frequency: 1000, gain: 0, q: 1 } }], + }, + outPath, + { trackId: "t" }, + ); + + const out = readWav(outPath); + expect(out.channels).toBe(2); + // A dropped tail chunk shows up here first. + expect(out.samples.length).toBe(frames * 2); + + // Probe across the whole clip and tightly around the 8 MiB seam. + const seam = (8 * 1024 * 1024) / 4; + const probes = [ + ...Array.from({ length: 60 }, (_, k) => Math.floor((k * (frames - 1)) / 59)), + ...[-2, -1, 0, 1, 2].map((d) => seam + d), + ].filter((i) => i >= 0 && i < frames); + for (const i of probes) { + // Two 16-bit steps of tolerance: the input was quantised on the way in + // and the output again on the way out. + expect(out.samples[i * 2]).toBeCloseTo(at(i), 3); + expect(out.samples[i * 2 + 1]).toBeCloseTo(-at(i), 3); + } + + // The ramp only ever rises, so this reads every sample and fails on a + // chunk reordered, duplicated or over-read anywhere in the file — not just + // at the points probed above. + let breaks = 0; + for (let i = 1; i < frames; i++) { + if ((out.samples[i * 2] ?? 0) < (out.samples[(i - 1) * 2] ?? 0)) breaks += 1; + } + expect(breaks).toBe(0); + }, 180_000); + it("renders a multi-effect chain including reverb", async () => { const input = join(dir, "in.wav"); tone(input); @@ -253,3 +386,25 @@ describe("browser render", () => { expect(readWav(outPath).samples.length).toBeGreaterThan(0); }, 180_000); }); + +/** + * ffmpeg exits 0 and writes a structurally valid but EMPTY wav whenever a + * clip's trim starts past the end of its source (`-ss 10 -t 5` on a 2 s file). + * That track used to reach `new OfflineAudioContext(ch, 0, rate)`, which throws + * — and the error travels past the mixer's per-track failure collector, so one + * mis-set `data-media-start` took the whole render down. + */ +describe("an empty track", () => { + it("is handed back untouched rather than failing the render", async () => { + const input = join(dir, "empty.wav"); + writeWav(input, new Float32Array(0), SR); + const output = join(dir, "out.wav"); + + // Returns the input path, the same contract as a chain with nothing enabled + // — and without paying for a browser to decide it. + await expect( + applyAudioFxChain(input, chainOf("peaking"), output, { trackId: "t" }), + ).resolves.toEqual({ path: input, envelopeBaked: false }); + expect(existsSync(output)).toBe(false); + }); +}); diff --git a/packages/engine/src/services/audioFxRender.ts b/packages/engine/src/services/audioFxRender.ts index 1d53156621..bca39030b4 100644 --- a/packages/engine/src/services/audioFxRender.ts +++ b/packages/engine/src/services/audioFxRender.ts @@ -20,6 +20,8 @@ import { getAudioFxRuntimeScript } from "@hyperframes/core/audio-fx-runtime"; import { enabledAudioFxNodes, type HfAudioFxChain } from "@hyperframes/core/audio-fx"; import { serializeAutomation, type HfAutomation } from "@hyperframes/core/audio-automation"; import { acquireBrowser } from "./browserManager.js"; +import { createEnvelopeWalker } from "./audioVolumeEnvelope.js"; +import type { AudioVolumeKeyframe } from "./audioMixer.types.js"; export class AudioFxRenderError extends Error { constructor(message: string) { @@ -172,10 +174,66 @@ function interleave(planes: readonly Float32Array[]): Float32Array { return out; } +/** + * Bytes of PCM per CDP message. + * + * The whole track used to cross in a single `page.evaluate` pair — ~184 MB of + * base64 for a 3-minute stereo 48 kHz clip, in one WebSocket frame each way. + * puppeteer-core caps its frames at 256 MB and the browser pool does not use + * the pipe transport, so stereo past ~8.7 minutes failed the render outright; + * V8's max string length is a second wall not far beyond it. Chunking bounds + * both, and bounds the peak Node-side allocation with them: the mixer renders + * tracks concurrently, so every track's payload was live at once. + * + * 8 MiB encodes to ~11 MB of base64. A multiple of 4, so a chunk boundary + * never falls inside a float. + */ +const TRANSFER_BYTES = 8 * 1024 * 1024; + +/** The page-side handover buffers, named off `window` so each step can find them. */ +interface AudioFxPageIo { + in: Uint8Array[][]; + out: Float32Array[]; +} + +interface AudioFxWindow { + __HF_AUDIO_FX?: { + render(p: Float32Array[], r: number, c: string, a?: string): Promise; + }; + __HF_FX_IO?: AudioFxPageIo; +} + +/** + * Multiply every channel by the envelope, in place, one gain per frame. + * + * Frame-outer rather than plane-outer on purpose: the walker's cursor only + * moves forward, so restarting each channel at t=0 would hand the second one + * the tail gain for its whole length. + */ +function applyEnvelopeToPlanes( + planes: readonly Float32Array[], + sampleRate: number, + gainAt: (time: number) => number, +): void { + const frames = planes[0]?.length ?? 0; + for (let frame = 0; frame < frames; frame += 1) { + const gain = gainAt(frame / sampleRate); + for (const plane of planes) plane[frame] = (plane[frame] ?? 0) * gain; + } +} + /** * Run a chain over `inputWav`, writing `outputWav`. Resolves to the path to use - * downstream: `outputWav` when the chain did something, `inputWav` untouched - * when the chain was empty. + * downstream — `outputWav` when the chain did something, `inputWav` untouched + * when the chain was empty — plus whether the volume envelope was baked here. + * + * The envelope is applied to the float samples the chain produced, BEFORE + * `writeWav` quantises them. Leaving it to the mixer's second pass meant a + * chain that overshoots full scale was destructively clipped to ±1 and only + * then ducked, so a track the lane pulls 12 dB down still rendered the + * distortion — which preview, working in float throughout, never had. + * `writeWav`'s clamp stays: it is the correct last resort for a signal that is + * still hot after the duck. * * Failure is fatal to the caller rather than a soft per-track warning: quietly * rendering the dry signal ships a mix that sounds plausible and is not what @@ -185,27 +243,44 @@ export async function applyAudioFxChain( inputWav: string, chain: HfAudioFxChain, outputWav: string, - options: { trackId: string; signal?: AbortSignal; automation?: HfAutomation }, -): Promise { - if (enabledAudioFxNodes(chain).length === 0) return inputWav; + options: { + trackId: string; + signal?: AbortSignal; + automation?: HfAutomation; + envelope?: { keyframes: AudioVolumeKeyframe[]; trackStart: number; baseVolume: number }; + }, +): Promise<{ path: string; envelopeBaked: boolean }> { + if (enabledAudioFxNodes(chain).length === 0) return { path: inputWav, envelopeBaked: false }; if (!existsSync(inputWav)) { throw new AudioFxRenderError(`Audio FX input is missing: ${inputWav}`); } const { samples, sampleRate, channels } = readWav(inputWav); const planes = deinterleave(samples, channels); + // An empty track has nothing to process — and an OfflineAudioContext of zero + // length throws, which is fatal for the WHOLE render rather than this track: + // the error travels past the mixer's per-track failure collector. ffmpeg + // writes an empty but structurally valid WAV whenever a clip's trim starts + // past the end of its source, so one mis-set `data-media-start` used to take + // the render down. Guarded here as well as in the runtime so an empty track + // never costs a browser. + if ((planes[0]?.length ?? 0) === 0) return { path: inputWav, envelopeBaked: false }; - // Audio processing needs no GPU or special capture mode; a plain sandboxed - // browser is enough, and the lease pool reuses one across tracks. - const lease = await acquireBrowser([ - "--no-sandbox", - "--autoplay-policy=no-user-gesture-required", - ]); + // Both resources are taken INSIDE the try that releases them. The lease used + // to be acquired above it, with the mkdtemp between — so a failure there + // (a full disk, a read-only tmpdir) leaked a pooled browser, and a pool with + // no leases left hangs every later render rather than failing one. const hostDir = mkdtempSync(join(tmpdir(), "hf-fx-host-")); + let lease: Awaited> | null = null; try { + // Audio processing needs no GPU or special capture mode; a plain sandboxed + // browser is enough, and the lease pool reuses one across tracks. + // Checked before the lease, not after: an already-cancelled track has no + // reason to take a browser out of the pool just to hand it straight back. if (options.signal?.aborted) { throw new AudioFxRenderError(`Audio FX cancelled for track ${options.trackId}`); } + lease = await acquireBrowser(["--no-sandbox", "--autoplay-policy=no-user-gesture-required"]); const page = await lease.browser.newPage(); try { // AudioWorklet is only exposed in a secure context, and about:blank is @@ -216,71 +291,119 @@ export async function applyAudioFxChain( await page.goto(pathToFileURL(hostPage).href, { waitUntil: "domcontentloaded" }); await page.addScriptTag({ content: getAudioFxRuntimeScript() }); - const rendered = (await page.evaluate( - async ([channelB64, rate, chainJson, automationJson]: [ - string[], - number, - string, - string, - ]) => { - const decode = (b64: string): Float32Array => { - const bin = atob(b64); - const bytes = new Uint8Array(bin.length); - for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); - return new Float32Array(bytes.buffer); - }; - const api = ( - window as unknown as { - __HF_AUDIO_FX?: { - render( - p: Float32Array[], - r: number, - c: string, - a?: string, - ): Promise; - }; + // Hand the input over a chunk at a time. The chunks stay separate byte + // arrays page-side rather than being concatenated into one string, so + // neither the frame cap nor V8's string limit sees the whole track. + await page.evaluate((count: number) => { + (window as unknown as AudioFxWindow).__HF_FX_IO = { + in: Array.from({ length: count }, (): Uint8Array[] => []), + out: [], + }; + }, planes.length); + + for (let p = 0; p < planes.length; p += 1) { + const plane = planes[p]; + if (!plane) continue; + const bytes = Buffer.from(plane.buffer, plane.byteOffset, plane.length * 4); + for (let at = 0; at < bytes.length; at += TRANSFER_BYTES) { + await page.evaluate( + ([index, b64]: [number, string]) => { + const bin = atob(b64); + const chunk = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) chunk[i] = bin.charCodeAt(i); + (window as unknown as AudioFxWindow).__HF_FX_IO?.in[index]?.push(chunk); + }, + [p, bytes.subarray(at, at + TRANSFER_BYTES).toString("base64")] as [number, string], + ); + } + } + + const outLengths = (await page.evaluate( + async ([rate, chainJson, automationJson]: [number, string, string]) => { + const w = window as unknown as AudioFxWindow; + const io = w.__HF_FX_IO; + if (!w.__HF_AUDIO_FX || !io) throw new Error("audio FX runtime failed to load"); + const inPlanes = io.in.map((chunks) => { + const bytes = new Uint8Array(chunks.reduce((n, c) => n + c.length, 0)); + let at = 0; + for (const chunk of chunks) { + bytes.set(chunk, at); + at += chunk.length; } - ).__HF_AUDIO_FX; - if (!api) throw new Error("audio FX runtime failed to load"); - const out = await api.render( - channelB64.map(decode), + return new Float32Array(bytes.buffer); + }); + // Dropped before the render allocates its own buffers, so the page + // does not hold two copies of the track at once. + io.in = []; + io.out = await w.__HF_AUDIO_FX.render( + inPlanes, rate, chainJson, automationJson || undefined, ); - const encode = (plane: Float32Array): string => { - const u8 = new Uint8Array(plane.buffer, plane.byteOffset, plane.length * 4); - let s = ""; - const CHUNK = 0x8000; - for (let i = 0; i < u8.length; i += CHUNK) { - s += String.fromCharCode.apply(null, Array.from(u8.subarray(i, i + CHUNK))); - } - return btoa(s); - }; - return out.map(encode); + return io.out.map((plane) => plane.length); }, [ - planes.map((plane) => - Buffer.from(plane.buffer, plane.byteOffset, plane.length * 4).toString("base64"), - ), sampleRate, JSON.stringify(chain), options.automation ? serializeAutomation(options.automation) : "", - ] as [string[], number, string, string], - )) as string[]; + ] as [number, string, string], + )) as number[]; - // byteOffset and byteLength matter: Node pools small allocations, so a - // short payload decodes into an 8 KiB pool and a view over the whole - // ArrayBuffer would read kilobytes of unrelated memory at the wrong length. - const outPlanes = rendered.map((b64) => { - const buf = Buffer.from(b64, "base64"); - return new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength)); - }); + const outPlanes: Float32Array[] = []; + for (let p = 0; p < outLengths.length; p += 1) { + const byteLength = (outLengths[p] ?? 0) * 4; + const parts: Buffer[] = []; + for (let at = 0; at < byteLength; at += TRANSFER_BYTES) { + const b64 = (await page.evaluate( + ([index, offset, limit]: [number, number, number]) => { + const plane = (window as unknown as AudioFxWindow).__HF_FX_IO?.out[index]; + if (!plane) return ""; + const u8 = new Uint8Array( + plane.buffer, + plane.byteOffset + offset, + Math.min(limit, plane.length * 4 - offset), + ); + let s = ""; + const CHUNK = 0x8000; + for (let i = 0; i < u8.length; i += CHUNK) { + // `apply` takes array-likes, so the subarray goes in as it is; + // Array.from boxed every byte of a 32 KiB window for nothing. + s += String.fromCharCode.apply( + null, + u8.subarray(i, i + CHUNK) as unknown as number[], + ); + } + return btoa(s); + }, + [p, at, TRANSFER_BYTES] as [number, number, number], + )) as string; + parts.push(Buffer.from(b64, "base64")); + } + // byteOffset and byteLength matter: Node pools small allocations, so a + // short payload decodes into an 8 KiB pool and a view over the whole + // ArrayBuffer would read kilobytes of unrelated memory at the wrong length. + const buf = Buffer.concat(parts); + outPlanes.push( + new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength)), + ); + } if (outPlanes.length === 0 || (outPlanes[0]?.length ?? 0) === 0) { throw new AudioFxRenderError(`Audio FX produced no samples for track ${options.trackId}`); } + // Null when the keyframes normalise away to nothing — then the mixer's + // own paths still own this track's gain, so say so rather than claiming + // a bake that never happened. + const gainAt = options.envelope + ? createEnvelopeWalker( + options.envelope.keyframes, + options.envelope.trackStart, + options.envelope.baseVolume, + ) + : null; + if (gainAt) applyEnvelopeToPlanes(outPlanes, sampleRate, gainAt); writeWav(outputWav, interleave(outPlanes), sampleRate, outPlanes.length); - return outputWav; + return { path: outputWav, envelopeBaked: gainAt !== null }; } finally { await page.close().catch(() => undefined); } @@ -291,7 +414,7 @@ export async function applyAudioFxChain( ); } finally { rmSync(hostDir, { recursive: true, force: true }); - await lease.release().catch(() => undefined); + await lease?.release().catch(() => undefined); } } diff --git a/packages/engine/src/services/audioMixer.test.ts b/packages/engine/src/services/audioMixer.test.ts index 957016d593..895fb0574e 100644 --- a/packages/engine/src/services/audioMixer.test.ts +++ b/packages/engine/src/services/audioMixer.test.ts @@ -44,11 +44,15 @@ vi.mock("../utils/runFfmpeg.js", async (importOriginal) => { // The FX render drives a headless browser; the mix only needs to know the // processed file exists and how long a tail the chain asked for. const { applyAudioFxChainMock } = vi.hoisted(() => ({ - applyAudioFxChainMock: vi.fn(async (_src: string, _chain: unknown, outPath: string) => { - const { writeFileSync } = await import("node:fs"); - writeFileSync(outPath, "stub"); - return outPath; - }), + applyAudioFxChainMock: vi.fn( + async (_src: string, _chain: unknown, outPath: string, options?: { envelope?: unknown }) => { + const { writeFileSync } = await import("node:fs"); + writeFileSync(outPath, "stub"); + // The real one bakes the volume envelope into its float output, so the + // mixer must not run its own pass afterwards. + return { path: outPath, envelopeBaked: Boolean(options?.envelope) }; + }, + ), })); vi.mock("./audioFxRender.js", async (importOriginal) => { @@ -212,6 +216,62 @@ describe("processCompositionAudio", () => { expect(filter).toContain("apad,atrim=0:8"); }); + it("hands the volume envelope to the FX pass instead of ducking the file after it", async () => { + // The FX pass writes 16-bit PCM, so a chain that overshoots full scale is + // clipped there. Ducking afterwards bakes that distortion in even though + // the lane pulls the track well down; the envelope has to travel into the + // FX pass and land on its float output. + const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); + const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); + tempDirs.push(baseDir, workDir); + writeFileSync(join(baseDir, "voice.wav"), "stub"); + + const result = await processCompositionAudio( + [ + { + id: "voice", + src: "voice.wav", + start: 2, + end: 5, + mediaStart: 0, + layer: 0, + volume: 0.4, + volumeKeyframes: [ + { time: 2, volume: 1 }, + { time: 5, volume: 0.25 }, + ], + type: "audio", + fxChain: JSON.stringify({ + version: 1, + nodes: [{ type: "peaking", id: "p", params: { frequency: 440, gain: 12, q: 1 } }], + }), + }, + ], + baseDir, + workDir, + join(baseDir, "out.m4a"), + 5, + ); + + expect(result.success).toBe(true); + expect(applyAudioFxChainMock).toHaveBeenCalledTimes(1); + expect(applyAudioFxChainMock.mock.calls[0]?.[3]).toMatchObject({ + envelope: { + keyframes: [ + { time: 2, volume: 1 }, + { time: 5, volume: 0.25 }, + ], + trackStart: 2, + baseVolume: 0.4, + }, + }); + + // And the mixer trusts that bake: unity gain, no second pass, no ffmpeg + // volume expression re-applying the same envelope on top of it. + const filter = capturedFilterScripts[capturedFilterScripts.length - 1]; + expect(filter).not.toContain(":eval=frame"); + }); + it("cuts at the clip boundary when the chain has no tail", async () => { const baseDir = mkdtempSync(join(tmpdir(), "hf-audio-base-")); const workDir = mkdtempSync(join(tmpdir(), "hf-audio-work-")); diff --git a/packages/engine/src/services/audioMixer.ts b/packages/engine/src/services/audioMixer.ts index 369562e328..9ef9cb602f 100644 --- a/packages/engine/src/services/audioMixer.ts +++ b/packages/engine/src/services/audioMixer.ts @@ -765,6 +765,12 @@ export async function processCompositionAudio( // be able to abort the in-flight ffmpeg runs before the finally-block removes // workDir out from under them. Chained off the caller's signal so external // cancellation still behaves as before. + // + // Every child that can outlive a sibling's failure has to be given THIS + // signal, not the caller's: the trim, the video extract and the download all + // took `signal`, so `internalController.abort()` cancelled nothing and the + // `rmSync(workDir)` on the next line ran while their ffmpeg children were + // still writing into it. const internalController = new AbortController(); const effectiveSignal = internalController.signal; if (signal) { @@ -795,7 +801,7 @@ export async function processCompositionAudio( if (isHttpUrl(srcPath)) { try { - srcPath = await downloadToTemp(srcPath, workDir); + srcPath = await downloadToTemp(srcPath, workDir, undefined, effectiveSignal); } catch (err: unknown) { failures.push( downloadFailure(err instanceof Error ? err.message : String(err), element.id), @@ -842,7 +848,7 @@ export async function processCompositionAudio( startTime: element.mediaStart, duration: element.end - element.start, }, - signal, + effectiveSignal, config, ); if (!extractResult.success) { @@ -868,7 +874,7 @@ export async function processCompositionAudio( trimmedPath, element.mediaStart, element.end - element.start, - signal, + effectiveSignal, config, ); if (!prepResult.success) { @@ -901,6 +907,27 @@ export async function processCompositionAudio( ) : null; + // Computed before the chain runs, not after: the FX pass bakes the + // envelope into its float output so the duck lands before the ±1 clamp + // in writeWav, instead of after it. + // + // A volume lane supersedes keyframes probed from the timeline: the two + // would fight, and the lane is the explicit one. `lint` warns when a + // track carries both. + const laneKeyframes = automation + ? volumeLaneKeyframes(automation, element.start, element.end - element.start) + : null; + const envelopeKeyframes = laneKeyframes ?? element.volumeKeyframes; + const envelope = + envelopeKeyframes && envelopeKeyframes.length > 0 + ? { + keyframes: envelopeKeyframes, + trackStart: element.start, + baseVolume: element.volume ?? 1.0, + } + : null; + + let bakedEnvelope = false; let tailSeconds = 0; if (element.fxChain) { // The chain is serialised into the attribute, the same way colour @@ -910,7 +937,7 @@ export async function processCompositionAudio( // The rendered WAV is longer than the input by exactly this much, so // the mix has to be told to let it through. tailSeconds = chainTailSeconds(chain, automation ?? undefined); - audioSrcPath = await applyAudioFxChain( + const fxResult = await applyAudioFxChain( audioSrcPath, chain, join(workDir, `${element.id}-fx.wav`), @@ -918,29 +945,24 @@ export async function processCompositionAudio( trackId: element.id, signal: effectiveSignal, ...(automation ? { automation } : {}), + ...(envelope ? { envelope } : {}), }, ); + audioSrcPath = fxResult.path; + bakedEnvelope = fxResult.envelopeBaked; } - // Primary volume-automation path: bake the envelope into the PCM samples - // (sample-accurate, no keyframe ceiling). If the WAV isn't the expected - // 16-bit PCM, fall back to the ffmpeg expression path by leaving the - // keyframes on the track for buildVolumeExpression to handle. - // - // A volume lane supersedes keyframes probed from the timeline: the two - // would fight, and the lane is the explicit one. `lint` warns when a - // track carries both. - const laneKeyframes = automation - ? volumeLaneKeyframes(automation, element.start, element.end - element.start) - : null; - const envelopeKeyframes = laneKeyframes ?? element.volumeKeyframes; - let bakedEnvelope = false; - if (envelopeKeyframes && envelopeKeyframes.length > 0) { + // Primary volume-automation path for a track the FX pass did not bake: + // multiply the envelope into the PCM samples (sample-accurate, no + // keyframe ceiling). If the WAV isn't the expected 16-bit PCM, fall + // back to the ffmpeg expression path by leaving the keyframes on the + // track for buildVolumeExpression to handle. + if (envelope && !bakedEnvelope) { bakedEnvelope = applyVolumeEnvelopeToWav( audioSrcPath, - envelopeKeyframes, - element.start, - element.volume ?? 1.0, + envelope.keyframes, + envelope.trackStart, + envelope.baseVolume, ); } tracks.push({ diff --git a/packages/engine/src/services/audioVolumeEnvelope.ts b/packages/engine/src/services/audioVolumeEnvelope.ts index 9d4d0aa730..08f8a18828 100644 --- a/packages/engine/src/services/audioVolumeEnvelope.ts +++ b/packages/engine/src/services/audioVolumeEnvelope.ts @@ -74,6 +74,40 @@ function parseWavLayout(buffer: Buffer): WavLayout | null { }; } +/** + * A gain lookup that walks forward through the envelope with a segment cursor, + * so a whole track costs O(N+M) rather than O(N×M). `interpolateVolumeGain` + * restarts from segment 0 on every call — fine for the preview path (once per + * RAF tick), not for a per-sample walk over 48k×duration frames. + * + * The cursor only ever advances, so callers must pass non-decreasing times. + * Returns null when the keyframes normalise to nothing, which the callers read + * as "no automation here". + */ +export function createEnvelopeWalker( + keyframes: AudioVolumeKeyframe[], + trackStart: number, + baseVolume: number, +): ((time: number) => number) | null { + const envelope = normaliseEnvelope(keyframes, trackStart, baseVolume); + const first = envelope[0]; + if (!first) return null; + + let segment = 0; + return (time: number): number => { + for (;;) { + const next = envelope[segment + 1]; + if (segment >= envelope.length - 2 || !next || time < next.time) break; + segment += 1; + } + const a = envelope[segment] ?? first; + const b = envelope[segment + 1] ?? a; + const span = b.time - a.time; + const progress = span <= 0 ? 0 : Math.min(1, Math.max(0, (time - a.time) / span)); + return a.volume + (b.volume - a.volume) * progress; + }; +} + /** * Multiply a prepared WAV's samples by a time-varying gain envelope in place. * @@ -86,8 +120,8 @@ export function applyVolumeEnvelopeToWav( trackStart: number, baseVolume: number, ): boolean { - const envelope = normaliseEnvelope(keyframes, trackStart, baseVolume); - if (envelope.length === 0) return false; + const gainAt = createEnvelopeWalker(keyframes, trackStart, baseVolume); + if (!gainAt) return false; try { const buffer = readFileSync(wavPath); @@ -99,21 +133,8 @@ export function applyVolumeEnvelopeToWav( const frameBytes = numChannels * bytesPerSample; const frameCount = Math.floor(dataSize / frameBytes); - // Maintain an incremental segment cursor so the per-frame envelope lookup - // is O(N+M) overall, not O(N×M). interpolateVolumeGain restarts from 0 on - // each call — fine for the preview path (one call per RAF tick) but not for - // the PCM path (one call per sample, 48k×duration frames total). - let segment = 0; for (let frame = 0; frame < frameCount; frame += 1) { - const time = frame / sampleRate; - while (segment < envelope.length - 2 && time >= envelope[segment + 1]!.time) segment += 1; - - const a = envelope[segment]!; - const b = envelope[segment + 1] ?? a; - const span = b.time - a.time; - const progress = span <= 0 ? 0 : Math.min(1, Math.max(0, (time - a.time) / span)); - const gain = a.volume + (b.volume - a.volume) * progress; - + const gain = gainAt(frame / sampleRate); const base = dataOffset + frame * frameBytes; for (let channel = 0; channel < numChannels; channel += 1) { const at = base + channel * bytesPerSample; diff --git a/packages/lint/src/rules/media.test.ts b/packages/lint/src/rules/media.test.ts index f2574aa178..8554bad69d 100644 --- a/packages/lint/src/rules/media.test.ts +++ b/packages/lint/src/rules/media.test.ts @@ -457,6 +457,27 @@ describe("audio_volume_double_automation", () => { } }); + it("does not blame the wrong element in a chained timeline", async () => { + // A chain has no semicolon until its very end, so a run that could cross `)` + // reached the `volume` in a LATER call and reported the element from an + // earlier one. Acting on the fixHint would have deleted #bgm's only real + // automation to fix a tween that is on #vo. + const res = await lintHyperframeHtml( + withScript( + LANE, + `gsap.timeline().to("#bgm", { duration: 0.6, x: 10 }).to("#vo", { volume: 1 });`, + ), + ); + expect(res.findings.some((f) => f.code === "audio_volume_double_automation")).toBe(false); + }); + + it("still catches a real tween further down the same call", async () => { + const res = await lintHyperframeHtml( + withScript(LANE, `gsap.timeline().to("#bgm", { duration: 0.6, ease: "none", volume: 0 });`), + ); + expect(res.findings.some((f) => f.code === "audio_volume_double_automation")).toBe(true); + }); + it("ignores a lane that automates something other than volume", async () => { const res = await lintHyperframeHtml( withScript( diff --git a/packages/lint/src/rules/media.ts b/packages/lint/src/rules/media.ts index 9efca4f6fe..32c10d8620 100644 --- a/packages/lint/src/rules/media.ts +++ b/packages/lint/src/rules/media.ts @@ -625,7 +625,14 @@ function findVolumeDoubleAutomationFindings(ctx: LintContext): HyperframeLintFin // the same call the runtime's own probe would pick up, and the rule only // warns, so a miss costs nothing. const escaped = escapeRegExp(id); - const tweened = new RegExp(`#${escaped}(?![\\w-])[^;]{0,200}?\\bvolume\\s*:`, "s").test(script); + // `[^;)]`, not `[^;]`: a chained timeline has no semicolon until the end of + // the whole chain, so a run that could cross `)` matched `volume` in a LATER + // `.to()` call and named the wrong element — and this rule's fixHint tells + // the author to delete their lane. Refusing to cross the closing paren keeps + // the match inside the call the selector belongs to. It costs a false + // negative when some other value in the same object is a call result, which + // is the safe direction for a warning that already only guesses. + const tweened = new RegExp(`#${escaped}(?![\\w-])[^;)]{0,200}?\\bvolume\\s*:`).test(script); if (!tweened) continue; findings.push({ code: "audio_volume_double_automation", diff --git a/packages/producer/src/services/render/stages/audioStage.test.ts b/packages/producer/src/services/render/stages/audioStage.test.ts index 62dc0ba1b9..b0b145a5c6 100644 --- a/packages/producer/src/services/render/stages/audioStage.test.ts +++ b/packages/producer/src/services/render/stages/audioStage.test.ts @@ -122,7 +122,27 @@ describe("runAudioStage", () => { const result = await runAudioStage(makeInput()); expect(result.hasAudio).toBe(false); expect(result.audioError).toMatch(/Audio FX failed for track bgm/); - expect(result.audioFailures).toBeUndefined(); + // And it is classified. This used to come back undefined, so the warning + // policy — which reads owner, retryability, reason and stage off this list + // — described the FATAL failure with strictly less detail than a single + // dropped track gets. + expect(result.audioFailures).toEqual([ + { + stage: "internal", + reason: "internal", + owner: "system", + retryable: false, + detail: "Audio FX failed for track bgm: browser launch failed", + }, + ]); + }); + + it("bounds the synthesised failure's detail", async () => { + // `detail` is contractually bounded diagnostic text; an ffmpeg-flavoured + // message can run to tens of kilobytes. + processCompositionAudioMock.mockRejectedValue(new Error("x".repeat(5_000))); + const result = await runAudioStage(makeInput()); + expect(result.audioFailures?.[0]?.detail.length).toBe(2_000); }); it("lets an abort keep its own shape rather than becoming an audio error", async () => { diff --git a/packages/producer/src/services/render/stages/audioStage.ts b/packages/producer/src/services/render/stages/audioStage.ts index bac3d3f552..7dd17cd133 100644 --- a/packages/producer/src/services/render/stages/audioStage.ts +++ b/packages/producer/src/services/render/stages/audioStage.ts @@ -84,12 +84,28 @@ export async function runAudioStage(input: AudioStageInput): Promise { ).toEqual(["fx.k1.frequency"]); }); }); + +describe("AudioFxGroup while the carve is measuring", () => { + /** + * `analyse` captures the chain and the automation before its fetch and decode, + * then rewrites the whole `data-fx-chain` from that snapshot. An effect added + * — or a knob committed — during those seconds landed first and was silently + * discarded when the analysis returned. Only the Analyse control was gated; + * every other control in the rack stayed live throughout. + */ + it("locks the rack, so an edit cannot be made against a snapshot that is moving", async () => { + // A fetch that never settles holds the panel in its analysing state, which + // is exactly the window the race lives in. + const hang = vi.fn(() => new Promise(() => {})); + vi.stubGlobal("fetch", hang); + // happy-dom has no Web Audio, and without a constructor `analyse` returns + // before it ever reaches the decode — closing the window under test. + vi.stubGlobal( + "OfflineAudioContext", + class { + decodeAudioData() { + return new Promise(() => {}); + } + }, + ); + try { + const bed = document.createElement("audio"); + bed.id = "bed"; + bed.setAttribute("src", "bed.wav"); + document.body.append(bed); + const voice = document.createElement("audio"); + voice.id = "narration"; + voice.setAttribute("src", "vo.wav"); + document.body.append(voice); + + const host = document.createElement("div"); + document.body.append(host); + await act(async () => { + createRoot(host).render( + , + ); + }); + + // The bed carves itself against its one candidate, which starts the + // decode — and the whole rack goes read-only until it lands. + expect(hang).toHaveBeenCalled(); + const controls = Array.from(host.querySelectorAll(".hf-fx-slider")); + expect(controls.length).toBeGreaterThan(0); + expect(controls.every((c) => c.disabled)).toBe(true); + } finally { + vi.unstubAllGlobals(); + } + }); +}); diff --git a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx index 33b9f162ce..5d132620e9 100644 --- a/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx +++ b/packages/studio/src/components/editor/propertyPanelAudioFxGroup.tsx @@ -11,6 +11,7 @@ import { useEffect, useState } from "react"; import { defaultAudioFxParams, HF_AUDIO_FX_ATTR, + HF_AUDIO_FX_DATA_KEY, mintAudioFxNodeId, parseAudioFxChain, serializeAudioFxChain, @@ -41,6 +42,7 @@ import { automatedTargetsOf, automationAttrValue, HF_AUDIO_AUTOMATION_ATTR, + HF_AUDIO_AUTOMATION_DATA_KEY, readPanelAutomation, resolveAutomationRange, withoutLane, @@ -109,7 +111,7 @@ export function AudioFxGroup({ onSetAttributeLive: (attr: string, value: string | null) => void | Promise; }) { const chain = ((): HfAudioFxChain => { - const raw = element.dataAttributes?.["fx-chain"]; + const raw = element.dataAttributes?.[HF_AUDIO_FX_DATA_KEY]; if (!raw) return { version: 1, nodes: [] }; try { return parseAudioFxChain(raw); @@ -120,7 +122,10 @@ export function AudioFxGroup({ } })(); - const automation = readPanelAutomation(element.dataAttributes?.["automation"], chain); + const automation = readPanelAutomation( + element.dataAttributes?.[HF_AUDIO_AUTOMATION_DATA_KEY], + chain, + ); const automatedTargets = automatedTargetsOf(automation); /** @@ -481,11 +486,24 @@ export function AudioFxGroup({ const analyse = async (active: HfCarveSettings | null = carve): Promise => { if (!active?.sources.length) return; const doc = element.element?.ownerDocument; + if (!doc) return; // Every named voice that is actually there with something to decode. A source // naming a deleted track is skipped rather than failing the whole analysis. - const voices = active.sources - .map((id) => doc?.getElementById(id) as HTMLAudioElement | null) - .filter((el): el is HTMLAudioElement => Boolean(el?.getAttribute("src"))); + // + // Read out to plain values here rather than carrying elements around: it is + // what lets the src and the start be non-null by construction downstream + // instead of by assertion. + const voices: { src: string; start: string | null }[] = []; + for (const id of active.sources) { + const el = doc.getElementById(id); + // By tag name, not `instanceof HTMLAudioElement`: these elements belong to + // the composition's iframe document, so the constructor they were made + // from is not this realm's and the instanceof is false for every one. + if (el?.tagName !== "AUDIO") continue; + const src = el.getAttribute("src"); + if (!src) continue; + voices.push({ src, start: el.getAttribute("data-start") }); + } if (voices.length === 0) return; setAnalysing(true); try { @@ -498,7 +516,7 @@ export function AudioFxGroup({ .webkitOfflineAudioContext; if (!Ctor) return; const decode = async (relative: string): Promise => { - const res = await fetch(new URL(relative, doc!.baseURI).href); + const res = await fetch(new URL(relative, doc.baseURI).href); return new Ctor(1, 1, DECODE_SAMPLE_RATE).decodeAudioData(await res.arrayBuffer()); }; const bedStart = clipStart(element.dataAttributes?.["start"]); @@ -508,9 +526,9 @@ export function AudioFxGroup({ // is also what lets the bands and the envelopes stay a single set: the chain is // fixed, so there is no per-voice filter to switch between. const decoded = await Promise.all( - voices.map(async (el) => ({ - samples: (await decode(el.getAttribute("src")!)).getChannelData(0), - offsetSeconds: clipStart(el.getAttribute("data-start")) - bedStart, + voices.map(async (voice) => ({ + samples: (await decode(voice.src)).getChannelData(0), + offsetSeconds: clipStart(voice.start) - bedStart, })), ); const voiceMix = mixCarveSources(decoded, DECODE_SAMPLE_RATE); @@ -609,6 +627,14 @@ export function AudioFxGroup({ return ( 0 ? serializeAutomation(automation) : ""; } -export { HF_AUDIO_AUTOMATION_ATTR, resolveAutomationRange }; +export { HF_AUDIO_AUTOMATION_ATTR, HF_AUDIO_AUTOMATION_DATA_KEY, resolveAutomationRange }; diff --git a/packages/studio/src/components/editor/propertyPanelFxControls.test.tsx b/packages/studio/src/components/editor/propertyPanelFxControls.test.tsx new file mode 100644 index 0000000000..e498875822 --- /dev/null +++ b/packages/studio/src/components/editor/propertyPanelFxControls.test.tsx @@ -0,0 +1,115 @@ +// @vitest-environment happy-dom +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { FxParamRow } from "./propertyPanelFxControls"; +import type { HfAudioFxNumberParam } from "@hyperframes/core/audio-fx"; + +(globalThis as unknown as { IS_REACT_ACT_ENVIRONMENT: boolean }).IS_REACT_ACT_ENVIRONMENT = true; + +afterEach(() => { + document.body.innerHTML = ""; +}); + +/** A frequency knob: wide range, so a clamp to the minimum is unmistakable. */ +const FREQUENCY: HfAudioFxNumberParam = { + kind: "number", + key: "frequency", + label: "Frequency", + min: 20, + max: 20000, + step: 1, + default: 1000, + unit: "Hz", +}; + +/** React tracks its own value on the node, so a plain assignment is ignored. */ +function setInputValue(input: HTMLInputElement, text: string): void { + const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")?.set; + setter?.call(input, text); + input.dispatchEvent(new Event("input", { bubbles: true })); +} + +function mount(value: number) { + const onChange = vi.fn(); + const onCommit = vi.fn(); + const host = document.createElement("div"); + document.body.append(host); + let root!: Root; + act(() => { + root = createRoot(host); + root.render( + , + ); + }); + const number = () => host.querySelector(".hf-fx-number")!; + const slider = () => host.querySelector(".hf-fx-slider")!; + const type = (text: string) => { + act(() => { + number().focus(); + setInputValue(number(), text); + }); + }; + /** A new value arrives through the prop — a carve, or an undo. */ + const receive = (next: number) => { + act(() => { + root.render( + , + ); + }); + }; + return { host, number, slider, onChange, onCommit, type, receive }; +} + +describe("FxParamRow number field", () => { + it("shows what is being typed instead of snapping back to the stored value", () => { + // Every keystroke is clamped and written live, and the live write does not + // refresh the prop — so a field bound to the committed value put the old + // number straight back. Typing 5000 wrote 20 on the first keystroke and the + // knob could not be typed into at all, only dragged. + const { number, type } = mount(1000); + type("5"); + expect(number().value).toBe("5"); + type("5000"); + expect(number().value).toBe("5000"); + }); + + it("does not write the parameter minimum while the field is empty", () => { + // `Number("") === 0` passes Number.isFinite, so select-all + Delete before + // retyping used to clamp to the minimum and live-write it — 20 Hz here. + const { onChange, type } = mount(1000); + type(""); + expect(onChange).not.toHaveBeenCalled(); + type("800"); + expect(onChange).toHaveBeenLastCalledWith("frequency", 800); + }); +}); + +describe("FxParamRow commit", () => { + it("stays quiet when a gesture changed nothing", () => { + // commit() hangs off pointerup, keyup and blur, all reachable without an + // edit. Firing then costs a source patch, a selection resync, a preview + // reload and an audio restart for a gesture that moved nothing. + const { slider, number, onCommit } = mount(1000); + act(() => slider().dispatchEvent(new Event("pointerup", { bubbles: true }))); + act(() => number().dispatchEvent(new Event("blur", { bubbles: true }))); + expect(onCommit).not.toHaveBeenCalled(); + }); + + it("does not re-persist a stale value over a change that arrived meanwhile", () => { + // The scenario: open an EQ row, run the carve (or press undo), which + // rewrites the chain so this row's value becomes 400. Then click the slider + // thumb and release without moving it. `latest` was seeded at mount and only + // written by an edit, so it still held 1000 — and the release wrote it back, + // undoing the carve with no gesture that looks like an edit. + const { slider, onCommit, type, receive } = mount(1000); + // An edit happened earlier in this row's life, so `latest` holds 1000. + type("1000"); + act(() => slider().dispatchEvent(new Event("pointerup", { bubbles: true }))); + onCommit.mockClear(); + + receive(400); + act(() => slider().dispatchEvent(new Event("pointerup", { bubbles: true }))); + expect(onCommit).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/studio/src/components/editor/propertyPanelFxControls.tsx b/packages/studio/src/components/editor/propertyPanelFxControls.tsx index 4fd3254bdb..63d7bfd6da 100644 --- a/packages/studio/src/components/editor/propertyPanelFxControls.tsx +++ b/packages/studio/src/components/editor/propertyPanelFxControls.tsx @@ -132,6 +132,26 @@ export function FxParamRow({ * write applied on the way down. */ const [pending, setPending] = useState(null); + /** + * What the number field is showing while it has focus. + * + * The field cannot be bound to the committed value: every keystroke is + * clamped into range and written live, and the live write does not refresh + * the prop — so React put the old number straight back and typing `5000` into + * a 20..20000 knob wrote 20 on the first keystroke and never got further. Held + * as TEXT, so a half-typed "-" or "" is a state the field can be in rather + * than a number to clamp and persist. + */ + const [typing, setTyping] = useState(null); + /** + * Did this gesture actually change anything? + * + * `commit()` hangs off pointerup, keyup and blur, all of which fire without + * an edit — clicking a slider thumb without moving it, or tabbing through the + * field. Committing then re-persisted whatever `latest` happened to hold and + * silently reverted any change that had arrived meanwhile. + */ + const edited = useRef(false); useEffect(() => { if (!dragging) setLocal(value); }, [value, dragging]); @@ -146,6 +166,7 @@ export function FxParamRow({ const p = param as HfAudioFxNumberParam; const next = Math.min(p.max, Math.max(p.min, raw)); latest.current = next; + edited.current = true; setLocal(next); onChange(param.key, next); }, @@ -154,6 +175,12 @@ export function FxParamRow({ const commit = useCallback(() => { setDragging(false); + // Nothing was edited, so there is nothing to persist. Without this a bare + // focus/blur — or a click on the slider thumb that never moved — fired a + // full persisting write: source patch, selection resync, preview reload and + // an audio restart, for a gesture that changed no value. + if (!edited.current) return; + edited.current = false; if (typeof latest.current === "number") setPending(latest.current); onCommit?.(param.key, latest.current); }, [onCommit, param.key]); @@ -228,13 +255,25 @@ export function FxParamRow({ min={param.min} max={param.max} step={param.step} - value={display(param, current)} + value={typing ?? display(param, current)} disabled={locked} + onFocus={() => setTyping(display(param, current))} onChange={(e) => { - const next = Number(e.target.value); + const text = e.target.value; + setTyping(text); + // An empty field, a lone "-", or a trailing "." are all states on the + // way to a number, not numbers. `Number("")` is 0, which passes + // Number.isFinite — so clearing the field to retype used to clamp to + // the parameter MINIMUM and write it live: 20 Hz on a cutoff, -40 dB + // on a gain. + if (text.trim() === "") return; + const next = Number(text); if (Number.isFinite(next)) handleNumber(next); }} - onBlur={commit} + onBlur={() => { + commit(); + setTyping(null); + }} onKeyDown={(e) => { if (e.key === "Enter") commit(); }} diff --git a/packages/studio/src/components/editor/propertyPanelFxSection.test.tsx b/packages/studio/src/components/editor/propertyPanelFxSection.test.tsx index 345d3574d9..c6a35307f7 100644 --- a/packages/studio/src/components/editor/propertyPanelFxSection.test.tsx +++ b/packages/studio/src/components/editor/propertyPanelFxSection.test.tsx @@ -163,6 +163,55 @@ describe("FxSection chain", () => { expect(next.nodes.map((n) => n.type)).toEqual(["reverb", "peaking"]); }); + it("keeps a half-typed value with its own effect across a reorder", () => { + // Rows used to be keyed `${type}-${index}`, so two effects of the same type + // kept their keys through a reorder and React reused each row where it + // stood. The controls hold real state — a number field mid-edit is held as + // text — so the buffer stayed at the position and landed on whichever + // effect moved into it. + const peaking = (id: string, frequency: number) => ({ + type: "peaking", + id, + enabled: true, + params: { ...defaultAudioFxParams("peaking"), frequency }, + }); + const a = peaking("pa", 400); + const b = peaking("pb", 1600); + const chainOfNodes = (...nodes: unknown[]): HfAudioFxChain => + ({ version: 1, nodes }) as HfAudioFxChain; + + const host = document.createElement("div"); + document.body.append(host); + const root = createRoot(host); + const render = (chain: HfAudioFxChain) => + act(() => { + root.render( + , + ); + }); + + render(chainOfNodes(a, b)); + // Only the first card is open, which is the one being edited. + const openFrequency = (): HTMLInputElement => + host.querySelector(".hf-fx-node .hf-fx-number")!; + + expect(openFrequency().value).toBe("400"); + typeInto(openFrequency(), "123"); + expect(openFrequency().value).toBe("123"); + + // The author moves that effect down; the other one takes the open slot. + render(chainOfNodes(b, a)); + + expect(openFrequency().value).toBe("1600"); + }); + it("cannot move the ends past themselves", () => { const { host } = mount({ chain: chainOf("peaking", "reverb") }); const ups = host.querySelectorAll('.hf-fx-move[title="Move up"]'); diff --git a/packages/studio/src/components/editor/propertyPanelFxSection.tsx b/packages/studio/src/components/editor/propertyPanelFxSection.tsx index ab696119af..33fda26891 100644 --- a/packages/studio/src/components/editor/propertyPanelFxSection.tsx +++ b/packages/studio/src/components/editor/propertyPanelFxSection.tsx @@ -795,7 +795,13 @@ export function FxSection({ handBuilt.map(({ node, i }) => { return ( [0]): void => { // Quiet: clicking the toggle used to reload the preview and restart every // playing track, while the same click on an effect parameter did not. diff --git a/plans/audio-fx-presets.md b/plans/audio-fx-presets.md new file mode 100644 index 0000000000..ac253e85f5 --- /dev/null +++ b/plans/audio-fx-presets.md @@ -0,0 +1,467 @@ +# Audio treatment presets — research and proposed catalog + +Written 2026-08-09. The ask: voice carve turned a pile of effects into one +understandable feature; what else can, using only the effects already shipped? + +This is a research + design doc. Nothing here is built. + +--- + +## 1. The governing precedent + +**`wiki/decisions/vst-studio-integration-cancelled.md`** is the most important +prior art, and it is a warning. A VST spectral-carve integration was built to +completion — 4-PR stack, published PyPI host — and then **cancelled**, because +of its standing costs: + +- a spawned WebSocket sidecar with its own lifecycle +- an external GPL repo with its own release cadence +- a non-bundled runtime dependency users had to install +- VST logic permanently in the render path, hard-failing renders on a missing + plugin + +Everything proposed here is the **inverse of all four**: preset values are +data, and the adaptive scripts reuse analysis code that already ships. No new +process, no new dependency, no new render-path failure mode. A preset that goes +wrong produces a chain the author can see and edit in the rack — the same chain +they could have built by hand. + +That is the argument for doing this at all, and the bar any proposal here has +to keep clearing. + +### What already exists, so this does not duplicate it + +- **`skills/hyperframes-audio`** teaches the effect *families* and when to + reach for each ("Reach for a family by the problem, not the name"), and + documents carve. It contains **no recipes and no named looks** — it explains + the why, and stops short of the one-click. This catalog is the complement, + not a rewrite. It also already fixes the canonical order this doc adopts: + *"corrective filtering goes early, character in the middle, and a limiter + last."* +- **`carveBandsToChain()`** already proves the mechanism: an analysis produces + an ordinary `HfAudioFxChain`, which the rack then owns like any other. +- **Demand is real and internal.** Cortex carries user asks for + [audio reprocessing for studio acoustics](https://heygen.slack.com/archives/C0A9ZHLSQFN/p1781063751377189), + [voice loudness and gain boosting](https://heygen.slack.com/archives/C07BR8QRE4T/p1769062551169729), + and auto-enhance across scenes. Nobody has built a preset catalog for it. + +--- + +## 2. What consumer tools actually ship + +Three distinct tiers, and only two are reachable for us. + +### Tier 1 — ML enhancement (NOT reachable) + +Adobe Podcast Enhance Speech and Descript Studio Sound are **one button** that +strips background noise, echo and room reverb, and rebuild the voice as if +recorded in a studio. Adobe's v3 does source separation and exposes a strength +slider. This is generative ML, not filtering — there is no arrangement of EQ, +compression or gating that reaches it. Denoising and de-reverberation are the +single most-requested thing in our own internal feedback, and we cannot do +them with this effect set. Any catalog that quietly implies otherwise is lying. + +### Tier 2 — intent panels over conventional DSP (the model to copy) + +Premiere Pro's **Essential Sound** panel is the closest analogue to what is +being asked for, and it is the design worth stealing. The author tags a clip by +*what it is* — Dialogue, Music, SFX, Ambience — and then gets a small set of +outcome-named controls rather than effects: + +- **Loudness**, with an Auto-Match that normalises to a broadcast target +- **Repair** — Reduce Noise, Reduce Rumble, DeHum, DeEss, Reduce Reverb, each a + single intensity slider +- **Clarity**, which boosts the bands that make consonants distinct +- **Creative**, for reverb/space + +Note the shape: **one noun per problem, one slider per noun, no frequencies on +screen.** That is exactly what carve does with its strength knob, and it is the +contract this catalog should hold to. + +### Tier 3 — character effects (fully reachable, and where consumer volume is) + +CapCut ships ~100 voice filters and ~100 "voice characters". The named ones +that recur across tools: Telephone, Megaphone, Radio/Radio Fuzz, Loudspeaker / +PA Speaker, Intercom, Lo-Fi, Cassette Tape, Cave, Echo, Broadcast, Retro. + +These are ordinary filter + saturation + bitcrush recipes and we can do all of +them. **The character half of CapCut's list we cannot do**: Chipmunk, Deep +Voice, Robot, Alien, Monster, Elf — every one of those is pitch, formant or +ring modulation, and the registry has no pitch shifter, no time-stretch and no +ring mod. Out of scope unless a new effect is added. + +--- + +## 3. What our 15 effects reach + +| Family | Effects | Reaches | +| --- | --- | --- | +| Filter | `highpass` `lowpass` `peaking` `lowshelf` `highshelf` | tone shaping, band isolation, rumble/mud removal, presence | +| Dynamics | `gain` `compressor` `limiter` `gate` | levelling, consistency, ceilings, room-tone gating | +| Nonlinear | `saturate` `bitcrush` | warmth, grit, distortion, digital degradation | +| Time | `delay` `reverb` `chorus` `phaser` | space, slap, width, wow/flutter, movement | + +Plus two force multipliers the catalog depends on: + +- **Automation lanes** on `gain` and on most effect parameters — so a preset can + be a *moving* treatment, not just a static one. +- **Offline analysis** (`powerSpectrum`, `windowDb`, `analyseCarveDynamics`, + `analyseCarveDuck`) already runs in the panel over decoded audio. This is what + makes the adaptive tier possible without new machinery. + +**Hard limits, stated once:** no noise reduction, no de-reverberation, no pitch +or formant shift, no time-stretch, no spectral repair, no stereo widening +beyond `chorus`, no true multiband compression (though stacked peaking + a +`mix`-blended compressor approximates it). + +--- + +## 4. Two mechanisms, and carve is already the second + +The user's instinct to say "presets/**scripts**" is the right split. + +### Static presets — a chain literal + +Mirrors `HF_COLOR_GRADING_PRESETS` exactly: named, fixed parameter values, +one click, fully editable afterwards. Suits anything whose right answer does +not depend on the audio — every character effect, every space, and the +corrective voice chains at a sensible default. + +### Adaptive scripts — measure, then generate + +The carve pattern: read the decoded samples, measure something, emit a chain +and/or automation lanes. **The machinery for this is already built and shipped** +— what carve does with a voice against a bed generalises: + +| Script | Reuses | Emits | +| --- | --- | --- | +| Voice carve *(shipped)* | `analyseCarveBands` | peaking cuts + per-band lanes | +| Auto-duck *(shipped, inside carve)* | `analyseCarveDuck` | one volume lane | +| De-esser | `analyseCarveDynamics`, re-parameterised (see §5e) | lane on a peaking cut | +| Leveller | `windowDb` walk | lane on a `gain` node | +| Tone match | `powerSpectrum` vs a target curve | 3–5 peaking nodes | + +This is the strongest argument in the doc: **none of them needs new DSP** — +only a different question asked of code that already runs. One (de-ess) needs +that code re-parameterised for a shorter timescale; the rest reuse it as-is. + +--- + +## 5. Proposed catalog + +Every value below is inside the registry's declared range. Node order is the +canonical one the skill already states: **gate → subtractive EQ → compressor → +presence EQ → saturation → limiter last.** + +### 5a. Voice (static) + +**`voice-clean` — "Clean Voice"** · the safe default, roughly Premiere's +Dialogue preset +``` +highpass frequency 80 q 0.707 poles 2 +peaking frequency 250 gain -3 q 1.2 (mud) +compressor threshold -20 ratio 3 attack 12 release 180 makeup 3 +peaking frequency 3000 gain +2.5 q 1.0 (presence / consonants) +limiter limit -1 attack 5 release 50 +``` + +**`voice-broadcast` — "Broadcast"** · denser, more forward +``` +highpass frequency 90 q 0.707 poles 2 +peaking frequency 400 gain -3 q 1.4 +compressor threshold -24 ratio 4 attack 8 release 150 makeup 5 +peaking frequency 2500 gain +3 q 0.9 +highshelf frequency 8000 gain +2 (air) +saturate type tanh threshold -12 output 0 (glue) +limiter limit -1 attack 5 release 60 +``` + +**`voice-warm` — "Close & Warm"** · intimate, less processed +``` +highpass frequency 70 q 0.707 poles 2 +lowshelf frequency 180 gain +2 +compressor threshold -18 ratio 2.5 attack 20 release 250 makeup 2 +peaking frequency 3000 gain +1.5 q 0.8 +limiter limit -1.5 +``` + +Frequency choices are the documented conventions: high-pass at 60–80 Hz for +voice, 150–300 Hz is where proximity-effect mud lives, and presence for +consonant intelligibility sits around 2–4 kHz. + +### 5b. Repair — honestly named (static) + +These are the *reachable half* of Premiere's Repair section. The names must not +promise noise removal. + +**`rumble-cut` — "Cut Rumble"** — `highpass` 100 Hz, poles 2. Two stacked nodes +for 24 dB/oct when a source is badly affected. + +**`room-gate` — "Quiet Between Phrases"** — `gate` threshold −45, range −18, +ratio 10, attack 2, release 180. **This gates, it does not denoise**: room tone +still sits under speech, it is only silenced in the gaps. Label it that way in +the UI or it will be mistaken for Tier 1. + +**`boom-tame` — "Tame Boominess"** — `peaking` 200 Hz, −4 dB, Q 1.4. + +**`harsh-tame` — "Soften Harshness"** — `peaking` 3.2 kHz, −3 dB, Q 1.6. Static +and broad on purpose; sibilance proper is a script (§5e), not a fixed cut. + +### 5c. Character (static) + +**`telephone` — "Telephone"** · the researched 300–3400 Hz band, steepened +``` +highpass frequency 300 q 0.707 poles 2 ×2 stacked (24 dB/oct) +lowpass frequency 3400 q 0.707 poles 2 ×2 stacked +peaking frequency 1200 gain +6 q 1.2 (the "honk") +peaking frequency 550 gain -4 q 1.0 (de-mud) +saturate type tanh threshold -18 output -2 (circuit colour) +``` +The registry maxes at 12 dB/oct per node (`poles: "2"`), so the classic +24 dB/oct skirts need **two stacked nodes each**. Worth encoding once here +rather than having every author rediscover it. + +**`radio-am` — "AM Radio"** — band 400–3000 Hz, `saturate` tanh threshold −15 +output −2, `bitcrush` bits 10 samples 1 mix 0.25. + +**`megaphone` — "Megaphone"** — highpass 500, lowpass 4000, `peaking` 1800 +8 +Q 1.5, `saturate` **hard** threshold −12 output −3, `delay` time 40 ms feedback +0.15 mix 0.15 (the horn's slap). + +**`lofi-tape` — "Tape"** — `lowpass` 6500, `lowshelf` 120 +2, `bitcrush` bits 12 +samples 2 mix 0.35, `saturate` tanh threshold −14, and **`chorus` delay 6, +depth 0.6, speed 0.4, mix 0.15** — a slow shallow chorus is exactly how you +fake wow and flutter, which is a genuinely nice use of an effect we have. + +**`pa-system` — "Tannoy / PA"** — band 350–3500, `peaking` 1500 +5 Q 1.2, +`saturate` tanh threshold −16, `reverb` size 0.5 damping 0.7 wet 0.25 dry 0.8. + +**`intercom` — "Intercom"** — band 500–3000, `peaking` 2000 +6 Q 2, +`bitcrush` bits 11 mix 0.3, `gate` threshold −40 range −30 (the squelch). + +### 5d. Space (static) + +**`room-tight`** — `reverb` size 0.25, damping 0.6, wet 0.18, dry 0.9 +**`room-natural`** — size 0.5, damping 0.5, wet 0.25, dry 0.85 +**`hall`** — size 0.9, damping 0.3, wet 0.4, dry 0.75 +**`slap-echo`** — `delay` time 110 ms, feedback 0.12, mix 0.22 +**`dub-throw`** — `delay` time 375 ms, feedback 0.55, mix 0.3 + +### 5e. Adaptive scripts + +**`de-ess` — "Soften Sibilance"** — *must be a script, not a preset.* A fixed +−6 dB at 7 kHz dulls the whole voice; a de-esser only acts when sibilance is +present. Run the carve pattern against the track's own 5–8 kHz band and emit a +`peaking` node at the measured centre with a **lane on its gain** that dips only +during sibilant windows. Sibilance centres around 5–6 kHz for lower voices, +7–8 kHz for higher — measure it rather than assuming. + +`analyseCarveDynamics` cannot be reused *unchanged* here, and this is the one +place in §5e that needs work rather than a new caller. Its hop is +`max(FRAME, length / POINT_BUDGET)` — 85 ms at best and ~150 ms on a +real-length track — while sibilants are 50–150 ms events, so at that resolution +the envelope cannot land on them. Its `ATTACK_S`/`RELEASE_S` ballistics are +tuned for musical ducking and are far too slow as well. Same machinery, same +FFT, but it needs a sibilance-scale hop and much faster attack/release. + +**`level-out` — "Even Out Levels"** — walk `windowDb`, emit a lane that lifts +quiet passages and holds loud ones, against a target range rather than a fixed +gain. + +**The lane must ride a `gain` node, not the volume lane.** `VOLUME_RANGE` is +0..1 and `normaliseEnvelope` clamps every keyframe into it, so a volume lane can +only ever attenuate — it cannot lift anything. A `gain` node spans −60..+12 dB +and is automatable, which is exactly what the audio skill means when it calls +`gain` "what an automation lane rides when a track has to move". So this script +emits both a chain (one `gain` node) and the lane addressing it. **Do not call this LUFS.** `windowDb` is plain RMS; platform +targets (−14 LUFS for Spotify/YouTube/TikTok/Instagram, −16 for Apple +Music/podcasts) are ITU-R BS.1770 K-weighted and gated. Either implement the +K-weighting prefilter — two biquads, entirely feasible offline with the filters +we have — or name the feature "even out" and cite the targets as context only. + +**`tone-match` — "Match Tone"** — `powerSpectrum` of the track against a target +curve (a built-in "broadcast voice" curve, or another clip in the project), +emitting 3–5 corrective `peaking` nodes. This is iZotope's Tonal Balance idea +at a tenth of the complexity, and `powerSpectrum` already returns exactly the +data it needs. + +--- + +## 5f. Module identity in the rack + +The rack is a 292 px column of stacked modules — which is, near enough, a +Eurorack case. Worth leaning into: a faceplate recognised by colour and +lettering before the label is read. Carve already gets a distinct treatment +(`hf-fx-carve-module`); this generalises it. + +**Direction chosen 2026-08-09: schematic** — picked from four mocked +alternatives (hardware silkscreen, vintage test equipment, risograph, +schematic). Recorded with its principles in `.impeccable.md` at the repo root. + +The rack is drawn as the signal path it is: numbered nodes on a dashed spine +running from an `IN` terminal to an `OUT`, leader-dotted dimension lines, +three-letter stage tags, family colour restricted to the node ring and the tag. + +**Why it won over the louder options:** it is the only one that *adds +information*. Chain order is load-bearing — a limiter first and a limiter last +are different sounds — and nothing in the panel communicated it. It is also the +quietest, which matters on a surface authors stare at while mixing. + +**What the drawing carries that the current rack cannot say:** + +| Drawn as | Instead of | +| --- | --- | +| `IN` names the source, `OUT` names the destination and the FX tail | nothing — the tail was invisible | +| Numbered nodes 01…06 | implicit top-to-bottom order | +| Bypassed node: ring drawn open, wire beside it solid and unbroken | a row at 50% opacity | +| Preset nodes gathered under a right-hand brace with its name | no grouping at all | +| An automated parameter's value reads **live at the playhead** and ticks while the transport runs, marked `~` | the stale seed the lane already replaced | +| A measuring module gets a second ring | nothing distinguishes carve from a hand-set chain | + +That last pair is the static-versus-script distinction from §4, delivered as +drawing rather than documentation. + +**Automated parameters show numbers, not shapes.** An earlier pass drew each +lane's envelope inside the module; that was dropped. The lane already has a home +in the timeline, and a second small drawing of it in the rack is decoration — +what the rack is missing is the *current* value. The panel already receives it: +`FxSectionProps.liveAutomationValues` exists precisely because "an automated +parameter's stored number is only the seed the lane replaced, so a rack that +shows it stands still while the carve is audibly working". So the row is an +ordinary parameter row whose number is live, `~` on the label saying it is +driven rather than set, and the marker moving with it. Stopped, the values go +grey and hold at the playhead. + +**The preset menu preview is the chain** — each row previews the nodes it will +draw as small connected rings in their family colours, so the shape you pick is +the shape you get. Smart entries read `MEASURES` instead. + +**The empty state keeps the wire.** `IN` and `OUT` remain, connected, with "no +effects, signal passes straight through" between them — which teaches that the +rack is a path before the author has added anything to it. + +**Not carried over from the first pass:** the 3 px coloured rail down each +module's left edge. It is the most overused device in dashboard UI, and +forcing each direction to find another answer is what separated them. + +**Family hues**, chosen to sit on `#0C0C0E` without competing with the studio's +`#3CE6AC` accent: + +| Family | Hue | Why | +| --- | --- | --- | +| Filter | `#4FA8FF` | the measuring family | +| Dynamics | `#FFB443` | grips the signal | +| Nonlinear | `#FF6B5C` | the only generative family | +| Time | `#B98CFF` | atmosphere, not control | +| Smart | `#3CE6AC` | the studio's own accent — reserved for modules that act on their own | + +**This needs almost no new plumbing.** `group` is already on every effect in the +registry, so hue and lettering are derived, not hand-assigned. And every module +already renders `data-fx-node=""`, so the whole identity layer is CSS on +an attribute that exists today. Only "Smart" is new, and carve already lives +there in spirit. + +**Type budget is the real decision.** Fonts must be self-hosted (no CDN). +Recommended: **two faces** — the UI sans plus one characterful display face +split across families. One face is too subtle at 11 px; five is a bundle cost +and starts reading as a collage on a surface authors stare at while mixing. + +**Preset menu**: grouped by the same families, so the colour picked in the menu +is the colour that appears in the rack. Each row carries the number of modules +it drops in — which quietly teaches that a preset *is* a chain. Smart entries +say "measures" instead of a count, which is the entire static-vs-adaptive +distinction delivered in one word. + +Open decisions: whether applying a preset appends or replaces (suggest append, +replace on modifier, re-apply swapping its own `fromPreset` nodes); the type +budget above; and whether character presets get drawn glyphs on their plates — +the biggest step toward "fun", and the only part needing artwork rather than +CSS. + +Mockup at the real panel width: published as an artifact, 2026-08-09. + +## 6. Architecture + +Mirror colour grading, because the author already understands that surface. + +```ts +export interface HfAudioFxPreset { + id: string; // "telephone" + label: string; // "Telephone" + family: "voice" | "repair" | "character" | "space"; + description: string; // one line, in the author's language + chain: HfAudioFxChain; // an ordinary chain — nothing special + automation?: HfAutomation; // for presets that move +} +``` + +Four notes: + +1. **A preset is just a chain.** Applying one writes `data-fx-chain` exactly as + hand-building would. It is inspectable, editable and undoable, and it cannot + introduce a failure mode the rack does not already have. +2. **Tag the nodes.** `HfAudioFxNode` already carries `fromCarve`; an analogous + `fromPreset: "telephone"` lets the panel show "Telephone (edited)" and lets a + re-apply replace its own nodes without touching hand-added ones. +3. **Fixed values in v1, no intensity knob.** Colour grading's presets are + fixed; carve earned its strength knob by being adaptive. Most effects carry a + `mix` parameter, which is the natural hook if intensity is wanted later — + note it, don't build it. +4. **Scripts need a separate registry** with an `analyse(samples, sampleRate) → + { chain, automation }` contract, because they cannot be static data. Carve is + the reference implementation. + +### Two things that will otherwise be filed as bugs + +- **Boost presets must end in a limiter.** `voice-broadcast` adds presence and + +5 dB makeup; without its limiter it will overshoot full scale. The + clip-before-duck fix (`55f852a72`) makes that survivable, but the measured + damage still scales with overshoot — 2.7× peak destroys 19% of samples. A + preset that ships hot is a bad preset regardless of what the render does + about it. +- **Worklet-backed presets play dry for a moment in preview.** Anything + containing `compressor`, `limiter`, `gate` or `bitcrush` waits for + `ensureAudioFxWorklets` before it can be built — so nearly every voice preset + is momentarily dry on first apply, then swaps in. Expected, not a defect. + +--- + +## 7. What to build first + +1. **Character presets** (§5c) — highest ratio of delight to risk. Pure + filter/saturation data, instantly recognisable, nothing to measure, and the + category consumer tools get the most use out of. +2. **Voice presets** (§5a/b) — the practical value, and the answer to the + internal asks already in cortex. +3. **`level-out`** — the most-wanted adaptive one, and the smallest script. +4. **`de-ess`**, then **`tone-match`** — both reuse carve's analysis directly. + +Deliberately not proposed: anything requiring a new effect. If pitch shifting +ever lands, the whole character half of CapCut's list opens up at once — but +that is a new DSP conversation, and this doc is explicitly the one that does not +need one. + +### If this ships + +`CLAUDE.md`'s skill-catalog rules apply: a preset catalog changes what +`/hyperframes-audio` covers, so the skill's `description:` and the surfaces +listed under "Skill catalog maintenance" have to move in lockstep. + +--- + +## Sources + +Internal: `wiki/decisions/vst-studio-integration-cancelled.md`; +`skills/hyperframes-audio/SKILL.md`; `packages/core/src/audioFx.ts` (ranges); +`packages/core/src/audioCarve.ts` (the script pattern); +`packages/core/src/colorGrading.ts` (the preset pattern). Cortex: +[studio acoustics reprocessing](https://heygen.slack.com/archives/C0A9ZHLSQFN/p1781063751377189), +[voice loudness and gain boosting](https://heygen.slack.com/archives/C07BR8QRE4T/p1769062551169729). + +External: +- [Adobe Podcast Enhance Speech guide](https://thepodcastconsultant.com/blog/adobe-podcast-enhance) and [Adobe vs Descript shootout](https://thepodcasthaven.com/adobe-speech-enhancement-vs-descript-studio-sound-a-shootout/) — Tier 1 boundary +- [Premiere Pro Essential Sound panel guide](https://josephnilo.com/blog/the-ultimate-guide-to-the-premiere-pro-essential-sound-panel/) and [Envato's dialogue walkthrough](https://photography.tutsplus.com/articles/how-to-use-the-essential-sound-panel-to-edit-dialogue-in-premiere-pro--cms-41936) — the intent-panel model +- [Rode's podcast processing guide](https://rode.com/en-us/about/news-info/a-guide-to-audio-processing-and-fx-for-podcasting), [Podigy on podcast EQ](https://www.podigy.co/podcasters-eq) and [iZotope on de-essing](https://www.izotope.com/en/learn/the-dos-and-donts-of-de-essing.html) — voice chain values and sibilance ranges +- [CapCut voice filters](https://www.capcut.com/tools/voice-filters) and [its full effect list](https://irda27987s-random-pages.fandom.com/wiki/All_Voice_Filters_and_Voice_Characters_in_App!_(Capcut)) — Tier 3 vocabulary +- [Telephone effect settings](https://voxbooster.com/blog/telephone-voice-effect-online/) and [Audiotent's walkthrough](https://www.audiotent.com/blogs/production-tips/create-telephone-vocal-effect) — the 300–3400 Hz band and its refinements +- [LUFS targets per platform 2026](https://www.forasoft.com/learn/audio-for-video/articles-audio/lufs-targets-per-platform-2026) and [podcast loudness standards](https://sone.app/blog/podcast-loudness-standards-2026-spotify-apple-youtube) — why `level-out` must not claim LUFS