From cbe18d66bd049da774f7a22882e676beaf122514 Mon Sep 17 00:00:00 2001 From: Akshar Patel <123344143+AksharP5@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:07:45 -0400 Subject: [PATCH 1/2] fix(sdk-recorder): retain microphone audio when sharing tab sound --- .../__tests__/audio-mixing.test.ts | 205 ++++++++++++++++++ packages/sdk-recorder/package.json | 1 + .../sdk-recorder/src/core/stream-manager.ts | 37 +++- packages/sdk-recorder/src/index.ts | 4 +- 4 files changed, 242 insertions(+), 5 deletions(-) create mode 100644 packages/sdk-recorder/__tests__/audio-mixing.test.ts diff --git a/packages/sdk-recorder/__tests__/audio-mixing.test.ts b/packages/sdk-recorder/__tests__/audio-mixing.test.ts new file mode 100644 index 00000000000..cb582c48e1e --- /dev/null +++ b/packages/sdk-recorder/__tests__/audio-mixing.test.ts @@ -0,0 +1,205 @@ +import { afterEach, beforeEach, expect, mock, test } from "bun:test"; +import { + buildCompositeStream, + type ManagedStreams, +} from "../src/core/stream-manager"; +import { CapRecorder } from "../src/index"; + +class Track { + stop = mock(() => {}); + + constructor(readonly kind: "audio" | "video") {} +} + +class Stream { + constructor(private readonly tracks: Track[]) {} + + getTracks() { + return this.tracks; + } + + getAudioTracks() { + return this.tracks.filter((track) => track.kind === "audio"); + } + + getVideoTracks() { + return this.tracks.filter((track) => track.kind === "video"); + } +} + +class Mixer { + static instances: Mixer[] = []; + readonly inputs: Track[] = []; + readonly output = new Track("audio"); + readonly destination = { stream: new Stream([this.output]) }; + resume = mock(async () => {}); + close = mock(async () => {}); + + constructor() { + Mixer.instances.push(this); + } + + createMediaStreamDestination() { + return this.destination; + } + + createMediaStreamSource(stream: Stream) { + return { + connect: () => { + this.inputs.push(...stream.getAudioTracks()); + }, + }; + } +} + +const globals = ["MediaStream", "AudioContext", "MediaRecorder", "navigator"]; +const descriptors = globals.map( + (name) => [name, Object.getOwnPropertyDescriptor(globalThis, name)] as const, +); + +const setGlobal = (name: string, value: unknown) => { + Object.defineProperty(globalThis, name, { configurable: true, value }); +}; + +const capture = (systemAudio: boolean, microphone: boolean) => { + const video = new Track("video"); + const system = new Track("audio"); + const mic = new Track("audio"); + const display = new Stream(systemAudio ? [video, system] : [video]); + const micStream = microphone ? new Stream([mic]) : null; + const streams = { + display, + camera: null, + microphone: micStream, + } as unknown as ManagedStreams; + setGlobal("navigator", { + mediaDevices: { + getDisplayMedia: async () => display, + getUserMedia: async () => micStream, + }, + }); + return { streams, video, system, mic }; +}; + +beforeEach(() => { + Mixer.instances = []; + setGlobal("MediaStream", Stream); + setGlobal("AudioContext", Mixer); +}); + +afterEach(() => { + for (const [name, descriptor] of descriptors) { + if (descriptor) Object.defineProperty(globalThis, name, descriptor); + else Reflect.deleteProperty(globalThis, name); + } +}); + +test("records both audio sources through one track and releases the mixer", async () => { + const { video, system, mic } = capture(true, true); + let input: MediaStream | undefined; + setGlobal( + "MediaRecorder", + class { + static isTypeSupported() { + return true; + } + + state = "recording"; + + constructor(stream: MediaStream) { + input = stream; + } + + start() {} + stop() { + this.state = "inactive"; + } + }, + ); + const recorder = new CapRecorder({ + publicKey: "test-public-key", + mode: "tab", + camera: { enabled: false }, + systemAudio: true, + }); + try { + await recorder.start(); + const mixer = Mixer.instances[0]; + expect(recorder.phase).toBe("recording"); + expect(input?.getVideoTracks()).toEqual([video]); + expect(input?.getAudioTracks()).toEqual([mixer?.output]); + expect(mixer?.inputs).toEqual([system, mic]); + } finally { + recorder.destroy(); + } + const mixer = Mixer.instances[0]; + expect(video.stop).toHaveBeenCalled(); + expect(system.stop).toHaveBeenCalled(); + expect(mic.stop).toHaveBeenCalled(); + expect(mixer?.output.stop).toHaveBeenCalled(); + expect(mixer?.close).toHaveBeenCalledTimes(1); +}); + +test.each([ + [false, false], + [true, false], + [false, true], +])( + "preserves direct audio tracks for system=%s microphone=%s", + async (systemAudio, microphone) => { + const { streams, video, system, mic } = capture(systemAudio, microphone); + const composite = await buildCompositeStream(streams); + expect(composite.getVideoTracks()).toEqual([video]); + expect(composite.getAudioTracks()).toEqual( + systemAudio ? [system] : microphone ? [mic] : [], + ); + expect(Mixer.instances).toHaveLength(0); + }, +); + +test.each(["encoder", "audio"] as const)( + "cleans up both audio sources when %s initialization fails", + async (failure) => { + const { video, system, mic } = capture(true, true); + if (failure === "audio") { + setGlobal( + "AudioContext", + class extends Mixer { + resume = mock(async () => { + throw new Error("Audio initialization failed"); + }); + }, + ); + } + setGlobal( + "MediaRecorder", + class { + static isTypeSupported() { + return true; + } + + constructor() { + throw new Error("Encoder initialization failed"); + } + }, + ); + const recorder = new CapRecorder({ + publicKey: "test-public-key", + mode: "tab", + camera: { enabled: false }, + systemAudio: true, + }); + await expect(recorder.start()).rejects.toThrow( + failure === "audio" + ? "Audio initialization failed" + : "Encoder initialization failed", + ); + expect(recorder.phase).toBe("error"); + expect(video.stop).toHaveBeenCalled(); + expect(system.stop).toHaveBeenCalled(); + expect(mic.stop).toHaveBeenCalled(); + const mixer = Mixer.instances[0]; + expect(mixer?.output.stop).toHaveBeenCalled(); + expect(mixer?.close).toHaveBeenCalledTimes(1); + }, +); diff --git a/packages/sdk-recorder/package.json b/packages/sdk-recorder/package.json index 7fef2785793..d318a0e4536 100644 --- a/packages/sdk-recorder/package.json +++ b/packages/sdk-recorder/package.json @@ -20,6 +20,7 @@ ], "scripts": { "build": "tsup", + "test": "bun test", "typecheck": "tsc --noEmit" }, "dependencies": {}, diff --git a/packages/sdk-recorder/src/core/stream-manager.ts b/packages/sdk-recorder/src/core/stream-manager.ts index 7c39f6b3ae7..f6a0d16907d 100644 --- a/packages/sdk-recorder/src/core/stream-manager.ts +++ b/packages/sdk-recorder/src/core/stream-manager.ts @@ -11,6 +11,7 @@ export interface ManagedStreams { display: MediaStream | null; camera: MediaStream | null; microphone: MediaStream | null; + audioMix?: { context: AudioContext; stream: MediaStream }; } export async function acquireStreams( @@ -69,16 +70,30 @@ export async function acquireStreams( } export function releaseStreams(streams: ManagedStreams) { - for (const stream of Object.values(streams)) { + const { audioMix } = streams; + for (const stream of [ + streams.display, + streams.camera, + streams.microphone, + audioMix?.stream, + ]) { if (stream) { for (const track of stream.getTracks()) { track.stop(); } } } + if (audioMix) { + void audioMix.context.close().catch((error) => { + console.error("CapRecorder: failed to close audio mixer:", error); + }); + streams.audioMix = undefined; + } } -export function buildCompositeStream(streams: ManagedStreams): MediaStream { +export async function buildCompositeStream( + streams: ManagedStreams, +): Promise { const tracks: MediaStreamTrack[] = []; if (streams.display) { @@ -102,5 +117,21 @@ export function buildCompositeStream(streams: ManagedStreams): MediaStream { } } - return new MediaStream(tracks); + const audioTracks = tracks.filter((track) => track.kind === "audio"); + if (audioTracks.length <= 1) return new MediaStream(tracks); + + const context = new AudioContext(); + const destination = context.createMediaStreamDestination(); + streams.audioMix = { context, stream: destination.stream }; + for (const track of audioTracks) { + context + .createMediaStreamSource(new MediaStream([track])) + .connect(destination); + } + await context.resume(); + + return new MediaStream([ + ...tracks.filter((track) => track.kind !== "audio"), + ...destination.stream.getAudioTracks(), + ]); } diff --git a/packages/sdk-recorder/src/index.ts b/packages/sdk-recorder/src/index.ts index 0f7eb10e1fb..193882099ed 100644 --- a/packages/sdk-recorder/src/index.ts +++ b/packages/sdk-recorder/src/index.ts @@ -14,10 +14,10 @@ import type { import { MultipartClient } from "./upload/multipart-client"; export type { + RecorderEventMap, RecorderOptions, RecorderPhase, RecordingResult, - RecorderEventMap, }; const DEFAULT_API_BASE = "https://cap.so"; @@ -124,7 +124,7 @@ export class CapRecorder { systemAudio: this.options.systemAudio, }); - const composite = buildCompositeStream(this.streams); + const composite = await buildCompositeStream(this.streams); if (composite.getTracks().length === 0) { throw new Error("No media tracks available"); } From fa64a45f448c1bc04c118af6dfb2e45c14e8c8d7 Mon Sep 17 00:00:00 2001 From: Akshar Patel <123344143+AksharP5@users.noreply.github.com> Date: Mon, 19 Oct 2026 22:21:29 -0400 Subject: [PATCH 2/2] fix(sdk-recorder): bound audio startup and leave mixing headroom --- .../__tests__/audio-mixing.test.ts | 66 ++++++++++++++++++- .../sdk-recorder/src/core/stream-manager.ts | 25 +++++-- 2 files changed, 85 insertions(+), 6 deletions(-) diff --git a/packages/sdk-recorder/__tests__/audio-mixing.test.ts b/packages/sdk-recorder/__tests__/audio-mixing.test.ts index cb582c48e1e..b9acd324ba2 100644 --- a/packages/sdk-recorder/__tests__/audio-mixing.test.ts +++ b/packages/sdk-recorder/__tests__/audio-mixing.test.ts @@ -30,8 +30,13 @@ class Stream { class Mixer { static instances: Mixer[] = []; readonly inputs: Track[] = []; + readonly connections: unknown[] = []; readonly output = new Track("audio"); readonly destination = { stream: new Stream([this.output]) }; + readonly mix = { + gain: { value: 1 }, + connect: mock(() => {}), + }; resume = mock(async () => {}); close = mock(async () => {}); @@ -43,16 +48,28 @@ class Mixer { return this.destination; } + createGain() { + return this.mix; + } + createMediaStreamSource(stream: Stream) { return { - connect: () => { + connect: (destination: unknown) => { this.inputs.push(...stream.getAudioTracks()); + this.connections.push(destination); }, }; } } -const globals = ["MediaStream", "AudioContext", "MediaRecorder", "navigator"]; +const globals = [ + "MediaStream", + "AudioContext", + "MediaRecorder", + "navigator", + "setTimeout", + "clearTimeout", +]; const descriptors = globals.map( (name) => [name, Object.getOwnPropertyDescriptor(globalThis, name)] as const, ); @@ -129,6 +146,9 @@ test("records both audio sources through one track and releases the mixer", asyn expect(input?.getVideoTracks()).toEqual([video]); expect(input?.getAudioTracks()).toEqual([mixer?.output]); expect(mixer?.inputs).toEqual([system, mic]); + expect(mixer?.connections).toEqual([mixer?.mix, mixer?.mix]); + expect(mixer?.mix.gain.value).toBe(0.5); + expect(mixer?.mix.connect).toHaveBeenCalledWith(mixer?.destination); } finally { recorder.destroy(); } @@ -157,6 +177,48 @@ test.each([ }, ); +test("times out pending audio startup and releases captured tracks", async () => { + const { video, system, mic } = capture(true, true); + setGlobal( + "AudioContext", + class extends Mixer { + resume = mock(() => new Promise(() => {})); + }, + ); + let deadline: (() => void) | undefined; + const timeout = mock((callback: () => void, delay: number) => { + expect(delay).toBe(10_000); + deadline = callback; + return 7; + }); + const clear = mock(() => {}); + setGlobal("setTimeout", timeout); + setGlobal("clearTimeout", clear); + const recorder = new CapRecorder({ + publicKey: "test-public-key", + mode: "tab", + camera: { enabled: false }, + systemAudio: true, + }); + try { + const starting = recorder.start(); + await new Promise((resolve) => setImmediate(resolve)); + expect(deadline).toBeDefined(); + deadline?.(); + await expect(starting).rejects.toThrow("Recording audio startup timed out"); + expect(recorder.phase).toBe("error"); + expect(clear).toHaveBeenCalledWith(7); + expect(video.stop).toHaveBeenCalled(); + expect(system.stop).toHaveBeenCalled(); + expect(mic.stop).toHaveBeenCalled(); + const mixer = Mixer.instances[0]; + expect(mixer?.output.stop).toHaveBeenCalled(); + expect(mixer?.close).toHaveBeenCalledTimes(1); + } finally { + recorder.destroy(); + } +}); + test.each(["encoder", "audio"] as const)( "cleans up both audio sources when %s initialization fails", async (failure) => { diff --git a/packages/sdk-recorder/src/core/stream-manager.ts b/packages/sdk-recorder/src/core/stream-manager.ts index f6a0d16907d..f93156d09a7 100644 --- a/packages/sdk-recorder/src/core/stream-manager.ts +++ b/packages/sdk-recorder/src/core/stream-manager.ts @@ -123,12 +123,29 @@ export async function buildCompositeStream( const context = new AudioContext(); const destination = context.createMediaStreamDestination(); streams.audioMix = { context, stream: destination.stream }; + const mix = context.createGain(); + mix.gain.value = 1 / audioTracks.length; + mix.connect(destination); for (const track of audioTracks) { - context - .createMediaStreamSource(new MediaStream([track])) - .connect(destination); + context.createMediaStreamSource(new MediaStream([track])).connect(mix); + } + let startupTimeout: ReturnType | undefined; + try { + await Promise.race([ + context.resume(), + new Promise((_, reject) => { + startupTimeout = setTimeout(() => { + reject( + new Error( + "Recording audio startup timed out. Check your audio device and try again.", + ), + ); + }, 10_000); + }), + ]); + } finally { + clearTimeout(startupTimeout); } - await context.resume(); return new MediaStream([ ...tracks.filter((track) => track.kind !== "audio"),