Seed the E2EE sender cryptor with the publish codec

.changeset/brown-rings-smile.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+When an SEI or AUD NALU precedes the slice, encrypt h264 frames from the correct offset. Those NALUs read as h265 slice types, so detection returned h265 and the unencrypted prefix ended inside the leading NALU. The slice header was encrypted and no error was raised. The frame bytes now take precedence over a caller-supplied codec. The sender cryptor is also seeded with the publish codec for h264 and h265. A fallback to VP8 handling now logs the first bytes of the frame.

src/e2ee/E2eeManager.test.ts

@@ -1,6 +1,9 @@
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { LogLevel, getWorkerLogLevelListenerCount, setLogLevel, workerLogger } from '../logger';
 import Room from '../room/Room';
+import { ParticipantEvent } from '../room/events';
+import { Track } from '../room/track/Track';
+import type { VideoCodec } from '../room/track/options';
 import { E2EEManager } from './E2eeManager';
 import { BaseKeyProvider } from './KeyProvider';
 
@@ -194,3 +197,43 @@ describe('E2EEManager GC cleanup', () => {
     },
   );
 });
+
+describe('E2EEManager sender codec seeding', () => {
+  function publishVideo(videoCodec: VideoCodec | undefined) {
+    const { room, worker, manager } = makeManager();
+    room.localParticipant.identity = 'me';
+    manager.setup(room);
+
+    const sender = {
+      createEncodedStreams: () => ({
+        readable: new ReadableStream(),
+        writable: new WritableStream(),
+      }),
+    } as unknown as RTCRtpSender;
+    const track = {
+      isLocal: true,
+      kind: Track.Kind.Video,
+      mediaStreamID: 'ms1',
+      publishOptions: { videoCodec },
+    };
+
+    room.localParticipant.emit(ParticipantEvent.LocalSenderCreated, sender, track as any);
+    manager.dispose();
+
+    return worker.postMessage.mock.calls.map(([m]) => m).find((m: any) => m?.kind === 'encode');
+  }
+
+  it('seeds the publish codec for h264', () => {
+    expect(publishVideo('h264')?.data.codec).toBe('h264');
+  });
+
+  it('seeds the publish codec for h265', () => {
+    expect(publishVideo('h265')?.data.codec).toBe('h265');
+  });
+
+  it('leaves vp8 unseeded so byte detection still decides', () => {
+    const encode = publishVideo('vp8');
+    expect(encode).toBeDefined();
+    expect(encode.data.codec).toBeUndefined();
+  });
+});

src/e2ee/E2eeManager.ts

@@ -620,11 +620,15 @@ export class E2EEManager
       if (!sender) this.log.warn('early return because sender is not ready');
       return;
     }
+    const isVideo = isVideoTrack(track);
+    // Safari exposes no per-frame codec info until trackInfo arrives. Seed only h264/h265:
+    // a wrong vp8 seed would skip the NALU path, a wrong NALU seed only costs a logged fallback.
+    const publishedCodec = isVideo ? track.publishOptions?.videoCodec : undefined;
     this.handleSender(
       sender,
       track.mediaStreamID,
-      undefined,
-      isVideoTrack(track)
+      publishedCodec === 'h264' || publishedCodec === 'h265' ? publishedCodec : undefined,
+      isVideo
         ? (track.publishOptions?.frameMetadata ?? track.publishOptions?.packetTrailer)
         : undefined,
     );

src/e2ee/worker/FrameCryptor.test.ts

@@ -1,6 +1,8 @@
 import { afterEach, describe, expect, it, vitest } from 'vitest';
 import { appendPacketTrailer, extractPacketTrailer } from '../../frameMetadata/frameMetadata';
 import type { FrameMetadataPublishOptions } from '../../frameMetadata/types';
+import { workerLogger } from '../../logger';
+import type { VideoCodec } from '../../room/track/options';
 import type { NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays';
 import { IV_LENGTH, KEY_PROVIDER_DEFAULTS } from '../constants';
 import { CryptorEvent } from '../events';
@@ -71,12 +73,14 @@ function prepareParticipantTestEncoder(
   participantIdentity: string,
   partialKeyProviderOptions: Partial<KeyProviderOptions>,
   packetTrailer?: FrameMetadataPublishOptions,
+  codec?: VideoCodec,
 ) {
   return prepareParticipantTest(
     'encode',
     participantIdentity,
     partialKeyProviderOptions,
     packetTrailer,
+    codec,
   );
 }
 
@@ -85,6 +89,7 @@ function prepareParticipantTest(
   participantIdentity: string,
   partialKeyProviderOptions: Partial<KeyProviderOptions>,
   packetTrailer?: FrameMetadataPublishOptions,
+  codec?: VideoCodec,
 ): {
   keys: ParticipantKeyHandler;
   cryptor: FrameCryptor;
@@ -110,7 +115,7 @@ function prepareParticipantTest(
     new ReadableStream(input),
     new WritableStream(output),
     'testTrack',
-    undefined,
+    codec,
     packetTrailer,
   );
 
@@ -306,83 +311,151 @@ describe('FrameCryptor', () => {
         );
         expect(Array.from(decryptedData)).toEqual(plainTextData);
       });
-    });
-
-    it('appends packet trailer after encryption', async () => {
-      vitest.useFakeTimers();
-      const now = new Date('2025-04-10T12:00:00.123Z');
-      vitest.setSystemTime(now);
-      try {
-        const { keys, input, output } = prepareParticipantTestEncoder(
-          participantIdentity,
-          {},
-          { timestamp: true, frameId: true },
-        );
-
-        await keys.setKey(await createKeyMaterialFromString('key1'), 1);
-
-        const frame = mockRTCEncodedVideoFrame(
-          new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
-        );
-
-        input.write(frame);
-        await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
-
-        const extracted = extractPacketTrailer(frame.data);
-        expect(extracted.metadata?.userTimestamp).toBeGreaterThanOrEqual(
-          BigInt(now.getTime()) * BigInt(1000),
-        );
-        expect(extracted.metadata?.frameId).toBe(1);
-
-        const frameTrailer = new Uint8Array(
-          extracted.data.buffer.slice(extracted.data.byteLength - 2),
-        );
-        expect(frameTrailer[0]).toEqual(IV_LENGTH);
-        expect(frameTrailer[1]).toEqual(1);
-      } finally {
-        vitest.useRealTimers();
-      }
-    });
-
-    it('does not append packet trailer after encryption when no trailer features are enabled', async () => {
-      vitest.useFakeTimers();
-      try {
-        const { keys, input, output } = prepareParticipantTestEncoder(
-          participantIdentity,
-          {},
-          { timestamp: false, frameId: false },
-        );
-
-        await keys.setKey(await createKeyMaterialFromString('key1'), 1);
-
-        const frame = mockRTCEncodedVideoFrame(
-          new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
-        );
+      it('logs the frame bytes once when NALU processing fails', async () => {
+        vitest.useFakeTimers();
+        const warn = vitest.spyOn(workerLogger, 'warn');
+        try {
+          const { keys, input, output } = prepareParticipantTestEncoder(
+            participantIdentity,
+            {},
+            undefined,
+            'h264',
+          );
+
+          await keys.setKey(await createKeyMaterialFromString('key1'), 1);
+
+          // No start code at offset 0, so findNALUIndices throws 'byte stream contains
+          // leading data' and the cryptor falls back to VP8 handling.
+          input.write(
+            mockRTCEncodedVideoFrame(new Uint8Array([9, 9, 9, 9, 1, 2, 3, 4, 5, 6, 7, 8])),
+          );
+          await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
+
+          expect(warn).toHaveBeenCalledWith(
+            'NALU processing failed, falling back to VP8 handling',
+            expect.objectContaining({
+              detectedCodec: 'h264',
+              frameType: 'key',
+              byteLength: 12,
+              firstBytes: expect.stringMatching(/^09 09 09 09 /),
+            }),
+          );
+
+          // same payload type, so the once-per-tuple guard must suppress the second log
+          input.write(
+            mockRTCEncodedVideoFrame(new Uint8Array([8, 8, 8, 8, 1, 2, 3, 4, 5, 6, 7, 8])),
+          );
+          await vitest.waitFor(() => expect(output.chunks).toHaveLength(2));
+
+          expect(warn).toHaveBeenCalledTimes(1);
+        } finally {
+          warn.mockRestore();
+          vitest.useRealTimers();
+        }
+      });
 
-        input.write(frame);
-        await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
+      it('appends packet trailer after encryption', async () => {
+        vitest.useFakeTimers();
+        const now = new Date('2025-04-10T12:00:00.123Z');
+        vitest.setSystemTime(now);
+        try {
+          const { keys, input, output } = prepareParticipantTestEncoder(
+            participantIdentity,
+            {},
+            { timestamp: true, frameId: true },
+          );
+
+          await keys.setKey(await createKeyMaterialFromString('key1'), 1);
+
+          const frame = mockRTCEncodedVideoFrame(
+            new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
+          );
+
+          input.write(frame);
+          await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
+
+          const extracted = extractPacketTrailer(frame.data);
+          expect(extracted.metadata?.userTimestamp).toBeGreaterThanOrEqual(
+            BigInt(now.getTime()) * BigInt(1000),
+          );
+          expect(extracted.metadata?.frameId).toBe(1);
+
+          const frameTrailer = new Uint8Array(
+            extracted.data.buffer.slice(extracted.data.byteLength - 2),
+          );
+          expect(frameTrailer[0]).toEqual(IV_LENGTH);
+          expect(frameTrailer[1]).toEqual(1);
+        } finally {
+          vitest.useRealTimers();
+        }
+      });
 
-        const extracted = extractPacketTrailer(frame.data);
-        expect(extracted.metadata).toBeUndefined();
-        expect(extracted.data.byteLength).toBe(frame.data.byteLength);
+      it('does not append packet trailer after encryption when no trailer features are enabled', async () => {
+        vitest.useFakeTimers();
+        try {
+          const { keys, input, output } = prepareParticipantTestEncoder(
+            participantIdentity,
+            {},
+            { timestamp: false, frameId: false },
+          );
+
+          await keys.setKey(await createKeyMaterialFromString('key1'), 1);
+
+          const frame = mockRTCEncodedVideoFrame(
+            new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
+          );
+
+          input.write(frame);
+          await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
+
+          const extracted = extractPacketTrailer(frame.data);
+          expect(extracted.metadata).toBeUndefined();
+          expect(extracted.data.byteLength).toBe(frame.data.byteLength);
+
+          const frameTrailer = new Uint8Array(frame.data.slice(frame.data.byteLength - 2));
+          expect(frameTrailer[0]).toEqual(IV_LENGTH);
+          expect(frameTrailer[1]).toEqual(1);
+        } finally {
+          vitest.useRealTimers();
+        }
+      });
 
-        const frameTrailer = new Uint8Array(frame.data.slice(frame.data.byteLength - 2));
-        expect(frameTrailer[0]).toEqual(IV_LENGTH);
-        expect(frameTrailer[1]).toEqual(1);
-      } finally {
-        vitest.useRealTimers();
-      }
-    });
+      it('appends only timestamp packet trailer metadata after encryption', async () => {
+        vitest.useFakeTimers();
+        const now = new Date('2025-04-10T12:00:00.123Z');
+        vitest.setSystemTime(now);
+        try {
+          const { keys, input, output } = prepareParticipantTestEncoder(
+            participantIdentity,
+            {},
+            { timestamp: true },
+          );
+
+          await keys.setKey(await createKeyMaterialFromString('key1'), 1);
+
+          const frame = mockRTCEncodedVideoFrame(
+            new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
+          );
+
+          input.write(frame);
+          await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
+
+          const extracted = extractPacketTrailer(frame.data);
+          expect(frame.data.byteLength - extracted.data.byteLength).toBe(15);
+          expect(extracted.metadata?.userTimestamp).toBeGreaterThanOrEqual(
+            BigInt(now.getTime()) * BigInt(1000),
+          );
+          expect(extracted.metadata?.frameId).toBe(0);
+        } finally {
+          vitest.useRealTimers();
+        }
+      });
 
-    it('appends only timestamp packet trailer metadata after encryption', async () => {
-      vitest.useFakeTimers();
-      const now = new Date('2025-04-10T12:00:00.123Z');
-      vitest.setSystemTime(now);
-      try {
+      it('appends only frame id packet trailer metadata after encryption', async () => {
         const { keys, input, output } = prepareParticipantTestEncoder(
           participantIdentity,
           {},
-          { timestamp: true },
+          { frameId: true },
         );
 
         await keys.setKey(await createKeyMaterialFromString('key1'), 1);
@@ -395,37 +468,11 @@ describe('FrameCryptor', () => {
         await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
 
         const extracted = extractPacketTrailer(frame.data);
-        expect(frame.data.byteLength - extracted.data.byteLength).toBe(15);
-        expect(extracted.metadata?.userTimestamp).toBeGreaterThanOrEqual(
-          BigInt(now.getTime()) * BigInt(1000),
-        );
-        expect(extracted.metadata?.frameId).toBe(0);
-      } finally {
-        vitest.useRealTimers();
-      }
-    });
-
-    it('appends only frame id packet trailer metadata after encryption', async () => {
-      const { keys, input, output } = prepareParticipantTestEncoder(
-        participantIdentity,
-        {},
-        { frameId: true },
-      );
-
-      await keys.setKey(await createKeyMaterialFromString('key1'), 1);
-
-      const frame = mockRTCEncodedVideoFrame(
-        new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]),
-      );
-
-      input.write(frame);
-      await vitest.waitFor(() => expect(output.chunks).toHaveLength(1));
-
-      const extracted = extractPacketTrailer(frame.data);
-      expect(frame.data.byteLength - extracted.data.byteLength).toBe(11);
-      expect(extracted.metadata).toEqual({
-        userTimestamp: BigInt(0),
-        frameId: 1,
+        expect(frame.data.byteLength - extracted.data.byteLength).toBe(11);
+        expect(extracted.metadata).toEqual({
+          userTimestamp: BigInt(0),
+          frameId: 1,
+        });
       });
     });
   });