E2EE support for Opus RED

package.json

@@ -55,6 +55,7 @@
     "@bufbuild/protobuf": "^1.3.0",
     "events": "^3.3.0",
     "loglevel": "^1.8.0",
+    "opus-red-parser": "^1.0.0",
     "sdp-transform": "^2.14.1",
     "ts-debounce": "^4.0.0",
     "typed-emitter": "^2.1.0",

src/e2ee/E2eeManager.ts

@@ -10,7 +10,7 @@ import { EngineEvent, ParticipantEvent, RoomEvent } from '../room/events';
 import LocalTrack from '../room/track/LocalTrack';
 import type RemoteTrack from '../room/track/RemoteTrack';
 import type { Track } from '../room/track/Track';
-import type { VideoCodec } from '../room/track/options';
+import type { AudioCodec, VideoCodec } from '../room/track/options';
 import type { BaseKeyProvider } from './KeyProvider';
 import { E2EE_FLAG } from './constants';
 import { type E2EEManagerCallbacks, EncryptionEvent, KeyProviderEvent } from './events';
@@ -21,14 +21,14 @@ import type {
   EncodeMessage,
   InitMessage,
   KeyInfo,
-  RTPVideoMapMessage,
+  RTPMapMessage,
   RatchetRequestMessage,
   RemoveTransformMessage,
   SetKeyMessage,
   SifTrailerMessage,
   UpdateCodecMessage,
 } from './types';
-import { isE2EESupported, isScriptTransformSupported, mimeTypeToVideoCodecString } from './utils';
+import { isE2EESupported, isScriptTransformSupported, mimeTypeToCodecString } from './utils';
 
 /**
  * @experimental
@@ -155,6 +155,9 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter<E2
     engine.on(EngineEvent.RTPVideoMapUpdate, (rtpMap) => {
       this.postRTPMap(rtpMap);
     });
+    engine.on(EngineEvent.RTPAudioMapUpdate, (rtpMap) => {
+      this.postRTPMap(rtpMap);
+    });
   }
 
   private setupEventListeners(room: Room, keyProvider: BaseKeyProvider) {
@@ -203,6 +206,8 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter<E2
         });
       });
     room.localParticipant.on(ParticipantEvent.LocalTrackPublished, async (publication) => {
+      console.log('sender info', publication.trackInfo);
+
       this.setupE2EESender(publication.track!, publication.track!.sender!);
     });
 
@@ -258,14 +263,14 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter<E2
     }
   }
 
-  private postRTPMap(map: Map<number, VideoCodec>) {
+  private postRTPMap(map: Map<number, VideoCodec> | Map<number, AudioCodec>) {
     if (!this.worker) {
       throw TypeError('could not post rtp map, worker is missing');
     }
     if (!this.room?.localParticipant.identity) {
       throw TypeError('could not post rtp map, local participant identity is missing');
     }
-    const msg: RTPVideoMapMessage = {
+    const msg: RTPMapMessage = {
       kind: 'setRTPMap',
       data: {
         map,
@@ -299,7 +304,7 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter<E2
       track.receiver,
       track.mediaStreamID,
       remoteId,
-      track.kind === 'video' ? mimeTypeToVideoCodecString(trackInfo.mimeType) : undefined,
+      mimeTypeToCodecString(trackInfo.mimeType),
     );
   }
 
@@ -320,7 +325,7 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter<E2
     receiver: RTCRtpReceiver,
     trackId: string,
     participantIdentity: string,
-    codec?: VideoCodec,
+    codec?: VideoCodec | AudioCodec,
   ) {
     if (!this.worker) {
       return;
@@ -386,7 +391,7 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter<E2
    * a frame encoder.
    *
    */
-  private handleSender(sender: RTCRtpSender, trackId: string, codec?: VideoCodec) {
+  private handleSender(sender: RTCRtpSender, trackId: string, codec?: VideoCodec | AudioCodec) {
     if (E2EE_FLAG in sender || !this.worker) {
       return;
     }

src/e2ee/constants.ts

@@ -24,6 +24,7 @@ export const UNENCRYPTED_BYTES = {
   key: 10,
   delta: 3,
   audio: 1, // frame.type is not set on audio, so this is set manually
+  red1: 5, // opus red with 1 redundancy frame
   empty: 0,
 } as const;
 

src/e2ee/types.ts

@@ -1,4 +1,4 @@
-import type { VideoCodec } from '../room/track/options';
+import type { AudioCodec, VideoCodec } from '../room/track/options';
 import type { BaseKeyProvider } from './KeyProvider';
 
 export interface BaseMessage {
@@ -23,10 +23,10 @@ export interface SetKeyMessage extends BaseMessage {
   };
 }
 
-export interface RTPVideoMapMessage extends BaseMessage {
+export interface RTPMapMessage extends BaseMessage {
   kind: 'setRTPMap';
   data: {
-    map: Map<number, VideoCodec>;
+    map: Map<number, VideoCodec> | Map<number, AudioCodec>;
     participantIdentity: string;
   };
 }
@@ -45,7 +45,7 @@ export interface EncodeMessage extends BaseMessage {
     readableStream: ReadableStream;
     writableStream: WritableStream;
     trackId: string;
-    codec?: VideoCodec;
+    codec?: VideoCodec | AudioCodec;
   };
 }
 
@@ -62,7 +62,7 @@ export interface UpdateCodecMessage extends BaseMessage {
   data: {
     participantIdentity: string;
     trackId: string;
-    codec: VideoCodec;
+    codec: VideoCodec | AudioCodec;
   };
 }
 
@@ -112,7 +112,7 @@ export type E2EEWorkerMessage =
   | ErrorMessage
   | EnableMessage
   | RemoveTransformMessage
-  | RTPVideoMapMessage
+  | RTPMapMessage
   | UpdateCodecMessage
   | RatchetRequestMessage
   | RatchetMessage

src/e2ee/utils.ts

@@ -1,5 +1,5 @@
-import { videoCodecs } from '../room/track/options';
-import type { VideoCodec } from '../room/track/options';
+import type { AudioCodec, VideoCodec } from '../room/track/options';
+import { isAudioCodec, isVideoCodec } from '../room/utils';
 import { ENCRYPTION_ALGORITHM } from './constants';
 
 export function isE2EESupported() {
@@ -116,12 +116,12 @@ export function createE2EEKey(): Uint8Array {
   return window.crypto.getRandomValues(new Uint8Array(32));
 }
 
-export function mimeTypeToVideoCodecString(mimeType: string) {
-  const codec = mimeType.split('/')[1].toLowerCase() as VideoCodec;
-  if (!videoCodecs.includes(codec)) {
-    throw Error(`Video codec not supported: ${codec}`);
+export function mimeTypeToCodecString(mimeType: string) {
+  const codec = mimeType.split('/')[1].toLowerCase() as VideoCodec | AudioCodec;
+  if (isVideoCodec(codec) || isAudioCodec(codec)) {
+    return codec;
   }
-  return codec;
+  throw Error(`Codec not supported: ${codec}`);
 }
 
 /**