diff --git a/.changeset/gorgeous-cameras-peel.md b/.changeset/gorgeous-cameras-peel.md deleted file mode 100644 index 4c0484ba5f..0000000000 --- a/.changeset/gorgeous-cameras-peel.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"livekit-client": patch ---- - -Disable opus RED when using E2EE diff --git a/package.json b/package.json index 08e992b4a0..b701160424 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "@bufbuild/protobuf": "^1.3.0", "events": "^3.3.0", "loglevel": "^1.8.0", + "opus-red-parser": "^1.0.0", "sdp-transform": "^2.14.1", "ts-debounce": "^4.0.0", "typed-emitter": "^2.1.0", diff --git a/src/e2ee/E2eeManager.ts b/src/e2ee/E2eeManager.ts index cc0fdac2c0..fd4335856e 100644 --- a/src/e2ee/E2eeManager.ts +++ b/src/e2ee/E2eeManager.ts @@ -10,7 +10,7 @@ import { EngineEvent, ParticipantEvent, RoomEvent } from '../room/events'; import LocalTrack from '../room/track/LocalTrack'; import type RemoteTrack from '../room/track/RemoteTrack'; import type { Track } from '../room/track/Track'; -import type { VideoCodec } from '../room/track/options'; +import type { AudioCodec, VideoCodec } from '../room/track/options'; import type { BaseKeyProvider } from './KeyProvider'; import { E2EE_FLAG } from './constants'; import { type E2EEManagerCallbacks, EncryptionEvent, KeyProviderEvent } from './events'; @@ -21,14 +21,14 @@ import type { EncodeMessage, InitMessage, KeyInfo, - RTPVideoMapMessage, + RTPMapMessage, RatchetRequestMessage, RemoveTransformMessage, SetKeyMessage, SifTrailerMessage, UpdateCodecMessage, } from './types'; -import { isE2EESupported, isScriptTransformSupported, mimeTypeToVideoCodecString } from './utils'; +import { isE2EESupported, isScriptTransformSupported, mimeTypeToCodecString } from './utils'; /** * @experimental @@ -155,6 +155,9 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter { this.postRTPMap(rtpMap); }); + engine.on(EngineEvent.RTPAudioMapUpdate, (rtpMap) => { + this.postRTPMap(rtpMap); + }); } private setupEventListeners(room: Room, keyProvider: BaseKeyProvider) { @@ -203,6 +206,8 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter { + console.log('sender info', publication.trackInfo); + this.setupE2EESender(publication.track!, publication.track!.sender!); }); @@ -258,14 +263,14 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter) { + private postRTPMap(map: Map | Map) { if (!this.worker) { throw TypeError('could not post rtp map, worker is missing'); } if (!this.room?.localParticipant.identity) { throw TypeError('could not post rtp map, local participant identity is missing'); } - const msg: RTPVideoMapMessage = { + const msg: RTPMapMessage = { kind: 'setRTPMap', data: { map, @@ -299,7 +304,7 @@ export class E2EEManager extends (EventEmitter as new () => TypedEventEmitter TypedEventEmitter TypedEventEmitter; + map: Map | Map; participantIdentity: string; }; } @@ -45,7 +45,7 @@ export interface EncodeMessage extends BaseMessage { readableStream: ReadableStream; writableStream: WritableStream; trackId: string; - codec?: VideoCodec; + codec?: VideoCodec | AudioCodec; }; } @@ -62,7 +62,7 @@ export interface UpdateCodecMessage extends BaseMessage { data: { participantIdentity: string; trackId: string; - codec: VideoCodec; + codec: VideoCodec | AudioCodec; }; } @@ -112,7 +112,7 @@ export type E2EEWorkerMessage = | ErrorMessage | EnableMessage | RemoveTransformMessage - | RTPVideoMapMessage + | RTPMapMessage | UpdateCodecMessage | RatchetRequestMessage | RatchetMessage diff --git a/src/e2ee/utils.ts b/src/e2ee/utils.ts index 31ad917104..0a11e469f1 100644 --- a/src/e2ee/utils.ts +++ b/src/e2ee/utils.ts @@ -1,5 +1,5 @@ -import { videoCodecs } from '../room/track/options'; -import type { VideoCodec } from '../room/track/options'; +import type { AudioCodec, VideoCodec } from '../room/track/options'; +import { isAudioCodec, isVideoCodec } from '../room/utils'; import { ENCRYPTION_ALGORITHM } from './constants'; export function isE2EESupported() { @@ -116,12 +116,12 @@ export function createE2EEKey(): Uint8Array { return window.crypto.getRandomValues(new Uint8Array(32)); } -export function mimeTypeToVideoCodecString(mimeType: string) { - const codec = mimeType.split('/')[1].toLowerCase() as VideoCodec; - if (!videoCodecs.includes(codec)) { - throw Error(`Video codec not supported: ${codec}`); +export function mimeTypeToCodecString(mimeType: string) { + const codec = mimeType.split('/')[1].toLowerCase() as VideoCodec | AudioCodec; + if (isVideoCodec(codec) || isAudioCodec(codec)) { + return codec; } - return codec; + throw Error(`Codec not supported: ${codec}`); } /** diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index e7f312c299..c3145aa0c6 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -1,9 +1,10 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ // TODO code inspired by https://github.com/webrtc/samples/blob/gh-pages/src/content/insertable-streams/endtoend-encryption/js/worker.js import { EventEmitter } from 'events'; +import * as red from 'opus-red-parser'; import type TypedEventEmitter from 'typed-emitter'; import { workerLogger } from '../../logger'; -import type { VideoCodec } from '../../room/track/options'; +import type { AudioCodec, VideoCodec } from '../../room/track/options'; import { ENCRYPTION_ALGORITHM, IV_LENGTH, UNENCRYPTED_BYTES } from '../constants'; import { CryptorError, CryptorErrorReason } from '../errors'; import { CryptorCallbacks, CryptorEvent } from '../events'; @@ -54,9 +55,9 @@ export class FrameCryptor extends BaseFrameCryptor { private keys: ParticipantKeyHandler; - private videoCodec?: VideoCodec; + private codec?: VideoCodec | AudioCodec; - private rtpMap: Map; + private rtpMap: Map; private keyProviderOptions: KeyProviderOptions; @@ -116,19 +117,21 @@ export class FrameCryptor extends BaseFrameCryptor { } /** - * Update the video codec used by the mediaStreamTrack + * Update the codec used by the mediaStreamTrack * @param codec */ - setVideoCodec(codec: VideoCodec) { - this.videoCodec = codec; + setCodec(codec: VideoCodec | AudioCodec) { + this.codec = codec; } /** * rtp payload type map used for figuring out codec of payload type when encoding * @param map */ - setRtpMap(map: Map) { - this.rtpMap = map; + setRtpMap(map: Map) { + for (const [key, val] of map) { + this.rtpMap.set(key, val); + } } setupTransform( @@ -136,11 +139,11 @@ export class FrameCryptor extends BaseFrameCryptor { readable: ReadableStream, writable: WritableStream, trackId: string, - codec?: VideoCodec, + codec?: VideoCodec | AudioCodec, ) { if (codec) { workerLogger.info('setting codec on cryptor to', { codec }); - this.videoCodec = codec; + this.codec = codec; } const transformFn = operation === 'encode' ? this.encodeFunction : this.decodeFunction; @@ -212,13 +215,6 @@ export class FrameCryptor extends BaseFrameCryptor { encodedFrame.timestamp, ); - // Thіs is not encrypted and contains the VP8 payload descriptor or the Opus TOC byte. - const frameHeader = new Uint8Array( - encodedFrame.data, - 0, - this.getUnencryptedBytes(encodedFrame), - ); - // Frame trailer contains the R|IV_LENGTH and key index const frameTrailer = new Uint8Array(2); @@ -233,29 +229,54 @@ export class FrameCryptor extends BaseFrameCryptor { // payload |IV...(length = IV_LENGTH)|R|IV_LENGTH|KID | // ---------+-------------------------+-+---------+---- try { - const cipherText = await crypto.subtle.encrypt( - { - name: ENCRYPTION_ALGORITHM, + if (this.getCodec(encodedFrame) === 'red') { + const { primaryBlock, redundancyBlocks } = red.split(encodedFrame.data); + console.log({ primaryBlock, redundancyBlocks }); + + const primaryBlockEncrypted = await this.encrypt( + primaryBlock.data.slice(1), + primaryBlock.data.slice(0, 1), + frameTrailer, iv, - additionalData: new Uint8Array(encodedFrame.data, 0, frameHeader.byteLength), - }, - encryptionKey, - new Uint8Array(encodedFrame.data, this.getUnencryptedBytes(encodedFrame)), - ); + encryptionKey, + ); + primaryBlock.data = primaryBlockEncrypted; + + const redundancyBlocksEncrypted = await Promise.all( + redundancyBlocks.map(async (block) => { + const blockEncrypted = await this.encrypt( + block.data.slice(1), + block.data.slice(0, 1), + frameTrailer, + iv, + encryptionKey, + ); + block.data = blockEncrypted; + block.header.blockLength = blockEncrypted.length; + return block; + }), + ); - const newData = new ArrayBuffer( - frameHeader.byteLength + cipherText.byteLength + iv.byteLength + frameTrailer.byteLength, - ); - const newUint8 = new Uint8Array(newData); + const encryptedRed = red.join({ + primaryBlock, + redundancyBlocks: redundancyBlocksEncrypted, + }); - newUint8.set(frameHeader); // copy first bytes. - newUint8.set(new Uint8Array(cipherText), frameHeader.byteLength); // add ciphertext. - newUint8.set(new Uint8Array(iv), frameHeader.byteLength + cipherText.byteLength); // append IV. - newUint8.set(frameTrailer, frameHeader.byteLength + cipherText.byteLength + iv.byteLength); // append frame trailer. + encodedFrame.data = encryptedRed; + return controller.enqueue(encodedFrame); + } else { + const encryptedFrame = await this.encrypt( + new Uint8Array(encodedFrame.data, this.getUnencryptedBytes(encodedFrame)), + new Uint8Array(encodedFrame.data, 0, this.getUnencryptedBytes(encodedFrame)), + frameTrailer, + iv, + encryptionKey, + ); - encodedFrame.data = newData; + encodedFrame.data = encryptedFrame.buffer; - return controller.enqueue(encodedFrame); + return controller.enqueue(encodedFrame); + } } catch (e: any) { // TODO: surface this to the app. workerLogger.error(e); @@ -268,6 +289,43 @@ export class FrameCryptor extends BaseFrameCryptor { } } + private async encrypt( + payload: Uint8Array, + unencryptedHeaderBytes: Uint8Array, + frameTrailer: Uint8Array, + iv: ArrayBuffer, + key: CryptoKey, + ) { + const cipherText = await crypto.subtle.encrypt( + { + name: ENCRYPTION_ALGORITHM, + iv, + additionalData: unencryptedHeaderBytes, + }, + key, + payload, + ); + const newData = new ArrayBuffer( + unencryptedHeaderBytes.byteLength + + cipherText.byteLength + + iv.byteLength + + frameTrailer.byteLength, + ); + const encryptedBytesWithTrailer = new Uint8Array(newData); + + encryptedBytesWithTrailer.set(unencryptedHeaderBytes); // copy first bytes. + encryptedBytesWithTrailer.set(new Uint8Array(cipherText), unencryptedHeaderBytes.byteLength); // add ciphertext. + encryptedBytesWithTrailer.set( + new Uint8Array(iv), + unencryptedHeaderBytes.byteLength + cipherText.byteLength, + ); // append IV. + encryptedBytesWithTrailer.set( + frameTrailer, + unencryptedHeaderBytes.byteLength + cipherText.byteLength + iv.byteLength, + ); // append frame trailer. + return encryptedBytesWithTrailer; + } + /** * Function that will be injected in a stream and will decrypt the given encoded frames. * @@ -356,42 +414,77 @@ export class FrameCryptor extends BaseFrameCryptor { // ---------+-------------------------+-+---------+---- try { - const frameHeader = new Uint8Array( - encodedFrame.data, - 0, - this.getUnencryptedBytes(encodedFrame), - ); - const frameTrailer = new Uint8Array(encodedFrame.data, encodedFrame.data.byteLength - 2, 2); + if (encodedFrame instanceof RTCEncodedAudioFrame) { + console.log('audio info', encodedFrame.getMetadata()); + } + if (this.getCodec(encodedFrame) === 'red') { + console.log('using red'); + const { primaryBlock, redundancyBlocks } = red.split(encodedFrame.data); - const ivLength = frameTrailer[0]; - const iv = new Uint8Array( - encodedFrame.data, - encodedFrame.data.byteLength - ivLength - frameTrailer.byteLength, - ivLength, - ); + const primaryFrame = this.extractFrameInfo(primaryBlock.data); - const cipherTextStart = frameHeader.byteLength; - const cipherTextLength = - encodedFrame.data.byteLength - - (frameHeader.byteLength + ivLength + frameTrailer.byteLength); - - const plainText = await crypto.subtle.decrypt( - { - name: ENCRYPTION_ALGORITHM, - iv, - additionalData: new Uint8Array(encodedFrame.data, 0, frameHeader.byteLength), - }, - ratchetOpts.encryptionKey ?? keySet!.encryptionKey, - new Uint8Array(encodedFrame.data, cipherTextStart, cipherTextLength), - ); + const primaryKey = + ratchetOpts.encryptionKey ?? this.keys.getKeySet(primaryFrame.keyIndex)?.encryptionKey; + if (!primaryKey) { + throw new TypeError( + `key missing for primary opus frame at index ${primaryFrame.keyIndex}`, + ); + } + + const primaryDecrypted = await this.decrypt( + primaryFrame.encryptedPayload.slice(1), + primaryFrame.encryptedPayload.slice(0, 1), + primaryFrame.iv, + primaryKey, + ); + primaryBlock.data = primaryDecrypted; + + const redundancyDecrypted = await Promise.all( + redundancyBlocks.map(async (block) => { + const redundancyFrame = this.extractFrameInfo(block.data); + const key = this.keys.getKeySet(redundancyFrame.keyIndex)?.encryptionKey; + if (!key) { + throw new TypeError( + `key missing for redundancy opus frame at index ${primaryFrame.keyIndex}`, + ); + } + const decryptedBlock = await this.decrypt( + redundancyFrame.encryptedPayload.slice(1), + redundancyFrame.encryptedPayload.slice(0, 1), + redundancyFrame.iv, + key, + ); + block.data = decryptedBlock; + block.header.blockLength = decryptedBlock.byteLength; + + return block; + }), + ); + + const decryptedFrame = red.join({ + primaryBlock: primaryBlock, + redundancyBlocks: redundancyDecrypted, + }); + + encodedFrame.data = decryptedFrame; + return encodedFrame; + } - const newData = new ArrayBuffer(frameHeader.byteLength + plainText.byteLength); - const newUint8 = new Uint8Array(newData); + const frameInfo = this.extractFrameInfo(new Uint8Array(encodedFrame.data)); + const headerLength = this.getUnencryptedBytes(encodedFrame); - newUint8.set(new Uint8Array(encodedFrame.data, 0, frameHeader.byteLength)); - newUint8.set(new Uint8Array(plainText), frameHeader.byteLength); + if (encodedFrame instanceof RTCEncodedAudioFrame) { + console.log('audio info', frameInfo, encodedFrame.getMetadata()); + } + + const decryptedData = await this.decrypt( + frameInfo.encryptedPayload.slice(headerLength), + frameInfo.encryptedPayload.slice(0, headerLength), + frameInfo.iv, + ratchetOpts?.encryptionKey ?? keySet!.encryptionKey, + ); - encodedFrame.data = newData; + encodedFrame.data = decryptedData.buffer; return encodedFrame; } catch (error: any) { @@ -443,12 +536,35 @@ export class FrameCryptor extends BaseFrameCryptor { } else { throw new CryptorError( `Decryption failed: ${error.message}`, - CryptorErrorReason.InvalidKey, + CryptorErrorReason.InternalError, ); } } } + private async decrypt( + payload: Uint8Array, + unencryptedHeaderBytes: Uint8Array, + iv: ArrayBuffer, + key: CryptoKey, + ) { + const plainText = await crypto.subtle.decrypt( + { + name: ENCRYPTION_ALGORITHM, + iv, + additionalData: unencryptedHeaderBytes, + }, + key, + payload, + ); + + const buffer = new ArrayBuffer(unencryptedHeaderBytes.byteLength + plainText.byteLength); + const decryptedBytes = new Uint8Array(buffer); + decryptedBytes.set(new Uint8Array(unencryptedHeaderBytes, 0)); + decryptedBytes.set(new Uint8Array(plainText), unencryptedHeaderBytes.byteLength); + return decryptedBytes; + } + /** * Construct the IV used for AES-GCM and sent (in plain) with the packet similar to * https://tools.ietf.org/html/rfc7714#section-8.1 @@ -490,9 +606,9 @@ export class FrameCryptor extends BaseFrameCryptor { } private getUnencryptedBytes(frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame): number { - if (isVideoFrame(frame)) { - let detectedCodec = this.getVideoCodec(frame) ?? this.videoCodec; + let detectedCodec = this.getCodec(frame) ?? this.codec; + if (isVideoFrame(frame)) { if (detectedCodec === 'av1' || detectedCodec === 'vp9') { throw new Error(`${detectedCodec} is not yet supported for end to end encryption`); } @@ -531,20 +647,47 @@ export class FrameCryptor extends BaseFrameCryptor { return UNENCRYPTED_BYTES[frame.type]; } else { + if (detectedCodec === 'red') { + return UNENCRYPTED_BYTES.red1; + } return UNENCRYPTED_BYTES.audio; } } + private extractFrameInfo(data: Uint8Array) { + const frameTrailer = data.slice(data.byteLength - 2); + const ivLength = frameTrailer[0]; + + const encryptedPayload = data.slice(0, data.byteLength - 2 - ivLength); + + const keyIndex = frameTrailer[1]; + + const iv = data.slice( + data.byteLength - ivLength - frameTrailer.byteLength, + data.byteLength - frameTrailer.byteLength, + ); + + return { encryptedPayload, keyIndex, iv }; + } + /** - * inspects frame payloadtype if available and maps it to the codec specified in rtpMap + * inspects frame payloadtype if available and maps it to the codec specified in rtpMap. + * falls back to this.codec */ - private getVideoCodec(frame: RTCEncodedVideoFrame): VideoCodec | undefined { + private getCodec( + frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame, + ): VideoCodec | AudioCodec | undefined { + // console.log('codec info', this.codec, this.rtpMap); + // if (isFireFox() || frame instanceof RTCEncodedAudioFrame) { + // return 'opus'; + // } if (this.rtpMap.size === 0) { - return undefined; + return this.codec; } + // @ts-expect-error payloadType is not yet part of the typescript definition and currently not supported in Safari const payloadType = frame.getMetadata().payloadType; - const codec = payloadType ? this.rtpMap.get(payloadType) : undefined; + const codec = payloadType ? this.rtpMap.get(payloadType) : this.codec; return codec; } } diff --git a/src/e2ee/worker/e2ee.worker.ts b/src/e2ee/worker/e2ee.worker.ts index 5d3b7eccef..d79bfa7ce7 100644 --- a/src/e2ee/worker/e2ee.worker.ts +++ b/src/e2ee/worker/e2ee.worker.ts @@ -1,4 +1,5 @@ import { workerLogger } from '../../logger'; +import type { AudioCodec, VideoCodec } from '../../room/track/options'; import { KEY_PROVIDER_DEFAULTS } from '../constants'; import { CryptorErrorReason } from '../errors'; import { CryptorEvent, KeyHandlerEvent } from '../events'; @@ -27,6 +28,8 @@ let sifTrailer: Uint8Array | undefined; let keyProviderOptions: KeyProviderOptions = KEY_PROVIDER_DEFAULTS; +const rtpMap = new Map(); + workerLogger.setDefaultLevel('info'); onmessage = (ev) => { @@ -85,14 +88,15 @@ onmessage = (ev) => { unsetCryptorParticipant(data.trackId); break; case 'updateCodec': - getTrackCryptor(data.participantIdentity, data.trackId).setVideoCodec(data.codec); + getTrackCryptor(data.participantIdentity, data.trackId).setCodec(data.codec); break; case 'setRTPMap': // this is only used for the local participant + for (const [key, val] of data.map) { + rtpMap.set(key, val); + } participantCryptors.forEach((cr) => { - if (cr.getParticipantIdentity() === data.participantIdentity) { - cr.setRtpMap(data.map); - } + cr.setRtpMap(rtpMap); }); break; case 'ratchetRequest': @@ -135,8 +139,8 @@ function getTrackCryptor(participantIdentity: string, trackId: string) { keyProviderOptions, sifTrailer, }); - setupCryptorErrorEvents(cryptor); + cryptor.setRtpMap(rtpMap); participantCryptors.push(cryptor); } else if (participantIdentity !== cryptor.getParticipantIdentity()) { // assign new participant id to track cryptor and pass in correct key handler diff --git a/src/room/PCTransport.ts b/src/room/PCTransport.ts index 62fa4fe5ed..05ba53be35 100644 --- a/src/room/PCTransport.ts +++ b/src/room/PCTransport.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'events'; -import type { MediaDescription } from 'sdp-transform'; +import type { MediaDescription, SessionDescription } from 'sdp-transform'; import { parse, write } from 'sdp-transform'; import { debounce } from 'ts-debounce'; import log from '../logger'; @@ -26,6 +26,7 @@ export const PCEvents = { NegotiationStarted: 'negotiationStarted', NegotiationComplete: 'negotiationComplete', RTPVideoPayloadTypes: 'rtpVideoPayloadTypes', + RTPAudioPayloadTypes: 'rtpAudioPayloadTypes', } as const; /** @internal */ @@ -145,14 +146,6 @@ export default class PCTransport extends EventEmitter { this.createAndSendOffer(); } else if (sd.type === 'answer') { this.emit(PCEvents.NegotiationComplete); - if (sd.sdp) { - const sdpParsed = parse(sd.sdp); - sdpParsed.media.forEach((media) => { - if (media.type === 'video') { - this.emit(PCEvents.RTPVideoPayloadTypes, media.rtp); - } - }); - } } } @@ -255,6 +248,7 @@ export default class PCTransport extends EventEmitter { }); await this.setMungedSDP(offer, write(sdpParsed)); + this.emitRTPMaps(sdpParsed); this.onOffer(offer); } @@ -267,6 +261,8 @@ export default class PCTransport extends EventEmitter { } }); await this.setMungedSDP(answer, write(sdpParsed)); + this.emitRTPMaps(sdpParsed); + return answer; } @@ -341,6 +337,16 @@ export default class PCTransport extends EventEmitter { throw new NegotiationError(msg); } } + + emitRTPMaps(sdpParsed: SessionDescription) { + sdpParsed.media.forEach((media) => { + if (media.type === 'video') { + this.emit(PCEvents.RTPVideoPayloadTypes, media.rtp); + } else if (media.type === 'audio') { + this.emit(PCEvents.RTPAudioPayloadTypes, media.rtp); + } + }); + } } function ensureAudioNackAndStereo( diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index 5272a5e353..997e587cf7 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -47,9 +47,10 @@ import type LocalTrack from './track/LocalTrack'; import type LocalVideoTrack from './track/LocalVideoTrack'; import type { SimulcastTrackInfo } from './track/LocalVideoTrack'; import { Track } from './track/Track'; -import type { TrackPublishOptions, VideoCodec } from './track/options'; +import type { AudioCodec, TrackPublishOptions, VideoCodec } from './track/options'; import { Mutex, + isAudioCodec, isVideoCodec, isWeb, sleep, @@ -451,6 +452,50 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit this.emit(EngineEvent.MediaTrackAdded, ev.track, ev.streams[0], ev.receiver); }; + this.publisher.once(PCEvents.RTPVideoPayloadTypes, (rtpTypes: MediaAttributes['rtp']) => { + const rtpMap = new Map(); + rtpTypes.forEach((rtp) => { + const codec = rtp.codec.toLowerCase(); + if (isVideoCodec(codec)) { + rtpMap.set(rtp.payload, codec); + } + }); + this.emit(EngineEvent.RTPVideoMapUpdate, rtpMap); + }); + + this.publisher.once(PCEvents.RTPAudioPayloadTypes, (rtpTypes: MediaAttributes['rtp']) => { + const rtpMap = new Map(); + rtpTypes.forEach((rtp) => { + const codec = rtp.codec.toLowerCase(); + if (isAudioCodec(codec)) { + rtpMap.set(rtp.payload, codec); + } + }); + this.emit(EngineEvent.RTPAudioMapUpdate, rtpMap); + }); + + this.subscriber.once(PCEvents.RTPVideoPayloadTypes, (rtpTypes: MediaAttributes['rtp']) => { + const rtpMap = new Map(); + rtpTypes.forEach((rtp) => { + const codec = rtp.codec.toLowerCase(); + if (isVideoCodec(codec)) { + rtpMap.set(rtp.payload, codec); + } + }); + this.emit(EngineEvent.RTPVideoMapUpdate, rtpMap); + }); + + this.subscriber.once(PCEvents.RTPAudioPayloadTypes, (rtpTypes: MediaAttributes['rtp']) => { + const rtpMap = new Map(); + rtpTypes.forEach((rtp) => { + const codec = rtp.codec.toLowerCase(); + if (isAudioCodec(codec)) { + rtpMap.set(rtp.payload, codec); + } + }); + this.emit(EngineEvent.RTPAudioMapUpdate, rtpMap); + }); + this.createDataChannels(); } @@ -1287,6 +1332,17 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit this.emit(EngineEvent.RTPVideoMapUpdate, rtpMap); }); + this.publisher.once(PCEvents.RTPAudioPayloadTypes, (rtpTypes: MediaAttributes['rtp']) => { + const rtpMap = new Map(); + rtpTypes.forEach((rtp) => { + const codec = rtp.codec.toLowerCase(); + if (isAudioCodec(codec)) { + rtpMap.set(rtp.payload, codec); + } + }); + this.emit(EngineEvent.RTPAudioMapUpdate, rtpMap); + }); + this.publisher.negotiate((e) => { cleanup(); reject(e); @@ -1411,6 +1467,7 @@ export type EngineEventCallbacks = { /** @internal */ trackSenderAdded: (track: Track, sender: RTCRtpSender) => void; rtpVideoMapUpdate: (rtpMap: Map) => void; + rtpAudioMapUpdate: (rtpMap: Map) => void; dcBufferStatusChanged: (isLow: boolean, kind: DataPacket_Kind) => void; participantUpdate: (infos: ParticipantInfo[]) => void; roomUpdate: (room: RoomModel) => void; diff --git a/src/room/events.ts b/src/room/events.ts index e136389ae2..92d40809f4 100644 --- a/src/room/events.ts +++ b/src/room/events.ts @@ -467,6 +467,7 @@ export enum EngineEvent { ActiveSpeakersUpdate = 'activeSpeakersUpdate', DataPacketReceived = 'dataPacketReceived', RTPVideoMapUpdate = 'rtpVideoMapUpdate', + RTPAudioMapUpdate = 'rtpAudioMapUpdate', DCBufferStatusChanged = 'dcBufferStatusChanged', ParticipantUpdate = 'participantUpdate', RoomUpdate = 'roomUpdate', diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index b9e81fd797..55de1fb395 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -649,7 +649,7 @@ export default class LocalParticipant extends Participant { disableDtx: !(opts.dtx ?? true), encryption: this.encryptionType, stereo: isStereo, - disableRed: this.isE2EEEnabled || !(opts.red ?? true), + disableRed: !(opts.red ?? true), }); // compute encodings and layers for video diff --git a/src/room/track/options.ts b/src/room/track/options.ts index ce8585fd04..506da7b647 100644 --- a/src/room/track/options.ts +++ b/src/room/track/options.ts @@ -282,8 +282,12 @@ const backupCodecs = ['vp8', 'h264'] as const; export const videoCodecs = ['vp8', 'h264', 'vp9', 'av1'] as const; +export const audioCodecs = ['opus', 'red'] as const; + export type VideoCodec = (typeof videoCodecs)[number]; +export type AudioCodec = (typeof audioCodecs)[number]; + export type BackupVideoCodec = (typeof backupCodecs)[number]; export function isBackupCodec(codec: string): codec is BackupVideoCodec { diff --git a/src/room/utils.ts b/src/room/utils.ts index 8049b2a562..386de96b5b 100644 --- a/src/room/utils.ts +++ b/src/room/utils.ts @@ -4,7 +4,7 @@ import { getBrowser } from '../utils/browserParser'; import { protocolVersion, version } from '../version'; import type LocalAudioTrack from './track/LocalAudioTrack'; import type RemoteAudioTrack from './track/RemoteAudioTrack'; -import { VideoCodec, videoCodecs } from './track/options'; +import { AudioCodec, VideoCodec, audioCodecs, videoCodecs } from './track/options'; import { getNewAudioContext } from './track/utils'; import type { LiveKitReactNativeInfo } from './types'; @@ -463,6 +463,10 @@ export function isVideoCodec(maybeCodec: string): maybeCodec is VideoCodec { return videoCodecs.includes(maybeCodec as VideoCodec); } +export function isAudioCodec(maybeCodec: string): maybeCodec is AudioCodec { + return audioCodecs.includes(maybeCodec as AudioCodec); +} + export function unwrapConstraint(constraint: ConstrainDOMString): string { if (typeof constraint === 'string') { return constraint; diff --git a/yarn.lock b/yarn.lock index 817ce5ba89..04583e68bf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4696,6 +4696,11 @@ optionator@^0.9.3: prelude-ls "^1.2.1" type-check "^0.4.0" +opus-red-parser@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/opus-red-parser/-/opus-red-parser-1.0.0.tgz#4007fb0cfb8c94c6422cfb73158025897377de8b" + integrity sha512-bPek9U/wYf5ki1znILBxL9bxBRAOUGL7E7LUNtqIh58lD4ly4BNQjsqfiZyunZ0Lm46k/gfJzvYQB3Fed6pBQw== + os-tmpdir@~1.0.2: version "1.0.2" resolved "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz"