feat: Codec Switching

GetStream · Nov 8, 2024 · ddf5f43 · ddf5f43
1 parent fae292a
commit ddf5f43
Show file tree

Hide file tree

Showing 3 changed files with 64 additions and 4 deletions.
diff --git a/packages/client/src/gen/video/sfu/event/events.ts b/packages/client/src/gen/video/sfu/event/events.ts
@@ -245,10 +245,24 @@ export interface SfuEvent {
          */
         participantMigrationComplete: ParticipantMigrationComplete;
       }
+    | {
+        oneofKind: 'codecNegotiationComplete';
+        /**
+         * CodecNegotiationComplete is sent to signal the completion of a codec negotiation.
+         * SDKs can safely stop previous transceivers
+         *
+         * @generated from protobuf field: stream.video.sfu.event.CodecNegotiationComplete codec_negotiation_complete = 26;
+         */
+        codecNegotiationComplete: CodecNegotiationComplete;
+      }
     | {
         oneofKind: undefined;
       };
 }
+/**
+ * @generated from protobuf message stream.video.sfu.event.CodecNegotiationComplete
+ */
+export interface CodecNegotiationComplete {}
 /**
  * @generated from protobuf message stream.video.sfu.event.ParticipantMigrationComplete
  */
@@ -961,6 +975,13 @@ class SfuEvent$Type extends MessageType<SfuEvent> {
         oneof: 'eventPayload',
         T: () => ParticipantMigrationComplete,
       },
+      {
+        no: 26,
+        name: 'codec_negotiation_complete',
+        kind: 'message',
+        oneof: 'eventPayload',
+        T: () => CodecNegotiationComplete,
+      },
     ]);
   }
 }
@@ -969,6 +990,16 @@ class SfuEvent$Type extends MessageType<SfuEvent> {
  */
 export const SfuEvent = new SfuEvent$Type();
 // @generated message type with reflection information, may provide speed optimized methods
+class CodecNegotiationComplete$Type extends MessageType<CodecNegotiationComplete> {
+  constructor() {
+    super('stream.video.sfu.event.CodecNegotiationComplete', []);
+  }
+}
+/**
+ * @generated MessageType for protobuf message stream.video.sfu.event.CodecNegotiationComplete
+ */
+export const CodecNegotiationComplete = new CodecNegotiationComplete$Type();
+// @generated message type with reflection information, may provide speed optimized methods
 class ParticipantMigrationComplete$Type extends MessageType<ParticipantMigrationComplete> {
   constructor() {
     super('stream.video.sfu.event.ParticipantMigrationComplete', []);

diff --git a/packages/client/src/rtc/Dispatcher.ts b/packages/client/src/rtc/Dispatcher.ts
@@ -42,6 +42,7 @@ const sfuEventKinds: { [key in SfuEventKinds]: undefined } = {
   callEnded: undefined,
   participantUpdated: undefined,
   participantMigrationComplete: undefined,
+  codecNegotiationComplete: undefined,
 };
 
 export const isSfuEvent = (

diff --git a/packages/client/src/rtc/Publisher.ts b/packages/client/src/rtc/Publisher.ts
@@ -16,7 +16,7 @@ import {
 import { getOptimalVideoCodec, getPreferredCodecs, isSvcCodec } from './codecs';
 import { trackTypeToParticipantStreamKey } from './helpers/tracks';
 import { CallingState, CallState } from '../store';
-import { PublishOptions } from '../types';
+import { PreferredCodec, PublishOptions } from '../types';
 import {
   enableHighQualityAudio,
   extractMid,
@@ -61,6 +61,7 @@ export class Publisher {
    * @internal
    */
   private readonly transceiverInitOrder: TrackType[] = [];
+  private readonly transceiverOrder: RTCRtpTransceiver[] = [];
   private readonly isDtxEnabled: boolean;
   private readonly isRedEnabled: boolean;
 
@@ -71,6 +72,8 @@ export class Publisher {
   private isIceRestarting = false;
   private sfuClient: StreamSfuClient;
 
+  private readonly dispatcher: Dispatcher;
+
   /**
    * Constructs a new `Publisher` instance.
    */
@@ -90,6 +93,7 @@ export class Publisher {
     this.state = state;
     this.isDtxEnabled = isDtxEnabled;
     this.isRedEnabled = isRedEnabled;
+    this.dispatcher = dispatcher;
     this.onUnrecoverableError = onUnrecoverableError;
 
     this.unsubscribeOnIceRestart = dispatcher.on('iceRestart', (iceRestart) => {
@@ -246,6 +250,7 @@ export class Publisher {
 
     this.logger('debug', `Added ${TrackType[trackType]} transceiver`);
     this.transceiverInitOrder.push(trackType);
+    this.transceiverOrder.push(transceiver);
     this.transceiverCache.set(trackType, transceiver);
     this.publishOptsForTrack.set(trackType, opts);
 
@@ -286,6 +291,24 @@ export class Publisher {
     await transceiver.sender.replaceTrack(track);
   };
 
+  publishTrack = (codec: PreferredCodec) => {
+    const currentTransceiver = this.transceiverCache.get(TrackType.VIDEO);
+    if (!currentTransceiver || !currentTransceiver.sender.track) return;
+    const track = currentTransceiver.sender.track;
+
+    const negotiationComplete = async () => {
+      this.dispatcher.off('codecNegotiationComplete', negotiationComplete);
+      this.logger('info', 'Codec negotiation complete');
+
+      await currentTransceiver.sender.replaceTrack(null);
+      currentTransceiver.stop();
+    };
+    this.dispatcher.on('codecNegotiationComplete', negotiationComplete);
+
+    const ms = new MediaStream([track]);
+    this.addTransceiver(TrackType.VIDEO, track, { preferredCodec: codec }, ms);
+  };
+
   /**
    * Stops publishing the given track type to the SFU, if it is currently being published.
    * Underlying track will be stopped and removed from the publisher.
@@ -586,11 +609,12 @@ export class Publisher {
     return this.pc
       .getTransceivers()
       .filter((t) => t.direction === 'sendonly' && t.sender.track)
-      .map<TrackInfo>((transceiver) => {
+      .map<TrackInfo | undefined>((transceiver) => {
         let trackType!: TrackType;
         this.transceiverCache.forEach((value, key) => {
           if (value === transceiver) trackType = key;
         });
+        if (!trackType) return;
         const track = transceiver.sender.track!;
         let optimalLayers: OptimalVideoLayer[];
         const isTrackLive = track.readyState === 'live';
@@ -627,17 +651,21 @@ export class Publisher {
         const isStereo = isAudioTrack && trackSettings.channelCount === 2;
         const transceiverInitIndex =
           this.transceiverInitOrder.indexOf(trackType);
+        const mid = this.transceiverOrder.indexOf(transceiver);
+
         return {
           trackId: track.id,
           layers: layers,
           trackType,
-          mid: extractMid(transceiver, transceiverInitIndex, sdp),
+          mid:
+            String(mid) ?? extractMid(transceiver, transceiverInitIndex, sdp),
           stereo: isStereo,
           dtx: isAudioTrack && this.isDtxEnabled,
           red: isAudioTrack && this.isRedEnabled,
           muted: !isTrackLive,
         };
-      });
+      })
+      .filter(Boolean) as TrackInfo[];
   };
 
   private computeLayers = (