From a59fc73bb989522266ecf704fd7fed616d760cc3 Mon Sep 17 00:00:00 2001 From: martincupela Date: Thu, 20 Jul 2023 15:21:05 +0200 Subject: [PATCH 01/26] feat(client): keep track of audio output level in call state and for each participant --- packages/client/src/Call.ts | 28 ++++++++++++++++++++ packages/client/src/events/participant.ts | 11 +++++++- packages/client/src/store/CallState.ts | 31 ++++++++++++++++++++++- packages/client/src/types.ts | 8 +++++- 4 files changed, 75 insertions(+), 3 deletions(-) diff --git a/packages/client/src/Call.ts b/packages/client/src/Call.ts index 9e9f8cd2e8..fbbf74a60e 100644 --- a/packages/client/src/Call.ts +++ b/packages/client/src/Call.ts @@ -1165,6 +1165,34 @@ export class Call { }); }; + /** + * Updates the audio output level for a given participant or for all the participants and the default call audio output level. + * @param level + * @param participant + */ + setAudioOutputLevel(level: number, participant?: StreamVideoParticipant) { + if (participant) { + this.state.updateParticipant(participant.sessionId, { + audioOutputLevel: level, + }); + } else if (this.sfuClient?.sessionId) { + this.state.updateParticipants( + this.state.participants.reduce( + (acc, p) => { + acc[p.sessionId] = { + audioOutputLevel: level, + }; + return acc; + }, + {}, + ), + ); + } + if (!participant) { + this.state.setDefaultAudioOutputLevel(level); + } + } + /** * Sets the `audioDeviceId` property of the [`localParticipant$`](./StreamVideoClient.md/#readonlystatestore)). * diff --git a/packages/client/src/events/participant.ts b/packages/client/src/events/participant.ts index f65a4e6aba..d1f3964401 100644 --- a/packages/client/src/events/participant.ts +++ b/packages/client/src/events/participant.ts @@ -23,6 +23,7 @@ export const watchParticipantJoined = (state: CallState) => { Object.assign>( participant, { + audioOutputLevel: state.defaultAudioOutputLevel, viewportVisibilityState: VisibilityState.UNKNOWN, }, ), @@ -61,7 +62,15 @@ export const watchTrackPublished = (state: CallState) => { // events, and instead, it would only provide the participant's information // once they start publishing a track. if (participant) { - state.updateOrAddParticipant(sessionId, participant); + state.updateOrAddParticipant( + sessionId, + Object.assign>( + participant, + { + audioOutputLevel: state.defaultAudioOutputLevel, + }, + ), + ); } else { state.updateParticipant(sessionId, (p) => ({ publishedTracks: [...p.publishedTracks, type].filter(unique), diff --git a/packages/client/src/store/CallState.ts b/packages/client/src/store/CallState.ts index 2eef0974c7..8c7b29434b 100644 --- a/packages/client/src/store/CallState.ts +++ b/packages/client/src/store/CallState.ts @@ -82,6 +82,10 @@ export enum CallingState { * @react You don't have to use this class directly, as we are exposing the state through Hooks. */ export class CallState { + /** + * The speaker volume level that is set, if `StreamVideoParticipant.defaultAudioOutputLevelSubject` is undefined. + */ + private defaultAudioOutputLevelSubject = new BehaviorSubject(1); /** * The raw call metadata object, as defined on the backend. * @@ -165,8 +169,13 @@ export class CallState { */ private callRecordingListSubject = new BehaviorSubject([]); - // Derived state + /** + * Emits the default audio output level value in form of decimal number in range of 0-1. + * The default value is assigned to each newly joined StreamVideoParticipant's audioOutputLevel property. + */ + defaultAudioOutputLevel$: Observable; + // Derived state /** * The time the call session actually started. * Useful for displaying the call duration. @@ -303,6 +312,8 @@ export class CallState { distinctUntilChanged(), ); + this.defaultAudioOutputLevel$ = + this.defaultAudioOutputLevelSubject.asObservable(); this.startedAt$ = this.startedAtSubject.asObservable(); this.participantCount$ = this.participantCountSubject.asObservable(); this.anonymousParticipantCount$ = @@ -473,6 +484,24 @@ export class CallState { return this.setCurrentValue(this.callingStateSubject, state); }; + /** + * Retrieves the current value of the default audio output level. + * + * @internal + */ + get defaultAudioOutputLevel() { + return this.getCurrentValue(this.defaultAudioOutputLevel$); + } + + /** + * Sets the current value of the default audio output level. + * + * @internal + */ + setDefaultAudioOutputLevel = (level: number) => { + return this.setCurrentValue(this.defaultAudioOutputLevelSubject, level); + }; + /** * The list of call recordings. */ diff --git a/packages/client/src/types.ts b/packages/client/src/types.ts index 73c8454ca1..d3d9dbbfd0 100644 --- a/packages/client/src/types.ts +++ b/packages/client/src/types.ts @@ -34,7 +34,13 @@ export enum DebounceType { export interface StreamVideoParticipant extends Participant { /** - * The participant's audio stream, if they are publishing audio and + * The speaker volume in range 0 - 1 set by the participant. + * If not set, then CallState.defaultAudioOutputLevel should be reflected. + */ + audioOutputLevel?: number; + + /** + * The participant's audio stream, if they are publishing audio, and * we have subscribed to it. */ audioStream?: MediaStream; From 6735037af62627c0607918c12c82819383e0fb35 Mon Sep 17 00:00:00 2001 From: martincupela Date: Thu, 20 Jul 2023 15:23:30 +0200 Subject: [PATCH 02/26] feat(react-bindings): add useDefaultAudioOutputLevel hook --- packages/react-bindings/src/hooks/call.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/react-bindings/src/hooks/call.ts b/packages/react-bindings/src/hooks/call.ts index 9d07a4ea60..9d7fa6f705 100644 --- a/packages/react-bindings/src/hooks/call.ts +++ b/packages/react-bindings/src/hooks/call.ts @@ -131,3 +131,14 @@ export const useCallStartedAt = () => { const { startedAt$ } = useCallState(); return useObservableValue(startedAt$); }; + +/** + * Utility hook providing the default audio output level in a given call + * Should be used to determine audio output level if the call is not joined yet. + * + * @category Call State + */ +export const useDefaultAudioOutputLevel = () => { + const { defaultAudioOutputLevel$ } = useCallState(); + return useObservableValue(defaultAudioOutputLevel$); +}; From 748ae42a36d255f985200b24d87012c00baba0d8 Mon Sep 17 00:00:00 2001 From: martincupela Date: Thu, 20 Jul 2023 15:27:56 +0200 Subject: [PATCH 03/26] feat(react-sdk): enable setting audio output volume to Audio component --- .../react-sdk/src/core/components/Audio/Audio.tsx | 12 +++++++++++- .../components/CallLayout/PaginatedGridLayout.tsx | 1 + .../components/ParticipantView/ParticipantView.tsx | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/react-sdk/src/core/components/Audio/Audio.tsx b/packages/react-sdk/src/core/components/Audio/Audio.tsx index 996469206b..6c31706cd6 100644 --- a/packages/react-sdk/src/core/components/Audio/Audio.tsx +++ b/packages/react-sdk/src/core/components/Audio/Audio.tsx @@ -12,10 +12,14 @@ export type AudioProps = DetailedHTMLProps< > & Pick & { sinkId?: string; + /** + * Value applied to underlying audio element to control the audio output level. + */ + volume?: number; }; // TODO: rename to BaseAudio -export const Audio = ({ audioStream, sinkId, ...rest }: AudioProps) => { +export const Audio = ({ audioStream, sinkId, volume, ...rest }: AudioProps) => { const audioRef = useRef(null); useEffect(() => { const $el = audioRef.current; @@ -31,10 +35,16 @@ export const Audio = ({ audioStream, sinkId, ...rest }: AudioProps) => { const $el = audioRef.current; if (!$el || !sinkId) return; + // HTMLMediaElement neither HTMLAudioElement in Typescript have prop setSinkId if (($el as any).setSinkId) { ($el as any).setSinkId(sinkId); } }, [sinkId]); + useEffect(() => { + if (typeof volume !== 'number' || !audioRef.current) return; + audioRef.current.volume = volume; + }, [audioRef, volume]); + return