From 9c378434c1a80dfb705e96baddb3e2aec62848f7 Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Mon, 16 Oct 2023 19:08:58 +0900 Subject: [PATCH 01/21] =?UTF-8?q?media=20access=20=E3=83=9C=E3=82=BF?= =?UTF-8?q?=E3=83=B3=E3=81=AE=E8=A1=A8=E7=A4=BA=E3=82=92=E8=BF=BD=E5=8A=A0?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app/actions.ts | 4 ++++ .../DevtoolsPane/MediaAccessButton.tsx | 22 +++++++++++++++++++ src/components/DevtoolsPane/index.tsx | 2 ++ 3 files changed, 28 insertions(+) create mode 100644 src/components/DevtoolsPane/MediaAccessButton.tsx diff --git a/src/app/actions.ts b/src/app/actions.ts index 645d0739..66c44829 100644 --- a/src/app/actions.ts +++ b/src/app/actions.ts @@ -1061,6 +1061,10 @@ async function setStatsReport( } } +export const mediaAccessAction = () => { + return async (_dispatch: Dispatch, _getState: () => SoraDevtoolsState): Promise => {}; +}; + export const connectSora = () => { return async (dispatch: Dispatch, getState: () => SoraDevtoolsState): Promise => { dispatch( diff --git a/src/components/DevtoolsPane/MediaAccessButton.tsx b/src/components/DevtoolsPane/MediaAccessButton.tsx new file mode 100644 index 00000000..cbcefe58 --- /dev/null +++ b/src/components/DevtoolsPane/MediaAccessButton.tsx @@ -0,0 +1,22 @@ +import React from 'react'; + +import { mediaAccessAction } from '@/app/actions'; +import { useAppDispatch } from '@/app/hooks'; + +export const MediaAccessButton: React.FC = () => { + const dispatch = useAppDispatch(); + const mediaAccess = (): void => { + dispatch(mediaAccessAction()); + }; + return ( +
+ +
+ ); +}; diff --git a/src/components/DevtoolsPane/index.tsx b/src/components/DevtoolsPane/index.tsx index 1503d131..ecbbf344 100644 --- a/src/components/DevtoolsPane/index.tsx +++ b/src/components/DevtoolsPane/index.tsx @@ -35,6 +35,7 @@ import { FakeVolumeForm } from './FakeVolumeForm'; import { ForwardingFilterForm } from './ForwardingFilterForm'; import { FrameRateForm } from './FrameRateForm'; import { LightAdjustmentForm } from './LightAdjustmentForm'; +import { MediaAccessButton } from './MediaAccessButton'; import { MediaProcessorsNoiseSuppressionForm } from './MediaProcessorsNoiseSuppressionForm'; import { MediaTypeForm } from './MediaTypeForm'; import { MetadataForm } from './MetadataForm'; @@ -477,6 +478,7 @@ export const DevtoolsPane: React.FC = () => {
+ From 595b05b3e025da7fbd96195ab5f7401852e5feca Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Tue, 17 Oct 2023 16:26:00 +0900 Subject: [PATCH 02/21] =?UTF-8?q?Sora=20=E3=81=A8=E3=81=AF=E6=8E=A5?= =?UTF-8?q?=E7=B6=9A=E3=81=9B=E3=81=9A=E3=81=AB=20gUM=20=E3=83=87=E3=83=90?= =?UTF-8?q?=E3=82=A4=E3=82=B9=E3=81=AE=E6=98=A0=E5=83=8F=E3=81=A8=E9=9F=B3?= =?UTF-8?q?=E5=A3=B0=E3=82=92=20video=20=E3=82=BF=E3=82=B0=E3=81=A7?= =?UTF-8?q?=E8=A1=A8=E7=A4=BA=E3=81=99=E3=82=8B=E6=A9=9F=E8=83=BD=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app/actions.ts | 137 +++++++++++++++++- src/app/slice.ts | 9 ++ .../DevtoolsPane/MediaAccessButton.tsx | 14 +- src/components/DevtoolsPane/index.tsx | 2 + src/components/Video/LocalVideo.tsx | 3 +- src/components/Video/LocalVideoTest.tsx | 47 ++++++ src/types.ts | 2 + 7 files changed, 206 insertions(+), 8 deletions(-) create mode 100644 src/components/Video/LocalVideoTest.tsx diff --git a/src/app/actions.ts b/src/app/actions.ts index 66c44829..222b4597 100644 --- a/src/app/actions.ts +++ b/src/app/actions.ts @@ -1061,8 +1061,141 @@ async function setStatsReport( } } -export const mediaAccessAction = () => { - return async (_dispatch: Dispatch, _getState: () => SoraDevtoolsState): Promise => {}; +export const testMediaAccess = () => { + return async (dispatch: Dispatch, getState: () => SoraDevtoolsState): Promise => { + const LOG_TITLE = 'MEDIA_CONSTRAINTS'; + const state = getState(); + const mediaStream = new MediaStream(); + const audioConstraints = createAudioConstraints({ + audio: state.audio && state.micDevice, + autoGainControl: state.autoGainControl, + noiseSuppression: state.noiseSuppression, + echoCancellation: state.echoCancellation, + echoCancellationType: state.echoCancellationType, + audioInput: state.audioInput, + }); + if (audioConstraints) { + dispatch( + slice.actions.setLogMessages({ + title: LOG_TITLE, + description: JSON.stringify({ audio: audioConstraints }), + }), + ); + dispatch( + slice.actions.setTimelineMessage( + createSoraDevtoolsTimelineMessage('audio-media-constraints', { audio: audioConstraints }), + ), + ); + const audioMediaStream = await navigator.mediaDevices.getUserMedia({ + audio: audioConstraints, + }); + let audioTrack = audioMediaStream.getAudioTracks()[0]; + dispatch( + slice.actions.setTimelineMessage( + createSoraDevtoolsMediaStreamTrackLog('start', audioTrack), + ), + ); + if (state.mediaProcessorsNoiseSuppression && NoiseSuppressionProcessor.isSupported()) { + if (state.noiseSuppressionProcessor === null) { + throw new Error( + "Failed to start NoiseSuppressionProcessor. NoiseSuppressionProcessor is 'null'", + ); + } + state.noiseSuppressionProcessor.stopProcessing(); + audioTrack = await state.noiseSuppressionProcessor.startProcessing(audioTrack); + } + dispatch( + slice.actions.setTimelineMessage( + createSoraDevtoolsTimelineMessage('succeed-audio-get-user-media'), + ), + ); + mediaStream.addTrack(audioTrack); + } + const videoConstraints = createVideoConstraints({ + aspectRatio: state.aspectRatio, + frameRate: state.frameRate, + resizeMode: state.resizeMode, + resolution: state.resolution, + video: state.video && state.cameraDevice, + videoInput: state.videoInput, + facingMode: state.facingMode, + }); + if (videoConstraints) { + dispatch( + slice.actions.setLogMessages({ + title: LOG_TITLE, + description: JSON.stringify({ video: videoConstraints }), + }), + ); + dispatch( + slice.actions.setTimelineMessage( + createSoraDevtoolsTimelineMessage('video-media-constraints', { video: videoConstraints }), + ), + ); + const videoMediaStream = await navigator.mediaDevices + .getUserMedia({ video: videoConstraints }) + .catch((error) => { + // video track の getUserMedia が失敗した場合には audio track が存在している可能性があるので止める + mediaStream.getTracks().forEach((t) => { + t.stop(); + }); + throw error; + }); + let videoTrack = videoMediaStream.getVideoTracks()[0]; + dispatch( + slice.actions.setTimelineMessage( + createSoraDevtoolsMediaStreamTrackLog('start', videoTrack), + ), + ); + if (state.lightAdjustment !== '' && LightAdjustmentProcessor.isSupported()) { + if (state.lightAdjustmentProcessor === null) { + throw new Error( + "Failed to start LightAdjustmentProcessor. LightAdjustmentProcessor is 'null'", + ); + } + const options = getLightAdjustmentOptions(state.lightAdjustment); + state.lightAdjustmentProcessor.stopProcessing(); + videoTrack = await state.lightAdjustmentProcessor.startProcessing(videoTrack, options); + } + if (state.blurRadius !== '' && VirtualBackgroundProcessor.isSupported()) { + if (state.virtualBackgroundProcessor === null) { + throw new Error( + "Failed to start VirtualBackgroundProcessor. VirtualBackgroundProcessor is 'null'", + ); + } + const options = { + blurRadius: getBlurRadiusNumber(state.blurRadius), + }; + state.virtualBackgroundProcessor.stopProcessing(); + videoTrack = await state.virtualBackgroundProcessor.startProcessing(videoTrack, options); + } + dispatch( + slice.actions.setTimelineMessage( + createSoraDevtoolsTimelineMessage('succeed-video-get-user-media'), + ), + ); + mediaStream.addTrack(videoTrack); + } + for (const track of mediaStream.getVideoTracks()) { + if (track.contentHint !== undefined) { + track.contentHint = state.videoContentHint; + } + track.enabled = state.videoTrack; + } + for (const track of mediaStream.getAudioTracks()) { + if (track.contentHint !== undefined) { + track.contentHint = state.audioContentHint; + } + track.enabled = state.audioTrack; + } + dispatch(slice.actions.setLocalTestMediaStream(mediaStream)); + }; +}; + +export const disposeTestMediaAccess = () => { + return async (dispatch: Dispatch, _getState: () => SoraDevtoolsState): Promise => { + dispatch(slice.actions.setLocalTestMediaStream(null)); + }; }; export const connectSora = () => { diff --git a/src/app/slice.ts b/src/app/slice.ts index 192490cc..3f8c7295 100644 --- a/src/app/slice.ts +++ b/src/app/slice.ts @@ -85,6 +85,7 @@ const initialState: SoraDevtoolsState = { datachannels: [], }, ignoreDisconnectWebSocket: '', + localTestMediaStream: null, logMessages: [], mediaProcessorsNoiseSuppression: false, mediaType: 'getUserMedia', @@ -402,6 +403,14 @@ export const slice = createSlice({ } state.soraContents.localMediaStream = action.payload; }, + setLocalTestMediaStream: (state, aciton: PayloadAction) => { + if (state.localTestMediaStream) { + state.localTestMediaStream.getTracks().forEach((track) => { + track.stop(); + }); + } + state.localTestMediaStream = aciton.payload; + }, setRemoteMediaStream: (state, action: PayloadAction) => { state.soraContents.remoteMediaStreams.push(action.payload); }, diff --git a/src/components/DevtoolsPane/MediaAccessButton.tsx b/src/components/DevtoolsPane/MediaAccessButton.tsx index cbcefe58..da0ceece 100644 --- a/src/components/DevtoolsPane/MediaAccessButton.tsx +++ b/src/components/DevtoolsPane/MediaAccessButton.tsx @@ -1,21 +1,25 @@ import React from 'react'; -import { mediaAccessAction } from '@/app/actions'; -import { useAppDispatch } from '@/app/hooks'; +import { disposeTestMediaAccess, testMediaAccess } from '@/app/actions'; +import { useAppDispatch, useAppSelector } from '@/app/hooks'; export const MediaAccessButton: React.FC = () => { const dispatch = useAppDispatch(); const mediaAccess = (): void => { - dispatch(mediaAccessAction()); + dispatch(testMediaAccess()); }; + const disposeMediaAccess = (): void => { + dispatch(disposeTestMediaAccess()); + }; + const localTestMediaStream = useAppSelector((state) => state.localTestMediaStream); return (
); diff --git a/src/components/DevtoolsPane/index.tsx b/src/components/DevtoolsPane/index.tsx index ecbbf344..2810e3df 100644 --- a/src/components/DevtoolsPane/index.tsx +++ b/src/components/DevtoolsPane/index.tsx @@ -4,6 +4,7 @@ import { Col, Collapse, Row } from 'react-bootstrap'; import { useAppSelector } from '@/app/hooks'; import { AlertMessages } from '@/components/AlertMessages'; import { LocalVideo } from '@/components/Video/LocalVideo'; +import { LocalVideoTest } from '@/components/Video/LocalVideoTest'; import { RemoteVideos } from '@/components/Video/RemoteVideos'; import { AspectRatioForm } from './AspectRatioForm'; @@ -485,6 +486,7 @@ export const DevtoolsPane: React.FC = () => {
+ {role === 'recvonly' || role === 'sendrecv' ? : null} diff --git a/src/components/Video/LocalVideo.tsx b/src/components/Video/LocalVideo.tsx index 51c17b50..2c845fb5 100644 --- a/src/components/Video/LocalVideo.tsx +++ b/src/components/Video/LocalVideo.tsx @@ -58,6 +58,7 @@ export const LocalVideo: React.FC = () => { const simulcast = useAppSelector((state) => state.simulcast); const spotlight = useAppSelector((state) => state.spotlight); const role = useAppSelector((state) => state.role); + const localMediaStream = useAppSelector((state) => state.localTestMediaStream); return (
@@ -85,7 +86,7 @@ export const LocalVideo: React.FC = () => {
) : null}
- {role !== 'recvonly' ? : null} + {localMediaStream === null && role !== 'recvonly' ? : null} ); diff --git a/src/components/Video/LocalVideoTest.tsx b/src/components/Video/LocalVideoTest.tsx new file mode 100644 index 00000000..baef62da --- /dev/null +++ b/src/components/Video/LocalVideoTest.tsx @@ -0,0 +1,47 @@ +import React, { useState } from 'react'; + +import { useAppSelector } from '@/app/hooks'; + +import { Video } from './Video'; +import { VolumeVisualizer } from './VolumeVisualizer'; + +const VideoBox: React.FC = () => { + const [height, setHeight] = useState(0); + const audio = useAppSelector((state) => state.audio); + const video = useAppSelector((state) => state.video); + const audioOutput = useAppSelector((state) => state.audioOutput); + const displayResolution = useAppSelector((state) => state.displayResolution); + const localTestMediaStream = useAppSelector((state) => state.localTestMediaStream); + const micDevice = useAppSelector((state) => state.micDevice); + if (audio === false && video === false) { + return null; + } + return ( + <> +
+
+
+
+ + ); +}; + +export const LocalVideoTest: React.FC = () => { + const localMediaStream = useAppSelector((state) => state.localTestMediaStream); + return ( +
+
{localMediaStream !== null && }
+
+ ); +}; diff --git a/src/types.ts b/src/types.ts index a693a227..72616c05 100644 --- a/src/types.ts +++ b/src/types.ts @@ -104,6 +104,7 @@ export type SoraDevtoolsState = { statsReport: RTCStats[]; datachannels: DataChannelConfiguration[]; }; + localTestMediaStream: MediaStream | null; ignoreDisconnectWebSocket: (typeof IGNORE_DISCONNECT_WEBSOCKET)[number]; logMessages: LogMessage[]; mediaProcessorsNoiseSuppression: boolean; @@ -383,6 +384,7 @@ export type DownloadReportParameters = Omit< | 'focusedSpotlightConnectionIds' | 'lightAdjustment' | 'lightAdjustmentProcessor' + | 'localTestMediaStream' | 'logMessages' | 'mediaProcessorsNoiseSuppression' | 'mute' From d7fed26fe2341b40a6525d3643fc0e3d22a903cf Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Wed, 18 Oct 2023 12:44:41 +0900 Subject: [PATCH 03/21] =?UTF-8?q?=E3=83=AD=E3=83=BC=E3=82=AB=E3=83=AB?= =?UTF-8?q?=E3=81=AE=E3=83=87=E3=83=90=E3=82=A4=E3=82=B9=E8=A1=A8=E7=A4=BA?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=81=A7=E3=83=87=E3=83=90=E3=82=A4?= =?UTF-8?q?=E3=82=B9=E5=88=87=E3=82=8A=E6=9B=BF=E3=81=88=E6=99=82=E3=81=AB?= =?UTF-8?q?=E8=A1=A8=E7=A4=BA=E3=82=92=E9=80=A3=E5=8B=95=E3=81=99=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app/actions.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/app/actions.ts b/src/app/actions.ts index 222b4597..6f85266d 100644 --- a/src/app/actions.ts +++ b/src/app/actions.ts @@ -1512,9 +1512,10 @@ export const setMediaDevices = () => { export const updateMediaStream = () => { return async (dispatch: Dispatch, getState: () => SoraDevtoolsState): Promise => { const state = getState(); - if (!state.soraContents.sora) { + if (!state.soraContents.sora && !state.localTestMediaStream) { return; } + const localMediaStream = state.soraContents.localMediaStream || state.localTestMediaStream; if (state.virtualBackgroundProcessor && state.virtualBackgroundProcessor.isProcessing()) { const originalTrack = state.virtualBackgroundProcessor.getOriginalTrack(); if (originalTrack) { @@ -1527,8 +1528,8 @@ export const updateMediaStream = () => { } state.virtualBackgroundProcessor.stopProcessing(); } else { - if (state.soraContents.localMediaStream) { - state.soraContents.localMediaStream.getVideoTracks().forEach((track) => { + if (localMediaStream) { + localMediaStream.getVideoTracks().forEach((track) => { track.stop(); dispatch( slice.actions.setTimelineMessage(createSoraDevtoolsMediaStreamTrackLog('stop', track)), @@ -1549,8 +1550,8 @@ export const updateMediaStream = () => { } state.noiseSuppressionProcessor.stopProcessing(); } else { - if (state.soraContents.localMediaStream) { - state.soraContents.localMediaStream.getAudioTracks().forEach((track) => { + if (localMediaStream) { + localMediaStream.getAudioTracks().forEach((track) => { track.stop(); dispatch( slice.actions.setTimelineMessage(createSoraDevtoolsMediaStreamTrackLog('stop', track)), @@ -1577,7 +1578,11 @@ export const updateMediaStream = () => { sender.replaceTrack(track); } }); - dispatch(slice.actions.setLocalMediaStream(mediaStream)); + if (state.soraContents.sora) { + dispatch(slice.actions.setLocalMediaStream(mediaStream)); + } else { + dispatch(slice.actions.setLocalTestMediaStream(mediaStream)); + } dispatch(slice.actions.setFakeContentsGainNode(gainNode)); }; }; From d03679ffd66bf05f32d664227024d52b68926795 Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Wed, 18 Oct 2023 16:24:49 +0900 Subject: [PATCH 04/21] =?UTF-8?q?media=20access=20=E5=AE=9F=E8=A1=8C?= =?UTF-8?q?=E4=B8=AD=E3=81=AB=20mediaType=20=E3=82=92=E5=A4=89=E6=9B=B4?= =?UTF-8?q?=E3=81=A7=E3=81=8D=E3=81=AA=E3=81=84=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/DevtoolsPane/MediaAccessButton.tsx | 4 ++++ src/components/DevtoolsPane/MediaTypeForm.tsx | 5 +++-- src/utils.ts | 7 +++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/components/DevtoolsPane/MediaAccessButton.tsx b/src/components/DevtoolsPane/MediaAccessButton.tsx index da0ceece..b44f4271 100644 --- a/src/components/DevtoolsPane/MediaAccessButton.tsx +++ b/src/components/DevtoolsPane/MediaAccessButton.tsx @@ -2,6 +2,7 @@ import React from 'react'; import { disposeTestMediaAccess, testMediaAccess } from '@/app/actions'; import { useAppDispatch, useAppSelector } from '@/app/hooks'; +import { isFormDisabled } from '@/utils'; export const MediaAccessButton: React.FC = () => { const dispatch = useAppDispatch(); @@ -11,7 +12,9 @@ export const MediaAccessButton: React.FC = () => { const disposeMediaAccess = (): void => { dispatch(disposeTestMediaAccess()); }; + const connectionStatus = useAppSelector((state) => state.soraContents.connectionStatus); const localTestMediaStream = useAppSelector((state) => state.localTestMediaStream); + const disabled = localTestMediaStream === null && isFormDisabled(connectionStatus); return (
{ name="media_access" defaultValue={localTestMediaStream === null ? 'media access' : 'dispose'} onClick={localTestMediaStream === null ? mediaAccess : disposeMediaAccess} + disabled={disabled} />
); diff --git a/src/components/DevtoolsPane/MediaTypeForm.tsx b/src/components/DevtoolsPane/MediaTypeForm.tsx index b26af658..d4c6956f 100644 --- a/src/components/DevtoolsPane/MediaTypeForm.tsx +++ b/src/components/DevtoolsPane/MediaTypeForm.tsx @@ -4,7 +4,7 @@ import { FormCheck, FormGroup } from 'react-bootstrap'; import { setMediaType } from '@/app/actions'; import { useAppDispatch, useAppSelector } from '@/app/hooks'; import { MEDIA_TYPES } from '@/constants'; -import { checkFormValue, isFormDisabled } from '@/utils'; +import { checkFormValue, isMediaTypeFormDisabled } from '@/utils'; import { TooltipFormLabel } from './TooltipFormLabel'; @@ -38,8 +38,9 @@ export const MediaTypeForm: React.FC = () => { const enabledMediacaptureRegion = typeof window !== 'undefined' && window.CropTarget !== undefined; const connectionStatus = useAppSelector((state) => state.soraContents.connectionStatus); + const localTestMediaStream = useAppSelector((state) => state.localTestMediaStream); const mediaType = useAppSelector((state) => state.mediaType); - const disabled = isFormDisabled(connectionStatus); + const disabled = isMediaTypeFormDisabled(localTestMediaStream, connectionStatus); const dispatch = useAppDispatch(); const onChange = (event: React.ChangeEvent): void => { if (checkFormValue(event.target.value, MEDIA_TYPES)) { diff --git a/src/utils.ts b/src/utils.ts index 0c19ca5e..bf74172c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -625,6 +625,13 @@ export function isFormDisabled( return connectionStatus === 'connected' || connectionStatus === 'connecting'; } +export function isMediaTypeFormDisabled( + localTestMediaStream: SoraDevtoolsState['localTestMediaStream'], + connectionStatus: SoraDevtoolsState['soraContents']['connectionStatus'], +): boolean { + return localTestMediaStream !== null || isFormDisabled(connectionStatus); +} + // track の設定情報を返す type GetMediaStreamTrackProperties = { id: MediaStreamTrack['id']; From d6cc470b1f245a7d800e197a7659e293fbd0de91 Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Wed, 18 Oct 2023 16:26:05 +0900 Subject: [PATCH 05/21] =?UTF-8?q?connect=20=E3=83=9C=E3=82=BF=E3=83=B3?= =?UTF-8?q?=E3=82=92=E6=8A=BC=E3=81=97=E3=81=9F=E6=99=82=E3=81=AB=20media?= =?UTF-8?q?=20access=20=E3=82=92=E5=AE=9F=E8=A1=8C=E4=B8=AD=E3=81=AE?= =?UTF-8?q?=E5=A0=B4=E5=90=88=E3=81=AF=E3=80=81=E4=B8=80=E6=97=A6=20MediaS?= =?UTF-8?q?tream=20=E3=82=92=E6=AD=A2=E3=82=81=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app/actions.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/app/actions.ts b/src/app/actions.ts index 6f85266d..b00b6f60 100644 --- a/src/app/actions.ts +++ b/src/app/actions.ts @@ -1209,6 +1209,10 @@ export const connectSora = () => { if (state.soraContents.sora) { await state.soraContents.sora.disconnect(); } + // media access でテスト表示している場合は停止する + if (state.localTestMediaStream !== null) { + dispatch(slice.actions.setLocalTestMediaStream(null)); + } // シグナリング候補のURLリストを作成する const signalingUrlCandidates = createSignalingURL( state.enabledSignalingUrlCandidates, From 3690a0cffa3a07333e902184d7c6f8f3d8bdb7e8 Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Wed, 18 Oct 2023 18:45:13 +0900 Subject: [PATCH 06/21] =?UTF-8?q?Sora=20=E6=8E=A5=E7=B6=9A=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=81=AE=E6=8E=A5=E7=B6=9A=E9=96=8B=E5=A7=8B=E5=89=8D?= =?UTF-8?q?=E3=81=AE=E7=8A=B6=E6=85=8B=E3=82=92=E5=A2=97=E3=82=84=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/constants.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/constants.ts b/src/constants.ts index 59a5018a..4df3a5eb 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -113,6 +113,7 @@ export const CONNECTION_STATUS = [ 'disconnecting', 'connected', 'connecting', + 'preparing', ] as const; export const ROLES = ['sendrecv', 'sendonly', 'recvonly'] as const; From f6617fbeee8b1cbb75554ad3bce96bbfdaf986f2 Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Wed, 18 Oct 2023 18:45:52 +0900 Subject: [PATCH 07/21] =?UTF-8?q?form=20=E6=9C=89=E5=8A=B9=E5=88=A4?= =?UTF-8?q?=E5=AE=9A=E3=81=AB=E8=BF=BD=E5=8A=A0=E3=81=97=E3=81=9F=E7=8A=B6?= =?UTF-8?q?=E6=85=8B=E3=82=92=E5=8A=A0=E3=81=88=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/utils.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index bf74172c..218665c6 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -622,14 +622,11 @@ export async function getDevices(): Promise { export function isFormDisabled( connectionStatus: SoraDevtoolsState['soraContents']['connectionStatus'], ): boolean { - return connectionStatus === 'connected' || connectionStatus === 'connecting'; -} - -export function isMediaTypeFormDisabled( - localTestMediaStream: SoraDevtoolsState['localTestMediaStream'], - connectionStatus: SoraDevtoolsState['soraContents']['connectionStatus'], -): boolean { - return localTestMediaStream !== null || isFormDisabled(connectionStatus); + return ( + connectionStatus === 'preparing' || + connectionStatus === 'connected' || + connectionStatus === 'connecting' + ); } // track の設定情報を返す From 196b6c71820c3840ec4a4ba8c74a7c668e5ac731 Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Wed, 18 Oct 2023 18:46:36 +0900 Subject: [PATCH 08/21] =?UTF-8?q?media=20access=20=E3=81=A7=E5=8F=96?= =?UTF-8?q?=E5=BE=97=E3=81=97=E3=81=9F=20MediaStream=20=E3=82=92=E3=81=9D?= =?UTF-8?q?=E3=81=AE=E3=81=BE=E3=81=BE=20Sora=20=E3=81=AE=E6=8E=A5?= =?UTF-8?q?=E7=B6=9A=E3=81=AB=E5=88=A9=E7=94=A8=E3=81=A7=E3=81=8D=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app/actions.ts | 59 ++++++++++--------- src/app/slice.ts | 9 --- .../DevtoolsPane/MediaAccessButton.tsx | 8 +-- src/components/DevtoolsPane/MediaTypeForm.tsx | 6 +- src/components/DevtoolsPane/index.tsx | 2 - src/components/Video/LocalVideo.tsx | 4 +- src/components/Video/LocalVideoTest.tsx | 47 --------------- src/types.ts | 1 - 8 files changed, 39 insertions(+), 97 deletions(-) delete mode 100644 src/components/Video/LocalVideoTest.tsx diff --git a/src/app/actions.ts b/src/app/actions.ts index b00b6f60..bc6e524c 100644 --- a/src/app/actions.ts +++ b/src/app/actions.ts @@ -1188,13 +1188,13 @@ export const testMediaAccess = () => { } track.enabled = state.audioTrack; } - dispatch(slice.actions.setLocalTestMediaStream(mediaStream)); + dispatch(slice.actions.setLocalMediaStream(mediaStream)); }; }; export const disposeTestMediaAccess = () => { return async (dispatch: Dispatch, _getState: () => SoraDevtoolsState): Promise => { - dispatch(slice.actions.setLocalTestMediaStream(null)); + dispatch(slice.actions.setLocalMediaStream(null)); }; }; @@ -1203,16 +1203,12 @@ export const connectSora = () => { dispatch( slice.actions.setTimelineMessage(createSoraDevtoolsTimelineMessage('start-connection')), ); - dispatch(slice.actions.setSoraConnectionStatus('connecting')); + dispatch(slice.actions.setSoraConnectionStatus('preparing')); const state = getState(); // 接続中の場合は切断する if (state.soraContents.sora) { await state.soraContents.sora.disconnect(); } - // media access でテスト表示している場合は停止する - if (state.localTestMediaStream !== null) { - dispatch(slice.actions.setLocalTestMediaStream(null)); - } // シグナリング候補のURLリストを作成する const signalingUrlCandidates = createSignalingURL( state.enabledSignalingUrlCandidates, @@ -1240,11 +1236,16 @@ export const connectSora = () => { }; } setSoraCallbacks(dispatch, getState, sora); - [mediaStream, gainNode] = await createMediaStream(dispatch, state).catch((error) => { - dispatch(slice.actions.setSoraErrorAlertMessage(error.toString())); - dispatch(slice.actions.setSoraConnectionStatus('disconnected')); - throw error; - }); + if (state.soraContents.localMediaStream) { + mediaStream = state.soraContents.localMediaStream; + } else { + [mediaStream, gainNode] = await createMediaStream(dispatch, state).catch((error) => { + dispatch(slice.actions.setSoraErrorAlertMessage(error.toString())); + dispatch(slice.actions.setSoraConnectionStatus('disconnected')); + throw error; + }); + } + dispatch(slice.actions.setSoraConnectionStatus('connecting')); await sora.connect(mediaStream); } else if (state.role === 'sendrecv') { sora = connection.sendrecv(state.channelId, null, connectionOptions); @@ -1256,16 +1257,21 @@ export const connectSora = () => { }; } setSoraCallbacks(dispatch, getState, sora); - [mediaStream, gainNode] = await createMediaStream(dispatch, state).catch((error) => { - dispatch(slice.actions.setSoraErrorAlertMessage(error.toString())); - dispatch(slice.actions.setSoraConnectionStatus('disconnected')); - throw error; - }); + if (state.soraContents.localMediaStream) { + mediaStream = state.soraContents.localMediaStream; + } else { + [mediaStream, gainNode] = await createMediaStream(dispatch, state).catch((error) => { + dispatch(slice.actions.setSoraErrorAlertMessage(error.toString())); + dispatch(slice.actions.setSoraConnectionStatus('disconnected')); + throw error; + }); + } await sora.connect(mediaStream); } else if (state.role === 'recvonly') { sora = connection.recvonly(state.channelId, null, connectionOptions); sora.metadata = metadata; setSoraCallbacks(dispatch, getState, sora); + dispatch(slice.actions.setSoraConnectionStatus('connecting')); await sora.connect(); } } catch (error) { @@ -1347,7 +1353,7 @@ export const connectSora = () => { // disconnect 時に stream を止めないためのハック sora.stream = null; dispatch(slice.actions.setSora(sora)); - if (mediaStream) { + if (mediaStream && state.soraContents.localMediaStream === null) { dispatch(slice.actions.setLocalMediaStream(mediaStream)); } if (gainNode) { @@ -1516,10 +1522,9 @@ export const setMediaDevices = () => { export const updateMediaStream = () => { return async (dispatch: Dispatch, getState: () => SoraDevtoolsState): Promise => { const state = getState(); - if (!state.soraContents.sora && !state.localTestMediaStream) { + if (!state.soraContents.localMediaStream) { return; } - const localMediaStream = state.soraContents.localMediaStream || state.localTestMediaStream; if (state.virtualBackgroundProcessor && state.virtualBackgroundProcessor.isProcessing()) { const originalTrack = state.virtualBackgroundProcessor.getOriginalTrack(); if (originalTrack) { @@ -1532,8 +1537,8 @@ export const updateMediaStream = () => { } state.virtualBackgroundProcessor.stopProcessing(); } else { - if (localMediaStream) { - localMediaStream.getVideoTracks().forEach((track) => { + if (state.soraContents.localMediaStream) { + state.soraContents.localMediaStream.getVideoTracks().forEach((track) => { track.stop(); dispatch( slice.actions.setTimelineMessage(createSoraDevtoolsMediaStreamTrackLog('stop', track)), @@ -1554,8 +1559,8 @@ export const updateMediaStream = () => { } state.noiseSuppressionProcessor.stopProcessing(); } else { - if (localMediaStream) { - localMediaStream.getAudioTracks().forEach((track) => { + if (state.soraContents.localMediaStream) { + state.soraContents.localMediaStream.getAudioTracks().forEach((track) => { track.stop(); dispatch( slice.actions.setTimelineMessage(createSoraDevtoolsMediaStreamTrackLog('stop', track)), @@ -1582,11 +1587,7 @@ export const updateMediaStream = () => { sender.replaceTrack(track); } }); - if (state.soraContents.sora) { - dispatch(slice.actions.setLocalMediaStream(mediaStream)); - } else { - dispatch(slice.actions.setLocalTestMediaStream(mediaStream)); - } + dispatch(slice.actions.setLocalMediaStream(mediaStream)); dispatch(slice.actions.setFakeContentsGainNode(gainNode)); }; }; diff --git a/src/app/slice.ts b/src/app/slice.ts index 3f8c7295..192490cc 100644 --- a/src/app/slice.ts +++ b/src/app/slice.ts @@ -85,7 +85,6 @@ const initialState: SoraDevtoolsState = { datachannels: [], }, ignoreDisconnectWebSocket: '', - localTestMediaStream: null, logMessages: [], mediaProcessorsNoiseSuppression: false, mediaType: 'getUserMedia', @@ -403,14 +402,6 @@ export const slice = createSlice({ } state.soraContents.localMediaStream = action.payload; }, - setLocalTestMediaStream: (state, aciton: PayloadAction) => { - if (state.localTestMediaStream) { - state.localTestMediaStream.getTracks().forEach((track) => { - track.stop(); - }); - } - state.localTestMediaStream = aciton.payload; - }, setRemoteMediaStream: (state, action: PayloadAction) => { state.soraContents.remoteMediaStreams.push(action.payload); }, diff --git a/src/components/DevtoolsPane/MediaAccessButton.tsx b/src/components/DevtoolsPane/MediaAccessButton.tsx index b44f4271..d144ce34 100644 --- a/src/components/DevtoolsPane/MediaAccessButton.tsx +++ b/src/components/DevtoolsPane/MediaAccessButton.tsx @@ -13,16 +13,16 @@ export const MediaAccessButton: React.FC = () => { dispatch(disposeTestMediaAccess()); }; const connectionStatus = useAppSelector((state) => state.soraContents.connectionStatus); - const localTestMediaStream = useAppSelector((state) => state.localTestMediaStream); - const disabled = localTestMediaStream === null && isFormDisabled(connectionStatus); + const localMediaStream = useAppSelector((state) => state.soraContents.localMediaStream); + const disabled = localMediaStream === null && isFormDisabled(connectionStatus); return (
diff --git a/src/components/DevtoolsPane/MediaTypeForm.tsx b/src/components/DevtoolsPane/MediaTypeForm.tsx index d4c6956f..ff5f2262 100644 --- a/src/components/DevtoolsPane/MediaTypeForm.tsx +++ b/src/components/DevtoolsPane/MediaTypeForm.tsx @@ -4,7 +4,7 @@ import { FormCheck, FormGroup } from 'react-bootstrap'; import { setMediaType } from '@/app/actions'; import { useAppDispatch, useAppSelector } from '@/app/hooks'; import { MEDIA_TYPES } from '@/constants'; -import { checkFormValue, isMediaTypeFormDisabled } from '@/utils'; +import { checkFormValue, isFormDisabled } from '@/utils'; import { TooltipFormLabel } from './TooltipFormLabel'; @@ -38,9 +38,9 @@ export const MediaTypeForm: React.FC = () => { const enabledMediacaptureRegion = typeof window !== 'undefined' && window.CropTarget !== undefined; const connectionStatus = useAppSelector((state) => state.soraContents.connectionStatus); - const localTestMediaStream = useAppSelector((state) => state.localTestMediaStream); + const localMediaStream = useAppSelector((state) => state.soraContents.localMediaStream); const mediaType = useAppSelector((state) => state.mediaType); - const disabled = isMediaTypeFormDisabled(localTestMediaStream, connectionStatus); + const disabled = localMediaStream !== null || isFormDisabled(connectionStatus); const dispatch = useAppDispatch(); const onChange = (event: React.ChangeEvent): void => { if (checkFormValue(event.target.value, MEDIA_TYPES)) { diff --git a/src/components/DevtoolsPane/index.tsx b/src/components/DevtoolsPane/index.tsx index 2810e3df..ecbbf344 100644 --- a/src/components/DevtoolsPane/index.tsx +++ b/src/components/DevtoolsPane/index.tsx @@ -4,7 +4,6 @@ import { Col, Collapse, Row } from 'react-bootstrap'; import { useAppSelector } from '@/app/hooks'; import { AlertMessages } from '@/components/AlertMessages'; import { LocalVideo } from '@/components/Video/LocalVideo'; -import { LocalVideoTest } from '@/components/Video/LocalVideoTest'; import { RemoteVideos } from '@/components/Video/RemoteVideos'; import { AspectRatioForm } from './AspectRatioForm'; @@ -486,7 +485,6 @@ export const DevtoolsPane: React.FC = () => {
- {role === 'recvonly' || role === 'sendrecv' ? : null} diff --git a/src/components/Video/LocalVideo.tsx b/src/components/Video/LocalVideo.tsx index 2c845fb5..c2666e96 100644 --- a/src/components/Video/LocalVideo.tsx +++ b/src/components/Video/LocalVideo.tsx @@ -58,7 +58,7 @@ export const LocalVideo: React.FC = () => { const simulcast = useAppSelector((state) => state.simulcast); const spotlight = useAppSelector((state) => state.spotlight); const role = useAppSelector((state) => state.role); - const localMediaStream = useAppSelector((state) => state.localTestMediaStream); + const localMediaStream = useAppSelector((state) => state.soraContents.localMediaStream); return (
@@ -86,7 +86,7 @@ export const LocalVideo: React.FC = () => {
) : null}
- {localMediaStream === null && role !== 'recvonly' ? : null} + {localMediaStream !== null && role !== 'recvonly' ? : null} ); diff --git a/src/components/Video/LocalVideoTest.tsx b/src/components/Video/LocalVideoTest.tsx deleted file mode 100644 index baef62da..00000000 --- a/src/components/Video/LocalVideoTest.tsx +++ /dev/null @@ -1,47 +0,0 @@ -import React, { useState } from 'react'; - -import { useAppSelector } from '@/app/hooks'; - -import { Video } from './Video'; -import { VolumeVisualizer } from './VolumeVisualizer'; - -const VideoBox: React.FC = () => { - const [height, setHeight] = useState(0); - const audio = useAppSelector((state) => state.audio); - const video = useAppSelector((state) => state.video); - const audioOutput = useAppSelector((state) => state.audioOutput); - const displayResolution = useAppSelector((state) => state.displayResolution); - const localTestMediaStream = useAppSelector((state) => state.localTestMediaStream); - const micDevice = useAppSelector((state) => state.micDevice); - if (audio === false && video === false) { - return null; - } - return ( - <> -
-
-
-
- - ); -}; - -export const LocalVideoTest: React.FC = () => { - const localMediaStream = useAppSelector((state) => state.localTestMediaStream); - return ( -
-
{localMediaStream !== null && }
-
- ); -}; diff --git a/src/types.ts b/src/types.ts index 72616c05..ba8f6a97 100644 --- a/src/types.ts +++ b/src/types.ts @@ -104,7 +104,6 @@ export type SoraDevtoolsState = { statsReport: RTCStats[]; datachannels: DataChannelConfiguration[]; }; - localTestMediaStream: MediaStream | null; ignoreDisconnectWebSocket: (typeof IGNORE_DISCONNECT_WEBSOCKET)[number]; logMessages: LogMessage[]; mediaProcessorsNoiseSuppression: boolean; From cc25d70ae54086ba387d6c39b923d618eaf496df Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Wed, 18 Oct 2023 19:20:59 +0900 Subject: [PATCH 09/21] =?UTF-8?q?media=20access=20=E3=83=9C=E3=82=BF?= =?UTF-8?q?=E3=83=B3=E3=81=AE=20disabled=20=E6=9D=A1=E4=BB=B6=E3=81=AB=20S?= =?UTF-8?q?ora=20=E3=81=AE=E6=8E=A5=E7=B6=9A=E7=8A=B6=E6=85=8B=E3=82=92?= =?UTF-8?q?=E5=8A=A0=E3=81=88=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/DevtoolsPane/MediaAccessButton.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/components/DevtoolsPane/MediaAccessButton.tsx b/src/components/DevtoolsPane/MediaAccessButton.tsx index d144ce34..9abdb4ad 100644 --- a/src/components/DevtoolsPane/MediaAccessButton.tsx +++ b/src/components/DevtoolsPane/MediaAccessButton.tsx @@ -13,8 +13,9 @@ export const MediaAccessButton: React.FC = () => { dispatch(disposeTestMediaAccess()); }; const connectionStatus = useAppSelector((state) => state.soraContents.connectionStatus); + const sora = useAppSelector((state) => state.soraContents.sora); const localMediaStream = useAppSelector((state) => state.soraContents.localMediaStream); - const disabled = localMediaStream === null && isFormDisabled(connectionStatus); + const disabled = sora !== null || isFormDisabled(connectionStatus); return (
Date: Thu, 19 Oct 2023 13:17:52 +0900 Subject: [PATCH 10/21] =?UTF-8?q?recvonly=20=E3=83=AD=E3=83=BC=E3=83=AB?= =?UTF-8?q?=E3=81=8C=E9=81=B8=E6=8A=9E=E3=81=95=E3=82=8C=E3=81=A6=E3=81=84?= =?UTF-8?q?=E3=82=8B=E6=99=82=E3=81=AF=20media=20access=20=E3=82=92?= =?UTF-8?q?=E7=84=A1=E5=8A=B9=E5=8C=96=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/DevtoolsPane/MediaAccessButton.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/components/DevtoolsPane/MediaAccessButton.tsx b/src/components/DevtoolsPane/MediaAccessButton.tsx index 9abdb4ad..f0b9f86c 100644 --- a/src/components/DevtoolsPane/MediaAccessButton.tsx +++ b/src/components/DevtoolsPane/MediaAccessButton.tsx @@ -15,7 +15,8 @@ export const MediaAccessButton: React.FC = () => { const connectionStatus = useAppSelector((state) => state.soraContents.connectionStatus); const sora = useAppSelector((state) => state.soraContents.sora); const localMediaStream = useAppSelector((state) => state.soraContents.localMediaStream); - const disabled = sora !== null || isFormDisabled(connectionStatus); + const role = useAppSelector((state) => state.role); + const disabled = role === 'recvonly' || sora !== null || isFormDisabled(connectionStatus); return (
Date: Thu, 19 Oct 2023 13:18:10 +0900 Subject: [PATCH 11/21] =?UTF-8?q?media=20access=20=E5=AE=9F=E8=A1=8C?= =?UTF-8?q?=E4=B8=AD=E3=81=AF=20role=20=E3=82=92=E5=A4=89=E6=9B=B4?= =?UTF-8?q?=E3=81=A7=E3=81=8D=E3=81=AA=E3=81=84=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/DevtoolsPane/RoleForm.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/components/DevtoolsPane/RoleForm.tsx b/src/components/DevtoolsPane/RoleForm.tsx index f7613e85..9f8d22e8 100644 --- a/src/components/DevtoolsPane/RoleForm.tsx +++ b/src/components/DevtoolsPane/RoleForm.tsx @@ -11,7 +11,8 @@ import { TooltipFormLabel } from './TooltipFormLabel'; export const RoleForm: React.FC = () => { const role = useAppSelector((state) => state.role); const connectionStatus = useAppSelector((state) => state.soraContents.connectionStatus); - const disabled = isFormDisabled(connectionStatus); + const localMediaStream = useAppSelector((state) => state.soraContents.localMediaStream); + const disabled = localMediaStream !== null || isFormDisabled(connectionStatus); const dispatch = useAppDispatch(); const onChange = (event: React.ChangeEvent): void => { if (checkFormValue(event.target.value, ROLES)) { From 33c2a592c094c13f92c12360dc49d8ccad6e2cae Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Thu, 19 Oct 2023 18:08:50 +0900 Subject: [PATCH 12/21] =?UTF-8?q?media=20access=20=E5=AE=9F=E8=A1=8C?= =?UTF-8?q?=E4=B8=AD=E3=81=AE=20device=20=E3=82=AA=E3=83=B3=E3=82=AA?= =?UTF-8?q?=E3=83=95=E3=81=AE=E5=88=87=E3=82=8A=E6=9B=BF=E3=81=88=E3=81=AB?= =?UTF-8?q?=E5=AF=BE=E5=BF=9C=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app/actions.ts | 64 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/src/app/actions.ts b/src/app/actions.ts index bc6e524c..49d6233e 100644 --- a/src/app/actions.ts +++ b/src/app/actions.ts @@ -1657,14 +1657,35 @@ export const setMicDevice = (micDevice: boolean) => { }, ); if (0 < mediaStream.getAudioTracks().length) { - await state.soraContents.sora.replaceAudioTrack( - state.soraContents.localMediaStream, - mediaStream.getAudioTracks()[0], - ); + if (state.soraContents.sora && state.soraContents.localMediaStream) { + // Sora 接続中の場合 + await state.soraContents.sora.replaceAudioTrack( + state.soraContents.localMediaStream, + mediaStream.getAudioTracks()[0], + ); + } else if (state.soraContents.localMediaStream) { + // Sora は未接続で media access での表示を行っている場合 + // 現在の AudioTrack を停止、削除してから、新しい AudioTrack を追加する + state.soraContents.localMediaStream.getAudioTracks().forEach((track) => { + track.enabled = false; + track.stop(); + state.soraContents.localMediaStream?.removeTrack(track); + }); + state.soraContents.localMediaStream.addTrack(mediaStream.getAudioTracks()[0]); + } dispatch(slice.actions.setFakeContentsGainNode(gainNode)); } - } else { + } else if (state.soraContents.sora && state.soraContents.localMediaStream) { + // Sora 接続中の場合 state.soraContents.sora.stopAudioTrack(state.soraContents.localMediaStream); + } else if (state.soraContents.localMediaStream) { + // Sora は未接続で media access での表示を行っている場合 + // localMediaStream の AudioTrack を停止して MediaStream から Track を削除する + state.soraContents.localMediaStream.getAudioTracks().forEach((track) => { + track.enabled = false; + track.stop(); + state.soraContents.localMediaStream?.removeTrack(track); + }); } dispatch(slice.actions.setMicDevice(micDevice)); }; @@ -1673,7 +1694,7 @@ export const setMicDevice = (micDevice: boolean) => { export const setCameraDevice = (cameraDevice: boolean) => { return async (dispatch: Dispatch, getState: () => SoraDevtoolsState): Promise => { const state = getState(); - if (!state.soraContents.localMediaStream || !state.soraContents.sora) { + if (!state.soraContents.localMediaStream && !state.soraContents.sora) { dispatch(slice.actions.setCameraDevice(cameraDevice)); return; } @@ -1715,14 +1736,35 @@ export const setCameraDevice = (cameraDevice: boolean) => { }, ); if (0 < mediaStream.getVideoTracks().length) { - state.soraContents.sora.replaceVideoTrack( - state.soraContents.localMediaStream, - mediaStream.getVideoTracks()[0], - ); + if (state.soraContents.sora && state.soraContents.localMediaStream) { + // Sora 接続中の場合 + state.soraContents.sora.replaceVideoTrack( + state.soraContents.localMediaStream, + mediaStream.getVideoTracks()[0], + ); + } else if (state.soraContents.localMediaStream) { + // Sora は未接続で media access での表示を行っている場合 + // 現在の VideoTrack を停止、削除してから、新しい VideoTrack を追加する + state.soraContents.localMediaStream.getVideoTracks().forEach((track) => { + track.enabled = false; + track.stop(); + state.soraContents.localMediaStream?.removeTrack(track); + }); + state.soraContents.localMediaStream.addTrack(mediaStream.getVideoTracks()[0]); + } dispatch(slice.actions.setFakeContentsGainNode(gainNode)); } - } else { + } else if (state.soraContents.sora && state.soraContents.localMediaStream) { + // Sora 接続中の場合 state.soraContents.sora.stopVideoTrack(state.soraContents.localMediaStream); + } else if (state.soraContents.localMediaStream) { + // Sora は未接続で media access での表示を行っている場合 + // localMediaStream の VideoTrack を停止して MediaStream から Track を削除する + state.soraContents.localMediaStream.getVideoTracks().forEach((track) => { + track.enabled = false; + track.stop(); + state.soraContents.localMediaStream?.removeTrack(track); + }); } dispatch(slice.actions.setCameraDevice(cameraDevice)); }; From 9eb91c06eb8eec81f75c994f8d97a53669b73b5a Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Thu, 19 Oct 2023 18:42:53 +0900 Subject: [PATCH 13/21] =?UTF-8?q?=E5=A4=89=E6=9B=B4=E5=B1=A5=E6=AD=B4?= =?UTF-8?q?=E3=81=AB=E8=BF=BD=E8=A8=98=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index e25dbb90..19be4bdd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,6 +11,8 @@ ## develop +- [ADD] Sora とは接続せず Audio / Video デバイスの表示確認を行う `media access` 機能を追加する + - @tnamao - [CHANGE] mediaType が getUserMedia 以外の場合は audioInput / videoInput のフォームを表示しないように修正する - @tnamao - [CHANGE] mediaType が getUserMedia 以外の場合は、copy URL をクリックした時にクリップボードにコピーする URL のパラメータに audioInput / videoInput を含めないように修正する From 1f56dd8c194628e5a3d40cf7d260d1b103c24464 Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Thu, 19 Oct 2023 19:29:38 +0900 Subject: [PATCH 14/21] =?UTF-8?q?=E5=A4=89=E6=9B=B4=E5=B1=A5=E6=AD=B4?= =?UTF-8?q?=E3=81=AB=E8=BF=BD=E8=A8=98=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 19be4bdd..e15764b5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,10 @@ ## develop - [ADD] Sora とは接続せず Audio / Video デバイスの表示確認を行う `media access` 機能を追加する + - 現状の設定項目を利用するため、`media access` の実行中は `role` や `mediaType` を disabled にする + - @tnamao +- [CHANGE] `media access` 機能で取得した MediaStream をそのまま Sora の接続に利用できるようにしたため、新たに `preparing` の状態を追加する + - `connecting` の状態は MediaStream を取得後、実際に Sora との接続処理を行う時の状態として意味を変更する - @tnamao - [CHANGE] mediaType が getUserMedia 以外の場合は audioInput / videoInput のフォームを表示しないように修正する - @tnamao From 6fd38cf0ee4d0d5982e3a522ff687266f8cc9cb9 Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Fri, 20 Oct 2023 15:17:50 +0900 Subject: [PATCH 15/21] =?UTF-8?q?media=20acess=20=E3=81=AE=E9=96=8B?= =?UTF-8?q?=E5=A7=8B=E3=81=A8=E7=B5=82=E4=BA=86=E3=81=AE=E3=83=9C=E3=82=BF?= =?UTF-8?q?=E3=83=B3=E3=82=92=E5=88=86=E3=81=91=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app/actions.ts | 4 +-- ...ccessButton.tsx => DisposeMediaButton.tsx} | 16 ++++------- .../DevtoolsPane/RequestMediaButton.tsx | 28 +++++++++++++++++++ src/components/DevtoolsPane/index.tsx | 6 ++-- 4 files changed, 40 insertions(+), 14 deletions(-) rename src/components/DevtoolsPane/{MediaAccessButton.tsx => DisposeMediaButton.tsx} (56%) create mode 100644 src/components/DevtoolsPane/RequestMediaButton.tsx diff --git a/src/app/actions.ts b/src/app/actions.ts index e00286c8..b432e1d3 100644 --- a/src/app/actions.ts +++ b/src/app/actions.ts @@ -1067,7 +1067,7 @@ async function setStatsReport( } } -export const testMediaAccess = () => { +export const requestMedia = () => { return async (dispatch: Dispatch, getState: () => SoraDevtoolsState): Promise => { const LOG_TITLE = 'MEDIA_CONSTRAINTS'; const state = getState(); @@ -1198,7 +1198,7 @@ export const testMediaAccess = () => { }; }; -export const disposeTestMediaAccess = () => { +export const disposeMedia = () => { return async (dispatch: Dispatch, _getState: () => SoraDevtoolsState): Promise => { dispatch(slice.actions.setLocalMediaStream(null)); }; diff --git a/src/components/DevtoolsPane/MediaAccessButton.tsx b/src/components/DevtoolsPane/DisposeMediaButton.tsx similarity index 56% rename from src/components/DevtoolsPane/MediaAccessButton.tsx rename to src/components/DevtoolsPane/DisposeMediaButton.tsx index f0b9f86c..41915923 100644 --- a/src/components/DevtoolsPane/MediaAccessButton.tsx +++ b/src/components/DevtoolsPane/DisposeMediaButton.tsx @@ -1,20 +1,16 @@ import React from 'react'; -import { disposeTestMediaAccess, testMediaAccess } from '@/app/actions'; +import { disposeMedia } from '@/app/actions'; import { useAppDispatch, useAppSelector } from '@/app/hooks'; import { isFormDisabled } from '@/utils'; -export const MediaAccessButton: React.FC = () => { +export const DisposeMediaButton: React.FC = () => { const dispatch = useAppDispatch(); - const mediaAccess = (): void => { - dispatch(testMediaAccess()); - }; - const disposeMediaAccess = (): void => { - dispatch(disposeTestMediaAccess()); + const onClick = (): void => { + dispatch(disposeMedia()); }; const connectionStatus = useAppSelector((state) => state.soraContents.connectionStatus); const sora = useAppSelector((state) => state.soraContents.sora); - const localMediaStream = useAppSelector((state) => state.soraContents.localMediaStream); const role = useAppSelector((state) => state.role); const disabled = role === 'recvonly' || sora !== null || isFormDisabled(connectionStatus); return ( @@ -23,8 +19,8 @@ export const MediaAccessButton: React.FC = () => { className="btn btn-secondary" type="button" name="media_access" - defaultValue={localMediaStream === null ? 'media access' : 'dispose'} - onClick={localMediaStream === null ? mediaAccess : disposeMediaAccess} + defaultValue="dispose media" + onClick={onClick} disabled={disabled} />
diff --git a/src/components/DevtoolsPane/RequestMediaButton.tsx b/src/components/DevtoolsPane/RequestMediaButton.tsx new file mode 100644 index 00000000..a4609bd0 --- /dev/null +++ b/src/components/DevtoolsPane/RequestMediaButton.tsx @@ -0,0 +1,28 @@ +import React from 'react'; + +import { requestMedia } from '@/app/actions'; +import { useAppDispatch, useAppSelector } from '@/app/hooks'; +import { isFormDisabled } from '@/utils'; + +export const RequestMediaButton: React.FC = () => { + const dispatch = useAppDispatch(); + const onClick = (): void => { + dispatch(requestMedia()); + }; + const connectionStatus = useAppSelector((state) => state.soraContents.connectionStatus); + const sora = useAppSelector((state) => state.soraContents.sora); + const role = useAppSelector((state) => state.role); + const disabled = role === 'recvonly' || sora !== null || isFormDisabled(connectionStatus); + return ( +
+ +
+ ); +}; diff --git a/src/components/DevtoolsPane/index.tsx b/src/components/DevtoolsPane/index.tsx index d79a822d..48ba87af 100644 --- a/src/components/DevtoolsPane/index.tsx +++ b/src/components/DevtoolsPane/index.tsx @@ -27,6 +27,7 @@ import { DataChannelForm } from './DataChannelForm'; import { DataChannelsForm } from './DataChannelsForm'; import { DisconnectButton } from './DisconnectButton'; import { DisplayResolutionForm } from './DisplayResolutionForm'; +import { DisposeMediaButton } from './DisposeMediaButton'; import { E2EEForm } from './E2EEForm'; import { EchoCancellationForm } from './EchoCancellationForm'; import { EchoCancellationTypeForm } from './EchoCancellationTypeForm'; @@ -35,7 +36,6 @@ import { FakeVolumeForm } from './FakeVolumeForm'; import { ForwardingFilterForm } from './ForwardingFilterForm'; import { FrameRateForm } from './FrameRateForm'; import { LightAdjustmentForm } from './LightAdjustmentForm'; -import { MediaAccessButton } from './MediaAccessButton'; import { MediaProcessorsNoiseSuppressionForm } from './MediaProcessorsNoiseSuppressionForm'; import { MediaTypeForm } from './MediaTypeForm'; import { MetadataForm } from './MetadataForm'; @@ -44,6 +44,7 @@ import { MultistreamForm } from './MultistreamForm'; import { NoiseSuppressionForm } from './NoiseSuppressionForm'; import { ReconnectForm } from './ReconnectForm'; import { ReloadDevicesButton } from './ReloadDevicesButton'; +import { RequestMediaButton } from './RequestMediaButton'; import { ResizeModeForm } from './ResizeModeForm'; import { ResolutionForm } from './ResolutionForm'; import { RoleForm } from './RoleForm'; @@ -482,7 +483,8 @@ export const DevtoolsPane: React.FC = () => {
- + + From 0c7cb9e406ca1091baefb49d5fc20c7b33bcda67 Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Fri, 20 Oct 2023 16:00:47 +0900 Subject: [PATCH 16/21] =?UTF-8?q?gUM=20=E4=BB=A5=E5=A4=96=E3=82=82=20reque?= =?UTF-8?q?st=20media=20=E3=81=A7=E3=83=AD=E3=83=BC=E3=82=AB=E3=83=AB?= =?UTF-8?q?=E8=A1=A8=E7=A4=BA=E3=81=A7=E3=81=8D=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app/actions.ts | 171 +++++++++++++++------------------------------ 1 file changed, 58 insertions(+), 113 deletions(-) diff --git a/src/app/actions.ts b/src/app/actions.ts index b432e1d3..76dec7e5 100644 --- a/src/app/actions.ts +++ b/src/app/actions.ts @@ -1069,130 +1069,75 @@ async function setStatsReport( export const requestMedia = () => { return async (dispatch: Dispatch, getState: () => SoraDevtoolsState): Promise => { - const LOG_TITLE = 'MEDIA_CONSTRAINTS'; + const LOG_TITLE = 'REQUEST_MEDIA'; const state = getState(); - const mediaStream = new MediaStream(); - const audioConstraints = createAudioConstraints({ - audio: state.audio && state.micDevice, - autoGainControl: state.autoGainControl, - noiseSuppression: state.noiseSuppression, - echoCancellation: state.echoCancellation, - echoCancellationType: state.echoCancellationType, - audioInput: state.audioInput, - }); - if (audioConstraints) { - dispatch( - slice.actions.setLogMessages({ - title: LOG_TITLE, - description: JSON.stringify({ audio: audioConstraints }), - }), - ); - dispatch( - slice.actions.setTimelineMessage( - createSoraDevtoolsTimelineMessage('audio-media-constraints', { audio: audioConstraints }), - ), - ); - const audioMediaStream = await navigator.mediaDevices.getUserMedia({ - audio: audioConstraints, + let mediaStream, gainNode; + try { + [mediaStream, gainNode] = await createMediaStream(dispatch, state).catch((error) => { + throw error; }); - let audioTrack = audioMediaStream.getAudioTracks()[0]; - dispatch( - slice.actions.setTimelineMessage( - createSoraDevtoolsMediaStreamTrackLog('start', audioTrack), - ), - ); - if (state.mediaProcessorsNoiseSuppression && NoiseSuppressionProcessor.isSupported()) { - if (state.noiseSuppressionProcessor === null) { - throw new Error( - "Failed to start NoiseSuppressionProcessor. NoiseSuppressionProcessor is 'null'", - ); + } catch (error) { + if (error instanceof Error) { + dispatch(slice.actions.setLogMessages({ title: LOG_TITLE, description: error.message })); + } + let originalTrack; + if (state.lightAdjustmentProcessor && state.lightAdjustmentProcessor.isProcessing()) { + originalTrack = state.lightAdjustmentProcessor.getOriginalTrack(); + state.lightAdjustmentProcessor.stopProcessing(); + } + if (state.virtualBackgroundProcessor && state.virtualBackgroundProcessor.isProcessing()) { + if (originalTrack === undefined) { + originalTrack = state.virtualBackgroundProcessor.getOriginalTrack(); } - state.noiseSuppressionProcessor.stopProcessing(); - audioTrack = await state.noiseSuppressionProcessor.startProcessing(audioTrack); + state.virtualBackgroundProcessor.stopProcessing(); } - dispatch( - slice.actions.setTimelineMessage( - createSoraDevtoolsTimelineMessage('succeed-audio-get-user-media'), - ), - ); - mediaStream.addTrack(audioTrack); - } - const videoConstraints = createVideoConstraints({ - aspectRatio: state.aspectRatio, - frameRate: state.frameRate, - resizeMode: state.resizeMode, - resolution: state.resolution, - video: state.video && state.cameraDevice, - videoInput: state.videoInput, - facingMode: state.facingMode, - }); - if (videoConstraints) { - dispatch( - slice.actions.setLogMessages({ - title: LOG_TITLE, - description: JSON.stringify({ video: videoConstraints }), - }), - ); - dispatch( - slice.actions.setTimelineMessage( - createSoraDevtoolsTimelineMessage('video-media-constraints', { video: videoConstraints }), - ), - ); - const videoMediaStream = await navigator.mediaDevices - .getUserMedia({ video: videoConstraints }) - .catch((error) => { - // video track の getUserMedia が失敗した場合には audio track が存在している可能性があるので止める - mediaStream.getTracks().forEach((t) => { - t.stop(); + if (originalTrack) { + originalTrack.stop(); + dispatch( + slice.actions.setTimelineMessage( + createSoraDevtoolsMediaStreamTrackLog('stop', originalTrack), + ), + ); + } else { + if (mediaStream) { + mediaStream.getVideoTracks().forEach((track) => { + track.stop(); + dispatch( + slice.actions.setTimelineMessage( + createSoraDevtoolsMediaStreamTrackLog('stop', track), + ), + ); }); - throw error; - }); - let videoTrack = videoMediaStream.getVideoTracks()[0]; - dispatch( - slice.actions.setTimelineMessage( - createSoraDevtoolsMediaStreamTrackLog('start', videoTrack), - ), - ); - if (state.lightAdjustment !== '' && LightAdjustmentProcessor.isSupported()) { - if (state.lightAdjustmentProcessor === null) { - throw new Error( - "Failed to start LightAdjustmentProcessor. LightAdjustmentProcessor is 'null'", - ); } - const options = getLightAdjustmentOptions(state.lightAdjustment); - state.lightAdjustmentProcessor.stopProcessing(); - videoTrack = await state.lightAdjustmentProcessor.startProcessing(videoTrack, options); } - if (state.blurRadius !== '' && VirtualBackgroundProcessor.isSupported()) { - if (state.virtualBackgroundProcessor === null) { - throw new Error( - "Failed to start VirtualBackgroundProcessor. VirtualBackgroundProcessor is 'null'", + + if (state.noiseSuppressionProcessor && state.noiseSuppressionProcessor.isProcessing()) { + const originalTrack = state.noiseSuppressionProcessor.getOriginalTrack(); + if (originalTrack) { + originalTrack.stop(); + dispatch( + slice.actions.setTimelineMessage( + createSoraDevtoolsMediaStreamTrackLog('stop', originalTrack), + ), ); } - const options = { - blurRadius: getBlurRadiusNumber(state.blurRadius), - }; - state.virtualBackgroundProcessor.stopProcessing(); - videoTrack = await state.virtualBackgroundProcessor.startProcessing(videoTrack, options); - } - dispatch( - slice.actions.setTimelineMessage( - createSoraDevtoolsTimelineMessage('succeed-video-get-user-media'), - ), - ); - mediaStream.addTrack(videoTrack); - } - for (const track of mediaStream.getVideoTracks()) { - if (track.contentHint !== undefined) { - track.contentHint = state.videoContentHint; + state.noiseSuppressionProcessor.stopProcessing(); + } else { + if (mediaStream) { + mediaStream.getAudioTracks().forEach((track) => { + track.stop(); + dispatch( + slice.actions.setTimelineMessage( + createSoraDevtoolsMediaStreamTrackLog('stop', track), + ), + ); + }); + } } - track.enabled = state.videoTrack; + throw error; } - for (const track of mediaStream.getAudioTracks()) { - if (track.contentHint !== undefined) { - track.contentHint = state.audioContentHint; - } - track.enabled = state.audioTrack; + if (gainNode) { + dispatch(slice.actions.setFakeContentsGainNode(gainNode)); } dispatch(slice.actions.setLocalMediaStream(mediaStream)); }; From 33c134ec000e8842cd8eee281ac2ffa9144718c8 Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Fri, 20 Oct 2023 16:05:53 +0900 Subject: [PATCH 17/21] =?UTF-8?q?recvonly=20=E3=81=AE=E6=99=82=E3=81=AF=20?= =?UTF-8?q?media=20access=20=E3=83=9C=E3=82=BF=E3=83=B3=E3=81=AF=E9=9D=9E?= =?UTF-8?q?=E8=A1=A8=E7=A4=BA=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/DevtoolsPane/index.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/components/DevtoolsPane/index.tsx b/src/components/DevtoolsPane/index.tsx index 48ba87af..709f33a4 100644 --- a/src/components/DevtoolsPane/index.tsx +++ b/src/components/DevtoolsPane/index.tsx @@ -483,8 +483,12 @@ export const DevtoolsPane: React.FC = () => {
- - + {role !== 'recvonly' ? ( + <> + + + + ) : null} From da498fe3aaff999038a583de7fd71d6bf39a3568 Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Fri, 20 Oct 2023 16:15:03 +0900 Subject: [PATCH 18/21] =?UTF-8?q?gUM=20gDM=20=E3=81=AA=E3=81=A9=E3=81=8C?= =?UTF-8?q?=E3=82=AD=E3=83=A3=E3=83=B3=E3=82=BB=E3=83=AB=E3=81=95=E3=82=8C?= =?UTF-8?q?=E3=81=9F=E6=99=82=E3=81=AB=E3=83=AD=E3=82=B0=E8=BF=BD=E5=8A=A0?= =?UTF-8?q?=E3=81=A8=E3=82=A2=E3=83=A9=E3=83=BC=E3=83=88=E3=82=92=E8=A1=A8?= =?UTF-8?q?=E7=A4=BA=E3=82=92=E8=A1=8C=E3=81=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app/actions.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/app/actions.ts b/src/app/actions.ts index 76dec7e5..8dd445e3 100644 --- a/src/app/actions.ts +++ b/src/app/actions.ts @@ -1078,7 +1078,15 @@ export const requestMedia = () => { }); } catch (error) { if (error instanceof Error) { - dispatch(slice.actions.setLogMessages({ title: LOG_TITLE, description: error.message })); + dispatch( + slice.actions.setLogMessages({ + title: LOG_TITLE, + description: JSON.stringify(error.message), + }), + ); + dispatch( + slice.actions.setAPIErrorAlertMessage(`Failed to get user devices. ${error.message}`), + ); } let originalTrack; if (state.lightAdjustmentProcessor && state.lightAdjustmentProcessor.isProcessing()) { From 0b877616642a87b80fa4ce22c77b43e172044b4f Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Fri, 20 Oct 2023 16:55:43 +0900 Subject: [PATCH 19/21] =?UTF-8?q?dispose=20media=20=E6=99=82=E3=81=AB=20fa?= =?UTF-8?q?ke=20=E3=82=84=E8=83=8C=E6=99=AF=E5=87=A6=E7=90=86=E3=81=AE?= =?UTF-8?q?=E5=81=9C=E6=AD=A2=E5=87=A6=E7=90=86=E3=82=92=E8=BF=BD=E5=8A=A0?= =?UTF-8?q?=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/app/actions.ts | 63 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/src/app/actions.ts b/src/app/actions.ts index 8dd445e3..bcee4c92 100644 --- a/src/app/actions.ts +++ b/src/app/actions.ts @@ -1152,7 +1152,68 @@ export const requestMedia = () => { }; export const disposeMedia = () => { - return async (dispatch: Dispatch, _getState: () => SoraDevtoolsState): Promise => { + return async (dispatch: Dispatch, getState: () => SoraDevtoolsState): Promise => { + const { + fakeContents, + soraContents, + lightAdjustmentProcessor, + noiseSuppressionProcessor, + virtualBackgroundProcessor, + } = getState(); + const { localMediaStream } = soraContents; + let originalTrack; + if (lightAdjustmentProcessor && lightAdjustmentProcessor.isProcessing()) { + originalTrack = lightAdjustmentProcessor.getOriginalTrack(); + lightAdjustmentProcessor.stopProcessing(); + } + if (virtualBackgroundProcessor && virtualBackgroundProcessor.isProcessing()) { + if (originalTrack === undefined) { + originalTrack = virtualBackgroundProcessor.getOriginalTrack(); + } + virtualBackgroundProcessor.stopProcessing(); + } + if (originalTrack !== undefined) { + originalTrack.stop(); + dispatch( + slice.actions.setTimelineMessage( + createSoraDevtoolsMediaStreamTrackLog('stop', originalTrack), + ), + ); + } else { + if (localMediaStream) { + localMediaStream.getVideoTracks().forEach((track) => { + track.stop(); + dispatch( + slice.actions.setTimelineMessage(createSoraDevtoolsMediaStreamTrackLog('stop', track)), + ); + }); + } + } + + if (noiseSuppressionProcessor && noiseSuppressionProcessor.isProcessing()) { + const originalTrack = noiseSuppressionProcessor.getOriginalTrack(); + if (originalTrack) { + originalTrack.stop(); + dispatch( + slice.actions.setTimelineMessage( + createSoraDevtoolsMediaStreamTrackLog('stop', originalTrack), + ), + ); + } + noiseSuppressionProcessor.stopProcessing(); + } else { + if (localMediaStream) { + localMediaStream.getAudioTracks().forEach((track) => { + track.stop(); + dispatch( + slice.actions.setTimelineMessage(createSoraDevtoolsMediaStreamTrackLog('stop', track)), + ); + }); + } + } + if (fakeContents.worker) { + fakeContents.worker.postMessage({ type: 'stop' }); + } dispatch(slice.actions.setLocalMediaStream(null)); }; }; From 40beab7491c5f4fd6412d9fc34efeb099950923d Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Fri, 20 Oct 2023 17:01:12 +0900 Subject: [PATCH 20/21] =?UTF-8?q?=E5=A4=89=E6=9B=B4=E5=B1=A5=E6=AD=B4?= =?UTF-8?q?=E3=82=92=E6=9B=B4=E6=96=B0=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e15764b5..3345ef88 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -11,10 +11,10 @@ ## develop -- [ADD] Sora とは接続せず Audio / Video デバイスの表示確認を行う `media access` 機能を追加する +- [ADD] Sora とは接続せず Audio / Video デバイスの表示確認と停止を行う `request media` `dispose media` 機能を追加する - 現状の設定項目を利用するため、`media access` の実行中は `role` や `mediaType` を disabled にする - @tnamao -- [CHANGE] `media access` 機能で取得した MediaStream をそのまま Sora の接続に利用できるようにしたため、新たに `preparing` の状態を追加する +- [CHANGE] `request media` 機能で取得した MediaStream をそのまま Sora の接続に利用できるようにしたため、新たに `preparing` の状態を追加する - `connecting` の状態は MediaStream を取得後、実際に Sora との接続処理を行う時の状態として意味を変更する - @tnamao - [CHANGE] mediaType が getUserMedia 以外の場合は audioInput / videoInput のフォームを表示しないように修正する From 03a6416ffb8804e5f200c5e1865399bfee4fdafc Mon Sep 17 00:00:00 2001 From: Takeshi NAMAO Date: Fri, 20 Oct 2023 17:34:15 +0900 Subject: [PATCH 21/21] =?UTF-8?q?=E5=A4=89=E6=9B=B4=E5=B1=A5=E6=AD=B4?= =?UTF-8?q?=E3=82=92=E6=9B=B4=E6=96=B0=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 3345ef88..387a065d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,7 +12,7 @@ ## develop - [ADD] Sora とは接続せず Audio / Video デバイスの表示確認と停止を行う `request media` `dispose media` 機能を追加する - - 現状の設定項目を利用するため、`media access` の実行中は `role` や `mediaType` を disabled にする + - 現状の設定項目を利用するため、`request media` の実行中は `role` や `mediaType` を disabled にする - @tnamao - [CHANGE] `request media` 機能で取得した MediaStream をそのまま Sora の接続に利用できるようにしたため、新たに `preparing` の状態を追加する - `connecting` の状態は MediaStream を取得後、実際に Sora との接続処理を行う時の状態として意味を変更する