From d25875669fff39b372e4a2e183f1ae19631f0a11 Mon Sep 17 00:00:00 2001 From: Taichiro Suzuki Date: Tue, 26 Mar 2024 17:13:45 +0900 Subject: [PATCH 1/5] wip --- packages/cdk/bin/generative-ai-use-cases.ts | 2 +- packages/web/src/App.tsx | 7 + packages/web/src/hooks/useFiles.ts | 2 +- packages/web/src/main.tsx | 5 + packages/web/src/pages/VideoAnalyzerPage.tsx | 187 +++++++++++++++++++ packages/web/src/prompts/claude.ts | 9 +- packages/web/src/prompts/index.ts | 5 + 7 files changed, 214 insertions(+), 3 deletions(-) create mode 100644 packages/web/src/pages/VideoAnalyzerPage.tsx diff --git a/packages/cdk/bin/generative-ai-use-cases.ts b/packages/cdk/bin/generative-ai-use-cases.ts index 56368db0..bf790e11 100644 --- a/packages/cdk/bin/generative-ai-use-cases.ts +++ b/packages/cdk/bin/generative-ai-use-cases.ts @@ -54,7 +54,7 @@ const anonymousUsageTracking: boolean = !!app.node.tryGetContext( ); const vpcId = app.node.tryGetContext('vpcId'); -if (typeof vpcId != 'undefined' && vpcId != null && typeof vpcId != 'string' ) { +if (typeof vpcId != 'undefined' && vpcId != null && typeof vpcId != 'string') { throw new Error('vpcId must be string or undefined'); } if (typeof vpcId == 'string' && !vpcId.match(/^vpc-/)) { diff --git a/packages/web/src/App.tsx b/packages/web/src/App.tsx index 3b888565..6598f1e1 100644 --- a/packages/web/src/App.tsx +++ b/packages/web/src/App.tsx @@ -17,6 +17,7 @@ import { PiX, PiRobot, PiUploadSimple, + PiVideoCamera, } from 'react-icons/pi'; import { Outlet } from 'react-router-dom'; import Drawer, { ItemProps } from './components/Drawer'; @@ -103,6 +104,12 @@ const items: ItemProps[] = [ icon: , display: 'usecase' as const, }, + { + label: '映像分析', + to: '/video', + icon: , + display: 'usecase' as const, + }, { label: '音声認識', to: '/transcribe', diff --git a/packages/web/src/hooks/useFiles.ts b/packages/web/src/hooks/useFiles.ts index dbb0f48b..be0de309 100644 --- a/packages/web/src/hooks/useFiles.ts +++ b/packages/web/src/hooks/useFiles.ts @@ -3,7 +3,7 @@ import useFileApi from './useFileApi'; import { UploadedFileType } from 'generative-ai-use-cases-jp'; import { produce } from 'immer'; -const extractBaseURL = (url: string) => { +export const extractBaseURL = (url: string) => { return url.split(/[?#]/)[0]; }; const useFilesState = create<{ diff --git a/packages/web/src/main.tsx b/packages/web/src/main.tsx index 251f771d..a6ac722e 100644 --- a/packages/web/src/main.tsx +++ b/packages/web/src/main.tsx @@ -16,6 +16,7 @@ import SummarizePage from './pages/SummarizePage'; import GenerateTextPage from './pages/GenerateTextPage'; import EditorialPage from './pages/EditorialPage'; import TranslatePage from './pages/TranslatePage'; +import VideoAnalyzerPage from './pages/VideoAnalyzerPage'; import NotFound from './pages/NotFound'; import KendraSearchPage from './pages/KendraSearchPage'; import RagPage from './pages/RagPage'; @@ -81,6 +82,10 @@ const routes: RouteObject[] = [ path: '/transcribe', element: , }, + { + path: '/video', + element: , + }, recognizeFileEnabled ? { path: '/file', diff --git a/packages/web/src/pages/VideoAnalyzerPage.tsx b/packages/web/src/pages/VideoAnalyzerPage.tsx new file mode 100644 index 00000000..d87be461 --- /dev/null +++ b/packages/web/src/pages/VideoAnalyzerPage.tsx @@ -0,0 +1,187 @@ +import React, { + useCallback, + useEffect, + useState, + useRef, + useMemo, +} from 'react'; +import { useLocation } from 'react-router-dom'; +import useChat from '../hooks/useChat'; +import useFileApi from '../hooks/useFileApi'; +import { UploadedFileType } from 'generative-ai-use-cases-jp'; +import { extractBaseURL } from '../hooks/useFiles'; +import { create } from 'zustand'; +import { getPrompter } from '../prompts'; +import { MODELS } from '../hooks/useModel'; +import Textarea from '../components/Textarea'; + +type StateType = { + content: string; + setContent: (c: string) => void; + clear: () => void; +}; + +const useVideoAnalyzerPageState = create((set) => { + const INIT_STATE = { + content: '', + }; + return { + ...INIT_STATE, + setContent: (c: string) => { + set(() => ({ + content: c, + })); + }, + clear: () => { + set(INIT_STATE); + }, + }; +}); + +const VideoAnalyzerPage: React.FC = () => { + const { content, setContent } = useVideoAnalyzerPageState(); + const [intervalId, setIntervalId] = useState(null); + const [mediaStream, setMediaStream] = useState(null); + const videoElement = useRef(null); + const callbackRef = React.useRef<() => void>(); + const { getSignedUrl, uploadFile } = useFileApi(); + const { pathname } = useLocation(); + const { getModelId, setModelId, loading, messages, postChat } = + useChat(pathname); + const modelId = getModelId(); + const prompter = useMemo(() => { + return getPrompter(modelId); + }, [modelId]); + + // もしマルチモーダルではないモデルが選択されていたら、マルチモーダルのモデルにする + useEffect(() => { + if (!MODELS.multiModalModelIds.includes(modelId)) { + setModelId(MODELS.multiModalModelIds[0]); + } + }, [modelId, setModelId]); + + // TODO: query string 対応 + + const startPreview = useCallback(async () => { + try { + if (videoElement.current) { + const stream = await navigator.mediaDevices.getUserMedia({ + video: true, + }); + videoElement.current.srcObject = stream; + videoElement.current.play(); + + setMediaStream(stream); + + const interval = setInterval(() => { + // streaming 中は処理をしない + if (loading) { + return; + } + + const canvas = document.createElement('canvas'); + canvas.width = videoElement.current.videoWidth; + canvas.height = videoElement.current.videoHeight; + const context = canvas.getContext('2d'); + context.drawImage( + videoElement.current, + 0, + 0, + canvas.width, + canvas.height + ); + // toDataURL() で返す値は以下の形式 (;base64, 以降のみを使う) + // ``` + // data:image/png;base64,<以下base64...> + // ``` + const imageBase64 = canvas + .toDataURL('image/png') + .split(';base64,')[1]; + + canvas.toBlob(async (blob) => { + const file = new File([blob], 'tmp.png', { type: 'image/png' }); + const signedUrl = (await getSignedUrl({ mediaFormat: 'png' })).data; + await uploadFile(signedUrl, { file }); + const baseUrl = extractBaseURL(signedUrl); + const uploadedFiles: UploadedFileType[] = [ + { + file, + s3Url: baseUrl, + base64EncodedImage: imageBase64, + }, + ]; + + postChat( + prompter.videoAnalyzerPrompt({ + content, + }), + true, // 履歴は考慮しない + undefined, + undefined, + undefined, + uploadedFiles + ); + }); + }, 10000); + setIntervalId(interval); + } + } catch (e) { + console.error('ウェブカメラにアクセスできませんでした:', e); + } + }, [ + videoElement, + setIntervalId, + content, + loading, + getSignedUrl, + postChat, + prompter, + uploadFile, + ]); + + const stopPreview = useCallback(() => { + if (mediaStream) { + mediaStream.getTracks().forEach((track) => track.stop()); + } + if (intervalId) { + clearInterval(intervalId); + } + }, [mediaStream, intervalId]); + + // Callback 関数を常に最新にしておく + useEffect(() => { + callbackRef.current = stopPreview; + }, [stopPreview]); + + // Unmount 時の処理 + useEffect(() => { + return () => { + if (callbackRef.current) { + callbackRef.current(); + callbackRef.current = undefined; + } + }; + }, []); + + const shownMessages = useMemo(() => { + return messages.reverse().filter((m) => m.role === 'assistant'); + }, [messages]); + + return ( +
+