From 8c00d96024e416bfce7fc45b8e47db674ab41bc8 Mon Sep 17 00:00:00 2001
From: Rohit Malhotra <rohitvinodmalhotra@gmail.com>
Date: Mon, 11 Nov 2024 13:22:17 -0500
Subject: [PATCH] Support displaying images/videos/pdfs in the workspace
 (#4898)

---
 .../_oh.app._index/code-editor-component.tsx  | 38 ++++++++++++++++---
 openhands/runtime/action_execution_server.py  | 29 ++++++++++++++
 2 files changed, 62 insertions(+), 5 deletions(-)

diff --git a/frontend/src/routes/_oh.app._index/code-editor-component.tsx b/frontend/src/routes/_oh.app._index/code-editor-component.tsx
index 8182805193a0..4cc26ad213a2 100644
--- a/frontend/src/routes/_oh.app._index/code-editor-component.tsx
+++ b/frontend/src/routes/_oh.app._index/code-editor-component.tsx
@@ -29,6 +29,10 @@ function CodeEditorCompoonent({
     if (selectedPath && value) modifyFileContent(selectedPath, value);
   };
 
+  const isBase64Image = (content: string) => content.startsWith("data:image/");
+  const isPDF = (content: string) => content.startsWith("data:application/pdf");
+  const isVideo = (content: string) => content.startsWith("data:video/");
+
   React.useEffect(() => {
     const handleSave = async (event: KeyboardEvent) => {
       if (selectedPath && event.metaKey && event.key === "s") {
@@ -62,16 +66,40 @@ function CodeEditorCompoonent({
     );
   }
 
+  const fileContent = modifiedFiles[selectedPath] || files[selectedPath];
+
+  if (isBase64Image(fileContent)) {
+    return (
+      <section className="flex flex-col relative items-center overflow-auto h-[90%]">
+        <img src={fileContent} alt={selectedPath} className="object-contain" />
+      </section>
+    );
+  }
+
+  if (isPDF(fileContent)) {
+    return (
+      <iframe
+        src={fileContent}
+        title={selectedPath}
+        width="100%"
+        height="100%"
+      />
+    );
+  }
+
+  if (isVideo(fileContent)) {
+    return (
+      <video controls src={fileContent} width="100%" height="100%">
+        <track kind="captions" label="English captions" />
+      </video>
+    );
+  }
   return (
     <Editor
       data-testid="code-editor"
       path={selectedPath ?? undefined}
       defaultValue=""
-      value={
-        selectedPath
-          ? modifiedFiles[selectedPath] || files[selectedPath]
-          : undefined
-      }
+      value={selectedPath ? fileContent : undefined}
       onMount={onMount}
       onChange={handleEditorChange}
       options={{ readOnly: isReadOnly }}
diff --git a/openhands/runtime/action_execution_server.py b/openhands/runtime/action_execution_server.py
index 883ea8e95401..2e060337a58e 100644
--- a/openhands/runtime/action_execution_server.py
+++ b/openhands/runtime/action_execution_server.py
@@ -7,7 +7,9 @@
 
 import argparse
 import asyncio
+import base64
 import io
+import mimetypes
 import os
 import shutil
 import tempfile
@@ -217,6 +219,33 @@ async def read(self, action: FileReadAction) -> Observation:
         working_dir = self.bash_session.workdir
         filepath = self._resolve_path(action.path, working_dir)
         try:
+            if filepath.lower().endswith(('.png', '.jpg', '.jpeg', '.bmp', '.gif')):
+                with open(filepath, 'rb') as file:
+                    image_data = file.read()
+                    encoded_image = base64.b64encode(image_data).decode('utf-8')
+                    mime_type, _ = mimetypes.guess_type(filepath)
+                    if mime_type is None:
+                        mime_type = 'image/png'  # default to PNG if mime type cannot be determined
+                    encoded_image = f'data:{mime_type};base64,{encoded_image}'
+
+                return FileReadObservation(path=filepath, content=encoded_image)
+            elif filepath.lower().endswith('.pdf'):
+                with open(filepath, 'rb') as file:
+                    pdf_data = file.read()
+                    encoded_pdf = base64.b64encode(pdf_data).decode('utf-8')
+                    encoded_pdf = f'data:application/pdf;base64,{encoded_pdf}'
+                return FileReadObservation(path=filepath, content=encoded_pdf)
+            elif filepath.lower().endswith(('.mp4', '.webm', '.ogg')):
+                with open(filepath, 'rb') as file:
+                    video_data = file.read()
+                    encoded_video = base64.b64encode(video_data).decode('utf-8')
+                    mime_type, _ = mimetypes.guess_type(filepath)
+                    if mime_type is None:
+                        mime_type = 'video/mp4'  # default to MP4 if MIME type cannot be determined
+                    encoded_video = f'data:{mime_type};base64,{encoded_video}'
+
+                return FileReadObservation(path=filepath, content=encoded_video)
+
             with open(filepath, 'r', encoding='utf-8') as file:
                 lines = read_lines(file.readlines(), action.start, action.end)
         except FileNotFoundError: