feat: upload and send images in chat

mlc-ai · Sep 29, 2024 · 713b227 · 713b227
1 parent 7953002
commit 713b227
Show file tree

Hide file tree

Showing 23 changed files with 169 additions and 69 deletions.
diff --git a/app/client/api.ts b/app/client/api.ts
@@ -1,4 +1,7 @@
-import { ChatCompletionFinishReason, CompletionUsage } from "@mlc-ai/web-llm";
+import {
+  ChatCompletionFinishReason,
+  CompletionUsage,
+} from "@neet-nestor/web-llm";
 import { CacheType, Model } from "../store";
 export const ROLES = ["system", "user", "assistant"] as const;
 export type MessageRole = (typeof ROLES)[number];
@@ -12,6 +15,10 @@ export interface MultimodalContent {
   image_url?: {
     url: string;
   };
+  dimension?: {
+    width: number;
+    height: number;
+  };
 }
 
 export interface RequestMessage {

diff --git a/app/client/mlcllm.ts b/app/client/mlcllm.ts
@@ -1,6 +1,9 @@
 import log from "loglevel";
 import { ChatOptions, LLMApi } from "./api";
-import { ChatCompletionFinishReason, CompletionUsage } from "@mlc-ai/web-llm";
+import {
+  ChatCompletionFinishReason,
+  CompletionUsage,
+} from "@neet-nestor/web-llm";
 
 export class MlcLLMApi implements LLMApi {
   private endpoint: string;

diff --git a/app/client/webllm.ts b/app/client/webllm.ts
@@ -12,10 +12,10 @@ import {
   WebWorkerMLCEngine,
   CompletionUsage,
   ChatCompletionFinishReason,
-} from "@mlc-ai/web-llm";
+} from "@neet-nestor/web-llm";
 
 import { ChatOptions, LLMApi, LLMConfig, RequestMessage } from "./api";
-import { LogLevel } from "@mlc-ai/web-llm";
+import { LogLevel } from "@neet-nestor/web-llm";
 import { fixMessage } from "../utils";
 import { DEFAULT_MODELS } from "../constant";
 

diff --git a/app/components/chat.module.scss b/app/components/chat.module.scss
@@ -477,6 +477,8 @@
   box-sizing: border-box;
   border-radius: 10px;
   border: rgba($color: #888, $alpha: 0.2) 1px solid;
+  height: auto;
+  display: block;
 }
 
 

diff --git a/app/components/chat.tsx b/app/components/chat.tsx
@@ -8,7 +8,6 @@ import React, {
   Fragment,
   RefObject,
   useContext,
-  ReactElement,
 } from "react";
 
 import ShareIcon from "../icons/share.svg";
@@ -21,16 +20,12 @@ import LoadingIcon from "../icons/three-dots.svg";
 import LoadingButtonIcon from "../icons/loading.svg";
 import PromptIcon from "../icons/prompt.svg";
 import MaxIcon from "../icons/max.svg";
-import BrainIcon from "../icons/brain.svg";
 import MinIcon from "../icons/min.svg";
 import ResetIcon from "../icons/reload.svg";
 import BreakIcon from "../icons/break.svg";
 import DeleteIcon from "../icons/clear.svg";
-import PinIcon from "../icons/pin.svg";
 import EditIcon from "../icons/rename.svg";
 import ConfirmIcon from "../icons/confirm.svg";
-import InfoIcon from "../icons/info.svg";
-import CancelIcon from "../icons/cancel.svg";
 import ImageIcon from "../icons/image.svg";
 
 import BottomIcon from "../icons/bottom.svg";
@@ -74,7 +69,6 @@ import {
   Modal,
   Popover,
   Selector,
-  Tooltip,
   showConfirm,
   showPrompt,
   showToast,
@@ -96,6 +90,8 @@ import { MultimodalContent } from "../client/api";
 import { Template, useTemplateStore } from "../store/template";
 import Image from "next/image";
 import { MLCLLMContext, WebLLMContext } from "../context";
+import EyeIcon from "../icons/eye.svg";
+import { ChatImage } from "../typing";
 
 export function ScrollDownToast(prop: { show: boolean; onclick: () => void }) {
   return (
@@ -481,7 +477,7 @@ function useScrollToBottom(
 
 export function ChatActions(props: {
   uploadImage: () => void;
-  setAttachImages: (images: string[]) => void;
+  setAttachImages: (images: ChatImage[]) => void;
   setUploading: (uploading: boolean) => void;
   scrollToBottom: () => void;
   showPromptSetting: () => void;
@@ -553,6 +549,7 @@ export function ChatActions(props: {
             title: m.name,
             value: m.name,
             family: m.family,
+            icon: isVisionModel(m.name) ? <EyeIcon /> : undefined,
           }))}
           onClose={() => setShowModelSelector(false)}
           onSelection={(s) => {
@@ -603,7 +600,7 @@ function _Chat() {
   const [hitBottom, setHitBottom] = useState(true);
   const isMobileScreen = useMobileScreen();
   const navigate = useNavigate();
-  const [attachImages, setAttachImages] = useState<string[]>([]);
+  const [attachImages, setAttachImages] = useState<ChatImage[]>([]);
   const [uploading, setUploading] = useState(false);
   const [showEditPromptModal, setShowEditPromptModal] = useState(false);
   const webllm = useContext(WebLLMContext)!;
@@ -971,15 +968,15 @@ function _Chat() {
           event.preventDefault();
           const file = item.getAsFile();
           if (file) {
-            const images: string[] = [];
+            const images: ChatImage[] = [];
             images.push(...attachImages);
             images.push(
-              ...(await new Promise<string[]>((res, rej) => {
+              ...(await new Promise<ChatImage[]>((res, rej) => {
                 setUploading(true);
-                const imagesData: string[] = [];
+                const imagesData: ChatImage[] = [];
                 compressImage(file, 256 * 1024)
-                  .then((dataUrl) => {
-                    imagesData.push(dataUrl);
+                  .then((imageData) => {
+                    imagesData.push(imageData);
                     setUploading(false);
                     res(imagesData);
                   })
@@ -1003,11 +1000,11 @@ function _Chat() {
   );
 
   async function uploadImage() {
-    const images: string[] = [];
+    const images: ChatImage[] = [];
     images.push(...attachImages);
 
     images.push(
-      ...(await new Promise<string[]>((res, rej) => {
+      ...(await new Promise<ChatImage[]>((res, rej) => {
         const fileInput = document.createElement("input");
         fileInput.type = "file";
         fileInput.accept =
@@ -1016,12 +1013,12 @@ function _Chat() {
         fileInput.onchange = (event: any) => {
           setUploading(true);
           const files = event.target.files;
-          const imagesData: string[] = [];
+          const imagesData: ChatImage[] = [];
           for (let i = 0; i < files.length; i++) {
             const file = event.target.files[i];
             compressImage(file, 256 * 1024)
-              .then((dataUrl) => {
-                imagesData.push(dataUrl);
+              .then((imageData) => {
+                imagesData.push(imageData);
                 if (
                   imagesData.length === 3 ||
                   imagesData.length === files.length
@@ -1225,7 +1222,11 @@ function _Chat() {
                                     newContent.push({
                                       type: "image_url",
                                       image_url: {
-                                        url: images[i],
+                                        url: images[i].url,
+                                      },
+                                      dimension: {
+                                        width: images[i].width,
+                                        height: images[i].height,
                                       },
                                     });
                                   }
@@ -1301,7 +1302,9 @@ function _Chat() {
                     {getMessageImages(message).length == 1 && (
                       <Image
                         className={styles["chat-message-item-image"]}
-                        src={getMessageImages(message)[0]}
+                        src={getMessageImages(message)[0].url}
+                        width={getMessageImages(message)[0].width}
+                        height={getMessageImages(message)[0].height}
                         alt=""
                       />
                     )}
@@ -1321,7 +1324,9 @@ function _Chat() {
                                 styles["chat-message-item-image-multi"]
                               }
                               key={index}
-                              src={image}
+                              src={image.url}
+                              width={image.width}
+                              height={image.height}
                               alt=""
                             />
                           );
@@ -1413,7 +1418,7 @@ function _Chat() {
                   <div
                     key={index}
                     className={styles["attach-image"]}
-                    style={{ backgroundImage: `url("${image}")` }}
+                    style={{ backgroundImage: `url("${image.url}")` }}
                   >
                     <div className={styles["attach-image-template"]}>
                       <DeleteImageButton

diff --git a/app/components/exporter.tsx b/app/components/exporter.tsx
@@ -437,7 +437,7 @@ export function ImagePreviewer(props: {
                 {getMessageImages(m).length == 1 && (
                   <img
                     key={i}
-                    src={getMessageImages(m)[0]}
+                    src={getMessageImages(m)[0].url}
                     alt="message"
                     className={styles["message-image"]}
                   />
@@ -451,7 +451,7 @@ export function ImagePreviewer(props: {
                       } as React.CSSProperties
                     }
                   >
-                    {getMessageImages(m).map((src, i) => (
+                    {getMessageImages(m).map(({ url: src }, i) => (
                       <img
                         key={i}
                         src={src}

diff --git a/app/components/home.tsx b/app/components/home.tsx
@@ -13,7 +13,7 @@ import {
   Route,
   useLocation,
 } from "react-router-dom";
-import { ServiceWorkerMLCEngine } from "@mlc-ai/web-llm";
+import { ServiceWorkerMLCEngine } from "@neet-nestor/web-llm";
 
 import MlcIcon from "../icons/mlc.svg";
 import LoadingIcon from "../icons/three-dots.svg";

diff --git a/app/components/settings.tsx b/app/components/settings.tsx
@@ -34,7 +34,7 @@ import { ErrorBoundary } from "./error";
 import { InputRange } from "./input-range";
 import { useNavigate } from "react-router-dom";
 import { nanoid } from "nanoid";
-import { LogLevel } from "@mlc-ai/web-llm";
+import { LogLevel } from "@neet-nestor/web-llm";
 import { WebLLMContext } from "../context";
 
 function EditPromptModal(props: { id: string; onClose: () => void }) {

diff --git a/app/components/template.tsx b/app/components/template.tsx
@@ -238,7 +238,7 @@ export function ContextPrompts(props: {
         const text = getMessageTextContent(context[i]);
         const newContext: MultimodalContent[] = [{ type: "text", text }];
         for (const img of images) {
-          newContext.push({ type: "image_url", image_url: { url: img } });
+          newContext.push({ type: "image_url", image_url: { url: img.url } });
         }
         context[i].content = newContext;
       }

diff --git a/app/components/ui-lib.module.scss b/app/components/ui-lib.module.scss
@@ -112,7 +112,10 @@
     align-items: center;
 
     .list-icon {
-      margin-right: 10px;
+      margin-left: 10px;
+      display: flex;
+      align-items: center;
+      justify-content: center;
     }
 
     .list-item-title {

diff --git a/app/components/ui-lib.tsx b/app/components/ui-lib.tsx
@@ -13,7 +13,7 @@ import MinIcon from "../icons/min.svg";
 import Locale from "../locales";
 
 import { createRoot } from "react-dom/client";
-import React, { HTMLProps, useEffect, useState } from "react";
+import React, { HTMLProps, ReactNode, useEffect, useState } from "react";
 import { IconButton } from "./button";
 
 export function Popover(props: {
@@ -95,7 +95,6 @@ export function ListItem(props: {
       onClick={props.onClick}
     >
       <div className={styles["list-header"]}>
-        {props.icon && <div className={styles["list-icon"]}>{props.icon}</div>}
         <div className={styles["list-item-title"]}>
           <div>{props.title}</div>
           {props.subTitle && (
@@ -104,6 +103,7 @@ export function ListItem(props: {
             </div>
           )}
         </div>
+        {props.icon && <div className={styles["list-icon"]}>{props.icon}</div>}
       </div>
       {props.children}
     </div>
@@ -489,6 +489,7 @@ export function Selector<T>(props: {
     subTitle?: string;
     value: T;
     family?: string;
+    icon?: JSX.Element;
   }>;
   defaultSelectedValue?: T;
   onSelection?: (selection: T[]) => void;
@@ -517,6 +518,7 @@ export function Selector<T>(props: {
                   props.onSelection?.([item.value]);
                   props.onClose?.();
                 }}
+                icon={item.icon}
               >
                 {selected ? (
                   <div

diff --git a/app/constant.ts b/app/constant.ts
@@ -65,7 +65,34 @@ Latex block format: $$e=mc^2$$
 `;
 
 export const DEFAULT_MODELS: ModelRecord[] = [
-  // Llama-3.2 1B
+  // Phi-3.5 Vision
+  {
+    name: "Phi-3.5-vision-instruct-q4f16_1-MLC",
+    display_name: "Phi",
+    provider: "Microsoft",
+    quantization: "q4f16",
+    family: "Phi 3.5 Vision",
+    recommended_config: {
+      temperature: 1,
+      presence_penalty: 0,
+      frequency_penalty: 0,
+      top_p: 1,
+    },
+  },
+  {
+    name: "Phi-3.5-vision-instruct-q4f32_1-MLC",
+    display_name: "Phi",
+    provider: "Microsoft",
+    quantization: "q4f32",
+    family: "Phi 3.5 Vision",
+    recommended_config: {
+      temperature: 1,
+      presence_penalty: 0,
+      frequency_penalty: 0,
+      top_p: 1,
+    },
+  },
+  // Llama-3.2
   {
     name: "Llama-3.2-1B-Instruct-q4f32_1-MLC",
     display_name: "Llama",
@@ -279,7 +306,7 @@ export const DEFAULT_MODELS: ModelRecord[] = [
   },
   {
     name: "Phi-3.5-mini-instruct-q4f16_1-MLC",
-    display_name: "Phi 3.5",
+    display_name: "Phi",
     provider: "Microsoft",
     quantization: "q4f16",
     family: "Phi 3.5 Mini",
@@ -292,7 +319,7 @@ export const DEFAULT_MODELS: ModelRecord[] = [
   },
   {
     name: "Phi-3.5-mini-instruct-q4f32_1-MLC",
-    display_name: "Phi 3.5",
+    display_name: "Phi",
     provider: "Microsoft",
     quantization: "q4f32",
     family: "Phi 3.5 Mini",