Resolve DOO-95 "Images live collab and export serialization dataurl f…

…or now" (#62)

* Images work in live collab.

* Images serialization and deserialization.

* Cleanup and tenant api.

* Fix tests/linting.

* Remove adjust skip.

client/src/components/lib/Canvas.tsx

@@ -391,6 +391,7 @@ export default function Canvas() {
         p1: { x: clientX - width / 2, y: clientY - height / 2 },
         p2: { x: clientX + width / 2, y: clientY + height / 2 },
       });
+      setWebsocketAction(pendingImageElementId, 'addCanvasShape');
       // Unselect current image and reset cursor
       setPendingImageElement('');
       setCursor('');

client/src/components/lib/SaveOpenDropDownMenu.tsx

@@ -8,6 +8,9 @@ import {
 } from '@/stores/CanvasElementsStore';
 import { createElement } from '@/lib/canvasElements/canvasElementUtils';
 import saveAs from 'file-saver';
+import { fileCache } from '@/lib/cache';
+import { BinaryFileData } from '@/types';
+import { commitImageToCache } from '@/lib/image';
 
 /**
  * Component that the save and load button with their functionality in the drop down menu in the canavas
@@ -35,6 +38,12 @@ export const SaveOpenDropDownMenu = () => {
     freehandPoints,
     p1,
     p2,
+    editCanvasElement,
+    textStrings,
+    isImagePlaceds,
+    freehandBounds,
+    angles,
+    fileIds,
   } = useCanvasElementStore([
     'setCanvasElementState',
     'allIds',
@@ -52,6 +61,12 @@ export const SaveOpenDropDownMenu = () => {
     'freehandPoints',
     'p1',
     'p2',
+    'editCanvasElement',
+    'textStrings',
+    'isImagePlaceds',
+    'freehandBounds',
+    'angles',
+    'fileIds',
   ]);
 
   /**
@@ -75,7 +90,10 @@ export const SaveOpenDropDownMenu = () => {
     reader.readAsText(file[0]);
 
     reader.onload = () => {
-      const state: CanvasElementState = JSON.parse(reader.result as string);
+      const { fileData, ...state } = JSON.parse(
+        reader.result as string,
+      ) as CanvasElementState & { fileData: Record<string, BinaryFileData> };
+
       const roughElements: Record<string, CanvasElement['roughElement']> = {};
 
       //create the roughElements
@@ -107,15 +125,36 @@ export const SaveOpenDropDownMenu = () => {
         ).roughElement;
       }
       state.roughElements = roughElements;
+      // Reset all fileIds, they will get get set to show the imageswhen they resolve.
+      const fileIds = state.fileIds;
+      state.fileIds = {};
       setCanvasElementState(state);
+
+      // Populate the cache and images asynchronously
+      Object.entries(fileIds).forEach(([elemId, fileId]) => {
+        const binary = fileId && fileData[fileId];
+        if (!binary) throw new Error('Failed to resolve saved binary images');
+
+        const imageElement = { id: elemId };
+        commitImageToCache(
+          {
+            ...fileData[fileId],
+            lastRetrieved: Date.now(),
+          },
+          imageElement,
+          // Will set fileIds, triggering a rerender. A placeholder
+          // will be shown in the mean time.
+          editCanvasElement,
+        );
+      });
     };
   };
 
   /**
    * Serializes the data and saves it to local disk
    */
   const handleSave = () => {
-    const serializedState = JSON.stringify({
+    const state = {
       allIds,
       types,
       strokeColors,
@@ -131,7 +170,14 @@ export const SaveOpenDropDownMenu = () => {
       freehandPoints,
       p1,
       p2,
-    });
+      textStrings,
+      isImagePlaceds,
+      freehandBounds,
+      angles,
+      fileIds,
+      fileData: fileCache.cache,
+    };
+    const serializedState = JSON.stringify(state);
 
     const blob = new Blob([serializedState], {
       type: 'text/plain;charset=utf-8',
@@ -140,7 +186,7 @@ export const SaveOpenDropDownMenu = () => {
   };
 
   return (
-    <React.Fragment>
+    <>
       <input
         type="file"
         ref={fileInputRef}
@@ -163,6 +209,6 @@ export const SaveOpenDropDownMenu = () => {
       >
         <DownloadIcon /> Save
       </DropdownMenu.Item>
-    </React.Fragment>
+    </>
   );
 };

client/src/components/lib/StableDiffusion/StableDiffusionSheet.tsx

@@ -36,13 +36,10 @@ const StableDiffusionSheet = () => {
       'addCanvasShape',
       'editCanvasElement',
     ]);
-  const { isUsingStableDiffusion, setIsUsingStableDiffusion, zoom, appHeight } =
-    useAppStore([
-      'isUsingStableDiffusion',
-      'setIsUsingStableDiffusion',
-      'zoom',
-      'appHeight',
-    ]);
+  const { isUsingStableDiffusion, setIsUsingStableDiffusion } = useAppStore([
+    'isUsingStableDiffusion',
+    'setIsUsingStableDiffusion',
+  ]);
   const { canvasColor, setTool } = useAppStore(['canvasColor', 'setTool']);
   const {
     selectedElementIds,
@@ -219,7 +216,6 @@ const StableDiffusionSheet = () => {
                   imageFile,
                   addCanvasShape,
                   editCanvasElement,
-                  { zoom, appHeight },
                   true,
                 );
                 // And let the user place the image

client/src/components/lib/ToolBar.tsx

@@ -61,11 +61,7 @@ const ToolButton = ({
   active: boolean;
   children?: React.ReactNode;
 }) => {
-  const { setTool, zoom, appHeight } = useAppStore([
-    'setTool',
-    'zoom',
-    'appHeight',
-  ]);
+  const { setTool } = useAppStore(['setTool']);
   const {
     removeCanvasElements,
     setSelectedElements,
@@ -119,7 +115,6 @@ const ToolButton = ({
             imageFile,
             addCanvasShape,
             editCanvasElement,
-            { zoom, appHeight },
             true,
           );
           // And let the user place the image

client/src/components/lib/ToolButtonSelector.tsx

@@ -88,6 +88,7 @@ const ToolButton = ({
     if (selectedElementId === undefined) {
       // If no element is selected, then set the tool options
       setToolOptions(customizabilityDict);
+      return;
     }
     const { roughElement, fillColor } = createElement(
       selectedElementId,

client/src/hooks/useMultiSelection.tsx

@@ -56,8 +56,14 @@ const getElementsWithinFrame = (
   const { elementIds, p1, p2, angles } = appState;
 
   // Destructure coordinates of the frame
-  const { x: frameX1, y: frameY1 } = frame.p1;
-  const { x: frameX2, y: frameY2 } = frame.p2;
+  let { x: frameX1, y: frameY1 } = frame.p1;
+  let { x: frameX2, y: frameY2 } = frame.p2;
+
+  // Adjust the coordinates so that x1 < x2 and y1 < y2
+  // this is needed in case we draw the frame from right to left or bottom to top and
+  // simplifies the logic of checking whether an element is within the frame
+  [frameX1, frameX2] = [frameX1, frameX2].sort((a, b) => a - b);
+  [frameY1, frameY2] = [frameY1, frameY2].sort((a, b) => a - b);
 
   // Filter element IDs based on whether their positions fall within the frame
   return elementIds.filter((id) => {

client/src/hooks/useSocket.tsx

@@ -7,6 +7,9 @@ import {
 import { createElement } from '@/lib/canvasElements/canvasElementUtils';
 import { useEffect, useRef } from 'react';
 import { useAuthStore } from '@/stores/AuthStore';
+import { fileCache } from '@/lib/cache';
+import { dataURLToFile } from '@/lib/bytes';
+import { commitImageToCache, isSupportedImageFile } from '@/lib/image';
 
 /**
  * Defines a hook that controls all socket related activities
@@ -49,6 +52,7 @@ export const useSocket = () => {
     p1,
     p2,
     textStrings,
+    fileIds,
   } = useCanvasElementStore([
     'addCanvasShape',
     'addCanvasFreehand',
@@ -74,6 +78,7 @@ export const useSocket = () => {
     'p1',
     'p2',
     'textStrings',
+    'fileIds',
   ]);
 
   const socket = useRef<WebsocketClient>();
@@ -104,8 +109,31 @@ export const useSocket = () => {
           angle: element.angle,
         },
       );
-      addCanvasShape(newElement);
-      pushCanvasHistory();
+
+      if (element.type === 'image' && element.imgDataURL) {
+        // Add the file to the cache, and set the image as placed
+        newElement.isImagePlaced = true;
+        const imageFile = dataURLToFile(element.imgDataURL);
+        if (!isSupportedImageFile(imageFile)) {
+          throw new Error('Unsupported image type.');
+        }
+        addCanvasShape(newElement);
+        commitImageToCache(
+          {
+            mimeType: imageFile.type,
+            id: element.id,
+            dataURL: element.imgDataURL,
+            created: Date.now(),
+            lastRetrieved: Date.now(),
+          },
+          newElement,
+          editCanvasElement,
+          false,
+        ).then(pushCanvasHistory);
+      } else {
+        addCanvasShape(newElement);
+        pushCanvasHistory();
+      }
     },
     addCanvasFreehand: (element: CanvasElement) => {
       const newElement = createElement(
@@ -132,6 +160,7 @@ export const useSocket = () => {
         },
         true,
       );
+
       addCanvasFreehand(newElement);
       pushCanvasHistory();
     },
@@ -202,7 +231,49 @@ export const useSocket = () => {
     }
   }, [roomID]);
 
-  // Send message once action gets set. Note: will be changed
+  const processWebsocketAction = async (id: string) => {
+    //Create element to send to other sockets in room
+    const element = createElement(
+      id,
+      p1[id].x,
+      p1[id].y,
+      p2[id].x,
+      p2[id].y,
+      types[id],
+      freehandPoints[id],
+      {
+        stroke: strokeColors[id],
+        fill: fillColors[id],
+        font: fontFamilies[id],
+        size: fontSizes[id],
+        bowing: bowings[id],
+        roughness: roughnesses[id],
+        strokeWidth: strokeWidths[id],
+        fillStyle: fillStyles[id],
+        strokeLineDash: strokeLineDashes[id],
+        opacity: opacities[id],
+        text: textStrings[id],
+        angle: angles[id],
+      },
+      true,
+    );
+
+    if (element.type === 'image') {
+      const imageFileId = fileIds[id];
+      const imageFile = fileCache.cache[imageFileId ?? ''];
+      if (imageFileId === undefined || imageFile === undefined) {
+        throw new Error('Image file not found for transmision');
+      }
+      const imgDataURL = imageFile.dataURL;
+      element.imgDataURL = imgDataURL;
+    }
+
+    delete element.roughElement;
+    socket.current?.sendMsgRoom(action, element);
+    setWebsocketAction('', '');
+  };
+
+  // Send message once action gets set.
   useEffect(() => {
     if (actionElementID === '') return;
 
@@ -218,36 +289,7 @@ export const useSocket = () => {
       setWebsocketAction('', '');
       return;
     }
-
-    //Create element to send to other sockets in room
-    const element = createElement(
-      actionElementID,
-      p1[actionElementID].x,
-      p1[actionElementID].y,
-      p2[actionElementID].x,
-      p2[actionElementID].y,
-      types[actionElementID],
-      freehandPoints[actionElementID],
-      {
-        stroke: strokeColors[actionElementID],
-        fill: fillColors[actionElementID],
-        font: fontFamilies[actionElementID],
-        size: fontSizes[actionElementID],
-        bowing: bowings[actionElementID],
-        roughness: roughnesses[actionElementID],
-        strokeWidth: strokeWidths[actionElementID],
-        fillStyle: fillStyles[actionElementID],
-        strokeLineDash: strokeLineDashes[actionElementID],
-        opacity: opacities[actionElementID],
-        text: textStrings[actionElementID],
-        angle: angles[actionElementID],
-      },
-      true,
-    );
-
-    delete element.roughElement;
-    socket.current?.sendMsgRoom(action, element);
-    setWebsocketAction('', '');
+    processWebsocketAction(actionElementID);
   }, [
     actionElementID,
     action,

client/src/lib/cache.ts

@@ -29,6 +29,10 @@ class FileCache {
     return this.#cache;
   }
 
+  public set cache(newCache: Record<string, BinaryFileData>) {
+    this.#cache = newCache;
+  }
+
   /**
    * Adds the provided file to the cache, corresponding
    * to the specified id.

client/src/lib/canvasElements/renderScene.ts

@@ -122,14 +122,23 @@ export const renderCanvasElements = (
       }
     } else if (type === 'image') {
       if (isImagePlaceds[id]) {
-        const [width, height] = [Math.abs(x2 - x1), Math.abs(y2 - y1)];
-
+        const [width, height] = [x2 - x1, y2 - y1];
         const imgFileId = fileIds[id];
         const img = imgFileId
           ? imageCache.cache.get(imgFileId)?.image
           : undefined;
         if (img !== undefined && !(img instanceof Promise)) {
           ctx.drawImage(img, x1, y1, width, height);
+          // const sX = Math.sign(width);
+          // const sY = Math.sign(height);
+          // ctx.scale(sX, sY);
+          // ctx.drawImage(
+          //   img,
+          //   sX * x1,
+          //   sY * y1,
+          //   Math.abs(width),
+          //   Math.abs(height),
+          // );
         } else {
           drawImagePlaceholder(width, height, ctx);
         }