Add explicit file length in form data for BLOBs (#156)

* Add explicit file length in form data for BLOBs * Transform Blob into Buffer at beginning to avoid misunderstandings; refactor/clarify method & parameter names * Allow pinning again * Fixed all those Storage bugs, jfc * Upgrade jest * Fix pinning * Don't upgrade npm * add File (polyfill) upload test * add File upload support, fix conditions when to upload, remove transformation to Buffer when uploading Files/Blobs * remove "form-data" dependency * add request error handling * use old blobToBuffer again * fix serialization * revert DEFAULT_API_V2 * Add compatibility with UInt8Array * Add node.js compatibility in blobToBuffer
aleph-im · Dec 22, 2023 · b86ef88 · b86ef88
1 parent 3e20a63
commit b86ef88
Show file tree

Hide file tree

Showing 8 changed files with 3,609 additions and 2,290 deletions.
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -22,9 +22,6 @@ jobs:
               with:
                   node-version: ${{ matrix.node-version }}
 
-            - name: Upgrade NPM
-              run: npm i -g npm
-
             - name: Install dependencies
               run: npm ci
 

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -63,9 +63,10 @@
         "eslint": "^7.32.0",
         "eslint-config-prettier": "^8.5.0",
         "eslint-plugin-prettier": "^4.0.0",
-        "jest": "^27.5.1",
+        "formdata-node": "^6.0.3",
+        "jest": "^29.7.0",
         "prettier": "^2.6.2",
-        "ts-jest": "^27.1.4",
+        "ts-jest": "^29.1.1",
         "ts-node": "^10.9.1",
         "typedoc": "^0.22.15",
         "typescript": "^4.6.3"

diff --git a/src/messages/create/publish.ts b/src/messages/create/publish.ts
@@ -1,7 +1,7 @@
 import shajs from "sha.js";
 
 import { BaseMessage, ItemType } from "../types";
-import axios from "axios";
+import axios, { AxiosError } from "axios";
 import FormDataNode from "form-data";
 import { getSocketPath, stripTrailingSlash } from "../../utils/url";
 
@@ -34,7 +34,7 @@ type PushResponse = {
 };
 
 type PushFileConfiguration = {
-    file: Buffer | Blob;
+    file: Buffer | Uint8Array;
     APIServer: string;
     storageEngine: ItemType;
 };
@@ -69,17 +69,26 @@ export async function PutContentToStorageEngine<T>(configuration: PutConfigurati
 }
 
 async function PushToStorageEngine<T>(configuration: PushConfiguration<T>): Promise<string> {
-    const response = await axios.post<PushResponse>(
-        `${stripTrailingSlash(configuration.APIServer)}/api/v0/${configuration.storageEngine.toLowerCase()}/add_json`,
-        configuration.content,
-        {
-            headers: {
-                "Content-Type": "application/json",
+    try {
+        const response = await axios.post<PushResponse>(
+            `${stripTrailingSlash(
+                configuration.APIServer,
+            )}/api/v0/${configuration.storageEngine.toLowerCase()}/add_json`,
+            configuration.content,
+            {
+                headers: {
+                    "Content-Type": "application/json",
+                },
+                socketPath: getSocketPath(),
             },
-            socketPath: getSocketPath(),
-        },
-    );
-    return response.data.hash;
+        );
+        return response.data.hash;
+    } catch (err) {
+        if (err instanceof AxiosError) {
+            console.error(err.response?.data);
+        }
+        throw err;
+    }
 }
 
 export async function PushFileToStorageEngine(configuration: PushFileConfiguration): Promise<string> {

diff --git a/src/messages/store/pin.ts b/src/messages/store/pin.ts
@@ -36,6 +36,7 @@ export async function Pin(spc: StorePinConfiguration): Promise<StoreMessage> {
         channel: spc.channel,
         fileHash: spc.fileHash,
         APIServer: spc.APIServer || DEFAULT_API_V2,
+        inlineRequested: true,
         storageEngine: ItemType.ipfs,
     });
 }
diff --git a/src/messages/store/publish.ts b/src/messages/store/publish.ts
@@ -6,8 +6,7 @@ import { RequireOnlyOne } from "../../utils/requiredOnlyOne";
 import { DEFAULT_API_V2 } from "../../global";
 import { MessageBuilder } from "../../utils/messageBuilder";
 import { stripTrailingSlash } from "../../utils/url";
-import FormData from "form-data";
-import axios from "axios";
+import axios, { AxiosError } from "axios";
 import { blobToBuffer, calculateSHA256Hash } from "./utils";
 
 /**
@@ -28,7 +27,7 @@ import { blobToBuffer, calculateSHA256Hash } from "./utils";
 type StorePublishConfiguration = {
     channel: string;
     account: Account;
-    fileObject?: Buffer | Blob;
+    fileObject?: Buffer | Blob | File | Uint8Array;
     fileHash?: string;
     storageEngine?: ItemType.ipfs | ItemType.storage;
     inlineRequested?: boolean;
@@ -56,39 +55,46 @@ export async function Publish({
     if (fileObject && fileHash) throw new Error("You can't pin a file and upload it at the same time.");
     if (fileHash && storageEngine !== ItemType.ipfs) throw new Error("You must choose ipfs to pin the file.");
 
-    const myHash = await getHash(fileObject, storageEngine, fileHash, APIServer);
-
+    let hash: string | undefined = fileHash;
+    if (!hash) {
+        const buffer = await processFileObject(fileObject);
+        hash = await getHash(buffer, storageEngine, fileHash, APIServer);
+        if (fileObject instanceof File) {
+            fileObject = new File([buffer], fileObject.name);
+        } else {
+            fileObject = new Blob([buffer]);
+        }
+    }
     const message = await createAndSendStoreMessage(
         account,
         channel,
-        myHash,
+        hash,
         storageEngine,
         APIServer,
         inlineRequested,
         sync,
-        fileObject,
+        fileObject as Blob | File | undefined,
     );
-
     return message;
 }
 
 async function getHash(
-    fileObject: Buffer | Blob | null | undefined,
+    buffer: Buffer | Uint8Array,
     storageEngine: ItemType,
     fileHash: string | undefined,
     APIServer: string,
 ) {
-    if (fileObject && storageEngine !== ItemType.ipfs) {
-        const hash = await processFileObject(fileObject);
+    if (buffer && storageEngine !== ItemType.ipfs) {
+        const hash = calculateSHA256Hash(buffer);
         if (hash === null || hash === undefined) {
             throw new Error("Cannot process file");
         }
         return hash;
-    } else if (fileObject && storageEngine === ItemType.ipfs) {
+    } else if (buffer && storageEngine === ItemType.ipfs) {
         return await PushFileToStorageEngine({
             APIServer,
             storageEngine,
-            file: fileObject,
+            file: buffer,
         });
     } else if (fileHash) {
         return fileHash;
@@ -100,18 +106,18 @@ async function getHash(
 async function createAndSendStoreMessage(
     account: Account,
     channel: string,
-    myHash: string,
+    fileHash: string,
     storageEngine: ItemType,
     APIServer: string,
     inlineRequested: boolean,
     sync: boolean,
-    fileObject: Buffer | Blob | undefined,
+    fileObject: Blob | File | undefined,
 ) {
     const timestamp = Date.now() / 1000;
     const storeContent: StoreContent = {
         address: account.address,
         item_type: storageEngine,
-        item_hash: myHash,
+        item_hash: fileHash,
         time: timestamp,
     };
 
@@ -130,8 +136,7 @@ async function createAndSendStoreMessage(
         inline: inlineRequested,
         APIServer,
     });
-
-    if (ItemType.ipfs === message.item_type) {
+    if (ItemType.ipfs == storageEngine && inlineRequested) {
         await SignAndBroadcast({
             message: message,
             account,
@@ -140,7 +145,7 @@ async function createAndSendStoreMessage(
     } else if (!fileObject) {
         throw new Error("You need to specify a File to upload or a Hash to pin.");
     } else {
-        return await sendMessage(
+        await sendMessage(
             {
                 message: message,
                 account,
@@ -154,13 +159,18 @@ async function createAndSendStoreMessage(
     return message;
 }
 
-async function processFileObject(fileObject: Blob | Buffer | null): Promise<string> {
-    if (!fileObject) throw new Error("fileObject is null");
+async function processFileObject(
+    fileObject: Blob | Buffer | File | Uint8Array | null | undefined,
+): Promise<Buffer | Uint8Array> {
+    if (!fileObject) {
+        throw new Error("fileObject is null");
+    }
 
-    if (fileObject instanceof Blob) {
-        fileObject = await blobToBuffer(fileObject);
+    if (fileObject instanceof Buffer || fileObject instanceof Uint8Array) {
+        return fileObject;
     }
-    return calculateSHA256Hash(fileObject);
+
+    return await blobToBuffer(fileObject);
 }
 
 type SignAndBroadcastConfiguration = {
@@ -170,7 +180,7 @@ type SignAndBroadcastConfiguration = {
     sync: boolean;
 };
 
-async function sendMessage(configuration: SignAndBroadcastConfiguration, fileObject: Blob | Buffer) {
+async function sendMessage(configuration: SignAndBroadcastConfiguration, file: Blob | File): Promise<any> {
     const form = new FormData();
     const metadata = {
         message: {
@@ -180,14 +190,25 @@ async function sendMessage(configuration: SignAndBroadcastConfiguration, fileObj
         sync: configuration.sync,
     };
 
-    form.append("file", fileObject);
+    form.append("file", file);
     form.append("metadata", JSON.stringify(metadata));
 
-    const response = await axios.post(`${stripTrailingSlash(configuration.APIServer)}/api/v0/storage/add_file`, form, {
-        headers: {
-            Accept: "application/json",
-            "Content-Type": "multipart/form-data",
-        },
-    });
-    return response.data;
+    try {
+        const response = await axios.post(
+            `${stripTrailingSlash(configuration.APIServer)}/api/v0/storage/add_file`,
+            form,
+            {
+                headers: {
+                    Accept: "application/json",
+                    "Content-Type": "multipart/form-data",
+                },
+            },
+        );
+        return response.data;
+    } catch (err) {
+        if (err instanceof AxiosError) {
+            console.error(err.response?.data);
+        }
+        throw err;
+    }
 }
diff --git a/src/messages/store/utils.ts b/src/messages/store/utils.ts
@@ -1,6 +1,12 @@
 import shajs from "sha.js";
 
-export async function blobToBuffer(blob: Blob): Promise<Buffer> {
+export async function blobToBuffer(blob: Blob | File): Promise<Buffer> {
+    const isBrowser = typeof FileReader !== "undefined";
+    if (!isBrowser) {
+        const arrayBuffer = await blob.arrayBuffer();
+        return Buffer.from(arrayBuffer);
+    }
+
     return new Promise<Buffer>((resolve, reject) => {
         const reader = new FileReader();
         reader.onload = () => {
@@ -10,6 +16,9 @@ export async function blobToBuffer(blob: Blob): Promise<Buffer> {
                 reject("Failed to convert Blob to Buffer.");
             }
         };
+        if (!(blob instanceof Blob)) {
+            console.log(blob);
+        }
         reader.readAsArrayBuffer(blob);
     });
 }