From 470ed55c23ec5d69bc842c097aa4b89365165881 Mon Sep 17 00:00:00 2001 From: Croos3r Date: Tue, 5 Nov 2024 22:29:02 +0900 Subject: [PATCH 1/8] deps: add react-dropzone for seamless file dropzone implementation and eciesjs for asymmetric encryption Took 9 hours 5 minutes --- front/package.json | 2 ++ yarn.lock | 35 +++++++++++++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/front/package.json b/front/package.json index 48a1295..056006e 100644 --- a/front/package.json +++ b/front/package.json @@ -48,10 +48,12 @@ "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", "cmdk": "1.0.0", + "eciesjs": "^0.4.10", "lucide-react": "^0.453.0", "next": "14.2.15", "react": "^18.3.1", "react-dom": "^18.3.1", + "react-dropzone": "^14.3.4", "sonner": "^1.7.0", "tailwind-merge": "^2.5.3", "tailwindcss-animate": "^1.0.7", diff --git a/yarn.lock b/yarn.lock index 70d6512..6d6dae3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8820,6 +8820,13 @@ __metadata: languageName: node linkType: hard +"attr-accept@npm:^2.2.4": + version: 2.2.4 + resolution: "attr-accept@npm:2.2.4" + checksum: 10c0/602d88b40cb039f1159b86e389ca4f908c13dba513753f7c511e69499ba6216c153519f31a484bac9c9efa633f8f6a4ec25b4f777bd55198f8cb2514cef04618 + languageName: node + linkType: hard + "available-typed-arrays@npm:^1.0.7": version: 1.0.7 resolution: "available-typed-arrays@npm:1.0.7" @@ -10684,7 +10691,7 @@ __metadata: languageName: node linkType: hard -"eciesjs@npm:^0.4.6, eciesjs@npm:^0.4.8": +"eciesjs@npm:^0.4.10, eciesjs@npm:^0.4.6, eciesjs@npm:^0.4.8": version: 0.4.10 resolution: "eciesjs@npm:0.4.10" dependencies: @@ -12083,6 +12090,15 @@ __metadata: languageName: node linkType: hard +"file-selector@npm:^2.1.0": + version: 2.1.0 + resolution: "file-selector@npm:2.1.0" + dependencies: + tslib: "npm:^2.7.0" + checksum: 10c0/6eccfba9e0ac9f2d91fd43678ce120661088bc5cee23a679f67101f979afff71171fc10f35d39916ce244edf1e151451091cee1692a8826d3a33eb4c81cdd4b8 + languageName: node + linkType: hard + "file-uri-to-path@npm:1.0.0": version: 1.0.0 resolution: "file-uri-to-path@npm:1.0.0" @@ -12313,6 +12329,7 @@ __metadata: class-variance-authority: "npm:^0.7.0" clsx: "npm:^2.1.1" cmdk: "npm:1.0.0" + eciesjs: "npm:^0.4.10" eslint: "npm:^8.57.1" eslint-config-next: "npm:14.2.15" eslint-plugin-perfectionist: "npm:^3.9.1" @@ -12325,6 +12342,7 @@ __metadata: prettier: "npm:^3.3.3" react: "npm:^18.3.1" react-dom: "npm:^18.3.1" + react-dropzone: "npm:^14.3.4" sonner: "npm:^1.7.0" storybook: "npm:^8.4.1" tailwind-merge: "npm:^2.5.3" @@ -16776,6 +16794,19 @@ __metadata: languageName: node linkType: hard +"react-dropzone@npm:^14.3.4": + version: 14.3.5 + resolution: "react-dropzone@npm:14.3.5" + dependencies: + attr-accept: "npm:^2.2.4" + file-selector: "npm:^2.1.0" + prop-types: "npm:^15.8.1" + peerDependencies: + react: ">= 16.8 || 18.0.0" + checksum: 10c0/e3e5dddd3bead7c6410bd3fccc3a87e93086ceac47526a2d35421ef7e11a9e59f47c8af8da5c4600a58ef238a5af87c751a71b6391d5c6f77f1f2857946c07cc + languageName: node + linkType: hard + "react-is@npm:^16.13.1": version: 16.13.1 resolution: "react-is@npm:16.13.1" @@ -18792,7 +18823,7 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2.0.0, tslib@npm:^2.0.1, tslib@npm:^2.0.3, tslib@npm:^2.1.0, tslib@npm:^2.3.1, tslib@npm:^2.4.0": +"tslib@npm:^2.0.0, tslib@npm:^2.0.1, tslib@npm:^2.0.3, tslib@npm:^2.1.0, tslib@npm:^2.3.1, tslib@npm:^2.4.0, tslib@npm:^2.7.0": version: 2.8.1 resolution: "tslib@npm:2.8.1" checksum: 10c0/9c4759110a19c53f992d9aae23aac5ced636e99887b51b9e61def52611732872ff7668757d4e4c61f19691e36f4da981cd9485e869b4a7408d689f6bf1f14e62 From b80691bd9d0226b0e42a6ced2a80de353be7c2e7 Mon Sep 17 00:00:00 2001 From: Croos3r Date: Tue, 5 Nov 2024 22:29:40 +0900 Subject: [PATCH 2/8] fix(shrink-eth-address): typo in function's name Took 38 seconds Took 3 minutes --- front/src/components/BedrockAccountMenu.tsx | 4 ++-- front/src/utils/ethereum.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/front/src/components/BedrockAccountMenu.tsx b/front/src/components/BedrockAccountMenu.tsx index 850e766..e8c400d 100644 --- a/front/src/components/BedrockAccountMenu.tsx +++ b/front/src/components/BedrockAccountMenu.tsx @@ -15,7 +15,7 @@ import { DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { SidebarMenuButton } from "@/components/ui/sidebar"; -import { shrinkEtAddress } from "@/utils/ethereum"; +import { shrinkEthAddress } from "@/utils/ethereum"; const BedrockAccountAvatar = () => { const account = useAccount(); @@ -28,7 +28,7 @@ const BedrockAccountAvatar = () => { {/*TODO: add default avatar picture*/}
- {ensName ?? shrinkEtAddress(account.address ?? "")} + {ensName ?? shrinkEthAddress(account.address ?? "")} {privyUser?.email?.address && {privyUser?.email?.address}}
diff --git a/front/src/utils/ethereum.ts b/front/src/utils/ethereum.ts index ac10ae2..560cef5 100644 --- a/front/src/utils/ethereum.ts +++ b/front/src/utils/ethereum.ts @@ -1,3 +1,3 @@ -export const shrinkEtAddress = (address: string) => { +export const shrinkEthAddress = (address: string) => { return `${address.slice(0, 6)}...${address.slice(-4)}`; }; From 12bad6fc591fdce7e165a5347e7f22395ee1a5d9 Mon Sep 17 00:00:00 2001 From: Croos3r Date: Tue, 5 Nov 2024 22:33:47 +0900 Subject: [PATCH 3/8] feat(file-upload): implemented logic for file upload and indexing - AlephService is now provided via BedrockService - AlephService contains new methods to create and fetch aleph data - shadcn ft. sonner Toaster is at the root of the app - EncryptionService now contains new method for encryption/decryption using ecies.js - Renamed useAccountStore to useBedrockAccountStore that now contains the bedrock service instead of the aleph one There is still a test dropzone that needs to be removed in order to display the files Took 45 seconds --- front/src/app/(drive)/page.tsx | 18 +++- front/src/app/layout.tsx | 6 +- .../hooks/useBedrockFileUploadDropzone.tsx | 33 ++++++ front/src/layouts/watchers.tsx | 7 +- front/src/services/aleph.ts | 54 +++++++++- front/src/services/bedrock.ts | 101 ++++++++++++++++++ front/src/services/encryption.ts | 13 ++- front/src/stores/account.ts | 19 ---- front/src/stores/bedrockAccount.ts | 19 ++++ 9 files changed, 237 insertions(+), 33 deletions(-) create mode 100644 front/src/hooks/useBedrockFileUploadDropzone.tsx create mode 100644 front/src/services/bedrock.ts delete mode 100644 front/src/stores/account.ts create mode 100644 front/src/stores/bedrockAccount.ts diff --git a/front/src/app/(drive)/page.tsx b/front/src/app/(drive)/page.tsx index be12a2a..fbbf7f0 100644 --- a/front/src/app/(drive)/page.tsx +++ b/front/src/app/(drive)/page.tsx @@ -1,13 +1,27 @@ +"use client"; + import { SidebarTrigger } from "@/components/ui/sidebar"; +import useBedrockFileUploadDropzone from "@/hooks/useBedrockFileUploadDropzone"; +import { useBedrockAccountStore } from "@/stores/bedrockAccount"; export default function Home() { + const { bedrockService } = useBedrockAccountStore(); + const { getInputProps, getRootProps } = useBedrockFileUploadDropzone({ + current_directory: "/", + bedrockService: bedrockService!, + }); + return ( - <> +
- +
+ +

Upload files here

+
+
); } diff --git a/front/src/app/layout.tsx b/front/src/app/layout.tsx index be86715..8d31999 100644 --- a/front/src/app/layout.tsx +++ b/front/src/app/layout.tsx @@ -2,10 +2,11 @@ import { type Metadata } from "next"; import localFont from "next/font/local"; import { ReactNode } from "react"; -import { Providers } from "@/layouts/providers"; - import "./globals.css"; +import { Toaster } from "@/components/ui/sonner"; +import { Providers } from "@/layouts/providers"; + const geistSans = localFont({ src: "./fonts/GeistVF.woff", variable: "--font-geist-sans", @@ -31,6 +32,7 @@ export default function RootLayout({ {children} + ); diff --git a/front/src/hooks/useBedrockFileUploadDropzone.tsx b/front/src/hooks/useBedrockFileUploadDropzone.tsx new file mode 100644 index 0000000..3dcfeb1 --- /dev/null +++ b/front/src/hooks/useBedrockFileUploadDropzone.tsx @@ -0,0 +1,33 @@ +import { useCallback } from "react"; +import { DropzoneOptions, useDropzone } from "react-dropzone"; +import { toast } from "sonner"; + +import BedrockService, { DirectoryPath } from "@/services/bedrock"; + +export type FileUploadOptions = { current_directory: DirectoryPath; bedrockService: BedrockService } & DropzoneOptions; + +export default function useBedrockFileUploadDropzone({ + current_directory, + bedrockService, + ...options +}: FileUploadOptions) { + const onDrop = useCallback( + async (acceptedFiles: File[]) => { + if (!bedrockService) return; + const fileInfos = await bedrockService.uploadFiles(current_directory, ...acceptedFiles); + let errors = 0; + fileInfos.map( + async (fileInfo) => + await bedrockService.saveFile(fileInfo).catch(() => { + errors += 1; + return toast.error(`Failed to save file ${fileInfo.path}`); + }), + ); + toast.success(`Successfully uploaded ${fileInfos.length - errors} files`); + }, + [current_directory, bedrockService], + ); + + // Placing the onDrop function first allows the user to override it in the hooks' options + return useDropzone({ onDrop, ...options }); +} diff --git a/front/src/layouts/watchers.tsx b/front/src/layouts/watchers.tsx index ef3da16..6a86dfd 100644 --- a/front/src/layouts/watchers.tsx +++ b/front/src/layouts/watchers.tsx @@ -6,14 +6,15 @@ import { useAccountEffect } from "wagmi"; import { wagmiConfig } from "@/config/wagmi"; import { AlephService, BEDROCK_MESSAGE } from "@/services/aleph"; -import { useAccountStore } from "@/stores/account"; +import BedrockService from "@/services/bedrock"; +import { useBedrockAccountStore } from "@/stores/bedrockAccount"; type WatchersProps = { children: ReactNode; }; export function Watchers({ children }: WatchersProps) { - const accountStore = useAccountStore(); + const accountStore = useBedrockAccountStore(); useAccountEffect({ async onConnect() { @@ -24,7 +25,7 @@ export function Watchers({ children }: WatchersProps) { return; } - accountStore.alephService = alephService; + accountStore.bedrockService = new BedrockService(alephService); }, onDisconnect() { accountStore.onDisconnect(); diff --git a/front/src/services/aleph.ts b/front/src/services/aleph.ts index 17be1ea..bcf1d94 100644 --- a/front/src/services/aleph.ts +++ b/front/src/services/aleph.ts @@ -1,6 +1,6 @@ import { AuthenticatedAlephHttpClient } from "@aleph-sdk/client"; import { ETHAccount, getAccountFromProvider, importAccountFromPrivateKey } from "@aleph-sdk/ethereum"; -import { ForgetMessage, ItemType, StoreMessage } from "@aleph-sdk/message"; +import { AggregateMessage, ForgetMessage, ItemType, PostMessage, StoreMessage } from "@aleph-sdk/message"; import { PrivateKey } from "eciesjs"; import { SignMessageReturnType } from "viem"; import web3 from "web3"; @@ -15,12 +15,9 @@ const ALEPH_GENERAL_CHANNEL = env.ALEPH_GENERAL_CHANNEL; export class AlephService { constructor( - /* eslint-disable-next-line no-unused-vars */ private account: ETHAccount, - /* eslint-disable-next-line no-unused-vars */ private subAccountClient: AuthenticatedAlephHttpClient, - // eslint-disable-next-line no-unused-vars - private encryptionPrivateKey: PrivateKey, + public encryptionPrivateKey: PrivateKey, ) {} static async initialize(hash: SignMessageReturnType) { @@ -119,4 +116,51 @@ export class AlephService { async deleteFiles(itemHashes: string[]): Promise { return this.subAccountClient.forget({ hashes: itemHashes }); } + + async createAggregate>(key: string, content: T): Promise> { + return this.subAccountClient.createAggregate({ key, content, channel: ALEPH_GENERAL_CHANNEL }); + } + + async fetchAggregate(key: string, schema: T) { + try { + return schema.parse(await this.subAccountClient.fetchAggregate(this.account.address, key)) as z.infer; + } catch (_) { + return schema.parse(undefined) as z.infer; + } + } + + async updateAggregate>( + key: string, + schema: S, + update_content: (content: T) => T, + ): Promise> { + const currentContent = await this.fetchAggregate(key, schema); + const newContent = update_content(currentContent); + + return this.createAggregate(key, newContent); + } + + async createPost>(type: string, content: T): Promise> { + return this.subAccountClient.createPost({ + postType: type, + content, + channel: ALEPH_GENERAL_CHANNEL, + }); + } + + async fetchPosts( + type: string, + schema: T, + addresses: string[] = [this.account.address], + hashes: string[] = [], + ) { + return schema.parse( + await this.subAccountClient.getPost({ + channels: [ALEPH_GENERAL_CHANNEL], + types: [type], + addresses, + hashes, + }), + ) as z.infer[]; + } } diff --git a/front/src/services/bedrock.ts b/front/src/services/bedrock.ts new file mode 100644 index 0000000..d61a645 --- /dev/null +++ b/front/src/services/bedrock.ts @@ -0,0 +1,101 @@ +import { toast } from "sonner"; +import { z } from "zod"; + +import { AlephService } from "@/services/aleph"; +import { EncryptionService } from "@/services/encryption"; + +export const FileEntrySchema = z.object({ + path: z.string().regex(/^(\/[A-Za-z0-9-_+.]+)+$/, { + message: "Invalid URI format. It should start with '/', contain valid path segments and do not end with a '/'.", + }), + post_hash: z.string().regex(/^[a-f0-9]{64}$/, { + message: "Invalid hash format. It should be a 64-character hexadecimal string.", + }), +}); + +export const FileMetaSchema = z.object({ + key: z.string().regex(/^[a-f0-9]{64}$/, { + message: "Invalid key format. It should be a 64-character hexadecimal string.", + }), + iv: z.string().regex(/^[a-f0-9]{32}$/, { + message: "Invalid IV format. It should be a 32-character hexadecimal string.", + }), + ipfs_hash: z.string().regex(/^[a-f0-9]{64}$/, { + message: "Invalid hash format. It should be a 64-character hexadecimal string.", + }), +}); + +export type FileEntry = z.infer; +export type FileMeta = z.infer; +export type FileFullInfos = FileEntry & FileMeta; +export type DirectoryPath = `${string}/`; + +export default class BedrockService { + constructor(private alephService: AlephService) {} + + async uploadFiles(directory_path: DirectoryPath, ...files: File[]): Promise[]> { + const results = await Promise.allSettled( + files + .map((file) => ({ + file, + key: EncryptionService.generateKey(), + iv: EncryptionService.generateIv(), + })) + .map(({ file, key, iv }) => ({ file: EncryptionService.encryptFile(file, key, iv), iv, key })) + .map(async ({ file, key, iv }, index) => { + const { item_hash } = await this.alephService.uploadFile(await file); + return { + key: key.toString("hex"), + iv: iv.toString("hex"), + ipfs_hash: item_hash, + path: `${directory_path}${files[index].name}`, + }; + }), + ); + + return results + .filter((result, index) => { + if (result.status === "rejected") { + console.error(`Failed to upload file: ${files[index].name}`, result.reason); + toast.error(`Failed to upload file: ${files[index].name}`); + return false; + } + return true; + }) + .map((result) => (result as PromiseFulfilledResult>).value); + } + + // The difference between this method and the uploadFiles method is that this method saves the file hash to blockchain so that it can be fetched later + async saveFile(fileInfos: Omit): Promise { + const { key, iv, ipfs_hash, path } = FileEntrySchema.omit({ post_hash: true }).and(FileMetaSchema).parse(fileInfos); // Validate the input because fileEntry can be built without using the parse method + + const { item_hash } = await this.alephService.createPost("bedrock_file", { + key: EncryptionService.encryptEcies(key, this.alephService.encryptionPrivateKey.publicKey.compressed), + iv: EncryptionService.encryptEcies(iv, this.alephService.encryptionPrivateKey.publicKey.compressed), + ipfs_hash, + }); + await this.alephService.updateAggregate( + "bedrock_file_entries", + z.array(FileEntrySchema).default([]), + (oldContent) => { + if (oldContent.some((entry) => entry.path === path)) throw new Error(`File already exists at path: ${path}`); + return [ + ...oldContent, + { + post_hash: item_hash, + path: EncryptionService.encryptEcies(path, this.alephService.encryptionPrivateKey.publicKey.compressed), + }, + ]; + }, + ); + } + + async fetchFileEntries(): Promise { + return (await this.alephService.fetchAggregate("bedrock_file_entries", z.array(FileEntrySchema).default([]))).map( + ({ post_hash, path }) => ({ + post_hash, + path: EncryptionService.decryptEcies(path, this.alephService.encryptionPrivateKey.secret), + }), + ); + } +} diff --git a/front/src/services/encryption.ts b/front/src/services/encryption.ts index 1780a37..90c83df 100644 --- a/front/src/services/encryption.ts +++ b/front/src/services/encryption.ts @@ -1,4 +1,5 @@ import { createCipheriv, createDecipheriv, randomBytes } from "crypto"; +import { decrypt as eciesDecrypt, encrypt as eciesEncrypt } from "eciesjs"; // Global settings const ALGORITHM = "aes-256-cbc"; @@ -14,13 +15,21 @@ export class EncryptionService { return encrypted; } - static decrypt(encryptedText: string, key: Buffer, iv: Buffer): string { + static decrypt(encryptedData: string, key: Buffer, iv: Buffer): string { const decipher = createDecipheriv(ALGORITHM, key, iv); - let decrypted = decipher.update(encryptedText, OUTPUT_ENCODING, INPUT_ENCODING); + let decrypted = decipher.update(encryptedData, OUTPUT_ENCODING, INPUT_ENCODING); decrypted += decipher.final(INPUT_ENCODING); return decrypted; } + static encryptEcies(data: string, key: Buffer): string { + return eciesEncrypt(key, Buffer.from(data, INPUT_ENCODING)).toString(OUTPUT_ENCODING); + } + + static decryptEcies(data: string, key: Buffer): string { + return eciesDecrypt(key, Buffer.from(data, OUTPUT_ENCODING)).toString(INPUT_ENCODING); + } + static async encryptFile(file: File, key: Buffer, iv: Buffer): Promise { const arrayBuffer = await file.arrayBuffer(); const bufferString = Buffer.from(arrayBuffer).toString("hex"); diff --git a/front/src/stores/account.ts b/front/src/stores/account.ts deleted file mode 100644 index 2b7bcfe..0000000 --- a/front/src/stores/account.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { create } from "zustand"; - -import { AlephService } from "@/services/aleph"; - -type AccountStoreState = { - alephService: AlephService | null; -}; - -type AccountStoreActions = { - onDisconnect: () => void; -}; - -export const useAccountStore = create((set) => ({ - alephService: null, - - onDisconnect: () => { - set({ alephService: null }); - }, -})); diff --git a/front/src/stores/bedrockAccount.ts b/front/src/stores/bedrockAccount.ts new file mode 100644 index 0000000..583492d --- /dev/null +++ b/front/src/stores/bedrockAccount.ts @@ -0,0 +1,19 @@ +import { create } from "zustand"; + +import BedrockService from "@/services/bedrock"; + +type AccountStoreState = { + bedrockService: BedrockService | null; +}; + +type AccountStoreActions = { + onDisconnect: () => void; +}; + +export const useBedrockAccountStore = create((set) => ({ + bedrockService: null, + + onDisconnect: () => { + set({ bedrockService: null }); + }, +})); From d628283c882dad1c91b257334cd0ba50035cc0b9 Mon Sep 17 00:00:00 2001 From: Croos3r Date: Tue, 5 Nov 2024 22:51:32 +0900 Subject: [PATCH 4/8] feat(file-upload): added some typing and constants unifying Took 17 minutes --- front/src/services/bedrock.ts | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/front/src/services/bedrock.ts b/front/src/services/bedrock.ts index d61a645..75e2efe 100644 --- a/front/src/services/bedrock.ts +++ b/front/src/services/bedrock.ts @@ -25,6 +25,11 @@ export const FileMetaSchema = z.object({ }), }); +export const FileEntriesSchema = z.array(FileEntrySchema).default([]); + +const FILE_ENTRIES_AGGREGATE_KEY = "bedrock_file_entries"; +const FILE_POST_TYPE = "bedrock_file"; + export type FileEntry = z.infer; export type FileMeta = z.infer; export type FileFullInfos = FileEntry & FileMeta; @@ -69,29 +74,25 @@ export default class BedrockService { async saveFile(fileInfos: Omit): Promise { const { key, iv, ipfs_hash, path } = FileEntrySchema.omit({ post_hash: true }).and(FileMetaSchema).parse(fileInfos); // Validate the input because fileEntry can be built without using the parse method - const { item_hash } = await this.alephService.createPost("bedrock_file", { + const { item_hash } = await this.alephService.createPost(FILE_POST_TYPE, { key: EncryptionService.encryptEcies(key, this.alephService.encryptionPrivateKey.publicKey.compressed), iv: EncryptionService.encryptEcies(iv, this.alephService.encryptionPrivateKey.publicKey.compressed), ipfs_hash, }); - await this.alephService.updateAggregate( - "bedrock_file_entries", - z.array(FileEntrySchema).default([]), - (oldContent) => { - if (oldContent.some((entry) => entry.path === path)) throw new Error(`File already exists at path: ${path}`); - return [ - ...oldContent, - { - post_hash: item_hash, - path: EncryptionService.encryptEcies(path, this.alephService.encryptionPrivateKey.publicKey.compressed), - }, - ]; - }, - ); + await this.alephService.updateAggregate(FILE_ENTRIES_AGGREGATE_KEY, FileEntriesSchema, (oldContent) => { + if (oldContent.some((entry) => entry.path === path)) throw new Error(`File already exists at path: ${path}`); + return [ + ...oldContent, + { + post_hash: item_hash, + path: EncryptionService.encryptEcies(path, this.alephService.encryptionPrivateKey.publicKey.compressed), + }, + ]; + }); } async fetchFileEntries(): Promise { - return (await this.alephService.fetchAggregate("bedrock_file_entries", z.array(FileEntrySchema).default([]))).map( + return (await this.alephService.fetchAggregate(FILE_ENTRIES_AGGREGATE_KEY, FileEntriesSchema)).map( ({ post_hash, path }) => ({ post_hash, path: EncryptionService.decryptEcies(path, this.alephService.encryptionPrivateKey.secret), From 0b2c34dcd35e1cb77d23d62d6d2528cf57967616 Mon Sep 17 00:00:00 2001 From: Croos3r Date: Wed, 6 Nov 2024 15:12:12 +0900 Subject: [PATCH 5/8] fix(file-upload): last bugs Took 5 hours 17 minutes --- front/src/app/(drive)/page.tsx | 10 ++ .../hooks/useBedrockFileUploadDropzone.tsx | 24 ++-- front/src/layouts/watchers.tsx | 2 +- front/src/services/aleph.ts | 19 +-- front/src/services/bedrock.ts | 115 ++++++++++++++---- front/src/stores/bedrockAccount.ts | 5 +- 6 files changed, 135 insertions(+), 40 deletions(-) diff --git a/front/src/app/(drive)/page.tsx b/front/src/app/(drive)/page.tsx index fbbf7f0..8098a53 100644 --- a/front/src/app/(drive)/page.tsx +++ b/front/src/app/(drive)/page.tsx @@ -1,5 +1,7 @@ "use client"; +import { useCallback } from "react"; /*TODO: REMOVE TESTING AFTER REVIEW*/ + import { SidebarTrigger } from "@/components/ui/sidebar"; import useBedrockFileUploadDropzone from "@/hooks/useBedrockFileUploadDropzone"; import { useBedrockAccountStore } from "@/stores/bedrockAccount"; @@ -11,6 +13,12 @@ export default function Home() { bedrockService: bedrockService!, }); + const onFetch = useCallback(async () => { + /*TODO: REMOVE TESTING AFTER REVIEW*/ + if (bedrockService === null) return; + console.log("fileEntries", await bedrockService!.fetchFileEntries()); + }, [bedrockService]); + return (
@@ -18,7 +26,9 @@ export default function Home() {
+ {/*TODO: REMOVE TESTING AFTER REVIEW*/}
+ {/*TODO: REMOVE TESTING AFTER REVIEW*/}

Upload files here

diff --git a/front/src/hooks/useBedrockFileUploadDropzone.tsx b/front/src/hooks/useBedrockFileUploadDropzone.tsx index 3dcfeb1..08e52a5 100644 --- a/front/src/hooks/useBedrockFileUploadDropzone.tsx +++ b/front/src/hooks/useBedrockFileUploadDropzone.tsx @@ -14,16 +14,20 @@ export default function useBedrockFileUploadDropzone({ const onDrop = useCallback( async (acceptedFiles: File[]) => { if (!bedrockService) return; - const fileInfos = await bedrockService.uploadFiles(current_directory, ...acceptedFiles); - let errors = 0; - fileInfos.map( - async (fileInfo) => - await bedrockService.saveFile(fileInfo).catch(() => { - errors += 1; - return toast.error(`Failed to save file ${fileInfo.path}`); - }), - ); - toast.success(`Successfully uploaded ${fileInfos.length - errors} files`); + const fileInfos = bedrockService.uploadFiles(current_directory, ...acceptedFiles); + toast.promise(fileInfos, { + loading: `Uploading ${acceptedFiles.length} files...`, + success: (uploadedFiles) => `Successfully uploaded ${uploadedFiles.length} files`, + error: (err) => `Failed to upload files: ${err}`, + }); + const awaitedFileInfos = await fileInfos; + if (awaitedFileInfos.length === 0) return; + const fileEntries = bedrockService.saveFiles(...awaitedFileInfos); + toast.promise(fileEntries, { + loading: `Saving ${awaitedFileInfos.length} files...`, + success: (savedFiles) => `Successfully saved ${savedFiles.length} files`, + error: (err) => `Failed to save files: ${err}`, + }); }, [current_directory, bedrockService], ); diff --git a/front/src/layouts/watchers.tsx b/front/src/layouts/watchers.tsx index 6a86dfd..532e358 100644 --- a/front/src/layouts/watchers.tsx +++ b/front/src/layouts/watchers.tsx @@ -25,7 +25,7 @@ export function Watchers({ children }: WatchersProps) { return; } - accountStore.bedrockService = new BedrockService(alephService); + accountStore.connect(new BedrockService(alephService)); }, onDisconnect() { accountStore.onDisconnect(); diff --git a/front/src/services/aleph.ts b/front/src/services/aleph.ts index bcf1d94..0d5e976 100644 --- a/front/src/services/aleph.ts +++ b/front/src/services/aleph.ts @@ -118,15 +118,20 @@ export class AlephService { } async createAggregate>(key: string, content: T): Promise> { - return this.subAccountClient.createAggregate({ key, content, channel: ALEPH_GENERAL_CHANNEL }); + return this.subAccountClient.createAggregate({ + key, + content, + channel: ALEPH_GENERAL_CHANNEL, + address: this.account.address, + }); } async fetchAggregate(key: string, schema: T) { - try { - return schema.parse(await this.subAccountClient.fetchAggregate(this.account.address, key)) as z.infer; - } catch (_) { - return schema.parse(undefined) as z.infer; - } + const { success, data, error } = schema.safeParse( + await this.subAccountClient.fetchAggregate(this.account.address, key).catch(() => {}), + ); + if (!success) throw new Error(`Invalid data from Aleph: ${error.message}`); + return data as z.infer; } async updateAggregate>( @@ -137,7 +142,7 @@ export class AlephService { const currentContent = await this.fetchAggregate(key, schema); const newContent = update_content(currentContent); - return this.createAggregate(key, newContent); + return await this.createAggregate(key, newContent); } async createPost>(type: string, content: T): Promise> { diff --git a/front/src/services/bedrock.ts b/front/src/services/bedrock.ts index 75e2efe..b3d295c 100644 --- a/front/src/services/bedrock.ts +++ b/front/src/services/bedrock.ts @@ -13,6 +13,13 @@ export const FileEntrySchema = z.object({ }), }); +export const EncryptedFileEntrySchema = z.object({ + path: z.string().regex(/^[a-f0-9]+$/, { message: "Invalid data, it should be encrypted and in hex format." }), + post_hash: z.string().regex(/^[a-f0-9]{64}$/, { + message: "Invalid hash format. It should be a 64-character hexadecimal string.", + }), +}); + export const FileMetaSchema = z.object({ key: z.string().regex(/^[a-f0-9]{64}$/, { message: "Invalid key format. It should be a 64-character hexadecimal string.", @@ -25,41 +32,71 @@ export const FileMetaSchema = z.object({ }), }); -export const FileEntriesSchema = z.array(FileEntrySchema).default([]); +export const EncryptedFileMetaSchema = z.object({ + key: z.string().regex(/^[a-f0-9]+$/, { message: "Invalid data, it should be encrypted and in hex format." }), + iv: z.string().regex(/^[a-f0-9]+$/, { message: "Invalid data, it should be encrypted and in hex format." }), + ipfs_hash: z.string().regex(/^[a-f0-9]+$/, { message: "Invalid data, it should be encrypted and in hex format." }), +}); + +export const FileFullInfosSchema = FileEntrySchema.merge(FileMetaSchema); +export const EncryptedFileFullInfosSchema = EncryptedFileEntrySchema.merge(EncryptedFileMetaSchema); + +export const FileEntriesSchema = z.object({ + files: z.array(FileEntrySchema).default([]), +}); + +export const EncryptedFileEntriesSchema = z.object({ + files: z.array(EncryptedFileEntrySchema).default([]), +}); const FILE_ENTRIES_AGGREGATE_KEY = "bedrock_file_entries"; const FILE_POST_TYPE = "bedrock_file"; export type FileEntry = z.infer; export type FileMeta = z.infer; -export type FileFullInfos = FileEntry & FileMeta; +export type EncryptedFileEntry = z.infer; +export type EncryptedFileMeta = z.infer; +export type FileFullInfos = z.infer; export type DirectoryPath = `${string}/`; export default class BedrockService { constructor(private alephService: AlephService) {} async uploadFiles(directory_path: DirectoryPath, ...files: File[]): Promise[]> { + const uploadedFiles = await this.fetchFileEntries(); const results = await Promise.allSettled( files - .map((file) => ({ + .map((file) => ({ file, path: `${directory_path}${file.name}` })) + .filter(({ path }) => { + if (this.fileExists(uploadedFiles, path)) { + console.error(`File already exists at path: ${path}`); + toast.error(`File already exists at path: ${path}`); + return false; + } + return true; + }) + .map(({ file, path }) => ({ file, key: EncryptionService.generateKey(), iv: EncryptionService.generateIv(), + path, })) - .map(({ file, key, iv }) => ({ file: EncryptionService.encryptFile(file, key, iv), iv, key })) - .map(async ({ file, key, iv }, index) => { + .map(({ file, key, iv, path }) => ({ file: EncryptionService.encryptFile(file, key, iv), iv, key, path })) + .map(async ({ file, key, iv, path }) => { const { item_hash } = await this.alephService.uploadFile(await file); return { key: key.toString("hex"), iv: iv.toString("hex"), ipfs_hash: item_hash, - path: `${directory_path}${files[index].name}`, + path, }; }), ); + if (results.length === 0) return Promise.reject("No files were uploaded"); + return results - .filter((result, index) => { + .filter((result, index): result is PromiseFulfilledResult> => { if (result.status === "rejected") { console.error(`Failed to upload file: ${files[index].name}`, result.reason); toast.error(`Failed to upload file: ${files[index].name}`); @@ -67,32 +104,68 @@ export default class BedrockService { } return true; }) - .map((result) => (result as PromiseFulfilledResult>).value); + .map((result) => result.value); } // The difference between this method and the uploadFiles method is that this method saves the file hash to blockchain so that it can be fetched later - async saveFile(fileInfos: Omit): Promise { - const { key, iv, ipfs_hash, path } = FileEntrySchema.omit({ post_hash: true }).and(FileMetaSchema).parse(fileInfos); // Validate the input because fileEntry can be built without using the parse method + async saveFiles(...fileInfos: Omit[]): Promise { + fileInfos = z.array(FileFullInfosSchema.omit({ post_hash: true })).parse(fileInfos); // Validate the input because fileEntry can be built without using the parse method + const fileEntries = ( + await Promise.allSettled( + fileInfos.map(async ({ path, ...rest }) => ({ post_hash: await this.postFile(rest), path })), + ) + ) + .filter((result): result is PromiseFulfilledResult => result.status === "fulfilled") + .map((result) => result.value); + let newFiles: FileEntry[] = []; + await this.alephService.updateAggregate(FILE_ENTRIES_AGGREGATE_KEY, EncryptedFileEntriesSchema, ({ files }) => { + const decryptedFiles = this.decryptFilesPaths(files); + newFiles = fileEntries.filter(({ path }) => { + if (this.fileExists(decryptedFiles, path)) { + console.error(`File already exists at path: ${path}`); + toast.error(`File already exists at path: ${path}`); + return false; + } + return true; + }); + + return { + files: [ + ...files, + ...newFiles.map(({ post_hash, path }) => ({ + post_hash, + path: EncryptionService.encryptEcies(path, this.alephService.encryptionPrivateKey.publicKey.compressed), + })), + ], + }; + }); + if (newFiles.length === 0) return Promise.reject("No new files were saved"); + return newFiles; + } + + private async postFile({ key, iv, ipfs_hash }: Omit): Promise { const { item_hash } = await this.alephService.createPost(FILE_POST_TYPE, { key: EncryptionService.encryptEcies(key, this.alephService.encryptionPrivateKey.publicKey.compressed), iv: EncryptionService.encryptEcies(iv, this.alephService.encryptionPrivateKey.publicKey.compressed), ipfs_hash, }); - await this.alephService.updateAggregate(FILE_ENTRIES_AGGREGATE_KEY, FileEntriesSchema, (oldContent) => { - if (oldContent.some((entry) => entry.path === path)) throw new Error(`File already exists at path: ${path}`); - return [ - ...oldContent, - { - post_hash: item_hash, - path: EncryptionService.encryptEcies(path, this.alephService.encryptionPrivateKey.publicKey.compressed), - }, - ]; - }); + return item_hash; + } + + fileExists(files: FileEntry[], path: string): boolean { + return files.some((entry) => entry.path === path); + } + + private decryptFilesPaths(files: EncryptedFileEntry[]): FileEntry[] { + return files.map(({ post_hash, path }) => ({ + post_hash, + path: EncryptionService.decryptEcies(path, this.alephService.encryptionPrivateKey.secret), + })); } async fetchFileEntries(): Promise { - return (await this.alephService.fetchAggregate(FILE_ENTRIES_AGGREGATE_KEY, FileEntriesSchema)).map( + return (await this.alephService.fetchAggregate(FILE_ENTRIES_AGGREGATE_KEY, EncryptedFileEntriesSchema)).files.map( ({ post_hash, path }) => ({ post_hash, path: EncryptionService.decryptEcies(path, this.alephService.encryptionPrivateKey.secret), diff --git a/front/src/stores/bedrockAccount.ts b/front/src/stores/bedrockAccount.ts index 583492d..1e8a61b 100644 --- a/front/src/stores/bedrockAccount.ts +++ b/front/src/stores/bedrockAccount.ts @@ -8,11 +8,14 @@ type AccountStoreState = { type AccountStoreActions = { onDisconnect: () => void; + connect: (bedrockService: BedrockService) => void; }; export const useBedrockAccountStore = create((set) => ({ bedrockService: null, - + connect(bedrockService: BedrockService) { + set({ bedrockService }); + }, onDisconnect: () => { set({ bedrockService: null }); }, From a8bc88c790c8f83de573dd4d44199db748ea2458 Mon Sep 17 00:00:00 2001 From: Croos3r Date: Wed, 6 Nov 2024 19:32:40 +0900 Subject: [PATCH 6/8] fix(file-upload): wrong naming for store hash for file meta Took 14 minutes --- front/src/services/bedrock.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/front/src/services/bedrock.ts b/front/src/services/bedrock.ts index b3d295c..3df2a6e 100644 --- a/front/src/services/bedrock.ts +++ b/front/src/services/bedrock.ts @@ -27,7 +27,7 @@ export const FileMetaSchema = z.object({ iv: z.string().regex(/^[a-f0-9]{32}$/, { message: "Invalid IV format. It should be a 32-character hexadecimal string.", }), - ipfs_hash: z.string().regex(/^[a-f0-9]{64}$/, { + store_hash: z.string().regex(/^[a-f0-9]{64}$/, { message: "Invalid hash format. It should be a 64-character hexadecimal string.", }), }); @@ -35,7 +35,7 @@ export const FileMetaSchema = z.object({ export const EncryptedFileMetaSchema = z.object({ key: z.string().regex(/^[a-f0-9]+$/, { message: "Invalid data, it should be encrypted and in hex format." }), iv: z.string().regex(/^[a-f0-9]+$/, { message: "Invalid data, it should be encrypted and in hex format." }), - ipfs_hash: z.string().regex(/^[a-f0-9]+$/, { message: "Invalid data, it should be encrypted and in hex format." }), + store_hash: z.string().regex(/^[a-f0-9]+$/, { message: "Invalid data, it should be encrypted and in hex format." }), }); export const FileFullInfosSchema = FileEntrySchema.merge(FileMetaSchema); @@ -87,7 +87,7 @@ export default class BedrockService { return { key: key.toString("hex"), iv: iv.toString("hex"), - ipfs_hash: item_hash, + store_hash: item_hash, path, }; }), @@ -144,11 +144,11 @@ export default class BedrockService { return newFiles; } - private async postFile({ key, iv, ipfs_hash }: Omit): Promise { + private async postFile({ key, iv, store_hash }: Omit): Promise { const { item_hash } = await this.alephService.createPost(FILE_POST_TYPE, { key: EncryptionService.encryptEcies(key, this.alephService.encryptionPrivateKey.publicKey.compressed), iv: EncryptionService.encryptEcies(iv, this.alephService.encryptionPrivateKey.publicKey.compressed), - ipfs_hash, + store_hash, }); return item_hash; } From 12d92c9dd39b6a4d226530dbfde77af6ed8ae843 Mon Sep 17 00:00:00 2001 From: Croos3r Date: Wed, 6 Nov 2024 19:36:56 +0900 Subject: [PATCH 7/8] fix(file-upload): renamed bedrock account store hook to just account store Took 4 minutes --- front/src/app/(drive)/page.tsx | 4 ++-- front/src/layouts/watchers.tsx | 4 ++-- front/src/stores/bedrockAccount.ts | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/front/src/app/(drive)/page.tsx b/front/src/app/(drive)/page.tsx index 8098a53..f265fdc 100644 --- a/front/src/app/(drive)/page.tsx +++ b/front/src/app/(drive)/page.tsx @@ -4,10 +4,10 @@ import { useCallback } from "react"; /*TODO: REMOVE TESTING AFTER REVIEW*/ import { SidebarTrigger } from "@/components/ui/sidebar"; import useBedrockFileUploadDropzone from "@/hooks/useBedrockFileUploadDropzone"; -import { useBedrockAccountStore } from "@/stores/bedrockAccount"; +import { useAccountStore } from "@/stores/bedrockAccount"; export default function Home() { - const { bedrockService } = useBedrockAccountStore(); + const { bedrockService } = useAccountStore(); const { getInputProps, getRootProps } = useBedrockFileUploadDropzone({ current_directory: "/", bedrockService: bedrockService!, diff --git a/front/src/layouts/watchers.tsx b/front/src/layouts/watchers.tsx index 532e358..15dd95f 100644 --- a/front/src/layouts/watchers.tsx +++ b/front/src/layouts/watchers.tsx @@ -7,14 +7,14 @@ import { useAccountEffect } from "wagmi"; import { wagmiConfig } from "@/config/wagmi"; import { AlephService, BEDROCK_MESSAGE } from "@/services/aleph"; import BedrockService from "@/services/bedrock"; -import { useBedrockAccountStore } from "@/stores/bedrockAccount"; +import { useAccountStore } from "@/stores/bedrockAccount"; type WatchersProps = { children: ReactNode; }; export function Watchers({ children }: WatchersProps) { - const accountStore = useBedrockAccountStore(); + const accountStore = useAccountStore(); useAccountEffect({ async onConnect() { diff --git a/front/src/stores/bedrockAccount.ts b/front/src/stores/bedrockAccount.ts index 1e8a61b..a9ea149 100644 --- a/front/src/stores/bedrockAccount.ts +++ b/front/src/stores/bedrockAccount.ts @@ -11,7 +11,7 @@ type AccountStoreActions = { connect: (bedrockService: BedrockService) => void; }; -export const useBedrockAccountStore = create((set) => ({ +export const useAccountStore = create((set) => ({ bedrockService: null, connect(bedrockService: BedrockService) { set({ bedrockService }); From 13a0ea60ce9d73e36792cd25102faae02ef3e656 Mon Sep 17 00:00:00 2001 From: Croos3r Date: Wed, 6 Nov 2024 19:41:40 +0900 Subject: [PATCH 8/8] fix(file-upload): removed test components in drive page Took 5 minutes --- front/src/app/(drive)/page.tsx | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/front/src/app/(drive)/page.tsx b/front/src/app/(drive)/page.tsx index f265fdc..1adf09a 100644 --- a/front/src/app/(drive)/page.tsx +++ b/front/src/app/(drive)/page.tsx @@ -1,24 +1,8 @@ "use client"; -import { useCallback } from "react"; /*TODO: REMOVE TESTING AFTER REVIEW*/ - import { SidebarTrigger } from "@/components/ui/sidebar"; -import useBedrockFileUploadDropzone from "@/hooks/useBedrockFileUploadDropzone"; -import { useAccountStore } from "@/stores/bedrockAccount"; export default function Home() { - const { bedrockService } = useAccountStore(); - const { getInputProps, getRootProps } = useBedrockFileUploadDropzone({ - current_directory: "/", - bedrockService: bedrockService!, - }); - - const onFetch = useCallback(async () => { - /*TODO: REMOVE TESTING AFTER REVIEW*/ - if (bedrockService === null) return; - console.log("fileEntries", await bedrockService!.fetchFileEntries()); - }, [bedrockService]); - return (
@@ -26,12 +10,6 @@ export default function Home() {
- {/*TODO: REMOVE TESTING AFTER REVIEW*/} -
- {/*TODO: REMOVE TESTING AFTER REVIEW*/} - -

Upload files here

-
); }