Feature: File encryption (#65)

* Feature: Add encryptionKey to FileMetadata and remove from Bucket level

* Feature: Encrypt uploads and Decrypt downloads

* Feature: Use correct file encryption key in file invitation

integration_tests/sharing_interactions.spec.ts

@@ -164,7 +164,7 @@ describe('Users sharing data', () => {
     const user2Pk = Buffer.from(user2.identity.public.pubKey).toString('hex');
     const storage1 = new UserStorage(user1, TestStorageConfig);
     await storage1.initMailbox();
-    const storage2 = new UserStorage(user2, { ...TestStorageConfig, debugMode: true });
+    const storage2 = new UserStorage(user2, TestStorageConfig);
     await storage2.initMailbox();
 
     const { txtContent } = await uploadTxtContent(storage1);
@@ -309,7 +309,7 @@ describe('Users sharing data', () => {
 
     // authenticate new user to sync notifications
     const { user: user2 } = await authenticateAnonymousUser();
-    const storage2 = new UserStorage(user2, { ...TestStorageConfig, debugMode: false });
+    const storage2 = new UserStorage(user2, TestStorageConfig);
     await storage2.syncFromTempKey(shareResult.publicKeys[0].tempKey || '');
 
     await new Promise((resolve) => setTimeout(resolve, 1000));

packages/storage/package.json

@@ -55,6 +55,7 @@
     "event-emitter": "^0.3.5",
     "gun": "^0.2020.520",
     "lodash": "^4.17.20",
+    "multibase": "^3.1.0",
     "multihashes": "^3.1.0",
     "pino": "^6.11.0",
     "uuid": "^8.3.2",

packages/storage/src/metadata/gundbMetadataStore.spec.ts

@@ -31,14 +31,9 @@ describe('GunsdbMetadataStore', () => {
     const newSchema = await store.createBucket(bucket, dbId, bucketKey);
     expect(newSchema).to.containSubset({ dbId, slug: bucket });
 
-    // eslint-disable-next-line no-unused-expressions
-    expect(newSchema.encryptionKey).to.not.be.empty;
-
     // test find bucket data
     const foundSchema = await store.findBucket(bucket);
     expect(foundSchema).to.containSubset({ dbId, slug: bucket });
-    expect(Buffer.from(foundSchema?.encryptionKey || '').toString('hex')).to
-      .equal(Buffer.from(newSchema.encryptionKey).toString('hex'));
 
     // ensure list bucket returns all value on fresh initialization
     const newStore = await GundbMetadataStore.fromIdentity(username, password);
@@ -58,6 +53,7 @@ describe('GunsdbMetadataStore', () => {
       bucketSlug,
       dbId,
       path,
+      encryptionKey: '',
     };
 
     await store.upsertFileMetadata(fileMetadata);
@@ -78,6 +74,7 @@ describe('GunsdbMetadataStore', () => {
       bucketSlug: 'personal',
       dbId: 'something',
       path: '/home/case.png',
+      encryptionKey: '',
     };
 
     await store.upsertFileMetadata(fileMetadata);
@@ -104,6 +101,7 @@ describe('GunsdbMetadataStore', () => {
       dbId: 'something',
       path: '/home/case.png',
       sharedBy: 'sharers-pk',
+      encryptionKey: '',
     };
 
     await store.upsertSharedWithMeFile(sharedFileMetadata);

packages/storage/src/metadata/gundbMetadataStore.ts

@@ -21,13 +21,9 @@ if (isNode) {
   require('gun/lib/rindexed');
 }
 
-const crypto = require('crypto-browserify');
-
 // this is an hack to enable using IGunChainReference in async functions
 export type GunChainReference<Data> = Omit<IGunChainReference<Data>, 'then'>;
 
-// 32 bytes aes key + 16 bytes salt/IV + 32 bytes HMAC key
-const BucketEncryptionKeyLength = 32 + 16 + 32;
 const BucketMetadataCollection = 'BucketMetadata';
 const SharedFileMetadataCollection = 'SharedFileMetadata';
 const SharedByMeFileMetadataCollection = 'SharedByMeFileMetadata';
@@ -168,7 +164,6 @@ export class GundbMetadataStore implements UserMetadataStore {
 
     const schema: BucketMetadata = {
       dbId,
-      encryptionKey: crypto.randomBytes(BucketEncryptionKeyLength),
       slug: bucketSlug,
       bucketKey,
     };
@@ -601,10 +596,7 @@ export class GundbMetadataStore implements UserMetadataStore {
 
   private async encryptBucketSchema(schema: BucketMetadata): Promise<EncryptedMetadata> {
     return {
-      data: await this.encrypt(JSON.stringify({
-        ...schema,
-        encryptionKey: Buffer.from(schema.encryptionKey).toString('hex'),
-      })),
+      data: await this.encrypt(JSON.stringify(schema)),
     };
   }
 
@@ -614,9 +606,6 @@ export class GundbMetadataStore implements UserMetadataStore {
       throw new Error('Unknown bucket metadata');
     }
 
-    return {
-      ...gunschema,
-      encryptionKey: Buffer.from(gunschema.encryptionKey, 'hex'),
-    };
+    return gunschema;
   }
 }

packages/storage/src/metadata/metadataStore.ts

@@ -129,12 +129,6 @@ export interface BucketMetadata {
    * unique user specified bucket slug
    */
   slug: string;
-  /**
-   * An 80 bytes encryption key used to encrypt and decrypt buckets storage content.
-   *
-   * 32 bytes aes key + 16 bytes salt/IV + 32 bytes HMAC key
-   */
-  encryptionKey: Uint8Array;
   /**
    * Unique dbId provided by the user storage
    */
@@ -154,6 +148,13 @@ export interface FileMetadata {
   bucketSlug: string;
   dbId: string;
   path: string;
+  /**
+   * An 80 bytes encryption key used to encrypt and decrypt files content.
+   *
+   * 32 bytes aes key + 16 bytes salt/IV + 32 bytes HMAC key
+   * It stored as a base32 multibase encoded string
+   */
+  encryptionKey: string;
 }
 
 /**

packages/storage/src/sharing/sharing.ts

@@ -1,5 +1,6 @@
 import { FullPath, Invitation, InvitationStatus } from '../types';
-import { UserMetadataStore, BucketMetadata } from '../metadata/metadataStore';
+import { UserMetadataStore } from '../metadata/metadataStore';
+import { decodeFileEncryptionKey } from '../utils/fsUtils';
 
 /**
  * Makes invitation objects that could then be
@@ -14,18 +15,9 @@ export const createFileInvitations = async (
 ): Promise<Invitation[]> => {
   const invites:Invitation[] = [];
 
-  const buckets = [];
-  const enhancedPaths:FullPath[] = [];
-
   const bucketsAndEnhancedPaths = await Promise.all(paths.map(async (path) => {
-    // const b = await store.findBucket(path.bucket);
-    //
-    // if (!b) {
-    //   throw new Error('Unable to find bucket metadata');
-    // }
-
     const f = await store.findFileMetadata(path.bucket, path.dbId || '', path.path);
-    const encryptionKey = f?.bucketKey;
+    const encryptionKey = f?.encryptionKey;
     return [encryptionKey, {
       ...path,
       uuid: f?.uuid,
@@ -35,11 +27,11 @@ export const createFileInvitations = async (
   }));
 
   const keys = bucketsAndEnhancedPaths.map((o) => o[0] as string);
-  const keysCleaned: Uint8Array[] = keys.map((k) => {
+  const keysCleaned: string[] = keys.map((k) => {
     if (!k) {
-      throw new Error('Required encryption key not found');
+      throw new Error('Required encryption key for invitation not found');
     }
-    return new TextEncoder().encode(k);
+    return k;
   });
 
   pubkeys.forEach((pubkey) => {

packages/storage/src/types.ts

@@ -128,6 +128,8 @@ export interface OpenUuidFileResponse {
   entry: DirectoryEntry;
 }
 
+export type AddItemDataType = ReadableStream<Uint8Array> | ArrayBuffer | string | Blob;
+
 export interface AddItemFile {
   /**
    * path in the bucket where the file should be uploaded.
@@ -142,7 +144,7 @@ export interface AddItemFile {
    *
    */
   mimeType: string;
-  data: ReadableStream<Uint8Array> | ArrayBuffer | string | Blob;
+  data: AddItemDataType;
   /**
    * progress callback if provided will be called with bytes written to
    * remote while uploading the file.
@@ -291,7 +293,7 @@ export interface Invitation {
   invitationID?: string;
   status: InvitationStatus;
   itemPaths: FullPath[];
-  keys:Uint8Array[];
+  keys: string[];
 }
 
 /**

packages/storage/src/userStorage.spec.ts

@@ -18,11 +18,13 @@ import { BucketMetadata,
 import { makeAsyncIterableString } from './testHelpers';
 import { AddItemsEventData } from './types';
 import { UserStorage } from './userStorage';
+import { decodeFileEncryptionKey, generateFileEncryptionKey, newEncryptedDataWriter } from './utils/fsUtils';
 
 use(chaiAsPromised.default);
 use(chaiSubset.default);
 
 const mockIdentity: Identity = PrivateKey.fromRandom();
+const encryptionKey = generateFileEncryptionKey();
 
 const initStubbedStorage = (): { storage: UserStorage; mockBuckets: Buckets } => {
   const mockBuckets: Buckets = mock();
@@ -85,7 +87,14 @@ const initStubbedStorage = (): { storage: UserStorage; mockBuckets: Buckets } =>
             return Promise.resolve({ ...input, bucketSlug: 'myBucket', dbId: '', path: '/' });
           },
           findFileMetadata(bucketSlug, dbId, path): Promise<FileMetadata | undefined> {
-            return Promise.resolve({ uuid: 'generated-uuid', mimeType: 'generic/type', bucketSlug, dbId, path });
+            return Promise.resolve({
+              uuid: 'generated-uuid',
+              mimeType: 'generic/type',
+              encryptionKey,
+              bucketSlug,
+              dbId,
+              path,
+            });
           },
           findFileMetadataByUuid(): Promise<FileMetadata | undefined> {
             return Promise.resolve({
@@ -94,6 +103,7 @@ const initStubbedStorage = (): { storage: UserStorage; mockBuckets: Buckets } =>
               bucketKey: 'myBucketKey',
               dbId: 'mockThreadId',
               path: '/',
+              encryptionKey,
             });
           },
           setFilePublic(_metadata: FileMetadata): Promise<void> {
@@ -255,8 +265,11 @@ describe('UserStorage', () => {
     it('should return a valid stream of files data', async () => {
       const { storage, mockBuckets } = initStubbedStorage();
       const actualFileContent = "file.txt's file content";
+
       when(mockBuckets.pullPath('myBucketKey', '/file.txt', anything())).thenReturn(
-        makeAsyncIterableString(actualFileContent) as AsyncIterableIterator<Uint8Array>,
+        // eslint-disable-next-line @typescript-eslint/ban-ts-ignore
+        // @ts-ignore
+        newEncryptedDataWriter(actualFileContent, decodeFileEncryptionKey(encryptionKey)),
       );
 
       const result = await storage.openFile({ bucket: 'personal', path: '/file.txt' });
@@ -297,7 +310,9 @@ describe('UserStorage', () => {
       });
 
       when(mockBuckets.pullPath('myBucketKey', anyString(), anything())).thenReturn(
-        makeAsyncIterableString(actualFileContent) as AsyncIterableIterator<Uint8Array>,
+        // eslint-disable-next-line @typescript-eslint/ban-ts-ignore
+        // @ts-ignore
+        newEncryptedDataWriter(actualFileContent, decodeFileEncryptionKey(encryptionKey)),
       );
 
       const mockMembers = new Map<string, PathAccessRole>();

packages/storage/src/userStorage.ts

@@ -47,7 +47,11 @@ import { AcceptInvitationResponse,
   ShareViaPublicKeyResponse,
   TxlSubscribeResponse } from './types';
 import { validateNonEmptyArray } from './utils/assertUtils';
-import { isMetaFileName } from './utils/fsUtils';
+import { decodeFileEncryptionKey,
+  encodeFileEncryptionKey,
+  generateFileEncryptionKey,
+  isMetaFileName, newDecryptedDataReader,
+  newEncryptedDataWriter } from './utils/fsUtils';
 import { filePathFromIpfsPath,
   getParentPath,
   isTopLevelPath,
@@ -231,13 +235,22 @@ export class UserStorage {
    */
   public async createFolder(request: CreateFolderRequest): Promise<void> {
     const client = this.getUserBucketsClient();
+    const metadataStore = await this.getMetadataStore();
 
     const bucket = await this.getOrCreateBucket(client, request.bucket);
     const file = {
       path: `${sanitizePath(request.path.trimStart())}/.keep`,
       content: Buffer.from(''),
     };
 
+    await metadataStore.upsertFileMetadata({
+      uuid: v4(),
+      bucketKey: bucket.root?.key,
+      bucketSlug: bucket.slug,
+      dbId: bucket.dbId,
+      encryptionKey: generateFileEncryptionKey(),
+      path: file.path,
+    });
     await client.pushPath(bucket.root?.key || '', '.keep', file);
   }
 
@@ -404,8 +417,6 @@ export class UserStorage {
    * ```
    */
   public async notificationSubscribe(): Promise<NotificationSubscribeResponse> {
-    const client = this.getUserBucketsClient();
-    // const bucket = await this.(client, request.bucket);
     const emitter = ee();
 
     if (!this.mailbox) {
@@ -490,7 +501,10 @@ export class UserStorage {
     const fileMetadata = await metadataStore.findFileMetadata(bucket.slug, bucket.dbId, path);
 
     try {
-      const fileData = client.pullPath(bucket.root?.key || '', path, { progress: request.progress });
+      const fileData = newDecryptedDataReader(
+        client.pullPath(bucket.root?.key || '', path, { progress: request.progress }),
+        decodeFileEncryptionKey(fileMetadata?.encryptionKey || ''),
+      );
       return {
         stream: fileData,
         consumeStream: () => consumeStream(fileData),
@@ -556,7 +570,10 @@ export class UserStorage {
         fileMetadata.dbId,
       );
 
-      const fileData = client.pullPath(bucketKey, fileMetadata.path, { progress: request.progress });
+      const fileData = newDecryptedDataReader(
+        client.pullPath(bucketKey, fileMetadata.path, { progress: request.progress }),
+        decodeFileEncryptionKey(fileMetadata.encryptionKey || ''),
+      );
 
       const [fileEntryWithmembers] = await UserStorage.addMembersToPathItems(
         [fileEntry],
@@ -766,9 +783,15 @@ export class UserStorage {
             bucketKey: bucket.root?.key,
             bucketSlug: bucket.slug,
             dbId: bucket.dbId,
+            encryptionKey: generateFileEncryptionKey(),
             path,
           });
-          await client.pushPath(rootKey, path, file.data, { progress: file.progress });
+
+          const encryptedDataReader = newEncryptedDataWriter(
+            file.data,
+            decodeFileEncryptionKey(metadata.encryptionKey),
+          );
+          await client.pushPath(rootKey, path, encryptedDataReader, { progress: file.progress });
           // set file entry
           const existingFile = await client.listPath(rootKey, path);
           const [fileEntry] = UserStorage.parsePathItems(
@@ -836,7 +859,7 @@ export class UserStorage {
     const client = this.getUserBucketsClient();
     const metadataStore = await this.getMetadataStore();
 
-    const filesPaths = await Promise.all(invitation.itemPaths.map(async (fullPath) => {
+    const filesPaths = await Promise.all(invitation.itemPaths.map(async (fullPath, index) => {
       const fileMetadata = await metadataStore.upsertSharedWithMeFile({
         bucketKey: fullPath.bucketKey,
         bucketSlug: fullPath.bucket,
@@ -846,6 +869,7 @@ export class UserStorage {
         sharedBy: invitation.inviterPublicKey,
         uuid: fullPath.uuid,
         accepted: accept,
+        encryptionKey: invitation.keys[index],
         invitationId,
       });
 
@@ -1200,6 +1224,7 @@ export class UserStorage {
         dbId: fullPath.dbId || '',
         path: fullPath.path,
         sharedBy: this.user.identity.public.toString(),
+        encryptionKey: fileMetadata?.encryptionKey || '',
         uuid: fileMetadata?.uuid,
       });
     }
@@ -1249,7 +1274,6 @@ export class UserStorage {
     fullPaths: FullPath[],
   ): Promise<{ key: string; fullPath: FullPath; }[]> {
     this.logger.info({ fullPaths }, 'Normalizing full path');
-    const bucketCache = new Map<string, BucketMetadataWithThreads>();
     const store = await this.getMetadataStore();
     return Promise.all(fullPaths.map(async (fullPath) => {
       let rootKey: string;