Support for custom S3 endpoint

backend/db/S3Personal.ts

@@ -1,11 +1,16 @@
 import AWS from "aws-sdk";
 
-const s3Auth = (id:string, key:string) => {
+const s3Auth = (id:string, key:string, endpoint:string) => {
 
-    AWS.config.update({
+    const s3Config = {
+        endpoint: endpoint || "https://s3.amazonaws.com",
         accessKeyId: id,
-        secretAccessKey: key
-    });
+        secretAccessKey: key,
+        s3ForcePathStyle: !!endpoint,
+        signatureVersion: 'v4'
+    }
+
+    AWS.config.update(s3Config);
 
     const s3 = new AWS.S3();
 

backend/db/s3.ts

@@ -1,10 +1,15 @@
 import AWS from "aws-sdk";
 import env from "../enviroment/env";
 
-AWS.config.update({
+const s3Config = {
+    endpoint: env.s3Endpoint || "https://s3.amazonaws.com",
     accessKeyId: env.s3ID,
-    secretAccessKey: env.s3Key
-});
+    secretAccessKey: env.s3Key,
+    s3ForcePathStyle: !!env.s3Endpoint,
+    signatureVersion: 'v4'
+}
+
+AWS.config.update(s3Config);
 
 const s3 = new AWS.S3();
 

backend/enviroment/env.ts

@@ -13,6 +13,7 @@ export default {
     s3ID: process.env.S3_ID,
     s3Key: process.env.S3_KEY,
     s3Bucket: process.env.S3_BUCKET,
+    s3Endpoint: process.env.S3_ENDPOINT,
     useDocumentDB: process.env.USE_DOCUMENT_DB,
     documentDBBundle: process.env.DOCUMENT_DB_BUNDLE,
     sendgridKey: process.env.SENDGRID_KEY,
@@ -37,6 +38,7 @@ module.exports = {
     s3ID: process.env.S3_ID,
     s3Key: process.env.S3_KEY,
     s3Bucket: process.env.S3_BUCKET,
+    s3Endpoint: process.env.S3_ENDPOINT,
     useDocumentDB: process.env.USE_DOCUMENT_DB,
     documentDBBundle: process.env.DOCUMENT_DB_BUNDLE,
     sendgridKey: process.env.SENDGRID_KEY,

backend/models/user.ts

@@ -107,6 +107,9 @@ const userSchema = new mongoose.Schema({
         bucket: {
             type: String
         },
+        endpoint: {
+            type: String
+        },
         iv: {
             type: Buffer
         }
@@ -160,6 +163,7 @@ export interface UserInterface extends Document {
         id?: string,
         key?: string,
         bucket?: string,
+        endpoint?: string,
         iv?: Buffer,
     },
     storageData?: {
@@ -198,8 +202,8 @@ export interface UserInterface extends Document {
     decryptDriveIDandKey: () => Promise<{clientID: string, clientKey: string}>;
     encryptDriveTokenData: (token: Object) => Promise<void>;
     decryptDriveTokenData: () => Promise<any>;
-    encryptS3Data: (id: string, key: string, bucket: string) => Promise<void>;
-    decryptS3Data: () => Promise<{id: string, key: string, bucket: string}>
+    encryptS3Data: (id: string, key: string, bucket: string, endpoint: string) => Promise<void>;
+    decryptS3Data: () => Promise<{id: string, key: string, bucket: string, endpoint: string}>
 }
 
 const maxAgeAccess =  60 * 1000 * 20 + (1000 * 60);
@@ -524,7 +528,7 @@ userSchema.methods.decryptDriveTokenData = async function() {
    return tokenToObj;
 }
 
-userSchema.methods.encryptS3Data = async function(ID: string, key: string, bucket: string) {
+userSchema.methods.encryptS3Data = async function(ID: string, key: string, bucket: string, endpoint:string) {
 
     const iv = crypto.randomBytes(16);
 
@@ -535,12 +539,14 @@ userSchema.methods.encryptS3Data = async function(ID: string, key: string, bucke
     const encryptedS3ID = user.encryptToken(ID, encryptedKey, iv);
     const encryptedS3Key = user.encryptToken(key, encryptedKey, iv);
     const encryptedS3Bucket = user.encryptToken(bucket, encryptedKey, iv);
+    const encryptedS3Endpoint = user.encryptToken(endpoint, encryptedKey, iv);
 
     if (!user.s3Data) user.s3Data = {};
 
     user.s3Data!.id = encryptedS3ID;
     user.s3Data!.key = encryptedS3Key;
     user.s3Data!.bucket = encryptedS3Bucket;
+    user.s3Data!.endpoint = encryptedS3Endpoint;
     user.s3Data!.iv = iv;
     user.s3Enabled = true;
 
@@ -558,17 +564,20 @@ userSchema.methods.decryptS3Data = async function() {
     const encrytpedS3ID = user.s3Data?.id;
     const encryptedS3Key = user.s3Data?.key;
     const encryptedS3Bucket = user.s3Data?.bucket;
+    const encryptedS3Endpoint = user.s3Data?.endpoint;
 
     const decrytpedS3ID = user.decryptToken(encrytpedS3ID, encryptedKey, iv);
     const decryptedS3Key = user.decryptToken(encryptedS3Key, encryptedKey, iv);
     const decryptedS3Bucket = user.decryptToken(encryptedS3Bucket, encryptedKey, iv);
+    const decryptedS3Endpoint = user.decryptToken(encryptedS3Endpoint, encryptedKey, iv);
 
     //console.log("decrypted keys", decrytpedS3ID, decryptedS3Key, decryptedS3Bucket);
 
     return {
         id: decrytpedS3ID,
         key: decryptedS3Key,
         bucket: decryptedS3Bucket,
+        endpoint: decryptedS3Endpoint,
     }
 }
 

backend/services/ChunkService/S3Service.ts

@@ -165,7 +165,7 @@ class S3Service implements ChunkInterface {
 
             const s3Data = await user.decryptS3Data();
             //console.log("s3 data", s3Data)
-            return {s3Storage: s3Auth(s3Data.id, s3Data.key), bucket: s3Data.bucket};
+            return {s3Storage: s3Auth(s3Data.id, s3Data.key, s3Data.endpoint), bucket: s3Data.bucket};
         } else {
 
             return {s3Storage: s3, bucket: env.s3Bucket};
@@ -178,7 +178,7 @@ class S3Service implements ChunkInterface {
 
             const s3Data = await user.decryptS3Data();
             //console.log("s3 data", s3Data)
-            return {s3Storage: s3Auth(s3Data.id, s3Data.key), bucket: s3Data.bucket};
+            return {s3Storage: s3Auth(s3Data.id, s3Data.key, s3Data.endpoint), bucket: s3Data.bucket};
         } else {
 
             return {s3Storage: s3, bucket: env.s3Bucket};

backend/services/ChunkService/utils/awaitUploadStreamS3.ts

@@ -1,13 +1,13 @@
 import s3 from "../../../db/s3";
 import s3Personal from "../../../db/S3Personal";
 
-const awaitUploadStreamS3 = (params: any, personalFile: boolean, s3Data: {id: string, key: string, bucket: string}) => {
+const awaitUploadStreamS3 = (params: any, personalFile: boolean, s3Data: {id: string, key: string, bucket: string, endpoint:string}) => {
 
     return new Promise((resolve, reject) => {
 
         if (personalFile) {
 
-            const s3PersonalAuth = s3Personal(s3Data.id, s3Data.key);
+            const s3PersonalAuth = s3Personal(s3Data.id, s3Data.key, s3Data.endpoint);
 
             s3PersonalAuth.upload(params, (err: any, data: any) => {
 

backend/services/ChunkService/utils/calculateS3Size.ts

@@ -1,8 +1,8 @@
 import s3Auth from "../../../db/S3Personal";
 
-const calculateS3Size = async(id:string, key:string, bucket:string) => {
+const calculateS3Size = async(id:string, key:string, bucket:string, endpoint:string) => {
 
-    const s3Storage = s3Auth(id, key);
+    const s3Storage = s3Auth(id, key, endpoint);
 
     const params = {
         Bucket: bucket

backend/services/ChunkService/utils/createThumbnailS3.ts

@@ -17,7 +17,7 @@ const getS3Auth = async (file: FileInterface, user: UserInterface) => {
 
         const s3Data = await user.decryptS3Data();
         //console.log("s3 data", s3Data)
-        return {s3Storage: s3Auth(s3Data.id, s3Data.key), bucket: s3Data.bucket};
+        return {s3Storage: s3Auth(s3Data.id, s3Data.key, s3Data.endpoint), bucket: s3Data.bucket};
     } else {
 
         return {s3Storage: s3, bucket: env.s3Bucket};

backend/services/ChunkService/utils/getPrevIVS3.ts

@@ -13,7 +13,7 @@ const getPrevIV = (start: number, key: string, isPersonal: boolean, user: UserIn
 
             const params: any = {Bucket: s3Data.bucket, Key: key, Range: `bytes=${start}-${start + 15}`};
 
-            const s3Storage = s3Auth(s3Data.id, s3Data.key);
+            const s3Storage = s3Auth(s3Data.id, s3Data.key, s3Data.endpoint);
 
             const stream = s3Storage.getObject(params).createReadStream();
 

backend/services/UserPersonalService/index.ts

@@ -20,12 +20,12 @@ class UserPeronsalService {
 
     addS3Storage = async(user: UserInterface, s3Data: any, uuid: string | undefined) => {
 
-        const {id, key, bucket} = s3Data;
+        const {id, key, bucket, endpoint} = s3Data;
 
-        user.storageDataPersonal!.storageSize = await calculateS3Size(id, key, bucket);
+        user.storageDataPersonal!.storageSize = await calculateS3Size(id, key, bucket, endpoint);
         user.personalStorageCanceledDate = undefined;
 
-        await user.encryptS3Data(id, key, bucket);
+        await user.encryptS3Data(id, key, bucket, endpoint);
 
         await user.save();
 

src/components/SettingsPage/index.js

@@ -36,6 +36,7 @@ class SettingsPageContainer extends React.Component {
             s3ID: "",
             s3Key: "",
             s3Bucket: "",
+            s3Endpoint: "",
             addGoogleAccountOpen: false,
             googleID: "",
             googleSecret: "",
@@ -227,6 +228,18 @@ class SettingsPageContainer extends React.Component {
         })
     }
 
+    onChangeS3Endpoint = (e) => {
+
+        const value = e.target.value;
+
+        this.setState(() => {
+            return {
+                ...this.state,
+                s3Endpoint: value
+            }
+        })
+    }
+
     onChangeGoogleID = (e) => {
 
         const value = e.target.value;
@@ -278,14 +291,15 @@ class SettingsPageContainer extends React.Component {
         const data = {
             id: this.state.s3ID,
             bucket: this.state.s3Bucket,
-            key: this.state.s3Key
+            key: this.state.s3Key,
+            endpoint: this.state.s3Endpoint
         }
 
         axios.post("/user-service/add-s3-storage", data).then((response) => {
 
             Swal.fire(
                 'S3 Account Added',
-                'Amazon S3 Account has been linked with myDrive',
+                'S3 compatible storage Account has been linked with myDrive',
                 'success'
               ).then(() => {
                   //window.location.assign(env.url);
@@ -337,7 +351,7 @@ class SettingsPageContainer extends React.Component {
 
         Swal.fire({
             title: 'Remove S3 Account?',
-            text: "This will unlink your Amazon S3 Account from myDrive.",
+            text: "This will unlink your S3 compatible storage Account from myDrive.",
             icon: 'warning',
             showCancelButton: true,
             confirmButtonColor: '#3085d6',
@@ -362,7 +376,7 @@ class SettingsPageContainer extends React.Component {
                 this.showS3Account()
                 Swal.fire(
                     'S3 Account Removed',
-                    'Your Amazon S3 Account has been unlinked from myDrive',
+                    'Your S3 compatible storage Account has been unlinked from myDrive',
                     'success'
                 )
 
@@ -1152,7 +1166,7 @@ class SettingsPageContainer extends React.Component {
           <div class="elem__control--settings">
             <div class="control__title">
               <div class="double__title">
-                <p>Amazon S3</p>
+                <p>S3 compatible storage</p>
                 <span>{!this.state.loaded ? "Loading..." : this.state.userDetails.s3Enabled ? this.state.userDetails.s3Data.bucket : "S3 Not Enabled"}</span>
               </div>
             </div>
@@ -1371,7 +1385,7 @@ class SettingsPageContainer extends React.Component {
             <div class="inner__modal">
                 <div class="password__modal">
                     <div class="head__password">
-                        <h2>{!this.state.loaded ? "Loading..." : this.state.userDetails.s3Enabled ? "Edit Amazon S3 Account" : "Add Amazon S3 Account"}</h2>
+                        <h2>{!this.state.loaded ? "Loading..." : this.state.userDetails.s3Enabled ? "Edit S3 compatible storage Account" : "Add S3 compatible storage Account"}</h2>
                         <div class="close__modal">
                             <a onClick={this.showS3Account}><img src="/assets/close.svg" alt="close"/></a>
                         </div>
@@ -1387,6 +1401,9 @@ class SettingsPageContainer extends React.Component {
                             <div class="group__password" style={!this.state.loaded ? {} : this.state.userDetails.s3Enabled ? {display:"none"} : {display:"block"}}>
                                 <input value={this.state.s3Key} onChange={this.onChangeS3Key} type="password" placeholder="S3 Key"/>
                             </div>
+                            <div class="group__password" style={!this.state.loaded ? {} : this.state.userDetails.s3Enabled ? {display:"none"} : {display:"block"}}>
+                                <input value={this.state.s3Endpoint} onChange={this.onChangeS3Endpoint} placeholder="S3 endpoint (leave empty for default endpoint)" />
+                            </div>
                             <div class="password__submit">
                                 <input type="submit" value={!this.state.loaded ? "Loading..." : this.state.userDetails.s3Enabled ? "Remove Account" : "Add Account"}/>
                             </div>