Merge pull request #7 from mind-network/dev

v1.0.3
mind-network · Sep 8, 2023 · 48b73dd · 48b73dd
2 parents d9bac35 + 11d23a5
commit 48b73dd
Show file tree

Hide file tree

Showing 12 changed files with 1,023 additions and 220 deletions.
diff --git a/README.md b/README.md
@@ -63,6 +63,8 @@ Full doc: [https://mind-network.gitbook.io/mind-lake-sdk](https://mind-network.g
     * Initial Release
 * v1.0.1
     * Add IPFS support
+* v1.0.3
+    * Change connectors to be class
 
 ## License
 

diff --git a/examples/use_case_arweave.ipynb b/examples/use_case_arweave.ipynb
@@ -230,12 +230,12 @@
    ],
    "source": [
     "dataPack1 = DataPack(env.walletPrivateKey)\n",
-    "result = dataPack1.loadFromCSVFileByDefineColumn(fn_local_csv, \n",
-    "    [\n",
+    "localFile1 = DataPack.LocalFileConnector(columns=[\n",
     "        DataPack.Column('uid', DataPack.DataType.int4, False),\n",
     "        DataPack.Column('wallet_address', DataPack.DataType.text, True),\n",
     "        DataPack.Column('register_date', DataPack.DataType.timestamp, True)\n",
     "    ])\n",
+    "result = dataPack1.loadFrom(localFile1, fn_local_csv)\n",
     "assert result, result.message\n",
     "df = pandas.DataFrame(dataPack1.data)\n",
     "df"
@@ -272,7 +272,8 @@
     "print(\"dropTable result:\", result.message)\n",
     "\n",
     "# upload from local csv into mindlake\n",
-    "result = dataPack1.saveToMindLake('a_new_table', mindlake)\n",
+    "mindLakeConnector = DataPack.MindLakeConnector(mindlake)\n",
+    "result = dataPack1.saveTo(mindLakeConnector, 'a_new_table')\n",
     "assert result, result.message\n",
     "print(\"a_new_table has been saved to MindLake.\")"
    ]
@@ -362,8 +363,8 @@
    ],
    "source": [
     "dataPack2 = DataPack(env.walletPrivateKey)\n",
-    "# make sure if you complete Step 1, so \"a_new_table\" exists in mindlake \n",
-    "result = dataPack2.loadFromMindByQuery('select * from \"a_new_table\"', mindlake)\n",
+    "# make sure if you complete Step 1, so \"a_new_table\" exists in mindlake\n",
+    "result = dataPack2.loadFrom(mindLakeConnector, 'select * from \"a_new_table\"')\n",
     "assert result, result.message\n",
     "df = pandas.DataFrame(dataPack2.data)\n",
     "df"
@@ -388,24 +389,25 @@
      "text": [
       "test_table_encrypted.csv has been saved to Arweave.\n",
       "\n",
-      "The Arweave ID is: sMboxrf5U9Z-FM_-woSfiVerz0Lzpa1_OCmRX09Jgn0 \n",
+      "The Arweave ID is: DwyQeo8B7RdoJlqWNGm1WHiKVh-ua5BmclKu9pdX29w \n",
       "\n",
-      "You can check on Arweave: https://viewblock.io/arweave/tx/sMboxrf5U9Z-FM_-woSfiVerz0Lzpa1_OCmRX09Jgn0\n",
+      "You can check on Arweave: https://viewblock.io/arweave/tx/DwyQeo8B7RdoJlqWNGm1WHiKVh-ua5BmclKu9pdX29w\n",
       "\n",
-      "And the content on Arweave: https://arseed.web3infra.dev/sMboxrf5U9Z-FM_-woSfiVerz0Lzpa1_OCmRX09Jgn0\n",
+      "And the content on Arweave: https://arseed.web3infra.dev/DwyQeo8B7RdoJlqWNGm1WHiKVh-ua5BmclKu9pdX29w\n",
       "\n",
       "The content is:\n",
       "uid,wallet_address,register_date\n",
-      "123,\\x9326bce223092712067ca9dd8d6600fd6d0a8e572261927932f0b222092249804635af229e5bfd2e8ed550fb44539c37fa5f4bc77e48499ce01ed9bec037ea7ddd,\\x992afbd2992fcc46071972b3069b44476bc2316cacbe12a5305839d60ab43dc674\n",
-      "124,\\x7726950c54dfd2470b9ef8929ee3c049af498bf99a56756f1824f0146fcda22df0bb392959e42a2c8639604ec301425e31844b8a7b847ae486abd29d6594c0ff7c,\\xcdf8811927b22d47aa10e2e8f367e986461d43242b9f81ff62564424d64f958235\n",
-      "125,\\x72ccfbbd92144aed8bd0763e30f63e68ac99f095a5fdcc3772ea4b35ead34ede8fc891e7b8e73ae5d07e25fc3b801e66bbea0a110e9047b3bccfdddeac8f7cbd51,\\xb8138ce9e9feaff5c610f424cc97dfb44d1809ad31ca830fd15039d9d3124679ac\n",
+      "123,\\x645039a18f2d0ecfda0049619267f6e4f658d3c8dea202cf2953eaee63894b0b3af0c159a58036cd81d29d4f07d63f75236243d1bbdfd447a90a68e901a62f4153,\\xc9bd351c80b4784cdb1e26b10a8064946eb62b1d53b2b1f42da3092bf6e2cd054f\n",
+      "124,\\x6e6190fd99e05a04ea63768f435b8913a350876a758869804ae6a8ca824091af8b89b821cb2f9f1a7b4bdf51e90e94d77e41dac7637bf1babcfa7f1ba94f103ff5,\\x69239ddf275ef971a8bd7051988b1f92dd784377df7b4989719322aab31eeb3cb6\n",
+      "125,\\x54c1c6f1bc35bba9a1903e7774ca3ba6646b2aaf0f4a4b92262f8932b92bfb8d3a1699ebbc6b391e3b19983aa7832be12d75d5bdfe128c094e61b7f0a1dbab2e2c,\\x045d590d0ef821fe691ee42e04d066338392adcaf50627e3335250e9886d1a7c02\n",
       "\n"
      ]
     }
    ],
    "source": [
-    "result = dataPack2.saveToArweave('test_table_encrypted.csv', 'ACNH')\n",
-    "# result = dataPack2.saveToArweave('test_table_encrypted.csv', 'AR', './arweave_wallet.json')\n",
+    "arweave1 = DataPack.ArweaveConnector('ACNH', ethWalletPrivateKey=env.walletPrivateKey)\n",
+    "# arweave = DataPack.ArweaveConnector('AR', arweaveWalletFile='./arweave_wallet.json')\n",
+    "result = dataPack2.saveTo(arweave1, 'test_table_encrypted.csv')\n",
     "assert result, result.message\n",
     "print(\"test_table_encrypted.csv has been saved to Arweave.\\n\")\n",
     "itemID = result.data\n",
@@ -504,7 +506,8 @@
    ],
    "source": [
     "dataPack3 = DataPack(env.walletPrivateKey)\n",
-    "result = dataPack3.loadFromArweave(itemID)\n",
+    "arweave2 = DataPack.ArweaveConnector()\n",
+    "result = dataPack3.loadFrom(arweave2, itemID)\n",
     "assert result, result.message\n",
     "print(\"You can see file on Arweave are encrypted, but datapack can decrypted it locally\")\n",
     "df = pandas.DataFrame(dataPack3.data)\n",
@@ -561,35 +564,35 @@
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>123</td>\n",
-       "      <td>\\x9c851ae97fd6eeb3ae2437c274274b3d773fcdee11f9...</td>\n",
-       "      <td>\\x44e6bdad517e634a985702320c26770a9275c79aa9b3...</td>\n",
+       "      <td>\\x3c364d7aed8d63b68701fd00feac41435ff456eb307d...</td>\n",
+       "      <td>\\x35bafd055143484471c52153385b7ee5eedc58b09021...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>124</td>\n",
-       "      <td>\\xaddba1e7648b04018dabaceb918cebb436dff5e57e61...</td>\n",
-       "      <td>\\xb5dac593d9436309272b5b56c16d4db24bab7d9e130a...</td>\n",
+       "      <td>\\xbe1d0f00778d9e15e08672f9494a22c11ef870758572...</td>\n",
+       "      <td>\\x25ceb7256dd9ab8ef563ae220c62df39cf13bf3a7d08...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>125</td>\n",
-       "      <td>\\x9afc9f24ec67f79105a340e939be6f1808f5a34495f2...</td>\n",
-       "      <td>\\x755479852b673a2183e1168734e78378fcf81413c3f2...</td>\n",
+       "      <td>\\xa5805d615740ead33874d7036246cc2aee7665901d3c...</td>\n",
+       "      <td>\\xb541751a94f6b5128af3a5939867531ed5736b0fd7c4...</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
        "   uid                                     wallet_address  \\\n",
-       "0  123  \\x9c851ae97fd6eeb3ae2437c274274b3d773fcdee11f9...   \n",
-       "1  124  \\xaddba1e7648b04018dabaceb918cebb436dff5e57e61...   \n",
-       "2  125  \\x9afc9f24ec67f79105a340e939be6f1808f5a34495f2...   \n",
+       "0  123  \\x3c364d7aed8d63b68701fd00feac41435ff456eb307d...   \n",
+       "1  124  \\xbe1d0f00778d9e15e08672f9494a22c11ef870758572...   \n",
+       "2  125  \\xa5805d615740ead33874d7036246cc2aee7665901d3c...   \n",
        "\n",
        "                                       register_date  \n",
-       "0  \\x44e6bdad517e634a985702320c26770a9275c79aa9b3...  \n",
-       "1  \\xb5dac593d9436309272b5b56c16d4db24bab7d9e130a...  \n",
-       "2  \\x755479852b673a2183e1168734e78378fcf81413c3f2...  "
+       "0  \\x35bafd055143484471c52153385b7ee5eedc58b09021...  \n",
+       "1  \\x25ceb7256dd9ab8ef563ae220c62df39cf13bf3a7d08...  \n",
+       "2  \\xb541751a94f6b5128af3a5939867531ed5736b0fd7c4...  "
       ]
      },
      "execution_count": 10,
@@ -598,7 +601,8 @@
     }
    ],
    "source": [
-    "result = dataPack3.saveToLocalFile('test_localtable_encrypted.csv', False)\n",
+    "localFile2 = DataPack.LocalFileConnector()\n",
+    "result = dataPack3.saveTo(localFile2, 'test_localtable_encrypted.csv')\n",
     "assert result, result.message\n",
     "print(\"test_localtable_encrypted.csv has been saved to local file.\")\n",
     "df = pandas.read_csv('test_localtable_encrypted.csv')\n",

diff --git a/examples/use_case_ipfs.ipynb b/examples/use_case_ipfs.ipynb
@@ -228,12 +228,12 @@
    ],
    "source": [
     "dataPack1 = DataPack(env.walletPrivateKey)\n",
-    "result = dataPack1.loadFromCSVFileByDefineColumn(fn_local_csv, \n",
-    "    [\n",
+    "localFile1 = DataPack.LocalFileConnector(columns=[\n",
     "        DataPack.Column('uid', DataPack.DataType.int4, False),\n",
     "        DataPack.Column('wallet_address', DataPack.DataType.text, True),\n",
     "        DataPack.Column('register_date', DataPack.DataType.timestamp, True)\n",
     "    ])\n",
+    "result = dataPack1.loadFrom(localFile1, fn_local_csv)\n",
     "assert result, result.message\n",
     "df = pandas.DataFrame(dataPack1.data)\n",
     "df"
@@ -270,7 +270,8 @@
     "print(\"dropTable result:\", result.message)\n",
     "\n",
     "# upload from local csv into mindlake\n",
-    "result = dataPack1.saveToMindLake('a_new_table', mindlake)\n",
+    "mindLakeConnector = DataPack.MindLakeConnector(mindlake)\n",
+    "result = dataPack1.saveTo(mindLakeConnector, 'a_new_table')\n",
     "assert result, result.message\n",
     "print(\"a_new_table has been saved to MindLake.\")"
    ]
@@ -360,8 +361,8 @@
    ],
    "source": [
     "dataPack2 = DataPack(env.walletPrivateKey)\n",
-    "# make sure if you complete Step 1, so \"a_new_table\" exists in mindlake \n",
-    "result = dataPack2.loadFromMindByQuery('select * from \"a_new_table\"', mindlake)\n",
+    "# make sure if you complete Step 1, so \"a_new_table\" exists in mindlake\n",
+    "result = dataPack2.loadFrom(mindLakeConnector, 'select * from \"a_new_table\"')\n",
     "assert result, result.message\n",
     "df = pandas.DataFrame(dataPack2.data)\n",
     "df"
@@ -386,24 +387,25 @@
      "text": [
       "test_table_encrypted.csv has been saved to IPFS.\n",
       "\n",
-      "The IPFS ID is: QmYxCi1BBhbch496SDVP58VesPuTnraPgu57S84b1XiteM \n",
+      "The IPFS ID is: QmU1nri3draq7SAHVAjSx91dEVCw1e1LGdv1NaiepGvTBm \n",
       "\n",
-      "You can check on local IPFS http service: http://127.0.0.1:8080/ipfs/QmYxCi1BBhbch496SDVP58VesPuTnraPgu57S84b1XiteM\n",
+      "You can check on local IPFS http service: http://127.0.0.1:8080/ipfs/QmU1nri3draq7SAHVAjSx91dEVCw1e1LGdv1NaiepGvTBm\n",
       "\n",
-      "And the content on IPFS: http://127.0.0.1:8080/ipfs/QmYxCi1BBhbch496SDVP58VesPuTnraPgu57S84b1XiteM/test_table_encrypted.csv\n",
+      "And the content on IPFS: http://127.0.0.1:8080/ipfs/QmU1nri3draq7SAHVAjSx91dEVCw1e1LGdv1NaiepGvTBm/test_table_encrypted.csv\n",
       "\n",
       "The content is:\n",
       "uid,wallet_address,register_date\n",
-      "123,\\x4810c8a24f2f22243d778860d9ee5180d5375d6cbd22561e568c23bce63f7516d1d2887dcb1993de0133e1d1e002c06b38b56b25d23acac51e24e740f1c6c5e163,\\xf9977c69c6a3d716e615b344f4f66953a4b3d764892a5960460aba34a6c7645943\n",
-      "124,\\xa7fca7f580180de2e92df9ddf9608c2442dcfa947ba23e3be37bf4764f981670016df19ca4fb3e7295479de368c45122f2cca35d619a402ac28d437cfd0188233f,\\xf06543016d08a89717d34fea65b66b4b8878ae8f411598c3ce3cbde0795a58d2c7\n",
-      "125,\\xd26a30809f578388dfc97a0262190094c086060a380e7b83a5f2e9bd9cd680a6e272c900921ca7c5824391e4f6bc6854e622f59c0d891152e861ff7ba62f542c40,\\xfa94a16e90f53456b9d053537fed04802514b15a3506055acbbd519e67de1d6e1c\n",
+      "123,\\x82148bcab7b5c8fc8f81ce8cb79b307d81bfde84e51fa87ae99d56b9985b8b9018b26eff1b96a38cf148ffe1b923b99be3908a7e6d27d57e78d769af6be82dc92d,\\x0776d01ca9ffa39ea95916108147db031025ce9e3ce7f6cba619041b9fe9c247ee\n",
+      "124,\\x43ad786aca3e01879ee6b7753032842658e6c24a302bf2986b70c247a36bf54724ec914002f930b13fa9de88b072502f887a930b655590090a21d0f6253ab9b5f4,\\xcf0cc270db2108338f9440a0fb810706e31a73b20284226d9bddc40649d66e43f7\n",
+      "125,\\xe49d212633890b86855d663d90ee71ed323b68f67a53fed837c88f037d5fdb4ef5456e596a88a10a6b3d9c347033e4c0c7e1f566017fed5c49d7546fbf76348c8d,\\xb8ec6581139a9c29ce58c85f094ec53e26a23047d6ea3c73edbdcf57500e9d5de7\n",
       "\n"
      ]
     }
    ],
    "source": [
     "# make sure you have started IPFS service on your local machine\n",
-    "result = dataPack2.saveToIPFS('test_table_encrypted.csv')\n",
+    "ipfs1 = DataPack.IPFSConnector()\n",
+    "result = dataPack2.saveTo(ipfs1, 'test_table_encrypted.csv')\n",
     "assert result, result.message\n",
     "print(\"test_table_encrypted.csv has been saved to IPFS.\\n\")\n",
     "ipfsHash = result.data\n",
@@ -502,7 +504,8 @@
    ],
    "source": [
     "dataPack3 = DataPack(env.walletPrivateKey)\n",
-    "result = dataPack3.loadFromIPFS(ipfsHash)\n",
+    "ipfs2 = DataPack.IPFSConnector()\n",
+    "result = dataPack3.loadFrom(ipfs2, ipfsHash)\n",
     "assert result, result.message\n",
     "print(\"You can see file on IPFS are encrypted, but datapack can decrypted it locally\")\n",
     "df = pandas.DataFrame(dataPack3.data)\n",
@@ -559,35 +562,35 @@
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td>123</td>\n",
-       "      <td>\\x1295126db9fb5f12c77ca17a7b2230509f05ee225d15...</td>\n",
-       "      <td>\\x3f7a44eb7cdd6c1ff9433b4b911a0a400af3aafdc6dd...</td>\n",
+       "      <td>\\x14bb21b7b3105cac85bc84383b3b0b84bf9d0503940d...</td>\n",
+       "      <td>\\x6aa4984ca3bd3682679308b2ef22cf6f12d42698fde0...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>1</th>\n",
        "      <td>124</td>\n",
-       "      <td>\\x3c9791f3a72caf6f8f4ed8d699c9efd63eccb60961b4...</td>\n",
-       "      <td>\\x83cd814f9a8f2a1e1684a62e64d87f488517526441b3...</td>\n",
+       "      <td>\\x3f87e2905e69ee36419ac126dee7eb23b1b7d3b5ba16...</td>\n",
+       "      <td>\\x9b8355a538a4b2f84824f55d75a098d984145de1371f...</td>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>2</th>\n",
        "      <td>125</td>\n",
-       "      <td>\\xe2137c5d87fbaa2fe39f2fea8731992ea7a194b90641...</td>\n",
-       "      <td>\\x550d9b3fc3d1edd7866990620a91ac5164c9fe9e30e0...</td>\n",
+       "      <td>\\x7901ba445c75ceb0016914f307707c84a26b655fe7fd...</td>\n",
+       "      <td>\\x7e91fca841863ca2ae32b2b1098b5191fdba7e0e0e6a...</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "text/plain": [
        "   uid                                     wallet_address  \\\n",
-       "0  123  \\x1295126db9fb5f12c77ca17a7b2230509f05ee225d15...   \n",
-       "1  124  \\x3c9791f3a72caf6f8f4ed8d699c9efd63eccb60961b4...   \n",
-       "2  125  \\xe2137c5d87fbaa2fe39f2fea8731992ea7a194b90641...   \n",
+       "0  123  \\x14bb21b7b3105cac85bc84383b3b0b84bf9d0503940d...   \n",
+       "1  124  \\x3f87e2905e69ee36419ac126dee7eb23b1b7d3b5ba16...   \n",
+       "2  125  \\x7901ba445c75ceb0016914f307707c84a26b655fe7fd...   \n",
        "\n",
        "                                       register_date  \n",
-       "0  \\x3f7a44eb7cdd6c1ff9433b4b911a0a400af3aafdc6dd...  \n",
-       "1  \\x83cd814f9a8f2a1e1684a62e64d87f488517526441b3...  \n",
-       "2  \\x550d9b3fc3d1edd7866990620a91ac5164c9fe9e30e0...  "
+       "0  \\x6aa4984ca3bd3682679308b2ef22cf6f12d42698fde0...  \n",
+       "1  \\x9b8355a538a4b2f84824f55d75a098d984145de1371f...  \n",
+       "2  \\x7e91fca841863ca2ae32b2b1098b5191fdba7e0e0e6a...  "
       ]
      },
      "execution_count": 10,
@@ -596,7 +599,8 @@
     }
    ],
    "source": [
-    "result = dataPack3.saveToLocalFile('test_localtable_encrypted.csv', False)\n",
+    "localFile2 = DataPack.LocalFileConnector()\n",
+    "result = dataPack3.saveTo(localFile2, 'test_localtable_encrypted.csv')\n",
     "assert result, result.message\n",
     "print(\"test_localtable_encrypted.csv has been saved to local file.\")\n",
     "df = pandas.read_csv('test_localtable_encrypted.csv')\n",