🚧 fix(wip): embedded magnet

Prismadic · Mar 13, 2024 · de455c2 · de455c2
1 parent ecced31
commit de455c2
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 66 deletions.
diff --git a/examples/0_embedded_cluster.ipynb b/examples/0_embedded_cluster.ipynb
@@ -77,8 +77,8 @@
     }
    ],
    "source": [
-    "from magnet.base import EmbeddedCluster\n",
-    "cluster = EmbeddedCluster()\n",
+    "from magnet.base import EmbeddedMagnet\n",
+    "cluster = EmbeddedMagnet()\n",
     "cluster.start()\n",
     "prism = cluster.create_prism()\n",
     "await prism.align()\n",

diff --git a/examples/0_prism.ipynb b/examples/0_prism.ipynb
@@ -10,51 +10,51 @@
      "output_type": "stream",
      "text": [
       "\u001b[92m🌊 SUCCESS: containerization engine connected\u001b[0m\n",
-      "\u001b[96m☕️ WAIT: <Container: 44d481784d64>\u001b[0m\n",
-      "\u001b[96m☕️ WAIT: magnet-embedded-nats container progressing with id 44d481784d64ff5586400116ed906562a4eb8c9c233743df90fce4621e0a3371\u001b[0m\n",
+      "\u001b[96m☕️ WAIT: <Container: 27f20e0d3448>\u001b[0m\n",
+      "\u001b[96m☕️ WAIT: magnet-embedded-nats container progressing with id 27f20e0d34480c799e2dd0762b1e383bc0633365e5c01a5cc8ae86f076ad3883\u001b[0m\n",
       "\u001b[94mℹ️ INFO: nats logs\u001b[0m\n",
-      "\u001b[33m 💜 ⣾: [1] 2024/03/13 15:01:57.594633 \u001b[0m\n",
-      "\u001b[91m 🤩 ⡿:  Starting nats-server\n",
-      "[1] 2024/03/13 15:01:57.594746 \u001b[0m\n",
-      "\u001b[34m 🧮 ⣷:    Version:  2.10.12\n",
-      "[1] 2024/03/13 15:01:57.594748 \u001b[0m\n",
-      "\u001b[31m 💙 ⣯:    Git:      [121169ea]\n",
-      "[1] 2024/03/13 15:01:57.594749 \u001b[0m\n",
-      "\u001b[95m 🧡 ⡿:    Name:     NDHQUMMB2CXQW4RROUZLRITI37R4ZZLZSPJ74JADXO52RFMZPKVWFE67\n",
-      "[1] 2024/03/13 15:01:57.594754 \u001b[0m\n",
-      "\u001b[31m 💚 ⢿:    Node:     mt6NMlxK\n",
-      "[1] 2024/03/13 15:01:57.594754 \u001b[0m\n",
-      "\u001b[34m 🍤 ⣻:    ID:       NDHQUMMB2CXQW4RROUZLRITI37R4ZZLZSPJ74JADXO52RFMZPKVWFE67\n",
-      "[1] 2024/03/13 15:01:57.594993 \u001b[0m\n",
-      "\u001b[36m 💙 ⣻:  Starting JetStream\n",
-      "[1] 2024/03/13 15:01:57.599277 \u001b[0m\n",
-      "\u001b[36m 🏁 ⣻:      _ ___ _____ ___ _____ ___ ___   _   __  __\n",
-      "[1] 2024/03/13 15:01:57.599288 \u001b[0m\n",
-      "\u001b[34m ❤️ ⢿:   _ | | __|_   _/ __|_   _| _ \\ __| /_\\ |  \\/  |\n",
-      "[1] 2024/03/13 15:01:57.599288 \u001b[0m\n",
-      "\u001b[33m ❤️ ⣾:  | || | _|  | | \\__ \\ | | |   / _| / _ \\| |\\/| |\n",
-      "[1] 2024/03/13 15:01:57.599289 \u001b[0m\n",
-      "\u001b[93m 🌈 ⣽:   \\__/|___| |_| |___/ |_| |_|_\\___/_/ \\_\\_|  |_|\n",
-      "[1] 2024/03/13 15:01:57.599290 \u001b[0m\n",
-      "\u001b[91m ❤️ ⣾:  \n",
-      "[1] 2024/03/13 15:01:57.599291 \u001b[0m\n",
-      "\u001b[34m 💛 ⣻:           https://docs.nats.io/jetstream\n",
-      "[1] 2024/03/13 15:01:57.599291 \u001b[0m\n",
-      "\u001b[36m 💛 ⣯:  \n",
-      "[1] 2024/03/13 15:01:57.599292 \u001b[0m\n",
-      "\u001b[32m 🐈 ⣯:  ---------------- JETSTREAM ----------------\n",
-      "[1] 2024/03/13 15:01:57.599294 \u001b[0m\n",
-      "\u001b[91m ❤️ ⣯:    Max Memory:      11.70 GB\n",
-      "[1] 2024/03/13 15:01:57.599295 \u001b[0m\n",
-      "\u001b[93m 📊 ⣯:    Max Storage:     12.16 GB\n",
-      "[1] 2024/03/13 15:01:57.599295 \u001b[0m\n",
-      "\u001b[92m 🌈 ⢿:    Store Directory: \"/tmp/nats/jetstream\"\n",
-      "[1] 2024/03/13 15:01:57.599296 \u001b[0m\n",
-      "\u001b[95m ❤️ ⣽:  -------------------------------------------\n",
-      "[1] 2024/03/13 15:01:57.600016 \u001b[0m\n",
-      "\u001b[95m 🌈 ⣷:  Listening for client connections on 0.0.0.0:4222\n",
-      "[1] 2024/03/13 15:01:57.600926 \u001b[0m\n",
-      "\u001b[35m 🏁 ⣷:  Server is ready\n",
+      "\u001b[33m 📊 ⣽: [1] 2024/03/13 15:13:00.411527 \u001b[0m\n",
+      "\u001b[95m 🏁 ⣾:  Starting nats-server\n",
+      "[1] 2024/03/13 15:13:00.411607 \u001b[0m\n",
+      "\u001b[34m 💙 ⣯:    Version:  2.10.12\n",
+      "[1] 2024/03/13 15:13:00.411608 \u001b[0m\n",
+      "\u001b[95m 💚 ⡿:    Git:      [121169ea]\n",
+      "[1] 2024/03/13 15:13:00.411609 \u001b[0m\n",
+      "\u001b[92m 🧮 ⢿:    Name:     NDKBXN7NLCYKB5AH6Q6IZZUXYKXBP6FFCNZWZWTDDCMZATZPITIEL32N\n",
+      "[1] 2024/03/13 15:13:00.411613 \u001b[0m\n",
+      "\u001b[93m 🐈 ⣯:    Node:     fhL5pDzE\n",
+      "[1] 2024/03/13 15:13:00.411614 \u001b[0m\n",
+      "\u001b[34m 🧠 ⣷:    ID:       NDKBXN7NLCYKB5AH6Q6IZZUXYKXBP6FFCNZWZWTDDCMZATZPITIEL32N\n",
+      "[1] 2024/03/13 15:13:00.411796 \u001b[0m\n",
+      "\u001b[94m 🤩 ⣽:  Starting JetStream\n",
+      "[1] 2024/03/13 15:13:00.415358 \u001b[0m\n",
+      "\u001b[34m 💜 ⣟:      _ ___ _____ ___ _____ ___ ___   _   __  __\n",
+      "[1] 2024/03/13 15:13:00.415376 \u001b[0m\n",
+      "\u001b[91m 🧡 ⣷:   _ | | __|_   _/ __|_   _| _ \\ __| /_\\ |  \\/  |\n",
+      "[1] 2024/03/13 15:13:00.415377 \u001b[0m\n",
+      "\u001b[34m 🧠 ⢿:  | || | _|  | | \\__ \\ | | |   / _| / _ \\| |\\/| |\n",
+      "[1] 2024/03/13 15:13:00.415378 \u001b[0m\n",
+      "\u001b[31m 🐈 ⣻:   \\__/|___| |_| |___/ |_| |_|_\\___/_/ \\_\\_|  |_|\n",
+      "[1] 2024/03/13 15:13:00.415378 \u001b[0m\n",
+      "\u001b[34m 🍤 ⣽:  \n",
+      "[1] 2024/03/13 15:13:00.415379 \u001b[0m\n",
+      "\u001b[33m 🤩 ⣾:           https://docs.nats.io/jetstream\n",
+      "[1] 2024/03/13 15:13:00.415380 \u001b[0m\n",
+      "\u001b[96m 💛 ⣯:  \n",
+      "[1] 2024/03/13 15:13:00.415381 \u001b[0m\n",
+      "\u001b[94m 🌊 ⢿:  ---------------- JETSTREAM ----------------\n",
+      "[1] 2024/03/13 15:13:00.415383 \u001b[0m\n",
+      "\u001b[36m 🍤 ⣽:    Max Memory:      11.70 GB\n",
+      "[1] 2024/03/13 15:13:00.415383 \u001b[0m\n",
+      "\u001b[93m ❤️ ⡿:    Max Storage:     12.16 GB\n",
+      "[1] 2024/03/13 15:13:00.415384 \u001b[0m\n",
+      "\u001b[94m 🐈 ⣽:    Store Directory: \"/tmp/nats/jetstream\"\n",
+      "[1] 2024/03/13 15:13:00.415385 \u001b[0m\n",
+      "\u001b[94m 🧠 ⢿:  -------------------------------------------\n",
+      "[1] 2024/03/13 15:13:00.415940 \u001b[0m\n",
+      "\u001b[94m 💜 ⣾:  Listening for client connections on 0.0.0.0:4222\n",
+      "[1] 2024/03/13 15:13:00.416200 \u001b[0m\n",
+      "\u001b[34m 💜 ⣽:  Server is ready\n",
       "\u001b[0m\n",
       "\u001b[92m🌊 SUCCESS: milvus server started\u001b[0m\n",
       "\u001b[96m☕️ WAIT: creating prism with embedded cluster\u001b[0m\n",
@@ -73,9 +73,9 @@
     {
      "data": {
       "text/plain": [
-       "[<nats.js.client.JetStreamContext at 0x30de46250>,\n",
-       " <nats.js.kv.KeyValue at 0x30de45a50>,\n",
-       " <nats.js.object_store.ObjectStore at 0x30de45790>]"
+       "[<nats.js.client.JetStreamContext at 0x165a42690>,\n",
+       " <nats.js.kv.KeyValue at 0x165a42450>,\n",
+       " <nats.js.object_store.ObjectStore at 0x165a42190>]"
       ]
      },
      "execution_count": 1,
@@ -85,9 +85,9 @@
    ],
    "source": [
     "from magnet.base import Prism\n",
-    "from magnet.base import EmbeddedCluster\n",
+    "from magnet.base import EmbeddedMagnet\n",
     "\n",
-    "cluster = EmbeddedCluster()\n",
+    "cluster = EmbeddedMagnet()\n",
     "cluster.start()\n",
     "prism = cluster.create_prism()\n",
     "await prism.align()\n",
@@ -180,7 +180,7 @@
      "text": [
       "\u001b[92m🌊 SUCCESS: pulsed to my_category on my_stream\u001b[0m\n",
       "\u001b[92m🌊 SUCCESS: pulsed to my_category on my_stream\u001b[0m\n",
-      "2024-03-13 15:02:05.503442+00:00 2024-03-13 15:02:06.508971+00:00 2024-03-13 15:02:07.516146+00:00\n"
+      "2024-03-13 15:13:07.805490+00:00 2024-03-13 15:13:08.813173+00:00 2024-03-13 15:13:09.816282+00:00\n"
      ]
     },
     {
@@ -223,8 +223,7 @@
       "\u001b[96m☕️ WAIT: connecting to 127.0.0.1\u001b[0m\n",
       "\u001b[92m🌊 SUCCESS: connected successfully to 127.0.0.1\u001b[0m\n",
       "\u001b[92m🌊 SUCCESS: test created\u001b[0m\n",
-      "\u001b[96m☕️ WAIT: loading test into memory, may take time\u001b[0m\n",
-      "\u001b[93m🚨 WARN: disconnected from 127.0.0.1\u001b[0m\n"
+      "\u001b[96m☕️ WAIT: loading test into memory, may take time\u001b[0m\n"
      ]
     }
    ],
@@ -261,7 +260,9 @@
       "\u001b[94mℹ️ INFO: encoding payload\n",
       "Payload(text='Hello', document='World again!')\u001b[0m\n",
       "\u001b[94mℹ️ INFO: indexing payload\u001b[0m\n",
-      "\u001b[91m☠️ FATAL: <MilvusException: (code=1, message=Field distances is not in the hit entity)>\u001b[0m\n"
+      "\u001b[92m🌊 SUCCESS: embedding indexed\n",
+      "Payload(text='Hello', document='World again!')\u001b[0m\n",
+      "\u001b[93m🚨 WARN: no more data\u001b[0m\n"
      ]
     }
    ],
@@ -281,23 +282,30 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "text/plain": [
-       "['test', 'bge_non_nlp']"
-      ]
-     },
-     "execution_count": 7,
-     "metadata": {},
-     "output_type": "execute_result"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[93m🚨 WARN: disconnected from 127.0.0.1\u001b[0m\n",
+      "\u001b[96m☕️ WAIT: stopping magnet-embedded-nats\u001b[0m\n",
+      "\u001b[92m🌊 SUCCESS: magnet-embedded-nats stopped\u001b[0m\n",
+      "\u001b[94mℹ️ INFO: removing magnet-embedded-nats\u001b[0m\n",
+      "\u001b[92m🌊 SUCCESS: embedded nats removed\u001b[0m\n",
+      "\u001b[92m🌊 SUCCESS: embedded milvus server stopped\u001b[0m\n",
+      "\u001b[93m🚨 WARN: container engine pruned\u001b[0m\n",
+      "\u001b[92m🌊 SUCCESS: embedded cluster cleaned up\u001b[0m\n"
+     ]
     }
    ],
    "source": [
     "mem.db.list_indices()\n",
-    "await mem.disconnect()"
+    "\n",
+    "await mem.disconnect()\n",
+    "cluster.stop()\n",
+    "cluster.cleanup()"
    ]
   }
  ],

diff --git a/magnet/base.py b/magnet/base.py
@@ -132,7 +132,7 @@ async def off(self):
         await self.nc.drain()
         _f('warn', f'disconnected from {self.config.host}')
 
-class EmbeddedCluster:
+class EmbeddedMagnet:
     def __init__(self):
         try:
             self.client = docker.from_env()