move nucliadb_node/binding to the root [sc-6670] (#1154)

.github/workflows/nucliadb.yml

@@ -59,7 +59,7 @@ jobs:
 
       - name: Install the package
         run: make -C nucliadb/ install-dev
-      
+
       - name: Install tikv
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://tiup-mirrors.pingcap.com/install.sh | sh -s -- v1.11.3
@@ -83,7 +83,7 @@ jobs:
       - name: Install rust binding for testing
         if: steps.filter.outputs.workflows == 'true'
         run: |
-          maturin build -m nucliadb_node/binding/Cargo.toml
+          maturin build -m nucliadb_node_binding/Cargo.toml
           pip install target/wheels/*_x86_64.whl
 
       - name: Run tests

.github/workflows/nucliadb_node.yml

@@ -16,6 +16,7 @@ on:
       - "nucliadb_core/**"
       - "nucliadb_telemetry/**"
       - "nucliadb_node/**"
+      - "nucliadb_node_binding/**"
       - "nucliadb_vectors/**"
       - "nucliadb_relations/**"
       - "nucliadb_telemetry/src/**"
@@ -35,6 +36,7 @@ on:
       - "nucliadb_core/**"
       - "nucliadb_telemetry/**"
       - "nucliadb_node/**"
+      - "nucliadb_node_binding/**"
       - "nucliadb_vectors/**"
       - "nucliadb_relations/**"
       - "nucliadb_telemetry/src/**"
@@ -194,7 +196,7 @@ jobs:
 
       - name: Install rust binding for testing
         run: |
-          maturin build -m nucliadb_node/binding/Cargo.toml
+          maturin build -m nucliadb_node_binding/Cargo.toml
           pip install target/wheels/*_x86_64.whl
 
       - uses: dorny/paths-filter@v2

.github/workflows/nucliadb_node_release.yml

@@ -6,13 +6,13 @@ on:
       - main
     paths:
       - ".github/workflows/nucliadb_node_release.yml"
-      - "nucliadb_node/binding/Cargo.toml"
+      - "nucliadb_node_binding/Cargo.toml"
   push:
     branches:
       - main
     paths:
       - ".github/workflows/nucliadb_node_release.yml"
-      - "nucliadb_node/binding/Cargo.toml"
+      - "nucliadb_node_binding/Cargo.toml"
 
 jobs:
   build:
@@ -82,7 +82,7 @@ jobs:
         uses: messense/maturin-action@v1
         with:
           command: sdist
-          args: -m nucliadb_node/binding/Cargo.toml --out dist
+          args: -m nucliadb_node_binding/Cargo.toml --out dist
           rust-toolchain: stable
 
       - name: build wheels
@@ -91,7 +91,7 @@ jobs:
           target: ${{ matrix.target }}
           manylinux: ${{ matrix.manylinux || 'auto' }}
           container: ${{ matrix.container }}
-          args: -m nucliadb_node/binding/Cargo.toml --release --out dist --interpreter ${{ matrix.interpreter || '3.7 3.8 3.9 3.10 3.11 pypy3.7 pypy3.8 pypy3.9' }}
+          args: -m nucliadb_node_binding/Cargo.toml --release --out dist --interpreter ${{ matrix.interpreter || '3.7 3.8 3.9 3.10 3.11 pypy3.7 pypy3.8 pypy3.9' }}
           rust-toolchain: stable
 
       - run: ${{ matrix.ls || 'ls -lh' }} dist/

.github/workflows/nucliadb_train.yml

@@ -74,7 +74,7 @@ jobs:
       - name: Install rust binding for testing
         if: steps.filter.outputs.workflows == 'true'
         run: |
-          maturin build -m nucliadb_node/binding/Cargo.toml
+          maturin build -m nucliadb_node_binding/Cargo.toml
           pip install target/wheels/*_x86_64.whl
 
       - name: Run tests

Cargo.toml

@@ -3,7 +3,7 @@
 members = [
   "nucliadb_core",
   "nucliadb_node",
-  "nucliadb_node/binding",
+  "nucliadb_node_binding",
   "nucliadb_paragraphs",
   "nucliadb_protos/rust",
   "nucliadb_relations",

Dockerfile.rust

@@ -25,7 +25,7 @@ RUN set -eux; \
     cargo --version; \
     rustc --version;
 
-RUN apt-get update -y && apt-get install --yes --no-install-recommends patchelf cmake && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 
+RUN apt-get update -y && apt-get install --yes --no-install-recommends patchelf cmake && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 RUN pip install maturin
 
@@ -41,5 +41,5 @@ COPY . /nucliadb
 WORKDIR /nucliadb
 
 RUN set -eux; \
-    cd nucliadb_node/binding; \
+    cd nucliadb_node_binding; \
     maturin build --release

Dockerfile.withbinding

@@ -26,7 +26,7 @@ RUN set -eux; \
     cargo --version; \
     rustc --version;
 
-RUN apt-get update -y && apt-get install --yes --no-install-recommends patchelf cmake && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* 
+RUN apt-get update -y && apt-get install --yes --no-install-recommends patchelf cmake && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 RUN pip install maturin
 
@@ -35,7 +35,7 @@ COPY . /nucliadb
 WORKDIR /nucliadb
 
 RUN set -eux; \
-    cd nucliadb_node/binding; \
+    cd nucliadb_node_binding; \
     maturin build --profile ${RUST_BUILD_PROFILE}
 
 

Makefile

@@ -66,6 +66,7 @@ python-code-lint:
 	make -C nucliadb_models/ format
 	make -C nucliadb_sdk/ format
 	make -C nucliadb_node/ format
+	make -C nucliadb_node_binding/ format
 	make -C nucliadb_utils/ format
 	make -C nucliadb/ format
 	make -C nucliadb_telemetry/ format
@@ -78,6 +79,7 @@ python-code-lint:
 	make -C nucliadb_client/ lint
 	make -C nucliadb_models/ lint
 	make -C nucliadb_node/ lint
+	make -C nucliadb_node_binding/ lint
 
 
 rust-code-lint:
@@ -138,12 +140,12 @@ debug-run-nucliadb-redis:
 
 build-node-binding:
 	rm -rf target/wheels/*
-	maturin build -m nucliadb_node/binding/Cargo.toml --release
+	maturin build -m nucliadb_node_binding/Cargo.toml --release
 	pip install target/wheels/nucliadb_node_binding-*.whl --force
 
 build-node-binding-debug:
 	rm -rf target/wheels/*
-	maturin build -m nucliadb_node/binding/Cargo.toml
+	maturin build -m nucliadb_node_binding/Cargo.toml
 	pip install target/wheels/nucliadb_node_binding-*.whl --force
 
 build-nucliadb-local:

nucliadb_node/binding/Cargo.toml → nucliadb_node_binding/Cargo.toml

@@ -10,9 +10,9 @@ crate-type = ["cdylib"]
 
 [dependencies]
 pyo3 = { version = "0.19.1", features = ["extension-module"] }
-nucliadb_node = { path = "../../nucliadb_node" }
-nucliadb_core= { path = "../../nucliadb_core" }
-nucliadb_telemetry = { path = "../../nucliadb_telemetry" }
+nucliadb_node = { path = "../nucliadb_node" }
+nucliadb_core= { path = "../nucliadb_core" }
+nucliadb_telemetry = { path = "../nucliadb_telemetry" }
 serde = { version = "1.0", features = ["derive"] }
 tokio = { version = "1", features = ["full"] }
 log = "0.4"

nucliadb_node_binding/Makefile

@@ -0,0 +1,15 @@
+.PHONY: install-dev
+install-dev:
+	pip install --upgrade pip wheel
+	cd .. && pip install \
+		-r test-requirements.txt \
+		-r code-requirements.txt
+	pip install -e .
+
+.PHONY: test
+test:
+	pytest -s --tb=native -v test.py
+
+.PHONY: build
+build:
+	maturin build

nucliadb_node/binding/pyproject.toml → nucliadb_node_binding/pyproject.toml

@@ -28,4 +28,4 @@ classifiers = [
     "Programming Language :: Rust",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
-]
+]

nucliadb_node_binding/test.py

@@ -0,0 +1,93 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at [email protected].
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import asyncio
+import os
+import shutil
+from contextlib import contextmanager
+from datetime import datetime
+from tempfile import mkdtemp
+
+from nucliadb_protos.nodereader_pb2 import SearchRequest, SearchResponse
+from nucliadb_protos.noderesources_pb2 import Resource
+from nucliadb_protos.nodewriter_pb2 import ShardCreated, NewShardRequest
+from nucliadb_protos.utils_pb2 import VectorSimilarity
+from nucliadb_protos.nodewriter_pb2 import OpStatus
+
+from nucliadb_node_binding import NodeReader, NodeWriter
+
+
+@contextmanager
+def temp_data_path():
+    if "DATA_PATH" in os.environ:
+        saved_path = os.environ["DATA_PATH"]
+    else:
+        saved_path = None
+
+    dir = mkdtemp()
+    try:
+        os.environ["DATA_PATH"] = dir
+        yield
+    finally:
+        shutil.rmtree(dir)
+        if saved_path is not None:
+            os.environ["DATA_PATH"] = saved_path
+        else:
+            del os.environ["DATA_PATH"]
+
+
+# TODO improve coverage (see what nucliadb/common/cluster/standalone/grpc_node_binding.py does)
+async def main():
+    with temp_data_path():
+        writer = NodeWriter()
+        reader = NodeReader()
+        req = NewShardRequest(kbid="test-kbid", similarity=VectorSimilarity.COSINE)
+
+        req = req.SerializeToString()
+        shard = writer.new_shard(req)
+
+        pb = ShardCreated()
+        pb.ParseFromString(bytes(shard))
+        assert pb.id is not None
+
+        resourcepb = Resource()
+        resourcepb.resource.uuid = "001"
+        resourcepb.resource.shard_id = pb.id
+        resourcepb.texts["field1"].text = "My lovely text"
+        resourcepb.status = Resource.ResourceStatus.PROCESSED
+        resourcepb.shard_id = pb.id
+        resourcepb.metadata.created.FromDatetime(datetime.now())
+        resourcepb.metadata.modified.FromDatetime(datetime.now())
+
+        pb_bytes = writer.set_resource(resourcepb.SerializeToString())
+        op_status = OpStatus()
+        op_status.ParseFromString(bytes(pb_bytes))
+        assert op_status.status == OpStatus.OK
+
+        searchpb = SearchRequest()
+        searchpb.shard = pb.id
+        searchpb.body = "text"
+        pbresult = reader.search(searchpb.SerializeToString())
+        pb = SearchResponse()
+        pb.ParseFromString(bytes(pbresult))
+
+        print(pb)
+
+
+asyncio.run(main())

nucliadb_vectors/src/data_point/mod.rs

@@ -449,8 +449,11 @@ impl DataPoint {
         }
 
         // Telling the OS our expected access pattern
-        nodes.advise(memmap2::Advice::WillNeed)?;
-        index.advise(memmap2::Advice::Sequential)?;
+        #[cfg(not(target_os = "windows"))]
+        {
+            nodes.advise(memmap2::Advice::WillNeed)?;
+            index.advise(memmap2::Advice::Sequential)?;
+        }
 
         Ok(DataPoint {
             journal,
@@ -482,8 +485,11 @@ impl DataPoint {
         let journal: Journal = serde_json::from_reader(journal)?;
 
         // Telling the OS our expected access pattern
-        nodes.advise(memmap2::Advice::WillNeed)?;
-        index.advise(memmap2::Advice::Sequential)?;
+        #[cfg(not(target_os = "windows"))]
+        {
+            nodes.advise(memmap2::Advice::WillNeed)?;
+            index.advise(memmap2::Advice::Sequential)?;
+        }
 
         Ok(DataPoint {
             journal,
@@ -558,8 +564,11 @@ impl DataPoint {
         }
 
         // Telling the OS our expected access pattern
-        nodes.advise(memmap2::Advice::WillNeed)?;
-        index.advise(memmap2::Advice::Sequential)?;
+        #[cfg(not(target_os = "windows"))]
+        {
+            nodes.advise(memmap2::Advice::WillNeed)?;
+            index.advise(memmap2::Advice::Sequential)?;
+        }
 
         Ok(DataPoint {
             journal,