From eb30db76dfc189533f09b2b96fce6033a2309c7e Mon Sep 17 00:00:00 2001 From: Tarek Ziade Date: Wed, 9 Aug 2023 15:42:28 +0200 Subject: [PATCH] Adding coverage to `nucliadb_node_binding` (#1173) * reorganized the test in the standard layout [sc-6677] * make the new action run manually for now * reactivate lints * fix the name --- .github/workflows/nucliadb_node_binding.yml | 55 +++ Cargo.lock | 382 ++++++++++++++++-- Makefile | 4 +- nucliadb_node_binding/Cargo.toml | 4 + nucliadb_node_binding/Makefile | 31 +- nucliadb_node_binding/cov.sh | 18 + nucliadb_node_binding/pyproject.toml | 7 + nucliadb_node_binding/test.py | 93 ----- nucliadb_node_binding/tests/__init__.py | 19 + nucliadb_node_binding/tests/conftest.py | 22 + .../tests/integration/test_indexing.py | 104 +++++ 11 files changed, 613 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/nucliadb_node_binding.yml create mode 100755 nucliadb_node_binding/cov.sh delete mode 100644 nucliadb_node_binding/test.py create mode 100644 nucliadb_node_binding/tests/__init__.py create mode 100644 nucliadb_node_binding/tests/conftest.py create mode 100644 nucliadb_node_binding/tests/integration/test_indexing.py diff --git a/.github/workflows/nucliadb_node_binding.yml b/.github/workflows/nucliadb_node_binding.yml new file mode 100644 index 0000000000..7ef7867a2f --- /dev/null +++ b/.github/workflows/nucliadb_node_binding.yml @@ -0,0 +1,55 @@ +name: nucliadb Node Binding (rust + python) + +on: workflow_dispatch + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +env: + RUSTFLAGS: -Dwarnings + RUST_BACKTRACE: 1 + CARGO_TERM_COLOR: always + API_VERSION: 1 + +jobs: + tests: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: [3.9] + + steps: + - name: Checkout the repository + uses: actions/checkout@v3 + + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + profile: minimal + components: llvm-tools-preview + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + + - uses: Swatinem/rust-cache@v1 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Install package + run: make -C nucliadb_node_binding/ install-dev + + - name: Run tests with coverage + run: make -C nucliadb_node_binding/ test-cov + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + files: ./nucliadb_node_binding/coverage.xml,./nucliadb_node_binding/coverage.lcov + flags: node-sidecar-binding diff --git a/Cargo.lock b/Cargo.lock index 6203c83835..a57c7d791f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,7 +179,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.28", ] [[package]] @@ -196,7 +196,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.28", ] [[package]] @@ -411,6 +411,17 @@ dependencies = [ "log", ] +[[package]] +name = "bstr" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" +dependencies = [ + "memchr", + "regex-automata 0.3.6", + "serde", +] + [[package]] name = "bumpalo" version = "3.12.0" @@ -435,6 +446,80 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +[[package]] +name = "camino" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo-config2" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d622c3fcbf7cbd550bd4df0cd519724857608af8c1d16599e5d42f9af4abef" +dependencies = [ + "cfg-expr", + "home", + "once_cell", + "serde", + "shell-escape", + "toml", +] + +[[package]] +name = "cargo-llvm-cov" +version = "0.5.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22d6cc7273ce5c9241a36728c7931aea581e8c69cabf321794eeca3f58919848" +dependencies = [ + "anyhow", + "camino", + "cargo-config2", + "cargo_metadata", + "duct", + "fs-err", + "glob", + "home", + "is-terminal", + "is_executable", + "lcov2cobertura", + "lexopt", + "opener", + "regex", + "rustc-demangle", + "serde", + "serde_json", + "shell-escape", + "termcolor", + "walkdir", +] + +[[package]] +name = "cargo-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cfa25e60aea747ec7e1124f238816749faa93759c6ff5b31f1ccdda137f4479" +dependencies = [ + "serde", +] + +[[package]] +name = "cargo_metadata" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7daec1a2a2129eeba1644b220b4647ec537b0b5d4bfd6876fcc5a540056b592" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cc" version = "1.0.79" @@ -447,6 +532,15 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fafee10a5dd1cffcb5cc560e0d0df8803d7355a2b12272e3557dee57314cb6e" +[[package]] +name = "cfg-expr" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b40ccee03b5175c18cde8f37e7d2a33bcef6f8ec8f7cc0d81090d1bb380949c9" +dependencies = [ + "smallvec", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -478,7 +572,7 @@ dependencies = [ "bitflags", "clap_derive", "clap_lex", - "indexmap", + "indexmap 1.9.3", "once_cell", "strsim", "termcolor", @@ -676,7 +770,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.13", + "syn 2.0.28", ] [[package]] @@ -693,7 +787,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.28", ] [[package]] @@ -738,7 +832,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" dependencies = [ "cfg-if", - "hashbrown", + "hashbrown 0.12.3", "lock_api", "once_cell", "parking_lot_core", @@ -821,6 +915,18 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65d09067bfacaa79114679b279d7f5885b53295b1e2cfb4e79c8e4bd3d633169" +[[package]] +name = "duct" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ae3fc31835f74c2a7ceda3aeede378b0ae2e74c8f1c36559fcc9ae2a4e7d3e" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + [[package]] name = "either" version = "1.8.1" @@ -849,6 +955,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.1" @@ -948,6 +1060,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0845fa252299212f0389d64ba26f34fa32cfe41588355f21ed507c59a0f64541" + [[package]] name = "fs2" version = "0.4.3" @@ -1030,7 +1148,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.28", ] [[package]] @@ -1080,6 +1198,12 @@ version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "gloo-timers" version = "0.2.6" @@ -1104,7 +1228,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -1120,6 +1244,12 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "heck" version = "0.4.1" @@ -1194,6 +1324,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "hostname" version = "0.3.1" @@ -1369,7 +1508,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", ] [[package]] @@ -1413,6 +1562,27 @@ version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +[[package]] +name = "is-terminal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes", + "rustix", + "windows-sys 0.48.0", +] + +[[package]] +name = "is_executable" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa9acdc6d67b75e626ad644734e8bc6df893d9cd2a834129065d3dd6158ea9c8" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "itertools" version = "0.10.5" @@ -1452,12 +1622,30 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lcov2cobertura" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf88eb39b4cae8a9c79b5052613fa2489e7bd0fb2a4e46c9d6c908b4c2de162d" +dependencies = [ + "anyhow", + "quick-xml", + "regex", + "rustc-demangle", +] + [[package]] name = "levenshtein_automata" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" +[[package]] +name = "lexopt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baff4b617f7df3d896f97fe922b64817f6cd9a756bb81d40f8883f2f66dcb401" + [[package]] name = "libc" version = "0.2.141" @@ -1516,7 +1704,7 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" dependencies = [ - "hashbrown", + "hashbrown 0.12.3", ] [[package]] @@ -1537,7 +1725,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] @@ -1661,6 +1849,15 @@ dependencies = [ "tempfile", ] +[[package]] +name = "normpath" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec60c60a693226186f5d6edf073232bfb6464ed97eb22cf3b01c1e8198fd97f5" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1760,6 +1957,7 @@ name = "nucliadb_node_binding" version = "0.8.0" dependencies = [ "bincode", + "cargo-llvm-cov", "log", "nucliadb_core", "nucliadb_node", @@ -1971,6 +2169,17 @@ version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +[[package]] +name = "opener" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c62dcb6174f9cb326eac248f07e955d5d559c272730b6c03e396b443b562788" +dependencies = [ + "bstr", + "normpath", + "winapi 0.3.9", +] + [[package]] name = "openssl" version = "0.10.50" @@ -1994,7 +2203,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.28", ] [[package]] @@ -2111,6 +2320,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "os_pipe" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae859aa07428ca9a929b936690f8b12dc5f11dd8c6992a18ca93919f28bc177" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "os_str_bytes" version = "6.5.0" @@ -2204,7 +2423,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 1.9.3", ] [[package]] @@ -2324,9 +2543,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -2475,11 +2694,20 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", +] + [[package]] name = "quote" -version = "1.0.26" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] @@ -2574,6 +2802,12 @@ dependencies = [ "regex-syntax 0.6.29", ] +[[package]] +name = "regex-automata" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" + [[package]] name = "regex-syntax" version = "0.4.2" @@ -2715,6 +2949,15 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.21" @@ -2774,6 +3017,9 @@ name = "semver" version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +dependencies = [ + "serde", +] [[package]] name = "sentry" @@ -2898,22 +3144,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.160" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.28", ] [[package]] @@ -2936,6 +3182,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2970,7 +3225,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.28", ] [[package]] @@ -2982,6 +3237,22 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shared_child" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0d94659ad3c2137fef23ae75b03d5241d633f8acded53d672decfa0e6e0caef" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "shell-escape" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f" + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -3047,9 +3318,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.13" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2", "quote", @@ -3218,7 +3489,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.28", ] [[package]] @@ -3343,7 +3614,7 @@ checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.28", ] [[package]] @@ -3403,6 +3674,40 @@ dependencies = [ "tracing", ] +[[package]] +name = "toml" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.19.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" +dependencies = [ + "indexmap 2.0.0", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tonic" version = "0.7.2" @@ -3471,7 +3776,7 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap", + "indexmap 1.9.3", "pin-project", "pin-project-lite", "rand", @@ -3738,7 +4043,7 @@ checksum = "20e8a505384e9309dc842520c6c9348f4b141dee06aaa845522727b1b99ca235" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.28", ] [[package]] @@ -3789,6 +4094,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +[[package]] +name = "walkdir" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.0" @@ -4116,6 +4431,15 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +[[package]] +name = "winnow" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acaaa1190073b2b101e15083c38ee8ec891b5e05cbee516521e94ec008f61e64" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.10.1" diff --git a/Makefile b/Makefile index bed21328a7..c9df58c662 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ python-code-lint: make -C nucliadb_models/ format make -C nucliadb_sdk/ format make -C nucliadb_node/ format -# make -C nucliadb_node_binding/ format + make -C nucliadb_node_binding/ format make -C nucliadb_utils/ format make -C nucliadb/ format make -C nucliadb_telemetry/ format @@ -79,7 +79,7 @@ python-code-lint: make -C nucliadb_client/ lint make -C nucliadb_models/ lint make -C nucliadb_node/ lint -# make -C nucliadb_node_binding/ lint + make -C nucliadb_node_binding/ lint rust-code-lint: diff --git a/nucliadb_node_binding/Cargo.toml b/nucliadb_node_binding/Cargo.toml index 85bdec2fd2..c05e40a1f2 100644 --- a/nucliadb_node_binding/Cargo.toml +++ b/nucliadb_node_binding/Cargo.toml @@ -17,6 +17,7 @@ serde = { version = "1.0", features = ["derive"] } tokio = { version = "1", features = ["full"] } log = "0.4" bincode = "1.3.3" +cargo-llvm-cov = "0.5.24" openssl = { version = "0.10", features = ["vendored"] } prost = "0.10" @@ -27,3 +28,6 @@ tracing-subscriber = { version = "0.3.11", features = [ "registry", "std", ] } + +[package.metadata.maturin] +python-source = "python" diff --git a/nucliadb_node_binding/Makefile b/nucliadb_node_binding/Makefile index 9bdc762329..a2586eec9c 100644 --- a/nucliadb_node_binding/Makefile +++ b/nucliadb_node_binding/Makefile @@ -1,14 +1,41 @@ +SHELL := /bin/bash + .PHONY: install-dev install-dev: pip install --upgrade pip wheel cd .. && pip install \ -r test-requirements.txt \ - -r code-requirements.txt + -r code-requirements.txt \ + -r nucliadb_node/requirements-sources.txt \ + -r nucliadb_node/requirements.txt \ + ./nucliadb_node/ pip install -e . +.PHONY: format +format: + isort --profile black tests + black . + +.PHONY: lint +lint: + flake8 tests --config=pyproject.toml + isort -c --profile black tests + black --check . + MYPYPATH=../mypy_stubs mypy --config-file=../mypy.ini . + .PHONY: test test: - pytest -s --tb=native -v test.py + pytest -s --tb=native -v tests + +# TODO rather complex set up, needs to be done globally +.PHONY: install-cov +install-cov: + rustup component add llvm-tools-preview + - cargo install cargo-llvm-cov + +.PHONY: test-cov +test-cov: + bash ./cov.sh .PHONY: build build: diff --git a/nucliadb_node_binding/cov.sh b/nucliadb_node_binding/cov.sh new file mode 100755 index 0000000000..55fb304749 --- /dev/null +++ b/nucliadb_node_binding/cov.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -ex + +cargo llvm-cov show-env --export-prefix >.env +cargo llvm-cov clean --workspace +rm -rf ../target/wheels/nucliadb_node_binding* +source .env + +maturin build +pip install ../target/wheels/nucliadb_node_binding*.whl + +pytest -rfE --cov=nucliadb_node_binding --cov-config=../.coveragerc -s --tb=native -v --cov-report term-missing:skip-covered --cov-report xml tests + +# generating coverage for the CI +cargo llvm-cov report --ignore-filename-regex="^(.*/nucliadb_vectors/).*|^(.*/nucliadb_texts/).*|^(.*/nucliadb_telemetry/).*|^(.*/nucliadb_relations/).*|^(.*/nucliadb_core/).*|^(.*/nucliadb_node/).*|^(.*/nucliadb_paragraphs/).*|^(.*/nucliadb_protos/)|^(.*/nucliadb/nucliadb/)" --lcov --output-path coverage.lcov + +# coverage for the console +cargo llvm-cov report --ignore-filename-regex="^(.*/nucliadb_vectors/).*|^(.*/nucliadb_texts/).*|^(.*/nucliadb_telemetry/).*|^(.*/nucliadb_relations/).*|^(.*/nucliadb_core/).*|^(.*/nucliadb_node/).*|^(.*/nucliadb_paragraphs/).*|^(.*/nucliadb_protos/)|^(.*/nucliadb/nucliadb/)" diff --git a/nucliadb_node_binding/pyproject.toml b/nucliadb_node_binding/pyproject.toml index 4bbe4bc82e..9ea46f8677 100644 --- a/nucliadb_node_binding/pyproject.toml +++ b/nucliadb_node_binding/pyproject.toml @@ -29,3 +29,10 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] + +[tool.pytest.ini_options] +asyncio_mode = "auto" + +[flake8] +max-line-length = 120 +extend-ignore = "E203" diff --git a/nucliadb_node_binding/test.py b/nucliadb_node_binding/test.py deleted file mode 100644 index 60218da76a..0000000000 --- a/nucliadb_node_binding/test.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (C) 2021 Bosutech XXI S.L. -# -# nucliadb is offered under the AGPL v3.0 and as commercial software. -# For commercial licensing, contact us at info@nuclia.com. -# -# AGPL: -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import asyncio -import os -import shutil -from contextlib import contextmanager -from datetime import datetime -from tempfile import mkdtemp - -from nucliadb_protos.nodereader_pb2 import SearchRequest, SearchResponse -from nucliadb_protos.noderesources_pb2 import Resource -from nucliadb_protos.nodewriter_pb2 import ShardCreated, NewShardRequest -from nucliadb_protos.utils_pb2 import VectorSimilarity -from nucliadb_protos.nodewriter_pb2 import OpStatus - -from nucliadb_node_binding import NodeReader, NodeWriter - - -@contextmanager -def temp_data_path(): - if "DATA_PATH" in os.environ: - saved_path = os.environ["DATA_PATH"] - else: - saved_path = None - - dir = mkdtemp() - try: - os.environ["DATA_PATH"] = dir - yield - finally: - shutil.rmtree(dir) - if saved_path is not None: - os.environ["DATA_PATH"] = saved_path - else: - del os.environ["DATA_PATH"] - - -# TODO improve coverage (see what nucliadb/common/cluster/standalone/grpc_node_binding.py does) -async def main(): - with temp_data_path(): - writer = NodeWriter() - reader = NodeReader() - req = NewShardRequest(kbid="test-kbid", similarity=VectorSimilarity.COSINE) - - req = req.SerializeToString() - shard = writer.new_shard(req) - - pb = ShardCreated() - pb.ParseFromString(bytes(shard)) - assert pb.id is not None - - resourcepb = Resource() - resourcepb.resource.uuid = "001" - resourcepb.resource.shard_id = pb.id - resourcepb.texts["field1"].text = "My lovely text" - resourcepb.status = Resource.ResourceStatus.PROCESSED - resourcepb.shard_id = pb.id - resourcepb.metadata.created.FromDatetime(datetime.now()) - resourcepb.metadata.modified.FromDatetime(datetime.now()) - - pb_bytes = writer.set_resource(resourcepb.SerializeToString()) - op_status = OpStatus() - op_status.ParseFromString(bytes(pb_bytes)) - assert op_status.status == OpStatus.OK - - searchpb = SearchRequest() - searchpb.shard = pb.id - searchpb.body = "text" - pbresult = reader.search(searchpb.SerializeToString()) - pb = SearchResponse() - pb.ParseFromString(bytes(pbresult)) - - print(pb) - - -asyncio.run(main()) diff --git a/nucliadb_node_binding/tests/__init__.py b/nucliadb_node_binding/tests/__init__.py new file mode 100644 index 0000000000..3b734776ac --- /dev/null +++ b/nucliadb_node_binding/tests/__init__.py @@ -0,0 +1,19 @@ +# Copyright (C) 2021 Bosutech XXI S.L. +# +# nucliadb is offered under the AGPL v3.0 and as commercial software. +# For commercial licensing, contact us at info@nuclia.com. +# +# AGPL: +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# diff --git a/nucliadb_node_binding/tests/conftest.py b/nucliadb_node_binding/tests/conftest.py new file mode 100644 index 0000000000..fe76dc701b --- /dev/null +++ b/nucliadb_node_binding/tests/conftest.py @@ -0,0 +1,22 @@ +# Copyright (C) 2021 Bosutech XXI S.L. +# +# nucliadb is offered under the AGPL v3.0 and as commercial software. +# For commercial licensing, contact us at info@nuclia.com. +# +# AGPL: +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +pytest_plugins = [ + "nucliadb_node.tests.fixtures", +] diff --git a/nucliadb_node_binding/tests/integration/test_indexing.py b/nucliadb_node_binding/tests/integration/test_indexing.py new file mode 100644 index 0000000000..d221afef72 --- /dev/null +++ b/nucliadb_node_binding/tests/integration/test_indexing.py @@ -0,0 +1,104 @@ +# Copyright (C) 2021 Bosutech XXI S.L. +# +# nucliadb is offered under the AGPL v3.0 and as commercial software. +# For commercial licensing, contact us at info@nuclia.com. +# +# AGPL: +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from datetime import datetime + +import pytest +from nucliadb_protos.nodereader_pb2 import ( + DocumentSearchRequest, + DocumentSearchResponse, + SearchRequest, + SearchResponse, +) +from nucliadb_protos.noderesources_pb2 import Resource +from nucliadb_protos.nodewriter_pb2 import NewShardRequest, OpStatus, ShardCreated +from nucliadb_protos.utils_pb2 import VectorSimilarity + +from nucliadb_node_binding import NodeReader, NodeWriter # type: ignore + + +class IndexNode: + def __init__(self): + self.writer = NodeWriter() + self.reader = NodeReader() + + def request(self, func, req_ob, resp_factory): + raw_resp = func(req_ob.SerializeToString()) + resp = resp_factory() + resp.ParseFromString(bytes(raw_resp)) + return resp + + def create_resource(self, shard_id): + resource = Resource() + resource.resource.uuid = "001" + resource.resource.shard_id = shard_id + resource.texts["field1"].text = "My lovely text" + resource.status = Resource.ResourceStatus.PROCESSED + resource.shard_id = shard_id + now = datetime.now() + resource.metadata.created.FromDatetime(now) + resource.metadata.modified.FromDatetime(now) + op_status = self.request(self.writer.set_resource, resource, OpStatus) + assert op_status.status == OpStatus.OK + return resource + + def create_shard(self, name): + # create a shard + shard_req = NewShardRequest( + kbid=name, similarity=VectorSimilarity.COSINE + ) # NOQA + pb = self.request(self.writer.new_shard, shard_req, ShardCreated) + shard_id = pb.id + assert shard_id is not None + return shard_id + + def call_search_api(self, name, request, resp_klass): + return self.request(getattr(self.reader, name), request, resp_klass) + + +@pytest.mark.asyncio +async def test_set_and_search(data_path): + cluster = IndexNode() + + shard_id = cluster.create_shard("test-kbid") + cluster.create_resource(shard_id) + + # search using the generic search API + searchpb = SearchRequest() + searchpb.shard = shard_id + searchpb.body = "lovely" + searchpb.document = True + pbresult = cluster.call_search_api("search", searchpb, SearchResponse) + assert pbresult.document.total == 1 + + +@pytest.mark.asyncio +async def test_set_and_document_search(data_path): + cluster = IndexNode() + shard_id = cluster.create_shard("test-kbid") + cluster.create_resource(shard_id) + + # search using the document search API + searchpb = DocumentSearchRequest() + searchpb.id = shard_id + searchpb.body = "lovely" + pbresult = cluster.call_search_api( + "document_search", searchpb, DocumentSearchResponse + ) + assert pbresult.total == 1