Fix workflows (#2222)

nuclia · Jun 6, 2024 · 0d03d9f · 0d03d9f
1 parent d39e682
commit 0d03d9f
Show file tree

Hide file tree

Showing 13 changed files with 40 additions and 101 deletions.
diff --git a/.github/.codecov.yml b/.github/.codecov.yml
@@ -13,16 +13,16 @@ flag_management:
   default_rules: # the rules that will be followed for any flag added, generally
     carryforward: true # needed for monorepos: total coverage is not uploaded at every commit.
   individual_flags:
-    - name: one
+    - name: standalone
       paths:
-        - nucliadb/nucliadb/one/**
+        - nucliadb/src/nucliadb/standalone/**
       carryforward: true
       statuses:
         - type: project
           target: 80%
     - name: ingest
       paths:
-        - nucliadb/nucliadb/ingest/**
+        - nucliadb/src/nucliadb/ingest/**
       carryforward: true
       statuses:
         - type: project
@@ -36,7 +36,7 @@ flag_management:
           target: 80%
     - name: train
       paths:
-        - nucliadb/nucliadb/train/**
+        - nucliadb/src/nucliadb/train/**
       carryforward: true
       statuses:
         - type: project
@@ -50,42 +50,42 @@ flag_management:
           target: 78%
     - name: search
       paths:
-        - nucliadb/nucliadb/search/**
+        - nucliadb/src/nucliadb/search/**
       carryforward: true
       statuses:
         - type: project
           target: 74%
     - name: writer
       paths:
-        - nucliadb/nucliadb/writer/**
+        - nucliadb/src/nucliadb/writer/**
       carryforward: true
       statuses:
         - type: project
           target: 83%
     - name: reader
       paths:
-        - nucliadb/nucliadb/reader/**
+        - nucliadb/src/nucliadb/reader/**
       carryforward: true
       statuses:
         - type: project
           target: 60%
     - name: sdk
       paths:
-        - nucliadb_sdk/nucliadb_sdk/**
+        - nucliadb_sdk/src/nucliadb_sdk/**
       carryforward: true
       statuses:
         - type: project
           target: 38%
     - name: dataset
       paths:
-        - nucliadb_dataset/nucliadb_dataset/**
+        - nucliadb_dataset/src/nucliadb_dataset/**
       carryforward: true
       statuses:
         - type: project
           target: 60%
     - name: node-sidecar
       paths:
-        - nucliadb_node/nucliadb_node/**
+        - nucliadb_sidecar/src/nucliadb_sidecar/**
       carryforward: true
       statuses:
         - type: project

diff --git a/.github/workflows/nucliadb_migrations.yml b/.github/workflows/nucliadb_migrations.yml
@@ -7,14 +7,14 @@ on:
     paths:
       - ".github/workflows/nucliadb_migrations.yml"
       - "nucliadb/migrations/**"
-      - "nucliadb/nucliadb/migrator/**"
+      - "nucliadb/src/nucliadb/migrator/**"
 
   push:
     branches:
       - main
       - ".github/workflows/nucliadb_migrations.yml"
       - "nucliadb/migrations/**"
-      - "nucliadb/nucliadb/migrator/**"
+      - "nucliadb/src/nucliadb/migrator/**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}

diff --git a/.github/workflows/nucliadb_search.yml b/.github/workflows/nucliadb_search.yml
@@ -8,9 +8,9 @@ on:
       - '.github/workflows/nucliadb_search.yml'
       - 'charts/nucliadb_search/**'
       - 'nucliadb_utils/**'
-      - 'nucliadb/nucliadb/ingest/**'
-      - 'nucliadb/nucliadb/search/**'
-      - 'nucliadb/nucliadb/common/**'
+      - 'nucliadb/src/nucliadb/ingest/**'
+      - 'nucliadb/src/nucliadb/search/**'
+      - 'nucliadb/src/nucliadb/common/**'
       - 'nucliadb_models/**'
       - 'nucliadb_telemetry/**'
       - 'nucliadb_protos/python/**'
@@ -29,9 +29,9 @@ on:
       - '.github/workflows/nucliadb_search.yml'
       - 'charts/nucliadb_search/**'
       - 'nucliadb_utils/**'
-      - 'nucliadb/nucliadb/ingest/**'
-      - 'nucliadb/nucliadb/search/**'
-      - 'nucliadb/nucliadb/common/**'
+      - 'nucliadb/src/nucliadb/ingest/**'
+      - 'nucliadb/src/nucliadb/search/**'
+      - 'nucliadb/src/nucliadb/common/**'
       - 'nucliadb_models/**'
       - 'nucliadb_telemetry/**'
       - 'nucliadb_protos/python/**'

diff --git a/.github/workflows/nucliadb_standalone.yml b/.github/workflows/nucliadb_standalone.yml
@@ -12,11 +12,11 @@ on:
     paths:
       - ".github/workflows/nucliadb_standalone.yml"
       - "charts/nucliadb/**"
-      - "nucliadb/nucliadb/ingest/**"
-      - "nucliadb/nucliadb/writer/**"
-      - "nucliadb/nucliadb/standalone/**"
-      - "nucliadb/nucliadb/reader/**"
-      - "nucliadb/nucliadb/search/**"
+      - "nucliadb/src/nucliadb/ingest/**"
+      - "nucliadb/src/nucliadb/writer/**"
+      - "nucliadb/src/nucliadb/standalone/**"
+      - "nucliadb/src/nucliadb/reader/**"
+      - "nucliadb/src/nucliadb/search/**"
       - "nucliadb_models/**"
       - "nucliadb_protos/**"
       - "nucliadb_node/**"
@@ -34,11 +34,11 @@ on:
     paths:
       - ".github/workflows/nucliadb_standalone.yml"
       - "charts/nucliadb/**"
-      - "nucliadb/nucliadb/ingest/**"
-      - "nucliadb/nucliadb/writer/**"
-      - "nucliadb/nucliadb/standalone/**"
-      - "nucliadb/nucliadb/reader/**"
-      - "nucliadb/nucliadb/search/**"
+      - "nucliadb/src/nucliadb/ingest/**"
+      - "nucliadb/src/nucliadb/writer/**"
+      - "nucliadb/src/nucliadb/standalone/**"
+      - "nucliadb/src/nucliadb/reader/**"
+      - "nucliadb/src/nucliadb/search/**"
       - "nucliadb_models/**"
       - "nucliadb_protos/**"
 

diff --git a/.gitignore b/.gitignore
@@ -82,9 +82,6 @@ nucliadb_performance/exports
 ### Others. Still in use?
 
 
-nucliadb/nucliadb/static/tmp
-nucliadb_ingest/ingest.key
-nucliadb_search/search.key
 host_key
 ingest.key
 

diff --git a/.licenserc.yaml b/.licenserc.yaml
@@ -86,7 +86,7 @@ header:
     - "**/__pycache__"
     - "nucliadb_telemetry/src/nucliadb_telemetry/tests/grpc/*"
     - "**/Makefile"
-    - "nucliadb/nucliadb/ingest/tests/orm/assets"
+    - "nucliadb/src/nucliadb/ingest/tests/orm/assets"
     - "VERSION"
     - "LICENSE"
     - "NOTICE"

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -32,4 +32,4 @@ The process is simple and fast. Upon your first pull request, you will be prompt
 
 1. Install Python == 3.12 with a virtualenv with your prefered tool (pyenv, conda, pipenv,...)
 2. Install NucliaDB Dev Dependencies `make install`
-3. Run `pytest nucliadb/nucliadb/tests`
+3. Run `pytest nucliadb/tests`
diff --git a/Makefile b/Makefile
@@ -118,7 +118,7 @@ build-sidecar:
 
 
 debug-test-nucliadb:
-	RUST_BACKTRACE=1 RUST_LOG=nucliadb_node=DEBUG,nucliadb_paragraphs_tantivy=DEBUG,nucliadb_fields_tantivy=DEBUG pytest nucliadb/nucliadb -sxv
+	RUST_BACKTRACE=1 RUST_LOG=nucliadb_node=DEBUG,nucliadb_paragraphs_tantivy=DEBUG,nucliadb_fields_tantivy=DEBUG pytest nucliadb/tests -sxv
 
 debug-run-nucliadb:
 	RUST_BACKTRACE=1 MAX_RECEIVE_MESSAGE_LENGTH=1024 RUST_LOG=nucliadb_node=DEBUG,nucliadb_paragraphs_tantivy=DEBUG,nucliadb_fields_tantivy=DEBUG nucliadb --maindb=data/maindb --blob=data/blob --node=data/node --zone=europe-1 --log=DEBUG

diff --git a/docs/internal/TESTING.md b/docs/internal/TESTING.md
diff --git a/nucliadb/Makefile b/nucliadb/Makefile
@@ -80,7 +80,7 @@ $(test_cov_subpackages_targets):
 # Generic implicit rules
 
 format-%:
-	cd .. && isort --profile black nucliadb/nucliadb/$*
+	cd .. && isort --profile black nucliadb/src/nucliadb/$*
 	black nucliadb/$*
 	ruff check --fix nucliadb/$* --config=../ruff.toml
 

diff --git a/nucliadb/tests/fixtures.py b/nucliadb/tests/fixtures.py
@@ -661,7 +661,7 @@ def driver_settings(request):
     """
     Allows dynamically loading the driver fixtures via env vars.
 
-    MAINDB_DRIVER=redis,local pytest nucliadb/nucliadb/tests/
+    MAINDB_DRIVER=redis,local pytest nucliadb/tests/
 
     Any test using the nucliadb fixture will be run twice, once with redis driver and once with local driver.
     """
@@ -672,7 +672,7 @@ def driver_lazy_fixtures(default_drivers: str = "redis"):
     """
     Allows running tests using maindb_driver for each supported driver type via env vars.
 
-    MAINDB_DRIVER=redis,local pytest nucliadb/nucliadb/ingest/tests/
+    MAINDB_DRIVER=redis,local pytest nucliadb/tests/ingest
 
     Any test using the maindb_driver fixture will be run twice, once with redis_driver and once with local_driver.
     """

diff --git a/nucliadb/tests/ingest/integration/ingest/test_ingest.py b/nucliadb/tests/ingest/integration/ingest/test_ingest.py
@@ -395,7 +395,7 @@ async def test_ingest_audit_stream_files_only(
 
     # Minimal assert to make sure we get the information from the node on the audit
     # gets from the sidecar to the audit report when adding or modifying a resource
-    # The values are hardcoded on nucliadb/nucliadb/ingest/orm/grpc_node_dummy.py
+    # The values are hardcoded on nucliadb/src/nucliadb/ingest/orm/grpc_node_dummy.py
 
     assert auditreq.kbid == knowledgebox_ingest
     assert auditreq.rid == rid
@@ -427,7 +427,7 @@ async def test_ingest_audit_stream_files_only(
 
     # Minimal assert to make sure we get the information from the node on the audit
     # gets from the sidecar to the audit report when adding or modifying a resource
-    # The values are hardcoded on nucliadb/nucliadb/ingest/orm/grpc_node_dummy.py
+    # The values are hardcoded on nucliadb/src/nucliadb/ingest/orm/grpc_node_dummy.py
 
     assert auditreq.kbid == knowledgebox_ingest
     assert auditreq.rid == rid
@@ -447,7 +447,7 @@ async def test_ingest_audit_stream_files_only(
 
     # Minimal assert to make sure we get the information from the node on the audit
     # gets from the sidecar to the audit report when adding or modifying a resource
-    # The values are hardcoded on nucliadb/nucliadb/ingest/orm/grpc_node_dummy.py
+    # The values are hardcoded on nucliadb/src/nucliadb/ingest/orm/grpc_node_dummy.py
 
     assert auditreq.kbid == knowledgebox_ingest
     assert auditreq.rid == rid
@@ -594,7 +594,7 @@ async def test_ingest_audit_stream_mixed(
 
     # Minimal assert to make sure we get the information from the node on the audit
     # gets from the sidecar to the audit report when adding or modifying a resource
-    # The values are hardcoded on nucliadb/nucliadb/ingest/orm/grpc_node_dummy.py
+    # The values are hardcoded on nucliadb/src/nucliadb/ingest/orm/grpc_node_dummy.py
 
     assert auditreq.kbid == kbid
     assert auditreq.rid == rid

diff --git a/nucliadb_node_binding/cov.sh b/nucliadb_node_binding/cov.sh
@@ -12,7 +12,7 @@ pip install ../target/wheels/nucliadb_node_binding*.whl
 pytest -rfE --cov=nucliadb_node_binding --cov-config=../.coveragerc -s --tb=native -v --cov-report term-missing:skip-covered --cov-report xml tests
 
 # generating coverage for the CI
-cargo llvm-cov report --ignore-filename-regex="^(.*/nucliadb_vectors/).*|^(.*/nucliadb_texts).*|^(.*/nucliadb_relations).*|^(.*/nucliadb_core/).*|^(.*/nucliadb_node/).*|^(.*/nucliadb_paragraphs/).*|^(.*/nucliadb_protos/)|^(.*/nucliadb/nucliadb/)" --lcov --output-path coverage.lcov
+cargo llvm-cov report --ignore-filename-regex="^(.*/nucliadb_vectors/).*|^(.*/nucliadb_texts).*|^(.*/nucliadb_relations).*|^(.*/nucliadb_core/).*|^(.*/nucliadb_node/).*|^(.*/nucliadb_paragraphs/).*|^(.*/nucliadb_protos/)|^(.*/nucliadb/src/nucliadb/)" --lcov --output-path coverage.lcov
 
 # coverage for the console
-cargo llvm-cov report --ignore-filename-regex="^(.*/nucliadb_vectors/).*|^(.*/nucliadb_texts).*|^(.*/nucliadb_relations).*|^(.*/nucliadb_core/).*|^(.*/nucliadb_node/).*|^(.*/nucliadb_paragraphs/).*|^(.*/nucliadb_protos/)|^(.*/nucliadb/nucliadb/)"
+cargo llvm-cov report --ignore-filename-regex="^(.*/nucliadb_vectors/).*|^(.*/nucliadb_texts).*|^(.*/nucliadb_relations).*|^(.*/nucliadb_core/).*|^(.*/nucliadb_node/).*|^(.*/nucliadb_paragraphs/).*|^(.*/nucliadb_protos/)|^(.*/nucliadb/src/nucliadb/)"