From 0d03d9feef62eac2dfe4caa8848179fa94d62715 Mon Sep 17 00:00:00 2001 From: Ferran Llamas Date: Thu, 6 Jun 2024 10:33:39 +0200 Subject: [PATCH] Fix workflows (#2222) --- .github/.codecov.yml | 20 +++---- .github/workflows/nucliadb_migrations.yml | 4 +- .github/workflows/nucliadb_search.yml | 12 ++-- .github/workflows/nucliadb_standalone.yml | 20 +++---- .gitignore | 3 - .licenserc.yaml | 2 +- CONTRIBUTING.md | 2 +- Makefile | 2 +- docs/internal/TESTING.md | 58 ------------------- nucliadb/Makefile | 2 +- nucliadb/tests/fixtures.py | 4 +- .../ingest/integration/ingest/test_ingest.py | 8 +-- nucliadb_node_binding/cov.sh | 4 +- 13 files changed, 40 insertions(+), 101 deletions(-) delete mode 100644 docs/internal/TESTING.md diff --git a/.github/.codecov.yml b/.github/.codecov.yml index 3ca245e84c..2ec0c2130f 100644 --- a/.github/.codecov.yml +++ b/.github/.codecov.yml @@ -13,16 +13,16 @@ flag_management: default_rules: # the rules that will be followed for any flag added, generally carryforward: true # needed for monorepos: total coverage is not uploaded at every commit. individual_flags: - - name: one + - name: standalone paths: - - nucliadb/nucliadb/one/** + - nucliadb/src/nucliadb/standalone/** carryforward: true statuses: - type: project target: 80% - name: ingest paths: - - nucliadb/nucliadb/ingest/** + - nucliadb/src/nucliadb/ingest/** carryforward: true statuses: - type: project @@ -36,7 +36,7 @@ flag_management: target: 80% - name: train paths: - - nucliadb/nucliadb/train/** + - nucliadb/src/nucliadb/train/** carryforward: true statuses: - type: project @@ -50,42 +50,42 @@ flag_management: target: 78% - name: search paths: - - nucliadb/nucliadb/search/** + - nucliadb/src/nucliadb/search/** carryforward: true statuses: - type: project target: 74% - name: writer paths: - - nucliadb/nucliadb/writer/** + - nucliadb/src/nucliadb/writer/** carryforward: true statuses: - type: project target: 83% - name: reader paths: - - nucliadb/nucliadb/reader/** + - nucliadb/src/nucliadb/reader/** carryforward: true statuses: - type: project target: 60% - name: sdk paths: - - nucliadb_sdk/nucliadb_sdk/** + - nucliadb_sdk/src/nucliadb_sdk/** carryforward: true statuses: - type: project target: 38% - name: dataset paths: - - nucliadb_dataset/nucliadb_dataset/** + - nucliadb_dataset/src/nucliadb_dataset/** carryforward: true statuses: - type: project target: 60% - name: node-sidecar paths: - - nucliadb_node/nucliadb_node/** + - nucliadb_sidecar/src/nucliadb_sidecar/** carryforward: true statuses: - type: project diff --git a/.github/workflows/nucliadb_migrations.yml b/.github/workflows/nucliadb_migrations.yml index 70985130b7..9077a224eb 100644 --- a/.github/workflows/nucliadb_migrations.yml +++ b/.github/workflows/nucliadb_migrations.yml @@ -7,14 +7,14 @@ on: paths: - ".github/workflows/nucliadb_migrations.yml" - "nucliadb/migrations/**" - - "nucliadb/nucliadb/migrator/**" + - "nucliadb/src/nucliadb/migrator/**" push: branches: - main - ".github/workflows/nucliadb_migrations.yml" - "nucliadb/migrations/**" - - "nucliadb/nucliadb/migrator/**" + - "nucliadb/src/nucliadb/migrator/**" concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/nucliadb_search.yml b/.github/workflows/nucliadb_search.yml index 66d6837b10..16b8779208 100644 --- a/.github/workflows/nucliadb_search.yml +++ b/.github/workflows/nucliadb_search.yml @@ -8,9 +8,9 @@ on: - '.github/workflows/nucliadb_search.yml' - 'charts/nucliadb_search/**' - 'nucliadb_utils/**' - - 'nucliadb/nucliadb/ingest/**' - - 'nucliadb/nucliadb/search/**' - - 'nucliadb/nucliadb/common/**' + - 'nucliadb/src/nucliadb/ingest/**' + - 'nucliadb/src/nucliadb/search/**' + - 'nucliadb/src/nucliadb/common/**' - 'nucliadb_models/**' - 'nucliadb_telemetry/**' - 'nucliadb_protos/python/**' @@ -29,9 +29,9 @@ on: - '.github/workflows/nucliadb_search.yml' - 'charts/nucliadb_search/**' - 'nucliadb_utils/**' - - 'nucliadb/nucliadb/ingest/**' - - 'nucliadb/nucliadb/search/**' - - 'nucliadb/nucliadb/common/**' + - 'nucliadb/src/nucliadb/ingest/**' + - 'nucliadb/src/nucliadb/search/**' + - 'nucliadb/src/nucliadb/common/**' - 'nucliadb_models/**' - 'nucliadb_telemetry/**' - 'nucliadb_protos/python/**' diff --git a/.github/workflows/nucliadb_standalone.yml b/.github/workflows/nucliadb_standalone.yml index 7ae600cb67..5c1b9a5a9b 100644 --- a/.github/workflows/nucliadb_standalone.yml +++ b/.github/workflows/nucliadb_standalone.yml @@ -12,11 +12,11 @@ on: paths: - ".github/workflows/nucliadb_standalone.yml" - "charts/nucliadb/**" - - "nucliadb/nucliadb/ingest/**" - - "nucliadb/nucliadb/writer/**" - - "nucliadb/nucliadb/standalone/**" - - "nucliadb/nucliadb/reader/**" - - "nucliadb/nucliadb/search/**" + - "nucliadb/src/nucliadb/ingest/**" + - "nucliadb/src/nucliadb/writer/**" + - "nucliadb/src/nucliadb/standalone/**" + - "nucliadb/src/nucliadb/reader/**" + - "nucliadb/src/nucliadb/search/**" - "nucliadb_models/**" - "nucliadb_protos/**" - "nucliadb_node/**" @@ -34,11 +34,11 @@ on: paths: - ".github/workflows/nucliadb_standalone.yml" - "charts/nucliadb/**" - - "nucliadb/nucliadb/ingest/**" - - "nucliadb/nucliadb/writer/**" - - "nucliadb/nucliadb/standalone/**" - - "nucliadb/nucliadb/reader/**" - - "nucliadb/nucliadb/search/**" + - "nucliadb/src/nucliadb/ingest/**" + - "nucliadb/src/nucliadb/writer/**" + - "nucliadb/src/nucliadb/standalone/**" + - "nucliadb/src/nucliadb/reader/**" + - "nucliadb/src/nucliadb/search/**" - "nucliadb_models/**" - "nucliadb_protos/**" diff --git a/.gitignore b/.gitignore index 66f20378fb..f981a02428 100644 --- a/.gitignore +++ b/.gitignore @@ -82,9 +82,6 @@ nucliadb_performance/exports ### Others. Still in use? -nucliadb/nucliadb/static/tmp -nucliadb_ingest/ingest.key -nucliadb_search/search.key host_key ingest.key diff --git a/.licenserc.yaml b/.licenserc.yaml index 1011210171..e07d2c2b5e 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -86,7 +86,7 @@ header: - "**/__pycache__" - "nucliadb_telemetry/src/nucliadb_telemetry/tests/grpc/*" - "**/Makefile" - - "nucliadb/nucliadb/ingest/tests/orm/assets" + - "nucliadb/src/nucliadb/ingest/tests/orm/assets" - "VERSION" - "LICENSE" - "NOTICE" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e1cdb72a50..cdaa68baab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,4 +32,4 @@ The process is simple and fast. Upon your first pull request, you will be prompt 1. Install Python == 3.12 with a virtualenv with your prefered tool (pyenv, conda, pipenv,...) 2. Install NucliaDB Dev Dependencies `make install` -3. Run `pytest nucliadb/nucliadb/tests` +3. Run `pytest nucliadb/tests` diff --git a/Makefile b/Makefile index c25b79158a..7ebf77e53b 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,7 @@ build-sidecar: debug-test-nucliadb: - RUST_BACKTRACE=1 RUST_LOG=nucliadb_node=DEBUG,nucliadb_paragraphs_tantivy=DEBUG,nucliadb_fields_tantivy=DEBUG pytest nucliadb/nucliadb -sxv + RUST_BACKTRACE=1 RUST_LOG=nucliadb_node=DEBUG,nucliadb_paragraphs_tantivy=DEBUG,nucliadb_fields_tantivy=DEBUG pytest nucliadb/tests -sxv debug-run-nucliadb: RUST_BACKTRACE=1 MAX_RECEIVE_MESSAGE_LENGTH=1024 RUST_LOG=nucliadb_node=DEBUG,nucliadb_paragraphs_tantivy=DEBUG,nucliadb_fields_tantivy=DEBUG nucliadb --maindb=data/maindb --blob=data/blob --node=data/node --zone=europe-1 --log=DEBUG diff --git a/docs/internal/TESTING.md b/docs/internal/TESTING.md deleted file mode 100644 index 4fbb0cfc7b..0000000000 --- a/docs/internal/TESTING.md +++ /dev/null @@ -1,58 +0,0 @@ -# Testing - -In order to test different scenarios we are splitting tests on this areas: - -- nucliadb/nucliadb/tests: [Workflow](../../.github/workflows/nucliadb.yml) - - - Storage: local - - MainDB: local - - Nodes: PyO3 binding - - TXN: Local - - Cache: Memory - -- nucliadb/nucliadb/ingest: [Workflow](../../.github/workflows/nucliadb_ingest.yml) - - - Storage: GCS - - MainDB: Redis/Tikv - - Nodes: Mock node - - TXN: Jetstream - - Cache: Nats - -- nucliadb/nucliadb/search: [Workflow](../../.github/workflows/nucliadb_search.yml) - - - Storage: GCS - - MainDB: Redis - - Nodes: 2 Nodes real cluster - - TXN: Jetstream - - Cache: Nats - -- nucliadb/nucliadb/one: [Workflow](../../.github/workflows/nucliadb_one.yml) - - - Storage: GCS - - MainDB: Redis - - Nodes: 2 Nodes real cluster - - TXN: Jetstream - - Cache: Nats - -- nucliadb/nucliadb/train: [Workflow](../../.github/workflows/nucliadb_train.yml) - - - Storage: GCS and local - - MainDB: Redis and local - - Nodes: Mock node and PyO3 binding - - TXN: Jetstream and Local - - Cache: Nats and Memory - -- nucliadb/nucliadb/writer: [Workflow](../../.github/workflows/nucliadb_writer.yml) - - - Storage: GCS - - MainDB: Redis - - Nodes: Mock node - - TXN: Jetstream - - Cache: Nats - -- nucliadb/nucliadb/reader: [Workflow](../../.github/workflows/nucliadb_reader.yml) - - Storage: GCS - - MainDB: Redis - - Nodes: Mock node - - TXN: Jetstream - - Cache: Nats diff --git a/nucliadb/Makefile b/nucliadb/Makefile index dbea01684c..63cf293488 100644 --- a/nucliadb/Makefile +++ b/nucliadb/Makefile @@ -80,7 +80,7 @@ $(test_cov_subpackages_targets): # Generic implicit rules format-%: - cd .. && isort --profile black nucliadb/nucliadb/$* + cd .. && isort --profile black nucliadb/src/nucliadb/$* black nucliadb/$* ruff check --fix nucliadb/$* --config=../ruff.toml diff --git a/nucliadb/tests/fixtures.py b/nucliadb/tests/fixtures.py index 30197e7138..6e2ff2f86c 100644 --- a/nucliadb/tests/fixtures.py +++ b/nucliadb/tests/fixtures.py @@ -661,7 +661,7 @@ def driver_settings(request): """ Allows dynamically loading the driver fixtures via env vars. - MAINDB_DRIVER=redis,local pytest nucliadb/nucliadb/tests/ + MAINDB_DRIVER=redis,local pytest nucliadb/tests/ Any test using the nucliadb fixture will be run twice, once with redis driver and once with local driver. """ @@ -672,7 +672,7 @@ def driver_lazy_fixtures(default_drivers: str = "redis"): """ Allows running tests using maindb_driver for each supported driver type via env vars. - MAINDB_DRIVER=redis,local pytest nucliadb/nucliadb/ingest/tests/ + MAINDB_DRIVER=redis,local pytest nucliadb/tests/ingest Any test using the maindb_driver fixture will be run twice, once with redis_driver and once with local_driver. """ diff --git a/nucliadb/tests/ingest/integration/ingest/test_ingest.py b/nucliadb/tests/ingest/integration/ingest/test_ingest.py index 4b35ffce47..1ebeab13c9 100644 --- a/nucliadb/tests/ingest/integration/ingest/test_ingest.py +++ b/nucliadb/tests/ingest/integration/ingest/test_ingest.py @@ -395,7 +395,7 @@ async def test_ingest_audit_stream_files_only( # Minimal assert to make sure we get the information from the node on the audit # gets from the sidecar to the audit report when adding or modifying a resource - # The values are hardcoded on nucliadb/nucliadb/ingest/orm/grpc_node_dummy.py + # The values are hardcoded on nucliadb/src/nucliadb/ingest/orm/grpc_node_dummy.py assert auditreq.kbid == knowledgebox_ingest assert auditreq.rid == rid @@ -427,7 +427,7 @@ async def test_ingest_audit_stream_files_only( # Minimal assert to make sure we get the information from the node on the audit # gets from the sidecar to the audit report when adding or modifying a resource - # The values are hardcoded on nucliadb/nucliadb/ingest/orm/grpc_node_dummy.py + # The values are hardcoded on nucliadb/src/nucliadb/ingest/orm/grpc_node_dummy.py assert auditreq.kbid == knowledgebox_ingest assert auditreq.rid == rid @@ -447,7 +447,7 @@ async def test_ingest_audit_stream_files_only( # Minimal assert to make sure we get the information from the node on the audit # gets from the sidecar to the audit report when adding or modifying a resource - # The values are hardcoded on nucliadb/nucliadb/ingest/orm/grpc_node_dummy.py + # The values are hardcoded on nucliadb/src/nucliadb/ingest/orm/grpc_node_dummy.py assert auditreq.kbid == knowledgebox_ingest assert auditreq.rid == rid @@ -594,7 +594,7 @@ async def test_ingest_audit_stream_mixed( # Minimal assert to make sure we get the information from the node on the audit # gets from the sidecar to the audit report when adding or modifying a resource - # The values are hardcoded on nucliadb/nucliadb/ingest/orm/grpc_node_dummy.py + # The values are hardcoded on nucliadb/src/nucliadb/ingest/orm/grpc_node_dummy.py assert auditreq.kbid == kbid assert auditreq.rid == rid diff --git a/nucliadb_node_binding/cov.sh b/nucliadb_node_binding/cov.sh index 28d9eac9d4..0085b23c47 100755 --- a/nucliadb_node_binding/cov.sh +++ b/nucliadb_node_binding/cov.sh @@ -12,7 +12,7 @@ pip install ../target/wheels/nucliadb_node_binding*.whl pytest -rfE --cov=nucliadb_node_binding --cov-config=../.coveragerc -s --tb=native -v --cov-report term-missing:skip-covered --cov-report xml tests # generating coverage for the CI -cargo llvm-cov report --ignore-filename-regex="^(.*/nucliadb_vectors/).*|^(.*/nucliadb_texts).*|^(.*/nucliadb_relations).*|^(.*/nucliadb_core/).*|^(.*/nucliadb_node/).*|^(.*/nucliadb_paragraphs/).*|^(.*/nucliadb_protos/)|^(.*/nucliadb/nucliadb/)" --lcov --output-path coverage.lcov +cargo llvm-cov report --ignore-filename-regex="^(.*/nucliadb_vectors/).*|^(.*/nucliadb_texts).*|^(.*/nucliadb_relations).*|^(.*/nucliadb_core/).*|^(.*/nucliadb_node/).*|^(.*/nucliadb_paragraphs/).*|^(.*/nucliadb_protos/)|^(.*/nucliadb/src/nucliadb/)" --lcov --output-path coverage.lcov # coverage for the console -cargo llvm-cov report --ignore-filename-regex="^(.*/nucliadb_vectors/).*|^(.*/nucliadb_texts).*|^(.*/nucliadb_relations).*|^(.*/nucliadb_core/).*|^(.*/nucliadb_node/).*|^(.*/nucliadb_paragraphs/).*|^(.*/nucliadb_protos/)|^(.*/nucliadb/nucliadb/)" +cargo llvm-cov report --ignore-filename-regex="^(.*/nucliadb_vectors/).*|^(.*/nucliadb_texts).*|^(.*/nucliadb_relations).*|^(.*/nucliadb_core/).*|^(.*/nucliadb_node/).*|^(.*/nucliadb_paragraphs/).*|^(.*/nucliadb_protos/)|^(.*/nucliadb/src/nucliadb/)"