From b195d9f9e3be6636cf825c8ee8d9cdbb8a1f1479 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 6 Jun 2024 11:36:05 +0200 Subject: [PATCH 1/3] Run all pre-checks in a single job --- .github/workflows/_component_prechecks.yml | 32 --------- .github/workflows/ci.yml | 34 +++------ .github/workflows/nucliadb.yml | 6 -- .github/workflows/nucliadb_dataset.yml | 9 --- .github/workflows/nucliadb_ingest.yml | 23 ------ .github/workflows/nucliadb_models.yml | 24 ------- .github/workflows/nucliadb_node.yml | 34 --------- .github/workflows/nucliadb_reader.yml | 23 ------ .github/workflows/nucliadb_sdk.yml | 9 --- .github/workflows/nucliadb_search.yml | 23 ------ .github/workflows/nucliadb_standalone.yml | 23 ------ .github/workflows/nucliadb_telemetry.yml | 9 --- .github/workflows/nucliadb_train.yml | 23 ------ .github/workflows/nucliadb_utils.yml | 10 --- .github/workflows/nucliadb_writer.yml | 23 ------ Makefile | 2 +- code-requirements.txt | 5 -- nucliadb/Makefile | 12 ---- nucliadb/setup.cfg | 16 ----- nucliadb_dataset/Makefile | 3 +- nucliadb_dataset/setup.cfg | 15 ---- nucliadb_models/Makefile | 4 +- nucliadb_models/setup.cfg | 16 ----- nucliadb_models/src/nucliadb_models/search.py | 12 ++-- nucliadb_node/Makefile | 3 +- nucliadb_node_binding/Makefile | 3 +- nucliadb_node_binding/setup.cfg | 3 - nucliadb_performance/Makefile | 4 +- nucliadb_performance/setup.cfg | 14 ---- nucliadb_protos/python/setup.cfg | 11 --- nucliadb_sdk/Makefile | 3 +- nucliadb_sdk/setup.cfg | 15 ---- .../src/nucliadb_sdk/v2/docstrings.py | 4 +- nucliadb_sidecar/Makefile | 3 +- nucliadb_sidecar/setup.cfg | 15 +--- nucliadb_telemetry/Makefile | 4 +- nucliadb_telemetry/setup.cfg | 16 ----- .../src/nucliadb_telemetry/fastapi/tracing.py | 4 +- nucliadb_utils/Makefile | 3 +- nucliadb_utils/setup.cfg | 19 +---- .../nucliadb_utils/tests/asyncbenchmark.py | 3 +- .../src/nucliadb_utils/tests/nats.py | 6 +- pdm.lock | 71 +++---------------- pyproject.toml | 1 - ruff.toml | 4 ++ 45 files changed, 51 insertions(+), 548 deletions(-) delete mode 100644 .github/workflows/_component_prechecks.yml delete mode 100644 .github/workflows/nucliadb_models.yml delete mode 100644 code-requirements.txt delete mode 100644 nucliadb_node_binding/setup.cfg delete mode 100644 nucliadb_performance/setup.cfg delete mode 100644 nucliadb_protos/python/setup.cfg diff --git a/.github/workflows/_component_prechecks.yml b/.github/workflows/_component_prechecks.yml deleted file mode 100644 index bc478a5bf0..0000000000 --- a/.github/workflows/_component_prechecks.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Component pre-checks - -on: - workflow_call: - inputs: - component: - required: true - type: string - python_version: - required: true - type: string - -jobs: - # Job to run pre-checks - pre_checks: - runs-on: ubuntu-latest - - steps: - - name: Checkout the repository - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ inputs.python_version }} - cache: "pip" - - - name: Install package - run: make -C ${{ inputs.component }}/ install-dev - - - name: Run pre-checks - run: make -C ${{ inputs.component }}/ lint diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41b07a7c6f..afff9c7702 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,31 +62,7 @@ jobs: run: git status && [[ -z "$(git status -s)" ]] pre-checks-python: - name: "Pre-checks: Node python lint" - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.12"] - - steps: - - name: Checkout the repository - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Run pre-checks - run: | - pip install -r code-requirements.txt - isort -c --profile black nucliadb_sidecar - cd nucliadb_sidecar - flake8 src/nucliadb_sidecar --config=setup.cfg - black --check . - - mypy-check: - name: "Pre-checks: Node mypy" + name: "Pre-checks: Python lint" runs-on: ubuntu-latest needs: - build-virtual-env @@ -103,7 +79,13 @@ jobs: key: venv-${{ github.sha }} fail-on-cache-miss: true - name: Run mypy - run: pdm run mypy nucliadb_sidecar + run: | + source .venv/bin/activate + for package in nucliadb nucliadb_utils nucliadb_telemetry nucliadb_sdk nucliadb_dataset nucliadb_models nucliadb_sidecar nucliadb_node_binding nucliadb_performance + do + make -C $package lint || ERROR=1 + done + exit $ERROR licenses: name: Check Licenses diff --git a/.github/workflows/nucliadb.yml b/.github/workflows/nucliadb.yml index 98572d3a8c..374398742a 100644 --- a/.github/workflows/nucliadb.yml +++ b/.github/workflows/nucliadb.yml @@ -23,12 +23,6 @@ concurrency: cancel-in-progress: true jobs: - pre-checks: - uses: ./.github/workflows/_component_prechecks.yml - with: - component: "nucliadb" - python_version: "3.12" - # Job to run Python tests tests: name: NucliaDBTests diff --git a/.github/workflows/nucliadb_dataset.yml b/.github/workflows/nucliadb_dataset.yml index 90b4a13365..c00fd9741f 100644 --- a/.github/workflows/nucliadb_dataset.yml +++ b/.github/workflows/nucliadb_dataset.yml @@ -13,15 +13,6 @@ concurrency: cancel-in-progress: true jobs: - pre-checks: - strategy: - matrix: - python-version: ["3.9", "3.12"] - uses: ./.github/workflows/_component_prechecks.yml - with: - python_version: "${{ matrix.python-version }}" - component: "nucliadb_dataset" - # Job to run tests tests: name: NucliaDBDatasetTests diff --git a/.github/workflows/nucliadb_ingest.yml b/.github/workflows/nucliadb_ingest.yml index a2c4aab24e..4fd2faf45d 100644 --- a/.github/workflows/nucliadb_ingest.yml +++ b/.github/workflows/nucliadb_ingest.yml @@ -18,29 +18,6 @@ env: TESTING_TIKV_LOCAL: "true" jobs: - # Job to run pre-checks - pre-checks: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.12"] - - steps: - - name: Checkout the repository - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install package - run: make -C nucliadb/ install-dev - - - name: Run pre-checks - run: make -C nucliadb/ lint - # Job to run tests tests: runs-on: ubuntu-latest diff --git a/.github/workflows/nucliadb_models.yml b/.github/workflows/nucliadb_models.yml deleted file mode 100644 index 92419656ed..0000000000 --- a/.github/workflows/nucliadb_models.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: nucliadb Models (py) - -on: - push: - branches: - - main - pull_request: - branches: - - main - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - pre-checks: - name: NucliaDBModelsTests - strategy: - matrix: - python-version: ["3.9", "3.12"] - uses: ./.github/workflows/_component_prechecks.yml - with: - python_version: "${{ matrix.python-version }}" - component: "nucliadb_models" diff --git a/.github/workflows/nucliadb_node.yml b/.github/workflows/nucliadb_node.yml index 29feabd90c..de21b8a9f5 100644 --- a/.github/workflows/nucliadb_node.yml +++ b/.github/workflows/nucliadb_node.yml @@ -69,39 +69,6 @@ env: GCP_SERVICE_ACCOUNT: "github-actions@nuclia-internal.iam.gserviceaccount.com" jobs: - pre-checks-python: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.12"] - - steps: - - name: Checkout the repository - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install package - run: make -C nucliadb_sidecar/ install-dev - - - name: Run pre-checks - run: make -C nucliadb_sidecar/ lint - - licenses: - name: Check Licenses - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: EmbarkStudios/cargo-deny-action@v1 - name: cargo deny - with: - log-level: warn - command: check licenses - format-rust: name: Rust code Format runs-on: ubuntu-latest @@ -168,7 +135,6 @@ jobs: tests-python: name: Python tests runs-on: ubuntu-latest - needs: [pre-checks-python, licenses] strategy: matrix: diff --git a/.github/workflows/nucliadb_reader.yml b/.github/workflows/nucliadb_reader.yml index a776aa59aa..6758ca0f05 100644 --- a/.github/workflows/nucliadb_reader.yml +++ b/.github/workflows/nucliadb_reader.yml @@ -17,29 +17,6 @@ env: IMAGE_NAME: reader jobs: - # Job to run pre-checks - pre-checks: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.9', '3.12'] - - steps: - - name: Checkout the repository - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - - name: Install package - run: make -C nucliadb/ install-dev - - - name: Run pre-checks - run: make -C nucliadb/ lint-reader - # Job to run tests tests: runs-on: ubuntu-latest diff --git a/.github/workflows/nucliadb_sdk.yml b/.github/workflows/nucliadb_sdk.yml index f959768f37..5baa85b67f 100644 --- a/.github/workflows/nucliadb_sdk.yml +++ b/.github/workflows/nucliadb_sdk.yml @@ -13,15 +13,6 @@ concurrency: cancel-in-progress: true jobs: - pre-checks: - strategy: - matrix: - python-version: ["3.9", "3.12"] - uses: ./.github/workflows/_component_prechecks.yml - with: - python_version: "${{ matrix.python-version }}" - component: "nucliadb_sdk" - # Job to run tests tests: name: NucliaDBSDKTests diff --git a/.github/workflows/nucliadb_search.yml b/.github/workflows/nucliadb_search.yml index 16b8779208..79484a8f30 100644 --- a/.github/workflows/nucliadb_search.yml +++ b/.github/workflows/nucliadb_search.yml @@ -53,29 +53,6 @@ env: GCP_SERVICE_ACCOUNT: "github-actions@nuclia-internal.iam.gserviceaccount.com" jobs: - # Job to run pre-checks - pre-checks: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.9', '3.10', '3.12'] - - steps: - - name: Checkout the repository - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - - name: Install package - run: make -C nucliadb/ install-dev - - - name: Run pre-checks - run: make -C nucliadb/ lint-search - # Job to run tests tests: runs-on: ubuntu-latest diff --git a/.github/workflows/nucliadb_standalone.yml b/.github/workflows/nucliadb_standalone.yml index 723af6e8d2..e666935d5a 100644 --- a/.github/workflows/nucliadb_standalone.yml +++ b/.github/workflows/nucliadb_standalone.yml @@ -47,29 +47,6 @@ permissions: contents: read # This is required for actions/checkout jobs: - # Job to run pre-checks - pre-checks: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.12"] - - steps: - - name: Checkout the repository - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install package - run: make -C nucliadb/ install-dev - - - name: Run pre-checks - run: make -C nucliadb/ lint-standalone - # Job to run tests tests: runs-on: ubuntu-latest diff --git a/.github/workflows/nucliadb_telemetry.yml b/.github/workflows/nucliadb_telemetry.yml index 36d905432a..c6bd8a1082 100644 --- a/.github/workflows/nucliadb_telemetry.yml +++ b/.github/workflows/nucliadb_telemetry.yml @@ -19,15 +19,6 @@ concurrency: cancel-in-progress: true jobs: - pre-checks: - strategy: - matrix: - python-version: ["3.9", "3.12"] - uses: ./.github/workflows/_component_prechecks.yml - with: - python_version: "${{ matrix.python-version }}" - component: "nucliadb_telemetry" - # Job to run tests tests: name: NucliaDBTelemetryTests diff --git a/.github/workflows/nucliadb_train.yml b/.github/workflows/nucliadb_train.yml index 5d6310e66b..a1e0e01359 100644 --- a/.github/workflows/nucliadb_train.yml +++ b/.github/workflows/nucliadb_train.yml @@ -18,29 +18,6 @@ env: IMAGE_NAME: nucliadb_train jobs: - # Job to run pre-checks - pre-checks: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.12"] - - steps: - - name: Checkout the repository - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install package - run: make -C nucliadb/ install-dev - - - name: Run pre-checks - run: make -C nucliadb/ lint-train - # Job to run tests tests: runs-on: ubuntu-latest diff --git a/.github/workflows/nucliadb_utils.yml b/.github/workflows/nucliadb_utils.yml index 4a16f82494..27858379d1 100644 --- a/.github/workflows/nucliadb_utils.yml +++ b/.github/workflows/nucliadb_utils.yml @@ -13,16 +13,6 @@ concurrency: cancel-in-progress: true jobs: - # Job to run pre-checks - pre-checks: - strategy: - matrix: - python-version: ["3.9", "3.12"] - uses: ./.github/workflows/_component_prechecks.yml - with: - python_version: "${{ matrix.python-version }}" - component: "nucliadb_utils" - # Job to run tests tests: name: NucliaDBUtilsTests diff --git a/.github/workflows/nucliadb_writer.yml b/.github/workflows/nucliadb_writer.yml index 9bc6e161a0..3bbe03b7fb 100644 --- a/.github/workflows/nucliadb_writer.yml +++ b/.github/workflows/nucliadb_writer.yml @@ -17,29 +17,6 @@ env: IMAGE_NAME: writer jobs: - # Job to run pre-checks - pre-checks: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.9', '3.12'] - - steps: - - name: Checkout the repository - uses: actions/checkout@v3 - - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: 'pip' - - - name: Install package - run: make -C nucliadb/ install-dev - - - name: Run pre-checks - run: make -C nucliadb/ lint-writer - # Job to run tests tests: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index 7ebf77e53b..d042fadc44 100644 --- a/Makefile +++ b/Makefile @@ -84,7 +84,7 @@ venv: ## Initializes an environment install: ## Install dependencies (on the active environment) pip install --upgrade pip wheel pip install "Cython==0.29.24" "grpcio-tools>=1.44.0,<1.63.0" - pip install -r code-requirements.txt -r test-requirements.txt + pip install -r test-requirements.txt pip install -e ./nucliadb_protos/python pip install -e ./nucliadb_telemetry pip install -e ./nucliadb_utils diff --git a/code-requirements.txt b/code-requirements.txt deleted file mode 100644 index b14624a61d..0000000000 --- a/code-requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -flake8==7.0.0 -ruff==0.3.5 -black==24.3.0 -isort==5.10.1 -pre-commit==2.20.0 diff --git a/nucliadb/Makefile b/nucliadb/Makefile index 63cf293488..69e3d022e2 100644 --- a/nucliadb/Makefile +++ b/nucliadb/Makefile @@ -21,7 +21,6 @@ install-dev: check-system pip install --upgrade pip wheel cd .. && pip install \ -r test-requirements.txt \ - -r code-requirements.txt \ -r nucliadb/requirements-sources.txt \ -r nucliadb/requirements.txt \ -r nucliadb/requirements-test.txt @@ -79,17 +78,6 @@ $(test_cov_subpackages_targets): # Generic implicit rules -format-%: - cd .. && isort --profile black nucliadb/src/nucliadb/$* - black nucliadb/$* - ruff check --fix nucliadb/$* --config=../ruff.toml - -lint-%: - ruff check src/nucliadb/$* --config=../ruff.toml - cd .. && isort -c --profile black nucliadb/src/nucliadb/$* - black --check src/nucliadb/$* - MYPYPATH=../mypy_stubs mypy --config-file=../mypy.ini src/nucliadb/$* - test-%: $(PYTEST) tests/$* diff --git a/nucliadb/setup.cfg b/nucliadb/setup.cfg index 900c27103c..f7ddc9d672 100644 --- a/nucliadb/setup.cfg +++ b/nucliadb/setup.cfg @@ -1,19 +1,3 @@ -[flake8] -max_line_length = 120 -no-accept-encodings = True -ignore = - E302 - W391 - E701 - W504 - F901 - E252 - W503 - E203 - BLK100 - F541 - E704 - [tool:pytest] # pytest is requiring this for async projects otherwise # we'd need to change all our fixtures diff --git a/nucliadb_dataset/Makefile b/nucliadb_dataset/Makefile index 4a345d1474..3a439bb566 100644 --- a/nucliadb_dataset/Makefile +++ b/nucliadb_dataset/Makefile @@ -3,7 +3,6 @@ install-dev: pip install --upgrade pip wheel cd .. && pip install \ -r test-requirements.txt \ - -r code-requirements.txt \ -r nucliadb_dataset/requirements-sources.txt \ -r nucliadb_dataset/requirements.txt \ -r nucliadb_dataset/requirements-test.txt @@ -16,7 +15,7 @@ format: .PHONY: lint lint: - flake8 . --config=setup.cfg + ruff check --config=../ruff.toml . cd .. && isort -c --profile black nucliadb_dataset black --check . MYPYPATH=../mypy_stubs mypy --config-file=../mypy.ini . diff --git a/nucliadb_dataset/setup.cfg b/nucliadb_dataset/setup.cfg index feb120e78e..5bcb67fcc9 100644 --- a/nucliadb_dataset/setup.cfg +++ b/nucliadb_dataset/setup.cfg @@ -1,18 +1,3 @@ -[flake8] -max_line_length = 120 -no-accept-encodings = True -ignore = - E302 - W391 - E701 - W504 - F901 - E252 - W503 - E203 - BLK100 - F541 - [zest.releaser] tag-format = nucliadb_dataset-{version} tag-message = NucliaDB Dataset {version} diff --git a/nucliadb_models/Makefile b/nucliadb_models/Makefile index b070b04591..47b4a4717a 100644 --- a/nucliadb_models/Makefile +++ b/nucliadb_models/Makefile @@ -3,7 +3,6 @@ install-dev: pip install --upgrade pip wheel cd .. && pip install \ -r test-requirements.txt \ - -r code-requirements.txt \ -r nucliadb_models/requirements-sources.txt \ -r nucliadb_models/requirements.txt pip install -e . @@ -15,8 +14,7 @@ format: .PHONY: lint lint: - flake8 . --config=setup.cfg + ruff check --config=../ruff.toml . cd .. && isort -c --profile black nucliadb_models black --check . MYPYPATH=../mypy_stubs mypy --config-file=../mypy.ini . - diff --git a/nucliadb_models/setup.cfg b/nucliadb_models/setup.cfg index 8444e0a6d3..31ad82b6a0 100644 --- a/nucliadb_models/setup.cfg +++ b/nucliadb_models/setup.cfg @@ -1,18 +1,2 @@ [aliases] test = pytest - -[flake8] -ignore = - E302 - W292 - W391 - E722 - E501 - W503 - E203 - F541 - E402 - E701 - -max-line-length = 110 - diff --git a/nucliadb_models/src/nucliadb_models/search.py b/nucliadb_models/src/nucliadb_models/search.py index 1bac5a7835..2a67977fb8 100644 --- a/nucliadb_models/src/nucliadb_models/search.py +++ b/nucliadb_models/src/nucliadb_models/search.py @@ -673,7 +673,7 @@ class MinScore(BaseModel): bm25: float = Field( default=0, title="Minimum bm25 score", - description="Minimum score used to filter bm25 index search. Check out the documentation for more information on how to use this parameter: https://docs.nuclia.dev/docs/docs/using/search/#minimum-score", + description="Minimum score used to filter bm25 index search. Check out the documentation for more information on how to use this parameter: https://docs.nuclia.dev/docs/docs/using/search/#minimum-score", # noqa: E501 ge=0, ) @@ -833,7 +833,7 @@ class ChatModel(BaseModel): generative_model: Optional[str] = Field( default=None, title="Generative model", - description="The generative model to use for the predict chat endpoint. If not provided, the model configured for the Knowledge Box is used.", + description="The generative model to use for the predict chat endpoint. If not provided, the model configured for the Knowledge Box is used.", # noqa: E501 ) max_tokens: Optional[int] = Field( @@ -859,7 +859,7 @@ class RephraseModel(BaseModel): generative_model: Optional[str] = Field( default=None, title="Generative model", - description="The generative model to use for the rephrase endpoint. If not provided, the model configured for the Knowledge Box is used.", + description="The generative model to use for the rephrase endpoint. If not provided, the model configured for the Knowledge Box is used.", # noqa: E501 ) @@ -1055,7 +1055,7 @@ class ChatRequest(BaseModel): generative_model: Optional[str] = Field( default=None, title="Generative model", - description="The generative model to use for the chat endpoint. If not provided, the model configured for the Knowledge Box is used.", + description="The generative model to use for the chat endpoint. If not provided, the model configured for the Knowledge Box is used.", # noqa: E501 ) max_tokens: Optional[Union[int, MaxTokens]] = Field( @@ -1132,7 +1132,7 @@ class SummarizeRequest(BaseModel): generative_model: Optional[str] = Field( default=None, title="Generative model", - description="The generative model to use for the summarization. If not provided, the model configured for the Knowledge Box is used.", + description="The generative model to use for the summarization. If not provided, the model configured for the Knowledge Box is used.", # noqa: E501 ) user_prompt: Optional[str] = Field( @@ -1146,7 +1146,7 @@ class SummarizeRequest(BaseModel): min_length=1, max_length=100, title="Resources", - description="Uids or slugs of the resources to summarize. If the resources are not found, they will be ignored.", + description="Uids or slugs of the resources to summarize. If the resources are not found, they will be ignored.", # noqa: E501 ) summary_kind: SummaryKind = Field( diff --git a/nucliadb_node/Makefile b/nucliadb_node/Makefile index 816fdae23d..6e402b40e5 100644 --- a/nucliadb_node/Makefile +++ b/nucliadb_node/Makefile @@ -3,7 +3,6 @@ install-dev: check-system pip install --upgrade pip wheel cd .. && pip install \ -r test-requirements.txt \ - -r code-requirements.txt \ -r nucliadb_sidecar/requirements-sources.txt \ -r nucliadb_sidecar/requirements.txt pip install -e . @@ -20,7 +19,7 @@ format: .PHONY: lint lint: - flake8 . --config=setup.cfg + ruff check --config=../ruff.toml . cd .. && isort -c --profile black nucliadb_sidecar black --check . MYPYPATH=../mypy_stubs mypy --config-file=../mypy.ini . diff --git a/nucliadb_node_binding/Makefile b/nucliadb_node_binding/Makefile index fd06a888d3..fdbc602899 100644 --- a/nucliadb_node_binding/Makefile +++ b/nucliadb_node_binding/Makefile @@ -5,7 +5,6 @@ install-dev: check-system pip install --upgrade pip wheel cd .. && pip install \ -r test-requirements.txt \ - -r code-requirements.txt \ -r nucliadb_node/requirements-sources.txt \ -r nucliadb_node/requirements.txt \ ./nucliadb_node/ @@ -23,7 +22,7 @@ format: .PHONY: lint lint: - flake8 tests --config=setup.cfg + ruff check --config=../ruff.toml . isort -c --profile black tests black --check tests MYPYPATH=../mypy_stubs mypy --config-file=../mypy.ini . diff --git a/nucliadb_node_binding/setup.cfg b/nucliadb_node_binding/setup.cfg deleted file mode 100644 index 8332de349f..0000000000 --- a/nucliadb_node_binding/setup.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[flake8] -max-line-length = 120 -extend-ignore = E203 diff --git a/nucliadb_performance/Makefile b/nucliadb_performance/Makefile index 179f83166f..24c3f4695b 100644 --- a/nucliadb_performance/Makefile +++ b/nucliadb_performance/Makefile @@ -46,7 +46,7 @@ install: # Install the package .PHONY: install-dev install-dev: # Install the package for development pip install --upgrade pip wheel - pip install -r ../test-requirements.txt -r ../code-requirements.txt -r requirements.txt + pip install -r ../test-requirements.txt -r requirements.txt pip install -e . @@ -57,7 +57,7 @@ format: # Format the code .PHONY: lint lint: # Run lint checks - flake8 . --config=setup.cfg + ruff check --config=../ruff.toml . cd .. && isort -c --profile black nucliadb_performance black --check . MYPYPATH=../mypy_stubs mypy --config-file=../mypy.ini . --show-traceback diff --git a/nucliadb_performance/setup.cfg b/nucliadb_performance/setup.cfg deleted file mode 100644 index 93c0013af8..0000000000 --- a/nucliadb_performance/setup.cfg +++ /dev/null @@ -1,14 +0,0 @@ -[flake8] -max_line_length = 120 -no-accept-encodings = True -ignore = - E302 - W391 - E701 - W504 - F901 - E252 - W503 - E203 - BLK100 - F541 diff --git a/nucliadb_protos/python/setup.cfg b/nucliadb_protos/python/setup.cfg deleted file mode 100644 index 2da56deee5..0000000000 --- a/nucliadb_protos/python/setup.cfg +++ /dev/null @@ -1,11 +0,0 @@ -[flake8] -ignore = - E302 - W292 - W391 - E722 - E501 - E203 - W503 - E203 -max-line-length = 110 diff --git a/nucliadb_sdk/Makefile b/nucliadb_sdk/Makefile index a144a33448..4f39c090dc 100644 --- a/nucliadb_sdk/Makefile +++ b/nucliadb_sdk/Makefile @@ -3,7 +3,6 @@ install-dev: pip install --upgrade pip wheel cd .. && pip install \ -r test-requirements.txt \ - -r code-requirements.txt \ -r nucliadb_sdk/requirements-sources.txt \ -r nucliadb_sdk/requirements.txt \ -r nucliadb_sdk/requirements-test.txt @@ -16,7 +15,7 @@ format: .PHONY: lint lint: - flake8 . --config=setup.cfg + ruff check --config=../ruff.toml . cd .. && isort -c --profile black nucliadb_sdk black --check . MYPYPATH=../mypy_stubs mypy --config-file=../mypy.ini . diff --git a/nucliadb_sdk/setup.cfg b/nucliadb_sdk/setup.cfg index f04756b6ad..f7ddc9d672 100644 --- a/nucliadb_sdk/setup.cfg +++ b/nucliadb_sdk/setup.cfg @@ -1,18 +1,3 @@ -[flake8] -max_line_length = 120 -no-accept-encodings = True -ignore = - E302 - W391 - E701 - W504 - F901 - E252 - W503 - E203 - BLK100 - F541 - [tool:pytest] # pytest is requiring this for async projects otherwise # we'd need to change all our fixtures diff --git a/nucliadb_sdk/src/nucliadb_sdk/v2/docstrings.py b/nucliadb_sdk/src/nucliadb_sdk/v2/docstrings.py index cf4c8ba1d5..f469cab65d 100644 --- a/nucliadb_sdk/src/nucliadb_sdk/v2/docstrings.py +++ b/nucliadb_sdk/src/nucliadb_sdk/v2/docstrings.py @@ -283,8 +283,8 @@ class Docstring(BaseModel): description="Get a summary of a document or a list of documents", code=""">>> summary = sdk.summarize(kbid="mykbid", resources=["uuid1"]).summary >>> print(summary) -'The document talks about Seville and its temperature. It also mentions the coldest month of the year, which is January.' # noqa -""", +'The document talks about Seville and its temperature. It also mentions the coldest month of the year, which is January.' +""", # noqa: E501 ), ], ) diff --git a/nucliadb_sidecar/Makefile b/nucliadb_sidecar/Makefile index ed9d18d954..2d4c93c565 100644 --- a/nucliadb_sidecar/Makefile +++ b/nucliadb_sidecar/Makefile @@ -3,7 +3,6 @@ install-dev: check-system pip install --upgrade pip wheel cd .. && pip install \ -r test-requirements.txt \ - -r code-requirements.txt \ -r nucliadb_sidecar/requirements-sources.txt \ -r nucliadb_sidecar/requirements.txt pip install -e . @@ -20,7 +19,7 @@ format: .PHONY: lint lint: - flake8 . --config=setup.cfg + ruff check --config=../ruff.toml . cd .. && isort -c --profile black nucliadb_sidecar black --check . MYPYPATH=../mypy_stubs mypy --config-file=../mypy.ini . diff --git a/nucliadb_sidecar/setup.cfg b/nucliadb_sidecar/setup.cfg index a4c1d0d9fc..9aa3832c3e 100644 --- a/nucliadb_sidecar/setup.cfg +++ b/nucliadb_sidecar/setup.cfg @@ -1,20 +1,7 @@ [aliases] test = pytest - -[flake8] -ignore = - E302 - W292 - W391 - E722 - E501 - W503 - E203 - F541 -max-line-length = 110 - [tool:pytest] # pytest is requiring this for async projects otherwise # we'd need to change all our fixtures -asyncio_mode=auto \ No newline at end of file +asyncio_mode=auto diff --git a/nucliadb_telemetry/Makefile b/nucliadb_telemetry/Makefile index b0f8b559b3..e854f2df11 100644 --- a/nucliadb_telemetry/Makefile +++ b/nucliadb_telemetry/Makefile @@ -6,7 +6,7 @@ protos: .PHONY: install-dev install-dev: pip install --upgrade pip wheel - pip install -r ../test-requirements.txt -r ../code-requirements.txt -r requirements.txt -r requirements-test.txt + pip install -r ../test-requirements.txt -r requirements.txt -r requirements-test.txt # utils dep required for test fixtures for nats # NEEDS TO BE DECOUPLED EVENTUALLY pip install -e ../nucliadb_protos/python @@ -20,7 +20,7 @@ format: .PHONY: lint lint: - flake8 . --config=setup.cfg + ruff check --config=../ruff.toml . cd .. && isort -c --profile black nucliadb_telemetry black --check . MYPYPATH=../mypy_stubs mypy --config-file=../mypy.ini . --show-traceback diff --git a/nucliadb_telemetry/setup.cfg b/nucliadb_telemetry/setup.cfg index a42eb3e8ab..8bb729180b 100644 --- a/nucliadb_telemetry/setup.cfg +++ b/nucliadb_telemetry/setup.cfg @@ -1,19 +1,3 @@ -[flake8] -ignore = - E302 - W292 - W391 - E722 - E501 - W503 - E203 - F541 - E402 - E704 - -max-line-length = 110 -exclude = src/nucliadb_telemetry/tests/grpc/hellostreamingworld_pb2_grpc.py,src/nucliadb_telemetry/tests/grpc/helloworld_pb2_grpc.py,src/nucliadb_telemetry/tests/grpc/hellostreamingworld_pb2.py,src/nucliadb_telemetry/tests/grpc/helloworld_pb2.py - [zest.releaser] tag-format = nucliadb_telemetry-{version} tag-message = NucliaDB Telemetry {version} diff --git a/nucliadb_telemetry/src/nucliadb_telemetry/fastapi/tracing.py b/nucliadb_telemetry/src/nucliadb_telemetry/fastapi/tracing.py index 6ce2a078c1..887dd8ca54 100644 --- a/nucliadb_telemetry/src/nucliadb_telemetry/fastapi/tracing.py +++ b/nucliadb_telemetry/src/nucliadb_telemetry/fastapi/tracing.py @@ -262,8 +262,8 @@ class OpenTelemetryMiddleware: requests it is invoked with. Args: app: The ASGI application callable to forward requests to. - default_span_details: Callback which should return a string and a tuple, representing the desired default span name and a - dictionary with any additional span attributes to set. + default_span_details: Callback which should return a string and a tuple, representing the desired default + span name and a dictionary with any additional span attributes to set. Optional: Defaults to get_default_span_details. server_request_hook: Optional callback which is called with the server span and ASGI scope object for every incoming request. diff --git a/nucliadb_utils/Makefile b/nucliadb_utils/Makefile index 6ad5522e7c..1f3e6f45ee 100644 --- a/nucliadb_utils/Makefile +++ b/nucliadb_utils/Makefile @@ -3,7 +3,6 @@ install-dev: pip install --upgrade pip wheel cd .. && pip install \ -r test-requirements.txt \ - -r code-requirements.txt \ -r nucliadb_utils/requirements-sources.txt \ -r nucliadb_utils/requirements.txt pip install -e .[cache,storages,fastapi,postgres] @@ -15,7 +14,7 @@ format: .PHONY: lint lint: - flake8 . --config=setup.cfg + ruff check --config=../ruff.toml . cd .. && isort -c --profile black nucliadb_utils black --check . MYPYPATH=../mypy_stubs mypy --config-file=../mypy.ini src diff --git a/nucliadb_utils/setup.cfg b/nucliadb_utils/setup.cfg index d112d9441c..c8e70f5dae 100644 --- a/nucliadb_utils/setup.cfg +++ b/nucliadb_utils/setup.cfg @@ -3,24 +3,7 @@ release = yes tag-format = nucliadb_utils-{version} tag-message = NucliaDB Utils {version} - -[flake8] -ignore = - E302 - W292 - W391 - E722 - E501 - W503 - E203 - F541 - E402 - E704 - -max-line-length = 110 - - [tool:pytest] # pytest is requiring this for async projects otherwise # we'd need to change all our fixtures -asyncio_mode=auto \ No newline at end of file +asyncio_mode=auto diff --git a/nucliadb_utils/src/nucliadb_utils/tests/asyncbenchmark.py b/nucliadb_utils/src/nucliadb_utils/tests/asyncbenchmark.py index a588029e0f..e6b36c047b 100644 --- a/nucliadb_utils/src/nucliadb_utils/tests/asyncbenchmark.py +++ b/nucliadb_utils/src/nucliadb_utils/tests/asyncbenchmark.py @@ -29,13 +29,12 @@ import pytest from pytest_benchmark.session import BenchmarkSession # type: ignore +from pytest_benchmark.stats import Metadata # type: ignore from pytest_benchmark.timers import compute_timer_precision # type: ignore from pytest_benchmark.utils import NameWrapper, format_time # type: ignore T = TypeVar("T") -from pytest_benchmark.stats import Metadata # type: ignore - class FixtureAlreadyUsed(Exception): ... # noqa diff --git a/nucliadb_utils/src/nucliadb_utils/tests/nats.py b/nucliadb_utils/src/nucliadb_utils/tests/nats.py index ff9608f8a2..a364c817da 100644 --- a/nucliadb_utils/src/nucliadb_utils/tests/nats.py +++ b/nucliadb_utils/src/nucliadb_utils/tests/nats.py @@ -160,7 +160,7 @@ def stop(self): if self.proc.returncode is not None: if self.debug: print( - "[\033[0;31mDEBUG\033[0;0m] Server listening on port {port} finished running already with exit {ret}".format( + "[\033[0;31mDEBUG\033[0;0m] Server listening on port {port} finished running already with exit {ret}".format( # noqa: E501 port=self.port, ret=self.proc.returncode ) ) @@ -195,7 +195,7 @@ def start_gnatsd(gnatsd: Gnatsd): # pragma: no cover response = httpclient.getresponse() if response.status == 200: break - except: + except Exception: retries += 1 time.sleep(0.1) @@ -250,7 +250,7 @@ def check(self): try: s.connect((self.host, int(self.get_port()))) return True - except: + except Exception: return False diff --git a/pdm.lock b/pdm.lock index 66c8c65e82..adca18dfdf 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:c279228e2ce9e2fa697c6a8525850279e9b0df6d0e025cfb30cc4772bc7327ed" +content_hash = "sha256:393a5694d80f4a0f884b0aa8658151e3eaedb36d8ff33678b7bbba3ea185803b" [[package]] name = "aiobotocore" @@ -888,22 +888,6 @@ files = [ {file = "filelock-3.14.0.tar.gz", hash = "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a"}, ] -[[package]] -name = "flake8" -version = "7.0.0" -requires_python = ">=3.8.1" -summary = "the modular source code checker: pep8 pyflakes and co" -groups = ["dev"] -dependencies = [ - "mccabe<0.8.0,>=0.7.0", - "pycodestyle<2.12.0,>=2.11.0", - "pyflakes<3.3.0,>=3.2.0", -] -files = [ - {file = "flake8-7.0.0-py2.py3-none-any.whl", hash = "sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3"}, - {file = "flake8-7.0.0.tar.gz", hash = "sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132"}, -] - [[package]] name = "frozenlist" version = "1.4.1" @@ -1782,17 +1766,6 @@ files = [ {file = "maturin-1.5.1.tar.gz", hash = "sha256:3dd834ece80edb866af18cbd4635e0ecac40139c726428d5f1849ae154b26dca"}, ] -[[package]] -name = "mccabe" -version = "0.7.0" -requires_python = ">=3.6" -summary = "McCabe checker, plugin for flake8" -groups = ["dev"] -files = [ - {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, - {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, -] - [[package]] name = "mdurl" version = "0.1.2" @@ -2147,7 +2120,7 @@ files = [ [[package]] name = "nucliadb" -version = "4.0.1" +version = "4.0.3" requires_python = ">=3.9, <4" editable = true path = "./nucliadb" @@ -2224,7 +2197,7 @@ files = [ [[package]] name = "nucliadb-models" -version = "4.0.1" +version = "4.0.3" requires_python = ">=3.9, <4" editable = true path = "./nucliadb_models" @@ -2246,7 +2219,7 @@ groups = ["dev"] [[package]] name = "nucliadb-protos" -version = "4.0.1" +version = "4.0.3" editable = true path = "./nucliadb_protos/python" summary = "protos for nucliadb" @@ -2261,7 +2234,7 @@ dependencies = [ [[package]] name = "nucliadb-sidecar" -version = "4.0.1" +version = "4.0.3" requires_python = ">=3.9, <4" editable = true path = "./nucliadb_sidecar" @@ -2283,7 +2256,7 @@ dependencies = [ [[package]] name = "nucliadb-telemetry" -version = "4.0.1" +version = "4.0.3" requires_python = ">=3.9" editable = true path = "./nucliadb_telemetry" @@ -2299,7 +2272,7 @@ dependencies = [ [[package]] name = "nucliadb-telemetry" -version = "4.0.1" +version = "4.0.3" extras = ["all"] requires_python = ">=3.9" summary = "NucliaDB Telemetry Library Python process" @@ -2315,7 +2288,7 @@ dependencies = [ "grpcio-tools<1.63.0,>=1.44.0", "grpcio<1.63.0,>=1.44.0", "nats-py[nkeys]>=2.5.0", - "nucliadb-telemetry==4.0.1", + "nucliadb-telemetry==4.0.3", "opentelemetry-api==1.21.0", "opentelemetry-exporter-jaeger==1.21.0", "opentelemetry-instrumentation-aiohttp-client>=0.42b0", @@ -2329,7 +2302,7 @@ dependencies = [ [[package]] name = "nucliadb-utils" -version = "4.0.1" +version = "4.0.3" requires_python = ">=3.9, <4" editable = true path = "./nucliadb_utils" @@ -2353,7 +2326,7 @@ dependencies = [ [[package]] name = "nucliadb-utils" -version = "4.0.1" +version = "4.0.3" extras = ["cache", "fastapi", "storages"] requires_python = ">=3.9, <4" summary = "" @@ -2366,7 +2339,7 @@ dependencies = [ "google-api-python-client>=2.37.0", "google-auth>=2.4.1", "lru-dict>=1.1.7", - "nucliadb-utils==4.0.1", + "nucliadb-utils==4.0.3", "oauth2client>=4.1.3", "orjson>=3.6.7", "redis>=4.3.4", @@ -2921,17 +2894,6 @@ files = [ {file = "pycares-4.4.0.tar.gz", hash = "sha256:f47579d508f2f56eddd16ce72045782ad3b1b3b678098699e2b6a1b30733e1c2"}, ] -[[package]] -name = "pycodestyle" -version = "2.11.1" -requires_python = ">=3.8" -summary = "Python style guide checker" -groups = ["dev"] -files = [ - {file = "pycodestyle-2.11.1-py2.py3-none-any.whl", hash = "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67"}, - {file = "pycodestyle-2.11.1.tar.gz", hash = "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f"}, -] - [[package]] name = "pycparser" version = "2.22" @@ -3053,17 +3015,6 @@ files = [ {file = "pydantic_settings-2.2.1.tar.gz", hash = "sha256:00b9f6a5e95553590434c0fa01ead0b216c3e10bc54ae02e37f359948643c5ed"}, ] -[[package]] -name = "pyflakes" -version = "3.2.0" -requires_python = ">=3.8" -summary = "passive checker of Python programs" -groups = ["dev"] -files = [ - {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"}, - {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"}, -] - [[package]] name = "pygments" version = "2.18.0" diff --git a/pyproject.toml b/pyproject.toml index dac9740996..628615f466 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,6 @@ dev = [ # Linting "mypy-protobuf>=3.6.0", "mypy==1.10", - "flake8==7.0.0", "ruff==0.3.5", "black==24.3.0", "isort==5.10.1", diff --git a/ruff.toml b/ruff.toml index b094901211..2a76cbcf84 100644 --- a/ruff.toml +++ b/ruff.toml @@ -19,4 +19,8 @@ ignore = [ exclude = [ "*.pyi", "nucliadb_protos/python/*", + "src/nucliadb_telemetry/tests/grpc/hellostreamingworld_pb2_grpc.py", + "src/nucliadb_telemetry/tests/grpc/helloworld_pb2_grpc.py", + "src/nucliadb_telemetry/tests/grpc/hellostreamingworld_pb2.py", + "src/nucliadb_telemetry/tests/grpc/helloworld_pb2.py", ] From 089fa4f1473f81904690b4b5a1a43f122b2d0922 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 6 Jun 2024 11:44:53 +0200 Subject: [PATCH 2/3] Fix telemetry tests --- .github/workflows/nucliadb_telemetry.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nucliadb_telemetry.yml b/.github/workflows/nucliadb_telemetry.yml index c6bd8a1082..241468b2a8 100644 --- a/.github/workflows/nucliadb_telemetry.yml +++ b/.github/workflows/nucliadb_telemetry.yml @@ -44,7 +44,7 @@ jobs: - name: Run tests run: | - pytest -rfE --cov=nucliadb_telemetry -s --tb=native -v --cov-report xml --cov-append nucliadb_telemetry/nucliadb_telemetry + pytest -rfE --cov=nucliadb_telemetry -s --tb=native -v --cov-report xml --cov-append nucliadb_telemetry/tests - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 From e3f482b726a22a491b266ac499f0f21a3c810aa2 Mon Sep 17 00:00:00 2001 From: Javier Torres Date: Thu, 6 Jun 2024 11:59:49 +0200 Subject: [PATCH 3/3] Cache pdm packages --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afff9c7702..a68ac77964 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,6 +124,7 @@ jobs: uses: pdm-project/setup-pdm@v3 with: python-version: "3.12" + cache: true - name: Install venv run: pdm sync -d - name: Upload virtualenv to cache