dev #2176
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| - dev | |
| push: | |
| branches: | |
| - main | |
| - dev | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # Pre-checks | |
| format-rust: | |
| name: "Pre-checks: Rust code Format" | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: rustfmt | |
| run: cargo fmt --check | |
| clippy-rust: | |
| name: "Pre-checks: Clippy lint" | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: rustup component add clippy | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: clippy | |
| run: cargo clippy | |
| nidx-format-rust: | |
| name: "nidx: Rust code Format" | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: rustfmt | |
| run: cd nidx && cargo fmt --check | |
| nidx-clippy-rust: | |
| name: "nidx: Clippy lint" | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: rustup component add clippy | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: nidx | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| - name: clippy | |
| run: cd nidx && cargo clippy | |
| updated-protos: | |
| name: "Pre-checks: Protos up-to-date" | |
| runs-on: ubuntu-24.04 | |
| permissions: write-all | |
| needs: | |
| - build-virtual-env | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12.5" | |
| - name: Restore venv | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Compile protos | |
| run: source .venv/bin/activate && make protos | |
| - name: Fail if something changed | |
| run: git status && [[ -z "$(git status -s)" ]] | |
| pre-checks-python: | |
| name: "Pre-checks: Python lint" | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - build-virtual-env | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up PDM | |
| uses: pdm-project/setup-pdm@v4 | |
| with: | |
| python-version: "3.12.5" | |
| - name: Restore venv | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Lint source code | |
| run: | | |
| source .venv/bin/activate | |
| for package in nucliadb nucliadb_utils nucliadb_telemetry nucliadb_sdk nucliadb_dataset nucliadb_models nucliadb_sidecar nucliadb_node_binding nucliadb_performance | |
| do | |
| make -C $package lint || ERROR=1 | |
| done | |
| exit $ERROR | |
| pre-checks-python-old: | |
| name: "Pre-checks: Python lint 3.9" | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: | | |
| . | |
| nidx | |
| - name: Set up PDM | |
| uses: pdm-project/setup-pdm@v4 | |
| with: | |
| python-version: "3.9.20" | |
| cache: true | |
| # Needed for nidx_protos (rust) | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| - name: Install venv | |
| run: pdm sync -d --clean --no-editable | |
| - name: Clean build | |
| run: find . -name build | xargs rm -r || true | |
| - name: Lint source code | |
| run: | | |
| source .venv/bin/activate | |
| for package in nucliadb nucliadb_utils nucliadb_telemetry nucliadb_sdk nucliadb_dataset nucliadb_models nucliadb_sidecar nucliadb_node_binding nucliadb_performance | |
| do | |
| make -C $package lint || ERROR=1 | |
| done | |
| exit $ERROR | |
| lint-helm: | |
| name: Lint all helm charts | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install Helm | |
| uses: azure/setup-helm@v4 | |
| with: | |
| version: v3.15.3 | |
| - name: Lint helm packages | |
| run: |- | |
| helm lint charts/* | |
| helm package charts/* | |
| check-licenses: | |
| name: Check Licenses | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check License Header | |
| uses: apache/skywalking-eyes/header@501a28d2fb4a9b962661987e50cf0219631b32ff | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: EmbarkStudios/cargo-deny-action@v1 | |
| name: cargo deny | |
| with: | |
| log-level: warn | |
| command: check licenses | |
| check-migrations: | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - build-virtual-env | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12.5" | |
| - name: Restore venv | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Check migrations | |
| run: .venv/bin/nucliadb-validate-migrations | |
| # Builds | |
| build-node-binaries: | |
| name: Build node binaries | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Compile | |
| run: cargo build --release --bin node_reader --bin node_writer | |
| - name: Upload binaries to cache | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: "target/release/node_*er" | |
| key: node-bin-${{ github.sha }} | |
| build-nidx-binary: | |
| name: Build nidx binary | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: nidx | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| - name: Compile | |
| run: cd nidx && cargo build --release | |
| - name: Upload binaries to cache | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: "nidx/target/release/nidx" | |
| key: nidx-bin-${{ github.sha }} | |
| build-virtual-env: | |
| name: Build virtual environment | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: | | |
| . | |
| nidx | |
| - name: Set up PDM | |
| uses: pdm-project/setup-pdm@v4 | |
| with: | |
| python-version: "3.12.5" | |
| cache: true | |
| # Needed for nidx_protos (rust) | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| - name: Install venv | |
| run: pdm sync -d --clean --no-editable | |
| - name: Upload virtualenv to cache | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ github.sha }} | |
| # Tests | |
| sidecar-tests: | |
| name: Sidecar tests | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - build-node-binaries | |
| - build-virtual-env | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12.5" | |
| - name: Restore venv | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Restore binaries | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: "target/release/node_*er" | |
| key: node-bin-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Build docker images for node | |
| run: mkdir builds && mv target/release/node_* builds && docker build -t europe-west4-docker.pkg.dev/nuclia-internal/nuclia/node:latest -f Dockerfile.node_prebuilt . | |
| - name: Run tests | |
| run: source .venv/bin/activate && make -C nucliadb_sidecar test-cov | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| file: ./nucliadb_sidecar/coverage.xml | |
| flags: nucliadb_sidecar | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| nucliadb-search-tests: | |
| name: NucliaDB search tests | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - build-node-binaries | |
| - build-virtual-env | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12.5" | |
| - name: Restore venv | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Restore binaries | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: "target/release/node_*er" | |
| key: node-bin-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Build docker images for node | |
| run: mkdir builds && mv target/release/node_* builds && docker build -t europe-west4-docker.pkg.dev/nuclia-internal/nuclia/node:latest -f Dockerfile.node_prebuilt . | |
| - name: Build sidecar image | |
| run: make build-sidecar | |
| - name: Run tests | |
| run: source .venv/bin/activate && make -C nucliadb test-cov-search | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| file: ./nucliadb/coverage.xml | |
| flags: nucliadb-search | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| nucliadb-search-tests-nidx: | |
| name: NucliaDB search tests (nidx) | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - build-node-binaries | |
| - build-nidx-binary | |
| - build-virtual-env | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12.5" | |
| - name: Restore venv | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Restore binaries | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: "target/release/node_*er" | |
| key: node-bin-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Build docker images for node | |
| run: mkdir builds && mv target/release/node_* builds && docker build -t europe-west4-docker.pkg.dev/nuclia-internal/nuclia/node:latest -f Dockerfile.node_prebuilt . | |
| - name: Build sidecar image | |
| run: make build-sidecar | |
| - name: Restore nidx binary | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: "nidx/target/release/nidx" | |
| key: nidx-bin-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Build docker images for nidx | |
| run: mv nidx/target/release/nidx builds && docker build -t nidx -f Dockerfile.nidx_prebuilt . | |
| - name: Run tests | |
| env: | |
| NIDX_ENABLED: 1 | |
| run: source .venv/bin/activate && make -C nucliadb test-cov-search | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| file: ./nucliadb/coverage.xml | |
| flags: nucliadb-search | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| node-rust-tests: | |
| name: Node Rust tests | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: test | |
| run: cargo test --workspace --all-features | |
| nidx-tests: | |
| name: nidx tests | |
| runs-on: ubuntu-24.04 | |
| services: | |
| postgres: | |
| image: postgres:15 | |
| env: | |
| POSTGRES_PASSWORD: postgres | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U postgres" | |
| --health-interval 1s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: nidx | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| - name: test | |
| env: | |
| DATABASE_URL: postgresql://postgres:postgres@localhost/postgres | |
| SQLX_OFFLINE: true | |
| run: cd nidx && cargo test --workspace --all-features | |
| nucliadb-tests: | |
| name: NucliaDB tests | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - build-virtual-env | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| maindb_driver: ["pg"] | |
| shard: [0, 1, 2] | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12.5" | |
| - name: Restore venv | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Run NucliaDB tests | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v3 | |
| env: | |
| TESTING_MAINDB_DRIVERS: ${{ matrix.maindb_driver }} | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: source .venv/bin/activate && make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 -n 4" || make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 --lf" | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| file: ./nucliadb/coverage.xml | |
| flags: nucliadb | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| nucliadb-components-tests: | |
| name: NucliaDB components tests | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - build-virtual-env | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| component: | |
| - ingest | |
| - reader | |
| - standalone | |
| - train | |
| - writer | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12.5" | |
| - name: Restore venv | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Run package tests | |
| run: source .venv/bin/activate && make -C nucliadb test-cov-${{ matrix.component }} | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| file: ./nucliadb/coverage.xml | |
| flags: nucliadb-${{ matrix.component }} | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| python-packages-tests: | |
| name: Python packages tests | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - build-virtual-env | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| package: | |
| - nucliadb_dataset | |
| - nucliadb_models | |
| - nucliadb_node_binding | |
| - nucliadb_sdk | |
| - nucliadb_telemetry | |
| - nucliadb_utils | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12.5" | |
| - name: Restore venv | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ github.sha }} | |
| fail-on-cache-miss: true | |
| - name: Run package tests | |
| run: source .venv/bin/activate && make -C ${{ matrix.package }} test-cov | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| file: ./${{ matrix.package }}/coverage.xml | |
| flags: ${{ matrix.package }} | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| # Depends on all tests to pass | |
| ci-passed: | |
| name: CI passed | |
| runs-on: ubuntu-24.04 | |
| needs: | |
| - format-rust | |
| - clippy-rust | |
| - updated-protos | |
| - pre-checks-python | |
| - check-licenses | |
| - check-migrations | |
| - lint-helm | |
| - sidecar-tests | |
| - nucliadb-search-tests | |
| - node-rust-tests | |
| - nucliadb-tests | |
| - nucliadb-components-tests | |
| - python-packages-tests | |
| - pre-checks-python-old | |
| steps: | |
| - name: CI passed | |
| run: echo "CI passed" | |
| deploy: | |
| name: Trigger deploy | |
| if: github.event_name == 'push' | |
| needs: | |
| - ci-passed | |
| uses: ./.github/workflows/deploy.yml | |
| secrets: inherit | |
| with: | |
| environment: ${{ github.ref == 'refs/heads/main' && 'stage' || 'dev' }} | |
| release: | |
| name: Trigger release | |
| if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| needs: | |
| - ci-passed | |
| uses: ./.github/workflows/release.yml | |
| secrets: inherit |