Add a vectors index resource wrapper to set resources in the proper vectorset #9267
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: nucliadb (py) | |
| env: | |
| COMPONENT: nucliadb | |
| TESTING_TIKV_LOCAL: "true" | |
| GCP_WORKLOAD_IDENTITY_PROVIDER: "projects/224545243904/locations/global/workloadIdentityPools/gh-nuclia/providers/gh-nuclia-provider" | |
| GCP_SERVICE_ACCOUNT: "[email protected]" | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| contents: write | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| pre-checks: | |
| uses: ./.github/workflows/_component_prechecks.yml | |
| with: | |
| component: "nucliadb" | |
| python_version: "3.11" | |
| # Job to run Python tests | |
| tests: | |
| name: NucliaDBTests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 6 | |
| matrix: | |
| python-version: ["3.11"] | |
| maindb_driver: ["tikv", "tikv_2", "tikv_3", "pg", "pg_2", "pg_3"] | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: "pip" | |
| - name: Install the package | |
| run: make -C nucliadb/ install-dev | |
| - name: Install tikv | |
| if: matrix.maindb_driver == 'tikv' || matrix.maindb_driver == 'tikv_2' || matrix.maindb_driver == 'tikv_3' | |
| run: sudo cp ./scripts/pingcap-com.pem /usr/local/share/ca-certificates/pingcap.crt && sudo update-ca-certificates && ./scripts/run-tikv.sh | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| workflows: | |
| - 'nucliadb_node/**' | |
| - 'nucliadb_texts2/**' | |
| - 'nucliadb_paragraphs2/**' | |
| - 'nucliadb_paragraphs3/**' | |
| - 'nucliadb_vectors/**' | |
| - 'nucliadb_relations2/**' | |
| - name: Calc git info | |
| id: git | |
| run: |- | |
| BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | |
| echo "branch=$BRANCH" >> $GITHUB_OUTPUT | |
| - name: Install rust binding for testing | |
| if: steps.filter.outputs.workflows == 'true' | |
| env: | |
| SECRET_KEY: ${{ secrets.RUST_BUILD_SERVER_SECRET }} | |
| COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }} | |
| BRANCH: ${{ steps.git.outputs.branch }} | |
| MATURIN: "true" | |
| ENV: '{"RUSTFLAGS": "--cfg tokio_unstable"}' | |
| run: | | |
| ./scripts/download-build.sh && pip install builds/wheels/*_x86_64.whl || ( | |
| echo "Failed building from the build server." && make build-node-binding-debug | |
| ) | |
| - name: Run PG tests 1/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'pg' | |
| env: | |
| TESTING_MAINDB_DRIVERS: pg | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=0 --num-shards=3" | |
| - name: Run PG tests 2/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'pg_2' | |
| env: | |
| TESTING_MAINDB_DRIVERS: pg | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=1 --num-shards=3" | |
| - name: Run PG tests 2/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'pg_3' | |
| env: | |
| TESTING_MAINDB_DRIVERS: pg | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=2 --num-shards=3" | |
| - name: Run Tikv tests 1/2 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'tikv' | |
| env: | |
| TESTING_MAINDB_DRIVERS: tikv | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=0 --num-shards=3" | |
| - name: Run Tikv tests 2/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'tikv_2' | |
| env: | |
| TESTING_MAINDB_DRIVERS: "tikv" | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=1 --num-shards=3" | |
| - name: Run Tikv tests 3/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'tikv_3' | |
| env: | |
| TESTING_MAINDB_DRIVERS: "tikv" | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=2 --num-shards=3" | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| file: ./coverage.xml | |
| flags: nucliadb | |
| push-dev: | |
| name: Build dev image and push | |
| runs-on: ubuntu-latest | |
| needs: tests | |
| if: github.event_name == 'pull_request' | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Get safe branch name | |
| id: branch_name | |
| run: |- | |
| BRANCH_NAME=`echo $GITHUB_HEAD_REF | sed 's$/$-$g'` | |
| echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT | |
| - name: Authenticate to Google Cloud | |
| id: gcp-auth | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| workload_identity_provider: "${{ env.GCP_WORKLOAD_IDENTITY_PROVIDER }}" | |
| service_account: "${{ env.GCP_SERVICE_ACCOUNT }}" | |
| token_format: access_token | |
| - name: Login to Google Artifact Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: europe-west4-docker.pkg.dev | |
| username: oauth2accesstoken | |
| password: ${{ steps.gcp-auth.outputs.access_token }} | |
| # We need to setup buildx to be able to cache with gha | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build and push to Google Artifact Repository | |
| uses: docker/build-push-action@v4 | |
| env: | |
| REGISTRY: europe-west4-docker.pkg.dev/nuclia-internal/private | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: ${{ env.REGISTRY }}/${{ env.COMPONENT }}:${{ steps.branch_name.outputs.branch_name }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=min |