fix conditional tikv dependency usage on telemetry (#1613) #6917
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: nucliadb (py) | |
| env: | |
| COMPONENT: nucliadb | |
| CONTAINER_REGISTRY: eu.gcr.io/${{ secrets.PROJECT_ID }} | |
| TESTING_TIKV_LOCAL: "true" | |
| permissions: | |
| contents: write | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| pre-checks: | |
| uses: ./.github/workflows/_component_prechecks.yml | |
| with: | |
| component: "nucliadb" | |
| python_version: "3.11" | |
| # Job to run Python tests | |
| tests: | |
| name: NucliaDBTests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 6 | |
| matrix: | |
| python-version: ["3.11"] | |
| maindb_driver: ["tikv", "tikv_2", "tikv_3", "pg", "pg_2", "pg_3"] | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: "pip" | |
| - name: Install the package | |
| run: make -C nucliadb/ install-dev | |
| - name: Install tikv | |
| if: matrix.maindb_driver == 'tikv' || matrix.maindb_driver == 'tikv_2' || matrix.maindb_driver == 'tikv_3' | |
| run: ./scripts/run-tikv.sh | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| workflows: | |
| - 'nucliadb_node/**' | |
| - 'nucliadb_fields_tantivy/**' | |
| - 'nucliadb_paragraphs_tantivy/**' | |
| - 'nucliadb_vectors/**' | |
| - 'nucliadb_relations/**' | |
| - name: Calc git info | |
| id: git | |
| run: |- | |
| BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | |
| echo "branch=$BRANCH" >> $GITHUB_OUTPUT | |
| - name: Install rust binding for testing | |
| if: steps.filter.outputs.workflows == 'true' | |
| env: | |
| SECRET_KEY: ${{ secrets.RUST_BUILD_SERVER_SECRET }} | |
| COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }} | |
| BRANCH: ${{ steps.git.outputs.branch }} | |
| MATURIN: "true" | |
| ENV: '{"RUSTFLAGS": "--cfg tokio_unstable"}' | |
| run: | | |
| ./scripts/download-build.sh && pip install builds/wheels/*_x86_64.whl || ( | |
| echo "Failed building from the build server." && make build-node-binding-debug | |
| ) | |
| - name: Run PG tests 1/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'pg' | |
| env: | |
| TESTING_MAINDB_DRIVERS: pg | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| pytest -rfE --cov=nucliadb -s --tb=native -v --cov-report xml --cov-append --benchmark-json benchmarks.json --shard-id=0 --num-shards=3 nucliadb/nucliadb/tests | |
| - name: Run PG tests 2/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'pg_2' | |
| env: | |
| TESTING_MAINDB_DRIVERS: pg | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| pytest -rfE --cov=nucliadb -s --tb=native -v --cov-report xml --cov-append --benchmark-json benchmarks.json --shard-id=1 --num-shards=3 nucliadb/nucliadb/tests | |
| - name: Run PG tests 2/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'pg_3' | |
| env: | |
| TESTING_MAINDB_DRIVERS: pg | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| pytest -rfE --cov=nucliadb -s --tb=native -v --cov-report xml --cov-append --benchmark-json benchmarks.json --shard-id=2 --num-shards=3 nucliadb/nucliadb/tests | |
| - name: Run Tikv tests 1/2 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'tikv' | |
| env: | |
| TESTING_MAINDB_DRIVERS: tikv | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| pytest -rfE --cov=nucliadb -s --tb=native -v --cov-report xml --cov-append --benchmark-json benchmarks.json --shard-id=0 --num-shards=3 nucliadb/nucliadb/tests | |
| - name: Run Tikv tests 2/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'tikv_2' | |
| env: | |
| TESTING_MAINDB_DRIVERS: "tikv" | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| pytest -rfE --cov=nucliadb -s --tb=native -v --cov-report xml --cov-append --benchmark-json benchmarks.json --shard-id=1 --num-shards=3 nucliadb/nucliadb/tests | |
| - name: Run Tikv tests 3/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'tikv_3' | |
| env: | |
| TESTING_MAINDB_DRIVERS: "tikv" | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| pytest -rfE --cov=nucliadb -s --tb=native -v --cov-report xml --cov-append --benchmark-json benchmarks.json --shard-id=2 --num-shards=3 nucliadb/nucliadb/tests | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| file: ./coverage.xml | |
| flags: nucliadb | |
| push: | |
| name: Build and push docker image | |
| runs-on: ubuntu-latest | |
| needs: tests | |
| if: github.event_name == 'push' | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| # We need to setup buildx to be able to cache with gha | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Get release version | |
| id: version_step | |
| run: |- | |
| python bump.py --build=${{github.run_number}} | |
| VERSION=`cat VERSION` | |
| HASH=`git rev-parse --short HEAD` | |
| echo "version_number=$VERSION" >> $GITHUB_OUTPUT | |
| echo "hash=$HASH" >> $GITHUB_OUTPUT | |
| - name: Build and push | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: | | |
| nuclia/nucliadb:latest | |
| nuclia/nucliadb:${{ steps.version_step.outputs.version_number }} | |
| nuclia/nucliadb:${{ steps.version_step.outputs.hash }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=min | |
| push-dev: | |
| name: Build dev image and push | |
| runs-on: ubuntu-latest | |
| needs: tests | |
| if: github.event_name == 'pull_request' | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Get safe branch name | |
| id: branch_name | |
| run: |- | |
| BRANCH_NAME=`echo $GITHUB_HEAD_REF | sed 's$/$-$g'` | |
| echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT | |
| - name: Setup gcloud CLI | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| credentials_json: "${{ secrets.GCP_CREDENTIALS }}" | |
| - name: "Set up Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@v1" | |
| - name: Configure Docker | |
| run: docker login -u oauth2accesstoken -p "$(gcloud auth application-default print-access-token)" https://eu.gcr.io | |
| # We need to setup buildx to be able to cache with gha | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build and push | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: ${{ env.CONTAINER_REGISTRY }}/${{ env.COMPONENT }}:${{ steps.branch_name.outputs.branch_name }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=min |