Fix storage uploads: use preferred chunk size #7530
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: nucliadb (py) | |
| env: | |
| COMPONENT: nucliadb | |
| CONTAINER_REGISTRY: eu.gcr.io/${{ secrets.PROJECT_ID }} | |
| TESTING_TIKV_LOCAL: "true" | |
| permissions: | |
| contents: write | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - main | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| pre-checks: | |
| uses: ./.github/workflows/_component_prechecks.yml | |
| with: | |
| component: "nucliadb" | |
| python_version: "3.11" | |
| # Job to run Python tests | |
| tests: | |
| name: NucliaDBTests | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 6 | |
| matrix: | |
| python-version: ["3.11"] | |
| maindb_driver: ["tikv", "tikv_2", "tikv_3", "pg", "pg_2", "pg_3"] | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: "pip" | |
| - name: Install the package | |
| run: make -C nucliadb/ install-dev | |
| - name: Install tikv | |
| if: matrix.maindb_driver == 'tikv' || matrix.maindb_driver == 'tikv_2' || matrix.maindb_driver == 'tikv_3' | |
| run: ./scripts/run-tikv.sh | |
| - uses: dorny/paths-filter@v2 | |
| id: filter | |
| with: | |
| filters: | | |
| workflows: | |
| - 'nucliadb_node/**' | |
| - 'nucliadb_texts/**' | |
| - 'nucliadb_texts2/**' | |
| - 'nucliadb_paragraphs/**' | |
| - 'nucliadb_vectors/**' | |
| - 'nucliadb_relations/**' | |
| - 'nucliadb_relations2/**' | |
| - name: Calc git info | |
| id: git | |
| run: |- | |
| BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | |
| echo "branch=$BRANCH" >> $GITHUB_OUTPUT | |
| - name: Install rust binding for testing | |
| if: steps.filter.outputs.workflows == 'true' | |
| env: | |
| SECRET_KEY: ${{ secrets.RUST_BUILD_SERVER_SECRET }} | |
| COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }} | |
| BRANCH: ${{ steps.git.outputs.branch }} | |
| MATURIN: "true" | |
| ENV: '{"RUSTFLAGS": "--cfg tokio_unstable"}' | |
| run: | | |
| ./scripts/download-build.sh && pip install builds/wheels/*_x86_64.whl || ( | |
| echo "Failed building from the build server." && make build-node-binding-debug | |
| ) | |
| - name: Run PG tests 1/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'pg' | |
| env: | |
| TESTING_MAINDB_DRIVERS: pg | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| pytest -rfE --cov=nucliadb -s --tb=native -v --cov-report xml --cov-append --benchmark-json benchmarks.json --shard-id=0 --num-shards=3 nucliadb/nucliadb/tests | |
| - name: Run PG tests 2/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'pg_2' | |
| env: | |
| TESTING_MAINDB_DRIVERS: pg | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| pytest -rfE --cov=nucliadb -s --tb=native -v --cov-report xml --cov-append --benchmark-json benchmarks.json --shard-id=1 --num-shards=3 nucliadb/nucliadb/tests | |
| - name: Run PG tests 2/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'pg_3' | |
| env: | |
| TESTING_MAINDB_DRIVERS: pg | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| pytest -rfE --cov=nucliadb -s --tb=native -v --cov-report xml --cov-append --benchmark-json benchmarks.json --shard-id=2 --num-shards=3 nucliadb/nucliadb/tests | |
| - name: Run Tikv tests 1/2 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'tikv' | |
| env: | |
| TESTING_MAINDB_DRIVERS: tikv | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| pytest -rfE --cov=nucliadb -s --tb=native -v --cov-report xml --cov-append --benchmark-json benchmarks.json --shard-id=0 --num-shards=3 nucliadb/nucliadb/tests | |
| - name: Run Tikv tests 2/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'tikv_2' | |
| env: | |
| TESTING_MAINDB_DRIVERS: "tikv" | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| pytest -rfE --cov=nucliadb -s --tb=native -v --cov-report xml --cov-append --benchmark-json benchmarks.json --shard-id=1 --num-shards=3 nucliadb/nucliadb/tests | |
| - name: Run Tikv tests 3/3 | |
| # These tests can be flaky, let's retry them... | |
| uses: nick-fields/retry@v2 | |
| if: matrix.maindb_driver == 'tikv_3' | |
| env: | |
| TESTING_MAINDB_DRIVERS: "tikv" | |
| with: | |
| max_attempts: 2 | |
| retry_on: error | |
| timeout_minutes: 10 | |
| command: | | |
| pytest -rfE --cov=nucliadb -s --tb=native -v --cov-report xml --cov-append --benchmark-json benchmarks.json --shard-id=2 --num-shards=3 nucliadb/nucliadb/tests | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| file: ./coverage.xml | |
| flags: nucliadb | |
| push-dev: | |
| name: Build dev image and push | |
| runs-on: ubuntu-latest | |
| needs: tests | |
| if: github.event_name == 'pull_request' | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| - name: Get safe branch name | |
| id: branch_name | |
| run: |- | |
| BRANCH_NAME=`echo $GITHUB_HEAD_REF | sed 's$/$-$g'` | |
| echo "branch_name=$BRANCH_NAME" >> $GITHUB_OUTPUT | |
| - name: Setup gcloud CLI | |
| uses: google-github-actions/auth@v1 | |
| with: | |
| credentials_json: "${{ secrets.GCP_CREDENTIALS }}" | |
| - name: "Set up Cloud SDK" | |
| uses: "google-github-actions/setup-gcloud@v1" | |
| - name: Configure Docker | |
| run: docker login -u oauth2accesstoken -p "$(gcloud auth application-default print-access-token)" https://eu.gcr.io | |
| # We need to setup buildx to be able to cache with gha | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build and push | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: true | |
| tags: ${{ env.CONTAINER_REGISTRY }}/${{ env.COMPONENT }}:${{ steps.branch_name.outputs.branch_name }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=min |