Nidx shards API & tests (#2609) #2122

Workflow file for this run

	name: CI

	on:
	pull_request:
	branches:
	- main
	- dev
	push:
	branches:
	- main
	- dev

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	# Pre-checks
	format-rust:
	name: "Pre-checks: Rust code Format"
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@v4
	- name: rustfmt
	run: cargo fmt --check

	clippy-rust:
	name: "Pre-checks: Clippy lint"
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@v4
	- run: rustup component add clippy
	- uses: Swatinem/rust-cache@v2
	- name: clippy
	run: cargo clippy

	nidx-format-rust:
	name: "nidx: Rust code Format"
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@v4
	- name: rustfmt
	run: cd nidx && cargo fmt --check

	nidx-clippy-rust:
	name: "nidx: Clippy lint"
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@v4
	- run: rustup component add clippy
	- uses: Swatinem/rust-cache@v2
	with:
	workspaces: nidx
	- name: Install Protoc
	uses: arduino/setup-protoc@v3
	- name: clippy
	run: cd nidx && cargo clippy

	updated-protos:
	name: "Pre-checks: Protos up-to-date"
	runs-on: ubuntu-24.04
	permissions: write-all
	needs:
	- build-virtual-env
	steps:
	- uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.12.5"
	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true
	- name: Compile protos
	run: source .venv/bin/activate && make protos
	- name: Fail if something changed
	run: git status && [[ -z "$(git status -s)" ]]

	pre-checks-python:
	name: "Pre-checks: Python lint"
	runs-on: ubuntu-24.04
	needs:
	- build-virtual-env
	steps:
	- uses: actions/checkout@v4
	- name: Set up PDM
	uses: pdm-project/setup-pdm@v4
	with:
	python-version: "3.12.5"
	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true
	- name: Lint source code
	run: \|
	source .venv/bin/activate
	for package in nucliadb nucliadb_utils nucliadb_telemetry nucliadb_sdk nucliadb_dataset nucliadb_models nucliadb_sidecar nucliadb_node_binding nucliadb_performance
	do
	make -C $package lint \|\| ERROR=1
	done
	exit $ERROR

	pre-checks-python-old:
	name: "Pre-checks: Python lint 3.9"
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	- name: Set up PDM
	uses: pdm-project/setup-pdm@v4
	with:
	python-version: "3.9.20"
	cache: true
	- name: Install venv
	run: pdm sync -d --clean --no-editable
	- name: Clean build
	run: find . -name build \| xargs rm -r \|\| true
	- name: Lint source code
	run: \|
	source .venv/bin/activate
	for package in nucliadb nucliadb_utils nucliadb_telemetry nucliadb_sdk nucliadb_dataset nucliadb_models nucliadb_sidecar nucliadb_node_binding nucliadb_performance
	do
	make -C $package lint \|\| ERROR=1
	done
	exit $ERROR

	lint-helm:
	name: Lint all helm charts
	runs-on: ubuntu-24.04

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Install Helm
	uses: azure/setup-helm@v4
	with:
	version: v3.15.3

	- name: Lint helm packages
	run: \|-
	helm lint charts/*
	helm package charts/*

	check-licenses:
	name: Check Licenses
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@v4
	- name: Check License Header
	uses: apache/skywalking-eyes/header@501a28d2fb4a9b962661987e50cf0219631b32ff
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	- uses: EmbarkStudios/cargo-deny-action@v1
	name: cargo deny
	with:
	log-level: warn
	command: check licenses

	check-migrations:
	runs-on: ubuntu-24.04
	needs:
	- build-virtual-env
	steps:
	- uses: actions/checkout@v4
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.12.5"
	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true
	- name: Check migrations
	run: .venv/bin/nucliadb-validate-migrations

	# Builds
	build-node-binaries:
	name: Build node binaries
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	- name: Compile
	run: cargo build --release --bin node_reader --bin node_writer
	- name: Upload binaries to cache
	uses: actions/cache/save@v4
	with:
	path: "target/release/node_*er"
	key: node-bin-${{ github.sha }}

	build-virtual-env:
	name: Build virtual environment
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	- name: Set up PDM
	uses: pdm-project/setup-pdm@v4
	with:
	python-version: "3.12.5"
	cache: true
	- name: Install venv
	run: pdm sync -d --clean --no-editable
	- name: Upload virtualenv to cache
	uses: actions/cache/save@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}

	# Tests
	sidecar-tests:
	name: Sidecar tests
	runs-on: ubuntu-24.04
	needs:
	- build-node-binaries
	- build-virtual-env
	steps:
	- uses: actions/checkout@v4
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.12.5"
	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true
	- name: Restore binaries
	uses: actions/cache/restore@v4
	with:
	path: "target/release/node_*er"
	key: node-bin-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Build docker images for node
	run: mkdir builds && mv target/release/node_* builds && docker build -t europe-west4-docker.pkg.dev/nuclia-internal/nuclia/node:latest -f Dockerfile.node_prebuilt .

	- name: Run tests
	run: source .venv/bin/activate && make -C nucliadb_sidecar test-cov

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v4
	with:
	file: ./nucliadb_sidecar/coverage.xml
	flags: nucliadb_sidecar
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	nucliadb-search-tests:
	name: NucliaDB search tests
	runs-on: ubuntu-24.04
	needs:
	- build-node-binaries
	- build-virtual-env
	steps:
	- uses: actions/checkout@v4
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.12.5"
	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true
	- name: Restore binaries
	uses: actions/cache/restore@v4
	with:
	path: "target/release/node_*er"
	key: node-bin-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Build docker images for node
	run: mkdir builds && mv target/release/node_* builds && docker build -t europe-west4-docker.pkg.dev/nuclia-internal/nuclia/node:latest -f Dockerfile.node_prebuilt .

	- name: Build sidecar image
	run: make build-sidecar

	- name: Run tests
	run: source .venv/bin/activate && make -C nucliadb test-cov-search

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v4
	with:
	file: ./nucliadb/coverage.xml
	flags: nucliadb-search
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	node-rust-tests:
	name: Node Rust tests
	runs-on: ubuntu-24.04
	steps:
	- uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	- name: test
	run: cargo test --workspace --all-features

	nidx-tests:
	name: nidx tests
	runs-on: ubuntu-24.04
	services:
	postgres:
	image: postgres:15
	env:
	POSTGRES_PASSWORD: postgres
	ports:
	- 5432:5432
	options: >-
	--health-cmd "pg_isready -U postgres"
	--health-interval 1s
	--health-timeout 5s
	--health-retries 10
	steps:
	- uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	with:
	workspaces: nidx
	- name: Install Protoc
	uses: arduino/setup-protoc@v3
	- name: test
	env:
	DATABASE_URL: postgresql://postgres:postgres@localhost/postgres
	SQLX_OFFLINE: true
	run: cd nidx && cargo test --workspace --all-features

	nucliadb-tests:
	name: NucliaDB tests
	runs-on: ubuntu-24.04
	needs:
	- build-virtual-env
	strategy:
	fail-fast: false
	matrix:
	maindb_driver: ["pg"]
	shard: [0, 1, 2]

	steps:
	- name: Checkout the repository
	uses: actions/checkout@v4

	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.12.5"

	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Run NucliaDB tests
	# These tests can be flaky, let's retry them...
	uses: nick-fields/retry@v3
	env:
	TESTING_MAINDB_DRIVERS: ${{ matrix.maindb_driver }}
	with:
	max_attempts: 2
	retry_on: error
	timeout_minutes: 10
	command: source .venv/bin/activate && make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 -n 4" \|\| make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 --lf"

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v4
	with:
	file: ./nucliadb/coverage.xml
	flags: nucliadb
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	nucliadb-components-tests:
	name: NucliaDB components tests
	runs-on: ubuntu-24.04
	needs:
	- build-virtual-env
	strategy:
	fail-fast: false
	matrix:
	component:
	- ingest
	- reader
	- standalone
	- train
	- writer

	steps:
	- name: Checkout the repository
	uses: actions/checkout@v4

	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.12.5"

	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Run package tests
	run: source .venv/bin/activate && make -C nucliadb test-cov-${{ matrix.component }}

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v4
	with:
	file: ./nucliadb/coverage.xml
	flags: nucliadb-${{ matrix.component }}
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	python-packages-tests:
	name: Python packages tests
	runs-on: ubuntu-24.04
	needs:
	- build-virtual-env
	strategy:
	fail-fast: false
	matrix:
	package:
	- nucliadb_dataset
	- nucliadb_models
	- nucliadb_node_binding
	- nucliadb_sdk
	- nucliadb_telemetry
	- nucliadb_utils

	steps:
	- name: Checkout the repository
	uses: actions/checkout@v4

	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.12.5"

	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Run package tests
	run: source .venv/bin/activate && make -C ${{ matrix.package }} test-cov

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v4
	with:
	file: ./${{ matrix.package }}/coverage.xml
	flags: ${{ matrix.package }}
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	# Depends on all tests to pass
	ci-passed:
	name: CI passed
	runs-on: ubuntu-24.04
	needs:
	- format-rust
	- clippy-rust
	- updated-protos
	- pre-checks-python
	- check-licenses
	- check-migrations
	- lint-helm
	- sidecar-tests
	- nucliadb-search-tests
	- node-rust-tests
	- nucliadb-tests
	- nucliadb-components-tests
	- python-packages-tests
	- pre-checks-python-old
	steps:
	- name: CI passed
	run: echo "CI passed"

	deploy:
	name: Trigger deploy
	if: github.event_name == 'push'
	needs:
	- ci-passed
	uses: ./.github/workflows/deploy.yml
	secrets: inherit
	with:
	environment: ${{ github.ref == 'refs/heads/main' && 'stage' \|\| 'dev' }}

	release:
	name: Trigger release
	if: github.event_name == 'push' && github.ref == 'refs/heads/main'
	needs:
	- ci-passed
	uses: ./.github/workflows/release.yml
	secrets: inherit

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Nidx shards API & tests (#2609) #2122

Workflow file

Nidx shards API & tests (#2609) #2122

Jobs

Run details

Workflow file for this run