Vectorsets with pinecone #952

Workflow file for this run

	name: CI

	on:
	pull_request:
	branches:
	- main
	push:
	branches:
	- main

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
	cancel-in-progress: true

	jobs:
	# Pre-checks
	format-rust:
	name: "Pre-checks: Rust code Format"
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: rustfmt
	run: cargo fmt --check

	clippy-rust:
	name: "Pre-checks: Clippy lint"
	runs-on: ubuntu-latest
	permissions: write-all
	steps:
	- uses: actions/checkout@v4
	- run: rustup component add clippy
	- uses: Swatinem/rust-cache@v2
	- uses: actions-rs/clippy-check@v1
	with:
	token: ${{ secrets.GITHUB_TOKEN }}
	args: --all-targets

	updated-protos:
	name: "Pre-checks: Protos up-to-date"
	runs-on: ubuntu-latest
	permissions: write-all
	needs:
	- build-virtual-env
	steps:
	- uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true
	- name: Compile protos
	run: source .venv/bin/activate && make protos
	- name: Fail if something changed
	run: git status && [[ -z "$(git status -s)" ]]

	pre-checks-python:
	name: "Pre-checks: Python lint"
	runs-on: ubuntu-latest
	needs:
	- build-virtual-env
	steps:
	- uses: actions/checkout@v4
	- name: Set up PDM
	uses: pdm-project/setup-pdm@v3
	with:
	python-version: "3.12"
	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true
	- name: Run mypy
	run: \|
	source .venv/bin/activate
	for package in nucliadb nucliadb_utils nucliadb_telemetry nucliadb_sdk nucliadb_dataset nucliadb_models nucliadb_sidecar nucliadb_node_binding nucliadb_performance
	do
	make -C $package lint \|\| ERROR=1
	done
	exit $ERROR

	lint-helm:
	name: Lint all helm charts
	runs-on: ubuntu-latest

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Install Helm
	uses: azure/setup-helm@v4
	with:
	version: v3.15.3

	- name: Lint helm packages
	run: \|-
	helm lint charts/*
	helm package charts/*

	check-licenses:
	name: Check Licenses
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- name: Check License Header
	uses: apache/skywalking-eyes/header@501a28d2fb4a9b962661987e50cf0219631b32ff
	env:
	GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	- uses: EmbarkStudios/cargo-deny-action@v1
	name: cargo deny
	with:
	log-level: warn
	command: check licenses

	check-migrations:
	runs-on: ubuntu-latest
	needs:
	- build-virtual-env
	steps:
	- uses: actions/checkout@v4
	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.12"
	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true
	- name: Check migrations
	run: .venv/bin/nucliadb-validate-migrations

	# Builds
	build-node-binaries:
	name: Build node binaries
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	- name: Compile
	run: cargo build --release --bin node_reader --bin node_writer
	- name: Upload binaries to cache
	uses: actions/cache/save@v4
	with:
	path: "target/release/node_*er"
	key: node-bin-${{ github.sha }}

	build-virtual-env:
	name: Build virtual environment
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	- name: Set up PDM
	uses: pdm-project/setup-pdm@v3
	with:
	python-version: "3.12"
	cache: true
	- name: Install venv
	run: pdm sync -d --clean --no-editable
	- name: Upload virtualenv to cache
	uses: actions/cache/save@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}

	# Tests
	sidecar-tests:
	name: Sidecar tests
	runs-on: ubuntu-latest
	needs:
	- build-node-binaries
	- build-virtual-env
	steps:
	- uses: actions/checkout@v4
	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.12"
	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true
	- name: Restore binaries
	uses: actions/cache/restore@v4
	with:
	path: "target/release/node_*er"
	key: node-bin-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Build docker images for node
	run: mkdir builds && mv target/release/node_* builds && docker build -t europe-west4-docker.pkg.dev/nuclia-internal/nuclia/node:latest -f Dockerfile.node_prebuilt .

	- name: Run tests
	run: source .venv/bin/activate && make -C nucliadb_sidecar test-cov

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./nucliadb_sidecar/coverage.xml
	flags: nucliadb_sidecar
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}


	nucliadb-search-tests:
	name: NucliaDB search tests
	runs-on: ubuntu-latest
	needs:
	- build-node-binaries
	- build-virtual-env
	steps:
	- uses: actions/checkout@v4
	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.12"
	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true
	- name: Restore binaries
	uses: actions/cache/restore@v4
	with:
	path: "target/release/node_*er"
	key: node-bin-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Build docker images for node
	run: mkdir builds && mv target/release/node_* builds && docker build -t europe-west4-docker.pkg.dev/nuclia-internal/nuclia/node:latest -f Dockerfile.node_prebuilt .

	- name: Build sidecar image
	run: make build-sidecar

	- name: Run tests
	run: source .venv/bin/activate && make -C nucliadb test-cov-search

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./nucliadb/coverage.xml
	flags: nucliadb-search
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	node-rust-tests:
	name: Node Rust tests
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v4
	- uses: Swatinem/rust-cache@v2
	- uses: actions-rs/cargo@v1
	env:
	RUST_BACKTRACE: 1
	name: cargo test
	with:
	command: test
	args: --workspace --all-features


	nucliadb-tests:
	name: NucliaDB tests
	runs-on: ubuntu-latest
	needs:
	- build-virtual-env
	strategy:
	fail-fast: false
	matrix:
	maindb_driver: ["pg"]
	shard: [0, 1, 2]

	steps:
	- name: Checkout the repository
	uses: actions/checkout@v4

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.12"

	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Run NucliaDB tests
	# These tests can be flaky, let's retry them...
	uses: nick-fields/retry@v2
	env:
	TESTING_MAINDB_DRIVERS: ${{ matrix.maindb_driver }}
	with:
	max_attempts: 2
	retry_on: error
	timeout_minutes: 10
	command: source .venv/bin/activate && make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 -n 4" \|\| make -C nucliadb test-cov-nucliadb pytest_extra_flags="--shard-id=${{ matrix.shard }} --num-shards=3 --lf"

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./nucliadb/coverage.xml
	flags: nucliadb
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}


	nucliadb-components-tests:
	name: NucliaDB components tests
	runs-on: ubuntu-latest
	needs:
	- build-virtual-env
	strategy:
	fail-fast: false
	matrix:
	component:
	- ingest
	- reader
	- standalone
	- train
	- writer

	steps:
	- name: Checkout the repository
	uses: actions/checkout@v4

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.12"

	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Run package tests
	run: source .venv/bin/activate && make -C nucliadb test-cov-${{ matrix.component }}

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./nucliadb/coverage.xml
	flags: nucliadb-${{ matrix.component }}
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}


	python-packages-tests:
	name: Python packages tests
	runs-on: ubuntu-latest
	needs:
	- build-virtual-env
	strategy:
	fail-fast: false
	matrix:
	package:
	- nucliadb_dataset
	- nucliadb_models
	- nucliadb_node_binding
	- nucliadb_sdk
	- nucliadb_telemetry
	- nucliadb_utils

	steps:
	- name: Checkout the repository
	uses: actions/checkout@v4

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.12"

	- name: Restore venv
	uses: actions/cache/restore@v4
	with:
	path: .venv
	key: venv-${{ github.sha }}
	fail-on-cache-miss: true

	- name: Run package tests
	run: source .venv/bin/activate && make -C ${{ matrix.package }} test-cov

	- name: Upload coverage to Codecov
	uses: codecov/codecov-action@v3
	with:
	file: ./${{ matrix.package }}/coverage.xml
	flags: ${{ matrix.package }}
	env:
	CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

	# Depends on all tests to pass
	ci-passed:
	name: CI passed
	runs-on: ubuntu-latest
	needs:
	- format-rust
	- clippy-rust
	- updated-protos
	- pre-checks-python
	- check-licenses
	- check-migrations
	- lint-helm
	- sidecar-tests
	- nucliadb-search-tests
	- node-rust-tests
	- nucliadb-tests
	- nucliadb-components-tests
	- python-packages-tests
	steps:
	- name: CI passed
	run: echo "CI passed"

	deploy:
	name: Trigger deploy
	if: github.event_name == 'push'
	needs:
	- ci-passed
	uses: ./.github/workflows/deploy.yml
	secrets: inherit

	release:
	name: Trigger release
	if: github.event_name == 'push'
	needs:
	- ci-passed
	uses: ./.github/workflows/release.yml
	secrets: inherit

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Vectorsets with pinecone #952

Workflow file

Vectorsets with pinecone #952

Jobs

Run details

Workflow file for this run