Skip to content
PyPI and Helm Releases

Fix hierarchy rag strategy and add test (#2273) #626

Sign in to view logs

Fix hierarchy rag strategy and add test (#2273)

Fix hierarchy rag strategy and add test (#2273) #626

Workflow file for this run

.github/workflows/release.yml at 5db3bc9
name: PyPI and Helm Releases
on:
push:
branches:
- main
jobs:
build_wheels:
name: Build wheels
runs-on: ubuntu-latest
steps:
- name: Generate a token
id: app-token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ secrets.GHAPP_ID_NUCLIABOT }}
private-key: ${{ secrets.PK_GHAPP_NUCLIABOT }}
owner: nuclia
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with:
python-version: 3.12
- name: Setup env
run: |
pip install wheel
pip install -U twine
- name: Set dependency versions in requirements
run: |
python bump.py --build=${{github.run_number}}
- uses: dorny/paths-filter@v2
id: component_changes
with:
filters: |
nucliadb_protos:
- 'nucliadb_protos/**'
nucliadb_utils:
- 'nucliadb_utils/**'
nucliadb_models:
- 'nucliadb_models/**'
nucliadb_sdk:
- 'nucliadb_sdk/**'
nucliadb_dataset:
- 'nucliadb_dataset/**'
nucliadb_telemetry:
- 'nucliadb_telemetry/**'
# Install all packages we are going to release
- name: Install all packages
run: |
pip install -e ./nucliadb_protos/python/
pip install -e ./nucliadb_telemetry/
pip install -e ./nucliadb_utils/
pip install -e ./nucliadb_models/
pip install -e ./nucliadb_sdk/
pip install -e ./nucliadb_dataset/
pip install -e ./nucliadb/
- name: Build wheels
run: pip wheel --no-deps -w dist ./nucliadb_protos/python/ ./nucliadb_utils/ ./nucliadb_models/ ./nucliadb_sdk/ ./nucliadb_telemetry ./nucliadb_dataset/ ./nucliadb/
# XXX wait for tests to run successfully in all dependencies before we actually release
- name: Wait for utils
uses: lewagon/[email protected]
if: steps.component_changes.outputs.nucliadb_utils == 'true'
with:
ref: main
check-name: NucliaDBUtilsTests (3.12)
repo-token: ${{ secrets.GITHUB_TOKEN }}
wait-interval: 10
- name: Wait for sdk tests
uses: lewagon/[email protected]
if: steps.component_changes.outputs.nucliadb_sdk == 'true'
with:
ref: main
check-name: NucliaDBSDKTests (3.12)
repo-token: ${{ secrets.GITHUB_TOKEN }}
wait-interval: 10
- name: Wait for dataset tests
uses: lewagon/[email protected]
if: steps.component_changes.outputs.nucliadb_dataset == 'true'
with:
ref: main
check-name: NucliaDBDatasetTests (3.12)
repo-token: ${{ secrets.GITHUB_TOKEN }}
wait-interval: 10
- name: Wait for telemetry tests
uses: lewagon/[email protected]
if: steps.component_changes.outputs.nucliadb_telemetry == 'true'
with:
ref: main
check-name: NucliaDBTelemetryTests (3.12)
repo-token: ${{ secrets.GITHUB_TOKEN }}
wait-interval: 10
- name: Wait for nucliadb tests
uses: lewagon/[email protected]
with:
ref: main
check-name: NucliaDBTests (3.12, pg)
repo-token: ${{ secrets.GITHUB_TOKEN }}
wait-interval: 10
- name: Upload to pypi
run: twine upload --verbose dist/*
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
# failing, disable for now
# - name: Run Nuclia SDK tests
# uses: peter-evans/repository-dispatch@v2
# with:
# token: ${{ steps.app-token.outputs.token }}
# repository: nuclia/nuclia.py
# event-type: test-stage
# client-payload: '{"component": "nucliadb", "commit": "${{ github.sha }}", "user": "${{ github.actor }}"}'
push-docker:
name: Build and push nucliadb docker image
runs-on: ubuntu-latest
needs: build_wheels
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
# We need to setup buildx to be able to cache with gha
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Get release version
id: version_step
run: |-
python bump.py --build=${{github.run_number}}
VERSION=`cat VERSION`
HASH=`git rev-parse --short HEAD`
echo "version_number=$VERSION" >> $GITHUB_OUTPUT
echo "hash=$HASH" >> $GITHUB_OUTPUT
- name: Build and push
uses: docker/build-push-action@v4
with:
context: .
platforms: linux/amd64,linux/arm64
push: true
tags: |
nuclia/nucliadb:latest
nuclia/nucliadb:${{ steps.version_step.outputs.version_number }}
nuclia/nucliadb:${{ steps.version_step.outputs.hash }}
cache-from: type=gha
cache-to: type=gha,mode=min
deploy:
name: Deploy Helm chart and trigger internal CI
runs-on: ubuntu-latest
needs: push-docker
steps:
- name: Generate a token
id: app-token
uses: actions/create-github-app-token@v1
with:
app-id: ${{ secrets.GHAPP_ID_NUCLIABOT }}
private-key: ${{ secrets.PK_GHAPP_NUCLIABOT }}
owner: nuclia
- name: Checkout repository
uses: actions/checkout@v3
- name: Calculate short sha
id: env-vars
run: |-
HASH=`git rev-parse --short HEAD`
echo "short_sha=$HASH" >> $GITHUB_OUTPUT
- name: Set helm package image
id: version_step
run: |-
python bump.py --build=${{github.run_number}}
DOCKER_VERSION=`cat VERSION`
# Helm package versioning does not support '.postX'. Convert to '+postX'
HELM_VERSION=`echo $DOCKER_VERSION | sed 's/.post/+post/'`
# Update chart with versions
echo "HELM_VERSION=$HELM_VERSION"
echo "DOCKER_VERSION=$DOCKER_VERSION"
sed -i.bak "s#99999.99999.99999#$HELM_VERSION#" ./charts/nucliadb/Chart.yaml
sed -i.bak "s#88888.88888.88888#$DOCKER_VERSION#" ./charts/nucliadb/Chart.yaml
echo "helm_version=$HELM_VERSION" >> $GITHUB_OUTPUT
- name: Configure Git
run: |
git config user.name "$GITHUB_ACTOR"
git config user.email "[email protected]"
- name: Install Helm
uses: azure/setup-helm@v3
with:
version: v3.4.0
- name: Push helm package
run: |-
helm lint charts/nucliadb
helm package charts/nucliadb
curl --data-binary "@nucliadb-${{ steps.version_step.outputs.helm_version }}.tgz" ${{ secrets.HELM_CHART_URL }}/api/charts
- name: Repository Dispatch
uses: peter-evans/repository-dispatch@v2
with:
token: ${{ steps.app-token.outputs.token }}
repository: nuclia/nucliadb_deploy
event-type: promote
client-payload: '{"component": "nucliadb_standalone", "chart-version": "${{ steps.version_step.outputs.helm_version }}" }'