nucliadb_search.yml

name: nucliadb Search (py)

on:
  pull_request:
    branches:
      - main
    paths:
      - '.github/workflows/nucliadb_search.yml'
      - 'charts/nucliadb_search/**'
      - 'nucliadb_utils/**'
      - 'nucliadb/src/nucliadb/ingest/**'
      - 'nucliadb/src/nucliadb/search/**'
      - 'nucliadb/src/nucliadb/common/**'
      - 'nucliadb_models/**'
      - 'nucliadb_telemetry/**'
      - 'nucliadb_protos/python/**'
      - 'nucliadb_node/**'
      - "nucliadb_sidecar/**"
      - 'nucliadb_vectors/**'
      - 'nucliadb_texts2/**'
      - 'nucliadb_paragraphs3/**'
      - 'nucliadb_core/**'
      - 'nucliadb_relations2/**'
  push:
    branches:
      - main
    paths:
      - '.github/workflows/nucliadb_search.yml'
      - 'charts/nucliadb_search/**'
      - 'nucliadb_utils/**'
      - 'nucliadb/src/nucliadb/ingest/**'
      - 'nucliadb/src/nucliadb/search/**'
      - 'nucliadb/src/nucliadb/common/**'
      - 'nucliadb_models/**'
      - 'nucliadb_telemetry/**'
      - 'nucliadb_protos/python/**'
      - 'nucliadb_node/**'
      - "nucliadb_sidecar/**"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false

permissions:
  id-token: write       # This is required for requesting the JWT
  contents: write       # This is required for actions/checkout
  pull-requests: write  # Used by the benchmark action to leave a comment

env:
  IMAGE_NAME: search
  GCP_WORKLOAD_IDENTITY_PROVIDER: "projects/224545243904/locations/global/workloadIdentityPools/gh-nuclia/providers/gh-nuclia-provider"
  GCP_SERVICE_ACCOUNT: "github-actions@nuclia-internal.iam.gserviceaccount.com"

jobs:
  # Job to run tests
  tests:
    runs-on: ubuntu-latest

    strategy:
      max-parallel: 3
      matrix:
        include:
          - maindb_driver: "pg"
            storage_backend: "s3"
            python-version: "3.12"
          - maindb_driver: "pg"
            storage_backend: "gcs"
            python-version: "3.12"

    steps:
      - name: Checkout the repository
        uses: actions/checkout@v3

      - name: Authenticate to Google Cloud
        id: gcp-auth
        uses: google-github-actions/auth@v2
        with:
          workload_identity_provider: "${{ env.GCP_WORKLOAD_IDENTITY_PROVIDER }}"
          service_account: "${{ env.GCP_SERVICE_ACCOUNT }}"
          token_format: access_token

      # setup docker, as we need to pull the node image to run the tests
      - name: Login to Google Artifact Registry
        uses: docker/login-action@v3
        with:
          registry: europe-west4-docker.pkg.dev
          username: oauth2accesstoken
          password: ${{ steps.gcp-auth.outputs.access_token }}

      - name: 'Set up Cloud SDK'
        uses: 'google-github-actions/setup-gcloud@v1'

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'

      - name: Install the package
        run: make -C nucliadb/ install-dev

      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            node_modified:
              - 'nucliadb_core/**'
              - 'nucliadb_node/src/**'
              - 'nucliadb_paragraphs3/**'
              - 'nucliadb_procs/**'
              - 'nucliadb_relations2/**'
              - 'nucliadb_telemetry/**'
              - 'nucliadb_texts2/**'
              - 'nucliadb_vectors/**'
            sidecar_modified:
              - 'nucliadb_sidecar/**'

      - name: Calc git info
        id: git
        run: |-
          BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
          echo "branch=$BRANCH" >> $GITHUB_OUTPUT

      - name: Install jq
        run: |
          sudo apt-get update -y || true
          sudo apt-get install -y jq

      - name: Build local Node for testing
        if: steps.filter.outputs.node_modified == 'true'
        env:
          SECRET_KEY: ${{ secrets.RUST_BUILD_SERVER_SECRET }}
          COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
          BRANCH: ${{ steps.git.outputs.branch }}
        run: |
          make build-node-debug

      - name: Build local Sidecar for testing
        if: steps.filter.outputs.sidecar_modified == 'true'
        run: |
          make build-sidecar

      - name: Run tests
        env:
          TESTING_MAINDB_DRIVER: ${{ matrix.maindb_driver }}
          TESTING_STORAGE_BACKEND: ${{ matrix.storage_backend }}
        run: |
          make -C nucliadb/ pytest_extra_flags="--benchmark-json ../benchmarks.json" test-cov-search

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          file: ./coverage.xml
          flags: search

      - name: Download previous benchmark data
        uses: actions/cache@v1
        with:
          path: ./cache
          key: ${{ runner.os }}-search-benchmark

      - name: Comment and Store benchmark result
        uses: benchmark-action/github-action-benchmark@v1
        if: github.event_name == 'push'
        with:
          tool: 'pytest'
          output-file-path: benchmarks.json
          external-data-json-path: ./cache/benchmark-data.json
          save-data-file: true
          comment-always: true
          github-token: ${{ secrets.GITHUB_TOKEN }}
          comment-on-alert: true
          fail-on-alert: true
          alert-comment-cc-users: '@nuclia/nuclia-db'

      - name: Comment benchmark result based on main
        uses: benchmark-action/github-action-benchmark@v1
        if: github.event_name != 'push'
        with:
          tool: 'pytest'
          output-file-path: benchmarks.json
          external-data-json-path: ./cache/benchmark-data.json
          save-data-file: false
          comment-always: true
          github-token: ${{ secrets.GITHUB_TOKEN }}
          comment-on-alert: true
          fail-on-alert: true
          alert-comment-cc-users: '@nuclia/nuclia-db'

      - name: Store on GH Pages
        uses: benchmark-action/github-action-benchmark@v1
        if: github.event_name == 'push'
        with:
          tool: 'pytest'
          output-file-path: benchmarks.json
          auto-push: true
          gh-repository: github.com/nuclia/nucliadb_performance
          gh-pages-branch: main
          github-token: ${{ secrets.PERFORMANCE_TOKEN }}

  # PR workflows END here
  # PUSH workflows contine with deploy activities

  build:
    name: Build image and push
    needs: tests
    uses: ./.github/workflows/_build-img-nucliadb.yml
    with:
      platforms: linux/amd64,linux/arm64
      image-name: search
      cache-gha-mode: min
    secrets:
      inherit
    if: github.event_name == 'push'

  deploy:
    name: Deploy Helm chart and trigger internal CI
    runs-on: ubuntu-latest
    needs: build
    if: github.event_name == 'push'

    steps:
      - name: Generate a token
        id: app-token
        uses: actions/create-github-app-token@v1
        with:
          app-id: ${{ secrets.GHAPP_ID_NUCLIABOT }}
          private-key: ${{ secrets.PK_GHAPP_NUCLIABOT }}
          owner: nuclia

      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Calculate short sha
        id: env-vars
        run: |-
          HASH=`git rev-parse --short HEAD`
          echo "short_sha=$HASH" >> $GITHUB_OUTPUT

      - name: Set helm package image
        id: version_step
        env:
          CONTAINER_REGISTRY: europe-west4-docker.pkg.dev/nuclia-internal/nuclia
        run: |-
          sed -i.bak "s#IMAGE_TO_REPLACE#$IMAGE_NAME:${{ steps.env-vars.outputs.short_sha }}#" ./charts/nucliadb_search/values.yaml
          sed -i.bak "s#CONTAINER_REGISTRY_TO_REPLACE#$CONTAINER_REGISTRY#" ./charts/nucliadb_search/values.yaml
          VERSION=`cat VERSION`
          VERSION_SHA=$VERSION+${{ steps.env-vars.outputs.short_sha }}
          sed -i.bak "s#99999.99999.99999#$VERSION_SHA#" ./charts/nucliadb_search/Chart.yaml
          echo "version_number=$VERSION_SHA" >> $GITHUB_OUTPUT

      - name: Configure Git
        run: |
          git config user.name "$GITHUB_ACTOR"
          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"

      - name: Install Helm
        uses: azure/setup-helm@v3
        with:
          version: v3.4.0

      - name: Push helm package
        run: |-
          helm lint charts/nucliadb_search
          helm package charts/nucliadb_search
          curl --data-binary "@nucliadb_search-${{ steps.version_step.outputs.version_number }}.tgz" ${{ secrets.HELM_CHART_URL }}/api/charts

      - name: Repository Dispatch
        uses: peter-evans/repository-dispatch@v2
        with:
          token: ${{ steps.app-token.outputs.token }}
          repository: nuclia/nucliadb_deploy
          event-type: promote
          client-payload: '{"component": "nucliadb_search", "chart-version": "${{ steps.version_step.outputs.version_number }}" }'