nucliadb_node.yml

name: nucliadb Node (rust + python)

on:
  pull_request:
    branches:
      - main
    paths:
      - ".github/workflows/nucliadb_node.yml"
      - "charts/nucliadb_node/**"
      - "Dockerfile.node"
      - "Dockerfile.node_sidecar"
      - "Cargo.lock"
      - "Cargo.toml"
      - "nucliadb_paragraphs/**"
      - "nucliadb_texts/**"
      - "nucliadb_texts2/**"
      - "nucliadb_core/**"
      - "nucliadb_telemetry/**"
      - "nucliadb_node/**"
      - "nucliadb_sidecar/**"
      - "nucliadb_vectors/**"
      - "nucliadb_paragraphs/**"
      - "nucliadb_core/**"
      - "nucliadb_relations/**"
      - 'nucliadb_relations2/**'
  push:
    branches:
      - main
    paths:
      - ".github/workflows/nucliadb_node.yml"
      - "charts/nucliadb_node/**"
      - "Dockerfile.node"
      - "Dockerfile.node_sidecar"
      - "Dockerfile.basenode"
      - "Cargo.lock"
      - "Cargo.toml"
      - "nucliadb_paragraphs/**"
      - "nucliadb_texts/**"
      - "nucliadb_texts2/**"
      - "nucliadb_core/**"
      - "nucliadb_telemetry/**"
      - "nucliadb_node/**"
      - "nucliadb_sidecar/**"
      - "nucliadb_node_binding/**"
      - "nucliadb_vectors/**"
      - "nucliadb_relations/**"
      - 'nucliadb_relations2/**'
      - "nucliadb_telemetry/src/**"
      - "nucliadb_utils/**"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  id-token: write   # This is required for requesting the JWT
  contents: read

env:
  RUSTFLAGS: -Dwarnings
  RUST_BACKTRACE: 1
  CARGO_TERM_COLOR: always
  COMPONENT: nucliadb-node
  CONTAINER_REGISTRY: europe-west4-docker.pkg.dev/nuclia-internal/nuclia
  IMAGE_NAME_BASE_NODE: basenode
  IMAGE_NAME_NODE: node
  IMAGE_NAME_NODE_SIDECAR: node_sidecar
  GCP_WORKLOAD_IDENTITY_PROVIDER: "projects/224545243904/locations/global/workloadIdentityPools/gh-nuclia/providers/gh-nuclia-provider"
  GCP_SERVICE_ACCOUNT: "github-actions@nuclia-internal.iam.gserviceaccount.com"

jobs:
  format-rust:
    name: Rust code Format
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly
          profile: minimal
          components: rustfmt
          override: true

      - uses: Swatinem/rust-cache@v2

      - name: rustfmt
        run: make fmt-check-package PACKAGE=nucliadb_node

  clippy-rust:
    name: Clippy lint
    runs-on: ubuntu-latest
    permissions: write-all
    steps:
      - uses: actions/checkout@v3

      - name: Check system
        run: make -C nucliadb_node/ check-system

      - run: rustup component add clippy

      - uses: Swatinem/rust-cache@v2

      - uses: actions-rs/clippy-check@v1
        env:
          RUSTFLAGS: --cfg tokio_unstable
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          args: -p nucliadb_node --tests

  tests-rust:
    name: Rust tests
    runs-on: ubuntu-latest
    needs: [clippy-rust, format-rust, licenses]

    steps:
      - uses: actions/checkout@v3

      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: "test_prebuild"

      - name: Check system
        run: make -C nucliadb_node/ check-system

      - uses: actions-rs/cargo@v1
        env:
          RUST_BACKTRACE: 1
          RUSTFLAGS: --cfg tokio_unstable
        name: cargo test
        with:
          command: test
          args: --workspace --all-features

  tests-python:
    name: Python tests
    runs-on: ubuntu-latest

    strategy:
      matrix:
        python-version: ["3.12"]

    steps:
      - name: Checkout the repository
        uses: actions/checkout@v3

      # setup docker, as we need to pull the node image to run the tests
      - name: Authenticate to Google Cloud
        id: gcp-auth
        uses: google-github-actions/auth@v2
        with:
          workload_identity_provider: "${{ env.GCP_WORKLOAD_IDENTITY_PROVIDER }}"
          service_account: "${{ env.GCP_SERVICE_ACCOUNT }}"
          token_format: access_token

      - name: Login to Google Artifact Registry
        uses: docker/login-action@v3
        with:
          registry: europe-west4-docker.pkg.dev
          username: oauth2accesstoken
          password: ${{ steps.gcp-auth.outputs.access_token }}

      - name: "Set up Cloud SDK"
        uses: "google-github-actions/setup-gcloud@v1"

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"

      - name: Install the package
        run: make -C nucliadb_sidecar/ install-dev

      - name: Calc git info
        id: git
        run: |-
          BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
          echo "branch=$BRANCH" >> $GITHUB_OUTPUT

      - name: Install jq
        run: |
          sudo apt-get update -y || true
          sudo apt-get install -y jq

      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            node_modified:
              - 'nucliadb_core/**'
              - 'nucliadb_node/src/**'
              - 'nucliadb_paragraphs/**'
              - 'nucliadb_paragraphs/**'
              - 'nucliadb_procs/**'
              - 'nucliadb_relations/**'
              - 'nucliadb_relations2/**'
              - 'nucliadb_telemetry/**'
              - 'nucliadb_texts/**'
              - 'nucliadb_texts2/**'
              - 'nucliadb_vectors/**'

      - name: Build local Node for testing
        if: steps.filter.outputs.node_modified == 'true'
        env:
          SECRET_KEY: ${{ secrets.RUST_BUILD_SERVER_SECRET }}
          COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
          BRANCH: ${{ steps.git.outputs.branch }}
        run: |
          make build-node-debug

      - name: Run tests
        run: make -C nucliadb_sidecar test-cov

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          file: ./coverage.xml
          flags: node-sidecar


  dockerfile-changes:
    name: Check if dockerfiles have changed
    runs-on: ubuntu-latest
    outputs:
      basenode: ${{ steps.changes.outputs.basenode_dockerfile }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Check if dockerfile have changed
        id: changes
        uses: dorny/paths-filter@v2
        with:
          filters: |
            basenode_dockerfile:
              - 'Dockerfile.basenode'


  build-base-rust:
    name: Build and push base image
    needs: [ dockerfile-changes, tests-rust ]
    uses: ./.github/workflows/_build-img-nucliadb.yml
    with:
      skip: ${{ ! needs.dockerfile-changes.outputs.basenode }}
      file: Dockerfile.basenode
      image-name: basenode
      cache-gha-mode: min
    secrets:
      inherit
    if: github.event_name == 'push'

  build-rust:
    name: Build and push index node docker image
    needs: build-base-rust
    uses: ./.github/workflows/_build-img-nucliadb.yml
    with:
      file: Dockerfile.node
      image-name: node
    secrets:
      inherit
    if: github.event_name == 'push'


  build-python:
    name: Build and push sidecar docker image
    needs: tests-python
    uses: ./.github/workflows/_build-img-nucliadb.yml
    with:
      context: .
      file: Dockerfile.node_sidecar
      image-name: node_sidecar
      cache-gha-mode: min
    secrets:
      inherit
    if: github.event_name == 'push'


  deploy:
    name: Deploy Helm chart and trigger internal CI
    runs-on: ubuntu-latest
    needs: [build-rust, build-python]
    if: github.event_name == 'push'

    steps:
      - name: Generate a token
        id: app-token
        uses: actions/create-github-app-token@v1
        with:
          app-id: ${{ secrets.GHAPP_ID_NUCLIABOT }}
          private-key: ${{ secrets.PK_GHAPP_NUCLIABOT }}
          owner: nuclia

      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Calculate short sha
        id: env-vars
        run: |-
          HASH=`git rev-parse --short HEAD`
          echo "short_sha=$HASH" >> $GITHUB_OUTPUT

      - name: Set helm package image
        id: version_step
        run: |-
          sed -i.bak "s#IMAGE_TO_REPLACE#$IMAGE_NAME_NODE:${{ steps.env-vars.outputs.short_sha }}#" ./charts/nucliadb_node/values.yaml
          sed -i.bak "s#IMAGE_SIDECAR_TO_REPLACE#$IMAGE_NAME_NODE_SIDECAR:${{ steps.env-vars.outputs.short_sha }}#" ./charts/nucliadb_node/values.yaml
          sed -i.bak "s#CONTAINER_REGISTRY_TO_REPLACE#$CONTAINER_REGISTRY#" ./charts/nucliadb_node/values.yaml
          VERSION=`cat VERSION`
          VERSION_SHA=$VERSION+${{ steps.env-vars.outputs.short_sha }}
          sed -i.bak "s#99999.99999.99999#$VERSION_SHA#" ./charts/nucliadb_node/Chart.yaml
          echo "version_number=$VERSION_SHA" >> $GITHUB_OUTPUT

      - name: Configure Git
        run: |
          git config user.name "$GITHUB_ACTOR"
          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"

      - name: Install Helm
        uses: azure/setup-helm@v3
        with:
          version: v3.4.0

      - name: Push helm package
        run: |-
          helm lint charts/nucliadb_node
          helm package charts/nucliadb_node
          curl --data-binary "@nucliadb_node-${{ steps.version_step.outputs.version_number }}.tgz" ${{ secrets.HELM_CHART_URL }}/api/charts

      - name: Repository Dispatch
        uses: peter-evans/repository-dispatch@v2
        with:
          token: ${{ steps.app-token.outputs.token }}
          repository: nuclia/nucliadb_deploy
          event-type: promote
          client-payload: '{"component": "nucliadb_node", "chart-version": "${{ steps.version_step.outputs.version_number }}" }'