nucliadb_node.yml

name: nucliadb Node (rust + python)

on:
  pull_request:
    branches:
      - main
    paths:
      - ".github/workflows/nucliadb_node.yml"
      - "charts/nucliadb_node/**"
      - "Dockerfile.node"
      - "Dockerfile.node_sidecar"
      - "Cargo.lock"
      - "Cargo.toml"
      - "nucliadb_paragraphs/**"
      - "nucliadb_texts/**"
      - "nucliadb_core/**"
      - "nucliadb_telemetry/**"
      - "nucliadb_node/**"
      - "nucliadb_vectors/**"
      - "nucliadb_texts/**"
      - "nucliadb_paragraphs/**"
      - "nucliadb_core/**"
      - "nucliadb_relations/**"
      - 'nucliadb_relations2/**'
  push:
    branches:
      - main
    paths:
      - ".github/workflows/nucliadb_node.yml"
      - "charts/nucliadb_node/**"
      - "Dockerfile.node"
      - "Dockerfile.node_sidecar"
      - "Dockerfile.basenode"
      - "Cargo.lock"
      - "Cargo.toml"
      - "nucliadb_paragraphs/**"
      - "nucliadb_texts/**"
      - "nucliadb_core/**"
      - "nucliadb_telemetry/**"
      - "nucliadb_node/**"
      - "nucliadb_node_binding/**"
      - "nucliadb_vectors/**"
      - "nucliadb_relations/**"
      - 'nucliadb_relations2/**'
      - "nucliadb_telemetry/src/**"
      - "nucliadb_utils/**"

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  RUSTFLAGS: -Dwarnings
  RUST_BACKTRACE: 1
  CARGO_TERM_COLOR: always
  COMPONENT: nucliadb-node
  CONTAINER_REGISTRY: eu.gcr.io/${{ secrets.PROJECT_ID }}
  IMAGE_NAME_BASE_NODE: basenode
  IMAGE_NAME_NODE: node
  IMAGE_NAME_NODE_SIDECAR: node_sidecar

jobs:
  pre-checks-python:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11"]

    steps:
      - name: Checkout the repository
        uses: actions/checkout@v3

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"

      - name: Install package
        run: make -C nucliadb_node/ install-dev

      - name: Run pre-checks
        run: make -C nucliadb_node/ lint

  licenses:
    name: Check Licenses
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: EmbarkStudios/cargo-deny-action@v1
        name: cargo deny
        with:
          log-level: warn
          command: check licenses

  format-rust:
    name: Rust code Format
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - uses: actions-rs/toolchain@v1
        with:
          toolchain: nightly
          profile: minimal
          components: rustfmt
          override: true

      - uses: Swatinem/rust-cache@v2

      - name: rustfmt
        run: make fmt-check-package PACKAGE=nucliadb_node

  clippy-rust:
    name: Clippy lint
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Check system
        run: make -C nucliadb_node/ check-system

      - run: rustup component add clippy

      - uses: Swatinem/rust-cache@v2

      - uses: actions-rs/clippy-check@v1
        env:
          RUSTFLAGS: --cfg tokio_unstable
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          args: -p nucliadb_node --tests

  tests-rust:
    name: Rust tests
    runs-on: ubuntu-latest
    needs: [clippy-rust, format-rust, licenses]

    steps:
      - uses: actions/checkout@v3

      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: "test_prebuild"

      - name: Check system
        run: make -C nucliadb_node/ check-system

      - uses: actions-rs/cargo@v1
        env:
          DEBUG: 1
          RUST_BACKTRACE: 1
          RUSTFLAGS: --cfg tokio_unstable
        name: cargo test
        with:
          command: test
          args: --workspace --all-features

  tests-python:
    name: Python tests
    runs-on: ubuntu-latest
    needs: [pre-checks-python, licenses]

    strategy:
      matrix:
        python-version: ["3.11"]

    steps:
      - name: Checkout the repository
        uses: actions/checkout@v3

      # Setup gcloud CLI
      - uses: google-github-actions/auth@v1
        with:
          credentials_json: "${{ secrets.GCP_CREDENTIALS }}"

      - name: "Set up Cloud SDK"
        uses: "google-github-actions/setup-gcloud@v1"

      # setup docker, as we need to pull the node image to run the tests
      - name: Configure Docker
        run: docker login -u oauth2accesstoken -p "$(gcloud auth application-default print-access-token)" https://eu.gcr.io

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"

      - name: Install the package
        run: make -C nucliadb_node/ install-dev
      
      - name: Calc git info
        id: git
        run: |-
          BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}
          echo "branch=$BRANCH" >> $GITHUB_OUTPUT

      - name: Install jq
        run: |
          sudo apt-get update -y
          sudo apt-get install -y jq

      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            node_modified:
              - 'nucliadb_core/**'
              - 'nucliadb_node/src/**'
              - 'nucliadb_paragraphs/**'
              - 'nucliadb_procs/**'
              - 'nucliadb_relations/**'
              - 'nucliadb_relations2/**'
              - 'nucliadb_telemetry/**'
              - 'nucliadb_texts/**'
              - 'nucliadb_vectors/**'

      - name: Build local Node for testing
        if: steps.filter.outputs.node_modified == 'true'
        env:
          SECRET_KEY: ${{ secrets.RUST_BUILD_SERVER_SECRET }}
          COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
          BRANCH: ${{ steps.git.outputs.branch }}
        run: |
          make build-node-debug

      - name: Run tests
        run: |
          pytest -rfE --cov=nucliadb_node -s --tb=native -v --cov-report xml --cov-append nucliadb_node/nucliadb_node

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          file: ./coverage.xml
          flags: node-sidecar


  dockerfile-changes:
    name: Check if dockerfiles have changed
    runs-on: ubuntu-latest
    outputs:
      basenode: ${{ steps.changes.outputs.basenode_dockerfile }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Check if dockerfile have changed
        id: changes
        uses: dorny/paths-filter@v2
        with:
          filters: |
            basenode_dockerfile:
              - 'Dockerfile.basenode'


  build-base-rust:
    name: Build and push base image
    needs: [ dockerfile-changes, tests-rust ]
    uses: ./.github/workflows/_build-img-nucliadb.yml
    with:
      skip: ${{ ! needs.dockerfile-changes.outputs.basenode }}
      file: Dockerfile.basenode
      image-name: basenode
      cache-gha-mode: min
    secrets:
      inherit
    if: github.event_name == 'push'

  build-rust:
    name: Build and push index node docker image
    needs: build-base-rust
    uses: ./.github/workflows/_build-img-nucliadb.yml
    with:
      file: Dockerfile.node
      image-name: node
    secrets:
      inherit
    if: github.event_name == 'push'


  build-python:
    name: Build and push sidecar docker image
    needs: tests-python
    uses: ./.github/workflows/_build-img-nucliadb.yml
    with:
      context: .
      file: Dockerfile.node_sidecar
      image-name: node_sidecar
      cache-gha-mode: min
    secrets:
      inherit
    if: github.event_name == 'push'


  deploy:
    name: Deploy Helm chart and trigger internal CI
    runs-on: ubuntu-latest
    needs: [build-rust, build-python]
    if: github.event_name == 'push'

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      - name: Calculate short sha
        id: env-vars
        run: |-
          HASH=`git rev-parse --short HEAD`
          echo "short_sha=$HASH" >> $GITHUB_OUTPUT

      - name: Set helm package image
        id: version_step
        run: |-
          sed -i.bak "s#IMAGE_TO_REPLACE#$IMAGE_NAME_NODE:${{ steps.env-vars.outputs.short_sha }}#" ./charts/nucliadb_node/values.yaml
          sed -i.bak "s#IMAGE_SIDECAR_TO_REPLACE#$IMAGE_NAME_NODE_SIDECAR:${{ steps.env-vars.outputs.short_sha }}#" ./charts/nucliadb_node/values.yaml
          sed -i.bak "s#CONTAINER_REGISTRY_TO_REPLACE#$CONTAINER_REGISTRY#" ./charts/nucliadb_node/values.yaml
          VERSION=`cat VERSION`
          VERSION_SHA=$VERSION+${{ steps.env-vars.outputs.short_sha }}
          sed -i.bak "s#99999.99999.99999#$VERSION_SHA#" ./charts/nucliadb_node/Chart.yaml
          echo "version_number=$VERSION_SHA" >> $GITHUB_OUTPUT

      - name: Configure Git
        run: |
          git config user.name "$GITHUB_ACTOR"
          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"

      - name: Install Helm
        uses: azure/setup-helm@v3
        with:
          version: v3.4.0

      - name: Push helm package
        run: |-
          helm lint charts/nucliadb_node
          helm package charts/nucliadb_node
          curl --data-binary "@nucliadb_node-${{ steps.version_step.outputs.version_number }}.tgz" ${{ secrets.HELM_CHART_URL }}/api/charts

      - name: Repository Dispatch
        uses: peter-evans/repository-dispatch@v2
        with:
          token: ${{ secrets.GH_CICD_PUBLIC }}
          repository: nuclia/nucliadb_deploy
          event-type: promote
          client-payload: '{"component": "nucliadb_node", "chart-version": "${{ steps.version_step.outputs.version_number }}" }'