Cleanup CI changes (#2309)

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: New CI
+name: CI
 
 on:
   pull_request:
@@ -81,24 +81,64 @@ jobs:
           done
           exit $ERROR
 
-  licenses:
+  lint-helm:
+    name: Lint all helm charts
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: v3.4.0
+
+      - name: Lint helm packages
+        run: |-
+          helm lint charts/*
+          helm package charts/*
+
+  check-licenses:
     name: Check Licenses
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
+      - name: Check License Header
+        uses: apache/skywalking-eyes/header@501a28d2fb4a9b962661987e50cf0219631b32ff
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - uses: EmbarkStudios/cargo-deny-action@v1
         name: cargo deny
         with:
           log-level: warn
           command: check licenses
 
+  check-migrations:
+    runs-on: ubuntu-latest
+    needs:
+      - build-virtual-env
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+      - name: Restore venv
+        uses: actions/cache/restore@v4
+        with:
+          path: .venv
+          key: venv-${{ github.sha }}
+          fail-on-cache-miss: true
+      - name: Check migrations
+        run: .venv/bin/nucliadb-validate-migrations
+
   # Builds
   build-node-binaries:
     name: Build node binaries
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-
       - uses: Swatinem/rust-cache@v2
       - name: Compile
         run: cargo build --release --bin node_reader --bin node_writer
@@ -326,6 +366,7 @@ jobs:
         package:
           - nucliadb_dataset
           - nucliadb_models
+          - nucliadb_node_binding
           - nucliadb_sdk
           - nucliadb_telemetry
           - nucliadb_utils
@@ -366,7 +407,9 @@ jobs:
       - clippy-rust
       - updated-protos
       - pre-checks-python
-      - licenses
+      - check-licenses
+      - check-migrations
+      - lint-helm
       - sidecar-tests
       - nucliadb-search-tests
       - node-rust-tests

.github/workflows/deploy.yml

@@ -54,12 +54,60 @@ jobs:
     secrets:
       inherit
 
+  deploy-nucliadb-shared:
+    name: Deploy nucliadb shared
+    runs-on: ubuntu-latest
+    steps:
+    - name: Generate a token
+      id: app-token
+      uses: actions/create-github-app-token@v1
+      with:
+        app-id: ${{ secrets.GHAPP_ID_NUCLIABOT }}
+        private-key: ${{ secrets.PK_GHAPP_NUCLIABOT }}
+        owner: nuclia
+
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Set helm package image
+      id: version_step
+      run: |-
+        VERSION=`cat VERSION`
+        VERSION_SHA=$VERSION-$(echo $GITHUB_SHA | cut -c1-6)
+        sed -i.bak "s#99999.99999.99999#$VERSION_SHA#" ./charts/nucliadb_shared/Chart.yaml
+        echo "version_number=$VERSION_SHA" >> $GITHUB_OUTPUT
+
+    - name: Configure Git
+      run: |
+        git config user.name "$GITHUB_ACTOR"
+        git config user.email "[email protected]"
+
+    - name: Install Helm
+      uses: azure/setup-helm@v3
+      with:
+        version: v3.4.0
+
+    - name: Push helm package
+      run: |-
+        helm lint charts/nucliadb_shared
+        helm package charts/nucliadb_shared
+        curl --data-binary "@nucliadb_shared-${{ steps.version_step.outputs.version_number }}.tgz" ${{ secrets.HELM_CHART_URL }}/api/charts
+
+    - name: Repository Dispatch
+      uses: peter-evans/repository-dispatch@v2
+      with:
+        token: ${{ steps.app-token.outputs.token }}
+        repository: nuclia/nucliadb_deploy
+        event-type: promote
+        client-payload: '{"component": "nucliadb_shared", "chart-version": "${{ steps.version_step.outputs.version_number }}" }'
+
   deploy-nucliadb-node:
     name: Deploy Node
     runs-on: ubuntu-latest
     needs:
       - build-python-images
       - build-node-image
+      - deploy-nucliadb-shared
     steps:
       - name: Generate a token
         id: app-token
@@ -118,6 +166,7 @@ jobs:
     runs-on: ubuntu-latest
     needs:
       - build-python-images
+      - deploy-nucliadb-shared
     strategy:
       # nucliadb_deploy pipeline has issues with too many parallel invocations
       max-parallel: 1

.github/workflows/e2e.yml

@@ -17,7 +17,7 @@ jobs:
   e2e-k8s-cluster:
     #
     # E2E tests run against a Kubernetes cluster.
-    # These tests setup microk8s and run NucliaDB with PostgreSQL.
+    # These tests setup microk8s and run NucliaDB standalone with PostgreSQL.
     # Chart configuration: e2e/conf/chart.values.yaml
     #
     name: E2E Kubernetes Cluster Tests
@@ -103,65 +103,3 @@ jobs:
               kubectl logs -n nucliadb pod/nucliadb-1 &&
               kubectl get pods -n nucliadb -l app=nucliadb-worker -o name | while read pod; do kubectl logs -n nucliadb $pod; done &&
               exit 1)
-
-  e2e-vm-cluster:
-    runs-on: ubuntu-latest
-    name: E2E VM Cluster Tests
-    strategy:
-      matrix:
-        ostypes: ["debian:bookworm-slim", "fedora:38"]
-
-    container:
-      image: ${{ matrix.ostypes }}
-      volumes:
-        - /opt/
-
-    services:
-      # Label used to access the service container
-      postgres:
-        # Docker Hub image
-        image: postgres
-        # Provide the password for postgres
-        env:
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: postgres
-        ports:
-          - 5432:5432
-        # Set health checks to wait until postgres has started
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-
-      - name: Install NucliaDB
-        # tests that installation script works
-        run: |
-          ./scripts/install-vm.sh
-
-      - name: Install local nucliadb
-        # Now, let's install the current version of nucliadb to
-        # run the tests against.
-        # Unfortunately, we're not testing against the released version
-        # of NucliaDB; however, it seems more important to test against
-        # the current branch.
-        run: |
-          bash -c 'source /opt/nucliadb/bin/activate && make -C nucliadb install-dev'
-
-      - name: Run Cluster and tests
-        env:
-          DRIVER: PG
-          FILE_BACKEND: LOCAL
-          DRIVER_PG_URL: postgresql://postgres:postgres@postgres:5432/postgres
-          NUCLIA_PUBLIC_URL: https://{zone}.stashify.cloud
-          cluster_discovery_mode: manual
-          cluster_discovery_manual_addresses: '["localhost:10009","localhost:10010"]'
-          NUA_API_KEY: ${{ secrets.NUA_API_KEY }}
-          NUCLIADB_DISABLE_ANALYTICS: "True"
-          ENVIRONMENT: "stage"
-          FLAG_SETTINGS_URL: https://raw.githubusercontent.com/nuclia/status/main/features-v2.json
-        run: bash e2e/run-e2e-tests.sh