ci: add airgap e2e test (#1034)

.github/workflows/airgap-e2e.yaml

@@ -0,0 +1,154 @@
+name: Elemental Airgap E2E tests with Rancher Manager
+
+on:
+  pull_request:
+    #test
+  workflow_dispatch:
+    inputs:
+      qase_run_id:
+        description: Case run ID where the results will be reported
+        required: false
+        type: string
+      cert-manager_version:
+        description: Version of cert-manager to use
+        type: string
+      destroy_runner:
+        description: Destroy the auto-generated self-hosted runner
+        default: true
+        type: boolean
+      operator_repo:
+        description: Elemental operator repository to use
+        type: string
+        default: oci://registry.opensuse.org/isv/rancher/elemental/dev/charts/rancher
+      os_to_test:
+        description: OS repository to test (dev/staging/stable)
+        type: string
+        default: dev
+      rancher_version:
+        description: Rancher Manager channel/version to use for installation
+        default: stable/latest
+        type: string
+
+jobs:
+  create-runner:
+    runs-on: ubuntu-latest
+    outputs:
+      uuid: ${{ steps.generator.outputs.uuid }}
+      runner: ${{ steps.generator.outputs.runner }}
+      public_dns: ${{ steps.dns.outputs.public_dns }}
+    steps:
+      # actions/checkout MUST come before auth
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Generate UUID and Runner hostname
+        id: generator
+        run: |
+          UUID=$(uuidgen)
+          echo "uuid=${UUID}" >> ${GITHUB_OUTPUT}
+          echo "runner=elemental-ci-${UUID}" >> ${GITHUB_OUTPUT}
+      - name: Authenticate to GCP
+        uses: google-github-actions/auth@v1
+        with:
+          credentials_json: ${{ secrets.GCP_CREDENTIALS }}
+      - name: Setup gcloud
+        uses: google-github-actions/setup-gcloud@v1
+      - name: Create runner
+        run: |
+          gcloud compute instances create ${{ steps.generator.outputs.runner }} \
+            --source-instance-template elemental-e2e-ci-runner-spot-x86-64-template-n2-standard-16-v4 \
+            --zone us-central1-a
+      - name: Create PAT token secret
+        run: |
+          echo -n ${{ secrets.SELF_HOSTED_RUNNER_PAT_TOKEN }} \
+            | gcloud secrets create PAT_TOKEN_${{ steps.generator.outputs.uuid }} --data-file=-
+      - name: Get public dns name in GCP
+        id: dns
+        run: |
+          # Do a timed out loop here, as gcloud can sometimes fail
+          typeset -i i=0
+          while true; do
+            # Get public IP
+            PUBLIC_IP=$(gcloud compute instances list 2> /dev/null \
+                        | awk '/${{ steps.generator.outputs.runner }}/ {print $6}')
+            # Exit if we reach the timeout or if IP is set
+            if (( ++i > 10 )) || [[ -n "${PUBLIC_IP}" ]]; then
+              break
+            fi
+            # Wait a little before retrying
+            sleep 2
+          done
+          # Get the public DNS
+          PUBLIC_DNS=$(host -l ${PUBLIC_IP} 2> /dev/null \
+                       | awk '{sub(/\.$/, ""); print $5}')
+          echo "public_dns=${PUBLIC_DNS}" >> ${GITHUB_OUTPUT}
+          # Raise an error if either IP and/or DNS are empty
+          if [[ -z "${PUBLIC_IP}" || -z "${PUBLIC_DNS}" ]]; then
+            echo "PUBLIC_IP and/or PUBLIC_DNS are empty!" >&2
+            false
+          fi
+  e2e:
+    needs: create-runner
+    runs-on: ${{ needs.create-runner.outputs.uuid }}
+    env:
+      ARCH: amd64
+      CERT_MANAGER_VERSION: v1.12.2
+      # Distribution to use to host Rancher Manager (K3s)
+      K8S_UPSTREAM_VERSION: 1.26.7
+      # QASE variables
+      QASE_API_TOKEN: ${{ secrets.qase_api_token }}
+      QASE_RUN_ID: ${{ inputs.qase_run_id }}
+      # For Rancher Manager
+      RANCHER_VERSION: stable/2.7.6
+      TIMEOUT_SCALE: 3
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: tests/go.mod
+      - name: Prepare the archive file to send to air-gapped nodes
+        run: |
+          cd tests && make e2e-prepare-archive
+      
+
+#      - name: Deploy a node to join Rancher manager
+#        if: inputs.test_type == 'ui'
+#        env:
+#          ISO_BOOT: ${{ inputs.iso_boot }} 
+#          VM_INDEX: 1
+#          VM_MEM: 8192
+#          HOST_MEMORY_RESERVED: 49152
+#        run: |
+#          cd tests && (
+#            # Removing 'downloads' is needed to avoid this error during 'make':
+#            # 'pattern all: open .../elemental/tests/cypress/downloads: permission denied'
+#            sudo rm -rf cypress/latest/downloads
+#            make e2e-ui-rancher
+#          )
+#
+#      - name: Configure Rancher & Libvirt
+#        if: inputs.test_type == 'cli'
+#        run: cd tests && make e2e-configure-rancher
+#  delete-runner:
+#    if: always() && needs.create-runner.result == 'success' && inputs.destroy_runner == true
+#    needs: [create-runner, e2e]
+#    runs-on: ubuntu-latest
+#    steps:
+#      # actions/checkout MUST come before auth
+#      - name: Checkout
+#        uses: actions/checkout@v3
+#      - name: Authenticate to GCP
+#        uses: google-github-actions/auth@v1
+#        with:
+#          credentials_json: ${{ secrets.credentials }}
+#      - name: Setup gcloud
+#        uses: google-github-actions/setup-gcloud@v1
+#      - name: Delete PAT token secret
+#        run: |
+#          gcloud --quiet secrets delete PAT_TOKEN_${{ needs.create-runner.outputs.uuid }}
+#      - name: Delete runner
+#        run: |
+#          gcloud --quiet compute instances delete ${{ needs.create-runner.outputs.runner }} \
+#            --delete-disks all \
+#            --zone ${{ inputs.zone }}

tests/Makefile

@@ -85,6 +85,8 @@ e2e-install-backup-restore: deps
 	ginkgo --label-filter install-backup-restore -r -v ./e2e
 e2e-iso-image: deps
 	ginkgo --label-filter iso-image -r -v ./e2e
+e2e-prepare-archive: deps
+	ginkgo --label-filter prepare-archive -r -v ./e2e
 e2e-ui-rancher: deps
 	ginkgo --label-filter ui -r -v ./e2e
 e2e-uninstall-operator:

tests/e2e/airgap_test.go

@@ -0,0 +1,29 @@
+/*
+Copyright © 2022 - 2023 SUSE LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e_test
+
+import (
+	"os/exec"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("E2E - Build the airgap archive", Label("prepare-archive"), func() {
+	It("Execute the script to build the archive", func() {
+		err := exec.Command("sudo", airgapBuildScript, k8sUpstreamVersion, certManagerVersion, rancherVersion).Run()
+		Expect(err).To(Not(HaveOccurred()))
+	})
+})

tests/e2e/install_test.go

@@ -175,7 +175,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() {
 				}
 
 				if clusterType == "hardened" {
-					flags = append(flags, "--version", CertManagerVersion)
+					flags = append(flags, "--version", certManagerVersion)
 				}
 
 				RunHelmCmdWithRetry(flags...)

tests/e2e/suite_test.go

@@ -30,6 +30,7 @@ import (
 )
 
 const (
+	airgapBuildScript     = "../scripts/build-airgap"
 	appYaml               = "../assets/hello-world_app.yaml"
 	backupYaml            = "../assets/backup.yaml"
 	clusterYaml           = "../assets/cluster.yaml"
@@ -60,7 +61,7 @@ var (
 	arch                  string
 	backupRestoreVersion  string
 	caType                string
-	CertManagerVersion    string
+	certManagerVersion    string
 	clusterName           string
 	clusterNS             string
 	clusterType           string
@@ -199,7 +200,7 @@ var _ = BeforeSuite(func() {
 	arch = os.Getenv("ARCH")
 	backupRestoreVersion = os.Getenv("BACKUP_RESTORE_VERSION")
 	caType = os.Getenv("CA_TYPE")
-	CertManagerVersion = os.Getenv("CERT_MANAGER_VERSION")
+	certManagerVersion = os.Getenv("CERT_MANAGER_VERSION")
 	clusterName = os.Getenv("CLUSTER_NAME")
 	clusterNS = os.Getenv("CLUSTER_NS")
 	clusterType = os.Getenv("CLUSTER_TYPE")

tests/scripts/build-airgap

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+set -e -x
+K3S_UPSTREAM_VERSION=$1
+CERT_MANAGER_VERSION=$2
+RANCHER_MANAGER_VERSION=$3
+# Variable(s) and default values
+
+export PATH=$PATH:/usr/local/bin
+
+mkdir -p /opt/rancher/{k3s_$K3S_UPSTREAM_VERSION,helm} /opt/rancher/images/{cert,rancher,registry,elemental}
+cd /opt/rancher/k3s_$K3S_UPSTREAM_VERSION/
+
+echo - Install packages
+zypper --no-refresh -n in zstd skopeo
+
+echo - Download k3s and rancher
+curl -#OL https://github.com/k3s-io/k3s/releases/download/v$K3S_UPSTREAM_VERSION%2Bk3s1/k3s-airgap-images-amd64.tar.zst
+curl -#OL https://github.com/k3s-io/k3s/releases/download/v$K3S_UPSTREAM_VERSION%2Bk3s1/k3s
+
+echo - Get the install script
+curl -sfL https://get.k3s.io/ -o install.sh
+
+echo - Get Helm Charts
+cd /opt/rancher/helm/
+
+echo - Add repos
+helm repo add jetstack https://charts.jetstack.io > /dev/null 2>&1
+helm repo add rancher-latest https://releases.rancher.com/server-charts/latest > /dev/null 2>&1
+helm repo update > /dev/null 2>&1
+
+echo - Get charts
+helm pull jetstack/cert-manager --version $CERT_MANAGER_VERSION > /dev/null 2>&1
+helm pull rancher-latest/rancher --version v$RANCHER_MANAGER_VERSION > /dev/null 2>&1
+
+echo - Get Images - Rancher/Elemental
+cd /opt/rancher/images/
+
+echo - Rancher image list
+curl -#L https://github.com/rancher/rancher/releases/download/v$RANCHER_MANAGER_VERSION/rancher-images.txt -o rancher/orig_rancher-images.txt
+
+echo - Shorten rancher list with a sort
+# fix library tags
+sed -i -e '0,/busybox/s/busybox/library\/busybox/' -e 's/registry/library\/registry/g' rancher/orig_rancher-images.txt
+
+# remove things that are not needed and overlapped
+sed -i -E '/neuvector|minio|gke|aks|eks|sriov|harvester|mirrored|longhorn|thanos|tekton|istio|multus|hyper|jenkins|windows/d' rancher/orig_rancher-images.txt
+
+# get latest version
+for i in $(cat rancher/orig_rancher-images.txt|awk -F: '{print $1}'); do
+  grep -w $i rancher/orig_rancher-images.txt | sort -Vr| head -1 >> rancher/version_unsorted.txt
+done
+
+# final sort
+cat rancher/version_unsorted.txt | sort -u > rancher/rancher-images.txt
+
+echo - Cert-manager image list
+helm template /opt/rancher/helm/cert-manager-$CERT_MANAGER_VERSION.tgz | awk '$1 ~ /image:/ {print $2}' | sed s/\"//g > cert/cert-manager-images.txt
+
+# get images
+echo - Skopeo - cert-manager
+for i in $(cat cert/cert-manager-images.txt); do
+  skopeo copy docker://$i docker-archive:cert/$(echo $i| awk -F/ '{print $3}'|sed 's/:/_/g').tar:$(echo $i| awk -F/ '{print $3}') > /dev/null 2>&1
+done
+
+echo - Skopeo - Rancher - be patient...
+for i in $(cat rancher/rancher-images.txt); do
+  skopeo copy docker://$i docker-archive:rancher/$(echo $i| awk -F/ '{print $2}'|sed 's/:/_/g').tar:$(echo $i| awk -F/ '{print $2}') > /dev/null 2>&1
+done
+
+curl -#L https://github.com/clemenko/rke_airgap_install/raw/main/registry.tar -o registry/registry.tar > /dev/null 2>&1
+
+cd /opt/rancher/
+echo - Compress all the things
+tar -I zstd -vcf /opt/airgap_rancher.zst $(ls) > /dev/null 2>&1
+
+echo "------------------------------------------------------------------"
+echo " to uncompress : "
+echo "   mkdir /opt/rancher"
+echo "   tar -I zstd -vxf airgap_rancher.zst -C /opt/rancher"
+echo "------------------------------------------------------------------"