Skip to content

Commit

Permalink
ci: add airgap e2e test (#1034)
Browse files Browse the repository at this point in the history
  • Loading branch information
juadk committed Oct 9, 2023
1 parent 0b8452e commit 8927c16
Show file tree
Hide file tree
Showing 6 changed files with 270 additions and 3 deletions.
154 changes: 154 additions & 0 deletions .github/workflows/airgap-e2e.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
name: Elemental Airgap E2E tests with Rancher Manager

on:
pull_request:
#test
workflow_dispatch:
inputs:
qase_run_id:
description: Case run ID where the results will be reported
required: false
type: string
cert-manager_version:
description: Version of cert-manager to use
type: string
destroy_runner:
description: Destroy the auto-generated self-hosted runner
default: true
type: boolean
operator_repo:
description: Elemental operator repository to use
type: string
default: oci://registry.opensuse.org/isv/rancher/elemental/dev/charts/rancher
os_to_test:
description: OS repository to test (dev/staging/stable)
type: string
default: dev
rancher_version:
description: Rancher Manager channel/version to use for installation
default: stable/latest
type: string

jobs:
create-runner:
runs-on: ubuntu-latest
outputs:
uuid: ${{ steps.generator.outputs.uuid }}
runner: ${{ steps.generator.outputs.runner }}
public_dns: ${{ steps.dns.outputs.public_dns }}
steps:
# actions/checkout MUST come before auth
- name: Checkout
uses: actions/checkout@v3
- name: Generate UUID and Runner hostname
id: generator
run: |
UUID=$(uuidgen)
echo "uuid=${UUID}" >> ${GITHUB_OUTPUT}
echo "runner=elemental-ci-${UUID}" >> ${GITHUB_OUTPUT}
- name: Authenticate to GCP
uses: google-github-actions/auth@v1
with:
credentials_json: ${{ secrets.GCP_CREDENTIALS }}
- name: Setup gcloud
uses: google-github-actions/setup-gcloud@v1
- name: Create runner
run: |
gcloud compute instances create ${{ steps.generator.outputs.runner }} \
--source-instance-template elemental-e2e-ci-runner-spot-x86-64-template-n2-standard-16-v4 \
--zone us-central1-a
- name: Create PAT token secret
run: |
echo -n ${{ secrets.SELF_HOSTED_RUNNER_PAT_TOKEN }} \
| gcloud secrets create PAT_TOKEN_${{ steps.generator.outputs.uuid }} --data-file=-
- name: Get public dns name in GCP
id: dns
run: |
# Do a timed out loop here, as gcloud can sometimes fail
typeset -i i=0
while true; do
# Get public IP
PUBLIC_IP=$(gcloud compute instances list 2> /dev/null \
| awk '/${{ steps.generator.outputs.runner }}/ {print $6}')
# Exit if we reach the timeout or if IP is set
if (( ++i > 10 )) || [[ -n "${PUBLIC_IP}" ]]; then
break
fi
# Wait a little before retrying
sleep 2
done
# Get the public DNS
PUBLIC_DNS=$(host -l ${PUBLIC_IP} 2> /dev/null \
| awk '{sub(/\.$/, ""); print $5}')
echo "public_dns=${PUBLIC_DNS}" >> ${GITHUB_OUTPUT}
# Raise an error if either IP and/or DNS are empty
if [[ -z "${PUBLIC_IP}" || -z "${PUBLIC_DNS}" ]]; then
echo "PUBLIC_IP and/or PUBLIC_DNS are empty!" >&2
false
fi
e2e:
needs: create-runner
runs-on: ${{ needs.create-runner.outputs.uuid }}
env:
ARCH: amd64
CERT_MANAGER_VERSION: v1.12.2
# Distribution to use to host Rancher Manager (K3s)
K8S_UPSTREAM_VERSION: 1.26.7
# QASE variables
QASE_API_TOKEN: ${{ secrets.qase_api_token }}
QASE_RUN_ID: ${{ inputs.qase_run_id }}
# For Rancher Manager
RANCHER_VERSION: stable/2.7.6
TIMEOUT_SCALE: 3
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install Go
uses: actions/setup-go@v3
with:
go-version-file: tests/go.mod
- name: Prepare the archive file to send to air-gapped nodes
run: |
cd tests && make e2e-prepare-archive

# - name: Deploy a node to join Rancher manager
# if: inputs.test_type == 'ui'
# env:
# ISO_BOOT: ${{ inputs.iso_boot }}
# VM_INDEX: 1
# VM_MEM: 8192
# HOST_MEMORY_RESERVED: 49152
# run: |
# cd tests && (
# # Removing 'downloads' is needed to avoid this error during 'make':
# # 'pattern all: open .../elemental/tests/cypress/downloads: permission denied'
# sudo rm -rf cypress/latest/downloads
# make e2e-ui-rancher
# )
#
# - name: Configure Rancher & Libvirt
# if: inputs.test_type == 'cli'
# run: cd tests && make e2e-configure-rancher
# delete-runner:
# if: always() && needs.create-runner.result == 'success' && inputs.destroy_runner == true
# needs: [create-runner, e2e]
# runs-on: ubuntu-latest
# steps:
# # actions/checkout MUST come before auth
# - name: Checkout
# uses: actions/checkout@v3
# - name: Authenticate to GCP
# uses: google-github-actions/auth@v1
# with:
# credentials_json: ${{ secrets.credentials }}
# - name: Setup gcloud
# uses: google-github-actions/setup-gcloud@v1
# - name: Delete PAT token secret
# run: |
# gcloud --quiet secrets delete PAT_TOKEN_${{ needs.create-runner.outputs.uuid }}
# - name: Delete runner
# run: |
# gcloud --quiet compute instances delete ${{ needs.create-runner.outputs.runner }} \
# --delete-disks all \
# --zone ${{ inputs.zone }}
2 changes: 2 additions & 0 deletions tests/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ e2e-install-backup-restore: deps
ginkgo --label-filter install-backup-restore -r -v ./e2e
e2e-iso-image: deps
ginkgo --label-filter iso-image -r -v ./e2e
e2e-prepare-archive: deps
ginkgo --label-filter prepare-archive -r -v ./e2e
e2e-ui-rancher: deps
ginkgo --label-filter ui -r -v ./e2e
e2e-uninstall-operator:
Expand Down
29 changes: 29 additions & 0 deletions tests/e2e/airgap_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
Copyright © 2022 - 2023 SUSE LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e_test

import (
"os/exec"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var _ = Describe("E2E - Build the airgap archive", Label("prepare-archive"), func() {
It("Execute the script to build the archive", func() {
err := exec.Command("sudo", airgapBuildScript, k8sUpstreamVersion, certManagerVersion, rancherVersion).Run()
Expect(err).To(Not(HaveOccurred()))
})
})
2 changes: 1 addition & 1 deletion tests/e2e/install_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() {
}

if clusterType == "hardened" {
flags = append(flags, "--version", CertManagerVersion)
flags = append(flags, "--version", certManagerVersion)
}

RunHelmCmdWithRetry(flags...)
Expand Down
5 changes: 3 additions & 2 deletions tests/e2e/suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
)

const (
airgapBuildScript = "../scripts/build-airgap"
appYaml = "../assets/hello-world_app.yaml"
backupYaml = "../assets/backup.yaml"
clusterYaml = "../assets/cluster.yaml"
Expand Down Expand Up @@ -60,7 +61,7 @@ var (
arch string
backupRestoreVersion string
caType string
CertManagerVersion string
certManagerVersion string
clusterName string
clusterNS string
clusterType string
Expand Down Expand Up @@ -199,7 +200,7 @@ var _ = BeforeSuite(func() {
arch = os.Getenv("ARCH")
backupRestoreVersion = os.Getenv("BACKUP_RESTORE_VERSION")
caType = os.Getenv("CA_TYPE")
CertManagerVersion = os.Getenv("CERT_MANAGER_VERSION")
certManagerVersion = os.Getenv("CERT_MANAGER_VERSION")
clusterName = os.Getenv("CLUSTER_NAME")
clusterNS = os.Getenv("CLUSTER_NS")
clusterType = os.Getenv("CLUSTER_TYPE")
Expand Down
81 changes: 81 additions & 0 deletions tests/scripts/build-airgap
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
#!/bin/bash

set -e -x
K3S_UPSTREAM_VERSION=$1
CERT_MANAGER_VERSION=$2
RANCHER_MANAGER_VERSION=$3
# Variable(s) and default values

export PATH=$PATH:/usr/local/bin

mkdir -p /opt/rancher/{k3s_$K3S_UPSTREAM_VERSION,helm} /opt/rancher/images/{cert,rancher,registry,elemental}
cd /opt/rancher/k3s_$K3S_UPSTREAM_VERSION/

echo - Install packages
zypper --no-refresh -n in zstd skopeo

echo - Download k3s and rancher
curl -#OL https://github.com/k3s-io/k3s/releases/download/v$K3S_UPSTREAM_VERSION%2Bk3s1/k3s-airgap-images-amd64.tar.zst
curl -#OL https://github.com/k3s-io/k3s/releases/download/v$K3S_UPSTREAM_VERSION%2Bk3s1/k3s

echo - Get the install script
curl -sfL https://get.k3s.io/ -o install.sh

echo - Get Helm Charts
cd /opt/rancher/helm/

echo - Add repos
helm repo add jetstack https://charts.jetstack.io > /dev/null 2>&1
helm repo add rancher-latest https://releases.rancher.com/server-charts/latest > /dev/null 2>&1
helm repo update > /dev/null 2>&1

echo - Get charts
helm pull jetstack/cert-manager --version $CERT_MANAGER_VERSION > /dev/null 2>&1
helm pull rancher-latest/rancher --version v$RANCHER_MANAGER_VERSION > /dev/null 2>&1

echo - Get Images - Rancher/Elemental
cd /opt/rancher/images/

echo - Rancher image list
curl -#L https://github.com/rancher/rancher/releases/download/v$RANCHER_MANAGER_VERSION/rancher-images.txt -o rancher/orig_rancher-images.txt

echo - Shorten rancher list with a sort
# fix library tags
sed -i -e '0,/busybox/s/busybox/library\/busybox/' -e 's/registry/library\/registry/g' rancher/orig_rancher-images.txt

# remove things that are not needed and overlapped
sed -i -E '/neuvector|minio|gke|aks|eks|sriov|harvester|mirrored|longhorn|thanos|tekton|istio|multus|hyper|jenkins|windows/d' rancher/orig_rancher-images.txt

# get latest version
for i in $(cat rancher/orig_rancher-images.txt|awk -F: '{print $1}'); do
grep -w $i rancher/orig_rancher-images.txt | sort -Vr| head -1 >> rancher/version_unsorted.txt
done

# final sort
cat rancher/version_unsorted.txt | sort -u > rancher/rancher-images.txt

echo - Cert-manager image list
helm template /opt/rancher/helm/cert-manager-$CERT_MANAGER_VERSION.tgz | awk '$1 ~ /image:/ {print $2}' | sed s/\"//g > cert/cert-manager-images.txt

# get images
echo - Skopeo - cert-manager
for i in $(cat cert/cert-manager-images.txt); do
skopeo copy docker://$i docker-archive:cert/$(echo $i| awk -F/ '{print $3}'|sed 's/:/_/g').tar:$(echo $i| awk -F/ '{print $3}') > /dev/null 2>&1
done

echo - Skopeo - Rancher - be patient...
for i in $(cat rancher/rancher-images.txt); do
skopeo copy docker://$i docker-archive:rancher/$(echo $i| awk -F/ '{print $2}'|sed 's/:/_/g').tar:$(echo $i| awk -F/ '{print $2}') > /dev/null 2>&1
done

curl -#L https://github.com/clemenko/rke_airgap_install/raw/main/registry.tar -o registry/registry.tar > /dev/null 2>&1

cd /opt/rancher/
echo - Compress all the things
tar -I zstd -vcf /opt/airgap_rancher.zst $(ls) > /dev/null 2>&1

echo "------------------------------------------------------------------"
echo " to uncompress : "
echo " mkdir /opt/rancher"
echo " tar -I zstd -vxf airgap_rancher.zst -C /opt/rancher"
echo "------------------------------------------------------------------"

0 comments on commit 8927c16

Please sign in to comment.