regression.yaml

name: regression

on:
  workflow_call:
    inputs:
      version_tag_old:
        description: 'The initial install version to use for upgrade scenarios'
        required: true
        type: string
      version_tag_new:
        description: 'The new version to use as the upgrade in upgrade scenarios and the initial version for fresh installs.'
        required: true
        type: string
      addon_package_url:
        description: 'The URL to the kURL add-on package to use as an override.'
        required: false
        type: string
      id:
        description: 'An arbitrary value used to identify the testing infrastructure. Using the same value twice will destroy the old infrastructure before creating.'
        required: false
        type: string
    secrets:
      TESTIM_ACCESS_TOKEN:
        required: true
      E2E_TESTIM_AWS_ACCESS_KEY_ID:
        required: true
      E2E_TESTIM_AWS_SECRET_ACCESS_KEY:
        required: true
      E2E_GH_PAT:
        required: true
      KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL:
        required: true

  workflow_dispatch:
    inputs:
      version_tag_old:
        description: 'The initial install version to use for upgrade scenarios'
        required: true
        type: string
      version_tag_new:
        description: 'The new version to use as the upgrade in upgrade scenarios and the initial version for fresh installs.'
        required: true
        type: string
      addon_package_url:
        description: 'The URL to the kURL add-on package to use as an override.'
        required: false
        type: string
      id:
        description: 'An arbitrary value used to identify the testing infrastructure. Using the same value twice will destroy the old infrastructure before creating.'
        required: false
        type: string

env:
  AWS_DEFAULT_REGION: us-east-1
  AWS_ACCESS_KEY_ID: ${{ secrets.E2E_TESTIM_AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_TESTIM_AWS_SECRET_ACCESS_KEY }}
  TF_VAR_testim_token: ${{ secrets.TESTIM_ACCESS_TOKEN }}
  WORKSPACE: automation-${{ github.event.inputs.id || inputs.id }}

jobs:
  jumpbox:
    runs-on: ubuntu-20.04
    container:
      image: hashicorp/terraform:1.0.11
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          repository: replicatedhq/kots-regression-automation
          token: ${{ secrets.E2E_GH_PAT }}
          path: automation
          ref: main
      - name: Download go_api artifact
        uses: actions/download-artifact@v3
        with:
          name: go_api
          path: ./automation/jumpbox/bin
      - name: Download kotsadm image
        uses: actions/download-artifact@v3
        with:
          name: kotsadm-image
          path: ./automation/jumpbox/docker-archive
      - name: Download migrations
        uses: actions/download-artifact@v3
        with:
          name: migrations-image
          path: ./automation/jumpbox/docker-archive
      - name: Make kotsadm airgap archive with minio image
        working-directory: automation/jumpbox
        run: tar czf ./kotsadm.tar.gz -C ./ ./docker-archive
      - name: Initialize terraform
        if: always()
        working-directory: automation/jumpbox
        run: |
          terraform init
          terraform workspace new "$WORKSPACE" || true
          terraform workspace select "$WORKSPACE"
      - name: Create Jumpbox Environment
        if: always()
        working-directory: automation/jumpbox
        run: terraform apply --auto-approve
      - name: Notify Slack
        if: failure() && github.ref_name == 'main'
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          fields: repo,ref,workflow
          custom_payload: |
            {
              attachments: [{
                color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
                text: `Create jumpbox: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}.\nRef: ${process.env.AS_REF}`,
              }]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }}

  tests:
    needs: jumpbox
    runs-on: ubuntu-20.04
    container:
      image: hashicorp/terraform:1.0.11
    strategy:
      fail-fast: false
      matrix:
        test: [
          {
            name: "type=embedded cluster, env=airgapped, phase=upgraded install, rbac=cluster admin",
            backend_config: "embedded-airgapped-upgrade-backend-config.tfvars",
            terraform_script: "embedded-airgapped-upgrade.sh",
            is_upgrade: "1"
          },
          {
            name: "type=embedded cluster, env=airgapped, phase=new install, rbac=cluster admin",
            backend_config: "embedded-airgapped-install-backend-config.tfvars",
            terraform_script: "embedded-airgapped-install.sh"
          },
          {
            name: "type=embedded cluster, env=online, phase=new install, rbac=cluster admin",
            backend_config: "embedded-online-install-backend-config.tfvars",
            terraform_script: "embedded-online-install.sh"
          },
          {
            name: "type=embedded cluster, env=online, phase=upgraded install, rbac=cluster admin",
            backend_config: "embedded-online-upgrade-backend-config.tfvars",
            terraform_script: "embedded-online-upgrade.sh",
            is_upgrade: "1"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=new install, rbac=cluster admin",
            backend_config: "existing-airgapped-install-admin-backend-config.tfvars",
            terraform_script: "existing-airgapped-install-admin.sh"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=new install, rbac=minimal rbac",
            backend_config: "existing-airgapped-install-minimum-backend-config.tfvars",
            terraform_script: "existing-airgapped-install-minimum.sh"
          },
          {
            name: "type=existing cluster, env=online, phase=upgraded install, rbac=cluster admin",
            backend_config: "existing-online-upgrade-admin-backend-config.tfvars",
            terraform_script: "existing-online-upgrade-admin.sh",
            is_upgrade: "1"
          },
          {
            name: "type=existing cluster, env=online, phase=upgraded install, rbac=minimal rbac",
            backend_config: "existing-online-upgrade-minimum-backend-config.tfvars",
            terraform_script: "existing-online-upgrade-minimum.sh",
            is_upgrade: "1"
          },
          {
            name: "type=existing cluster, env=online, phase=new install, rbac=cluster admin",
            backend_config: "existing-online-install-admin-backend-config.tfvars",
            terraform_script: "existing-online-install-admin.sh"
          },
          {
            name: "type=existing cluster, env=online, phase=new install, rbac=minimal rbac",
            backend_config: "existing-online-install-minimum-backend-config.tfvars",
            terraform_script: "existing-online-install-minimum.sh"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=cluster admin",
            backend_config: "existing-airgapped-upgrade-admin-backend-config.tfvars",
            terraform_script: "existing-airgapped-upgrade-admin.sh",
            is_upgrade: "1"
          },
          {
            name: "type=existing cluster, env=airgapped, phase=upgraded install, rbac=minimal rbac",
            backend_config: "existing-airgapped-upgrade-minimum-backend-config.tfvars",
            terraform_script: "existing-airgapped-upgrade-minimum.sh",
            is_upgrade: "1"
          },
          {
            name: "type=helmvm cluster, env=online, phase=new install, rbac=cluster admin",
            backend_config: "helmvm-online-install-backend-config.tfvars",
            terraform_script: "helmvm-online-install.sh"
          }
        ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          repository: replicatedhq/kots-regression-automation
          token: ${{ secrets.E2E_GH_PAT }}
          path: automation
          ref: main
      - name: Initialize terraform
        if: always()
        working-directory: automation/cluster
        run: |
          terraform init -backend-config ${{ matrix.test.backend_config }}
          terraform workspace new "$WORKSPACE" || true
          terraform workspace select "$WORKSPACE"
      - name: Destroy test environment before re-run
        if: always() && github.run_attempt > 1
        working-directory: automation/cluster
        run: ./${{ matrix.test.terraform_script }} destroy
      - name: Create test environment
        if: always()
        working-directory: automation/cluster
        run: |
          export CURRENT_TAG="${{ github.event.inputs.version_tag_old || inputs.version_tag_old }}"
          export NEXT_TAG="${{ github.event.inputs.version_tag_new || inputs.version_tag_new }}"

          if [ -n "${{ matrix.test.is_upgrade }}" ]; then
            export TF_VAR_kots_version_initial="${CURRENT_TAG:1}"
            export TF_VAR_kots_version_upgrade="${NEXT_TAG:1}"
          else
            export TF_VAR_kots_version_initial="${NEXT_TAG:1}"
          fi
          export TF_VAR_kots_addon_package_url="${{ github.event.inputs.addon_package_url || inputs.addon_package_url }}"
          export TF_VAR_testim_branch="master"
          ./${{ matrix.test.terraform_script }} apply
      - name: Wait for instance to be ready
        working-directory: automation/cluster
        run: |
          apk update && apk add aws-cli jq
          terraform output -raw control_plane_instance_id
          until [ -n "$(aws ec2 describe-instance-status --instance-ids "$(terraform output -raw control_plane_instance_id)" --filter Name=instance-status.reachability,Values=passed | jq -r '.InstanceStatuses[].InstanceStatus.Details[].Status')" ]
          do
          echo "waiting for instance to become ready" && sleep 1
          done
      - name: Run the test
        working-directory: automation/cluster
        run: |
          terraform output -raw jumpbox_private_key > ssh.pem
          chmod 600 ssh.pem
          ssh -i ssh.pem "ubuntu@$(terraform output -raw jumpbox_public_ip)" -oStrictHostKeyChecking=no -oServerAliveInterval=60 -oServerAliveCountMax=10 "ssh -tt ubuntu@$(terraform output -raw control_plane_private_ip) -oServerAliveInterval=60 -oServerAliveCountMax=10 -oConnectionAttempts=30 \"sudo /tmp/start.sh\""
      - name: Notify Slack
        if: always() && github.ref_name == 'main'
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          fields: repo,ref,workflow,commit
          custom_payload: |
            {
              attachments: [{
                color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
                text: `Test name: ${{ matrix.test.name }}.\nStatus: ${{ job.status }}. Workflow: ${process.env.AS_WORKFLOW}. Repo: ${process.env.AS_REPO}. Ref: ${process.env.AS_REF}\nCommit: ${process.env.AS_COMMIT}`,
              }]
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.KOTS_BUILD_STATUS_SLACK_WEBHOOK_URL }}
          MATRIX_CONTEXT: ${{ toJson(matrix) }}
  cleanup:
    needs: tests
    runs-on: ubuntu-20.04
    steps:
      - name: Trigger workspace cleanup
        uses: peter-evans/repository-dispatch@v2
        with:
          token: ${{ secrets.E2E_GH_PAT }}
          repository: replicatedhq/kots
          event-type: e2e-workspace-cleanup
          client-payload: '{"workspace": "automation-${{ github.event.inputs.id || inputs.id }}"}'