Native s390x builds and multi-arch testing (#1448)

- s390x images are now built natively
- ppc64le core-bpf is enabled
- VM creation is consolidated in GHA
- numerous improvements to ansible variables and multiarch VM creation

.github/actions/setup-vm-creds/action.yml

@@ -0,0 +1,109 @@
+name: Setup VM Credentials
+description: |
+  This action will setup the runner with the necessary credentials to create and
+  interact with VMs
+
+inputs:
+  gcp-ssh-key:
+    description:
+      The SSH private key to use for GCP
+  gcp-ssh-key-pub:
+    description:
+      The SSH public key to use for GCP
+  s390x-ssh-key:
+    description:
+      The SSH private key to use for s390x
+  ppc64le-ssh-key:
+    description:
+      The SSH private key to use for ppc64le
+  ppc64le-ssh-key-pub:
+    description:
+      The SSH public key to use for ppc64le
+  s390x-key:
+    description:
+      The API key to use for s390x
+  ppc64le-key:
+    description:
+      The API key to use for ppc64le
+  redhat-username:
+    description:
+      The username for registering redhat servers
+  redhat-password:
+    description:
+      The password for registering redhat servers
+  vm-type:
+    description:
+      The type of VMs to be created
+  job-tag:
+    description:
+      Optional job tag to add to the ID
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      run: |
+          mkdir -p /tmp/secret/stackrox-collector-e2e-tests
+          cp "$GOOGLE_APPLICATION_CREDENTIALS" /tmp/secret/stackrox-collector-e2e-tests/GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT
+
+          mkdir -p "$HOME/.ssh"
+          chmod 0700 "$HOME/.ssh"
+
+          function copy_secret_to_file() {
+            local secret="$1"
+            local destination="$2"
+            local perms="$3"
+
+            echo "$secret" > "$destination"
+            chmod "$perms" "$destination"
+          }
+
+          copy_secret_to_file "${{ inputs.gcp-ssh-key }}" "$HOME/.ssh/google_compute_engine" 0600
+          copy_secret_to_file "${{ inputs.gcp-ssh-key-pub }}" "$HOME/.ssh/google_compute_engine.pub" 0600
+          copy_secret_to_file "${{ inputs.s390x-ssh-key }}" "$HOME/.ssh/acs-s390x-rsa.prv" 0600
+          copy_secret_to_file "${{ inputs.ppc64le-ssh-key }}" "$HOME/.ssh/acs-ppc64le-rsa.prv" 0600
+
+          ls -lah $HOME/.ssh/
+
+    - shell: bash
+      run: |
+        #
+        # JOB_ID is a little odd in that it needs to be unique per vm_type,
+        # but GHA's run_id is only unique per job. Appending the vm_type
+        # makes it fit our requirements...
+        #
+        # but wait, there's more
+        #
+        # the job id is used both as part of a GCP instance name and label, but
+        # also as a group name in ansible, which means it has some restrictions:
+        #
+        # - must be alpha-numeric
+        # - must not contain underscores (because of GCP)
+        # - must not contain hyphens (because of ansible)
+        #
+        # vm_type may contain hyphens, so the id is normalized below
+        #
+        JOB_ID="${{ github.run_id }}${{ inputs.vm-type }}${{ inputs.job-tag }}"
+        NORM_JOB_ID="${JOB_ID//-/}"
+
+        {
+          echo "IBM_CLOUD_POWER_API_KEY=${{ inputs.ppc64le-key }}"
+          echo "IBM_CLOUD_S390X_API_KEY=${{ inputs.s390x-key }}"
+          echo "IBM_CLOUD_POWER_SSH_PUBLIC_KEY=${{ inputs.ppc64le-ssh-key-pub }}"
+          echo "JOB_ID=${NORM_JOB_ID}"
+          echo "BUILD_TYPE=ci"
+          echo "VM_TYPE=${{ inputs.vm-type }}"
+          echo "REDHAT_USERNAME=${{ inputs.redhat-username }}"
+          echo "REDHAT_PASSWORD=${{ inputs.redhat-password }}"
+        } >> "$GITHUB_ENV"
+
+    - shell: bash
+      run: |
+        python -m pip install -r "${{ github.workspace }}/ansible/requirements.txt"
+        ansible-galaxy collection install -r "${{ github.workspace }}/ansible/requirements.yml"
+        ansible-galaxy collection install -r "${{ github.workspace }}/ansible/ansible-collections.yml"
+
+    # Added workaround for ssh connection issue with power vm from ubuntu machine.
+    # Changing mtu works in resolving the issue
+    - shell: bash
+      run: sudo ifconfig eth0 mtu 1000 up
+

.github/workflows/collector-builder.yml

@@ -11,11 +11,12 @@ on:
     outputs:
       collector-builder-tag:
         description: The builder tag used by the build
-        value: ${{ jobs.build-builder-image.outputs.collector-builder-tag || 'master' }}
+        value: ${{ jobs.build-builder-image.outputs.collector-builder-tag || '3.16.x-195-g8f32e71fad' }}
 
 env:
   COLLECTOR_TAG: ${{ inputs.collector-tag }}
   DEFAULT_BUILDER_TAG: 3.16.x-195-g8f32e71fad
+  ANSIBLE_CONFIG: ${{ github.workspace }}/ansible/ansible.cfg
 
 jobs:
   builder-needs-rebuilding:
@@ -59,6 +60,7 @@ jobs:
 
     env:
       PLATFORM: linux/${{ matrix.arch }}
+      BUILD_TYPE: ci
 
     steps:
       - uses: actions/checkout@v3
@@ -71,6 +73,35 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+
+      - uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}'
+
+      - uses: 'google-github-actions/setup-gcloud@v1'
+
+      - uses: ./.github/actions/setup-vm-creds
+        with:
+          gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }}
+          gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }}
+          s390x-ssh-key: ${{ secrets.IBM_CLOUD_S390X_SSH_PRIVATE_KEY }}
+          ppc64le-ssh-key: ${{ secrets.IBM_CLOUD_POWER_SSH_PRIVATE_KEY }}
+          ppc64le-ssh-key-pub: ${{ secrets.IBM_CLOUD_POWER_SSH_PUBLIC_KEY }}
+          s390x-key: ${{ secrets.IBM_CLOUD_S390x_API_KEY }}
+          ppc64le-key: ${{ secrets.IBM_CLOUD_POWER_API_KEY }}
+          redhat-username: ${{ secrets.REDHAT_USERNAME }}
+          redhat-password: ${{ secrets.REDHAT_PASSWORD }}
+          vm-type: all
+          job-tag: builder
+
+      - name: Create Build VMs
+        if: matrix.arch == 's390x'
+        run: |
+          make -C "${{ github.workspace }}/ansible" create-build-vms
+
       - name: Define builder tag
         id: builder-tag
         run: |
@@ -84,29 +115,27 @@ jobs:
           echo "COLLECTOR_BUILDER_TAG=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_ENV"
           echo "collector-builder-tag=${COLLECTOR_BUILDER_TAG}" >> "$GITHUB_OUTPUT"
 
-      - name: Create Ansible Vars (inc. secrets)
+      - name: Create ansible vars
         run: |
           {
             echo "---"
             echo "stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}"
             echo "stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}"
             echo "rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}"
             echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}"
+            echo "collector_git_ref: ${{ github.ref }}"
+            echo "collector_git_sha: ${{ github.sha }}"
+            echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}"
           } > ${{ github.workspace }}/ansible/secrets.yml
 
-          if [[ "${RUNNER_DEBUG}" == "1" ]]; then
-            echo "ANSIBLE_STDOUT_CALLBACK=debug" >> "${GITHUB_ENV}"
-          fi
-
       - name: Build images
         if: |
           github.event_name == 'push' ||
           matrix.arch == 'amd64' ||
-          contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')
+          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch != 's390x')
         timeout-minutes: 480
         run: |
           ansible-galaxy install -r ansible/requirements.yml
-
           ansible-playbook \
             --connection local \
             -i localhost, \
@@ -115,6 +144,24 @@ jobs:
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-builder.yml
 
+      - name: Build s390x images
+        if: |
+          github.event_name == 'push' ||
+          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch == 's390x')
+        timeout-minutes: 480
+        run: |
+          ansible-playbook \
+            -i ansible/ci \
+            -e build_hosts='job_id_${{ env.JOB_ID }}' \
+            -e arch='${{ matrix.arch }}' \
+            -e @'${{ github.workspace }}/ansible/secrets.yml' \
+            ansible/ci-build-builder.yml
+
+      - name: Destroy VMs
+        if: always() && matrix.arch == 's390x'
+        run: |
+          make -C ansible destroy-vms
+
   create-multiarch-manifest:
     needs:
     - build-builder-image

.github/workflows/collector-slim.yml

@@ -23,6 +23,7 @@ env:
   COLLECTOR_TAG: ${{ inputs.collector-tag }}
   COLLECTOR_BUILDER_TAG: ${{ inputs.collector-builder-tag }}
   RHACS_ENG_IMAGE: quay.io/rhacs-eng/collector:${{ inputs.collector-tag }}
+  ANSIBLE_CONFIG: ${{ github.workspace }}/ansible/ansible.cfg
 
 jobs:
   build-collector-image:
@@ -42,54 +43,110 @@ jobs:
           submodules: true
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
+
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+
+      - uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}'
+
+      - uses: 'google-github-actions/setup-gcloud@v1'
+
+      - uses: ./.github/actions/setup-vm-creds
+        with:
+          gcp-ssh-key: ${{ secrets.GCP_SSH_KEY }}
+          gcp-ssh-key-pub: ${{ secrets.GCP_SSH_KEY_PUB }}
+          s390x-ssh-key: ${{ secrets.IBM_CLOUD_S390X_SSH_PRIVATE_KEY }}
+          ppc64le-ssh-key: ${{ secrets.IBM_CLOUD_POWER_SSH_PRIVATE_KEY }}
+          ppc64le-ssh-key-pub: ${{ secrets.IBM_CLOUD_POWER_SSH_PUBLIC_KEY }}
+          s390x-key: ${{ secrets.IBM_CLOUD_S390x_API_KEY }}
+          ppc64le-key: ${{ secrets.IBM_CLOUD_POWER_API_KEY }}
+          redhat-username: ${{ secrets.REDHAT_USERNAME }}
+          redhat-password: ${{ secrets.REDHAT_PASSWORD }}
+          vm-type: all
+          job-tag: builder
+
+      - name: Create Build VMs
+        if: matrix.arch == 's390x'
+        run: |
+          make -C "${{ github.workspace }}/ansible" create-build-vms
 
       - name: Checks PR, main and release branches
         run: |
           if [[ "${{ github.event_name }}" == 'pull_request' ]]; then
             echo "COLLECTOR_APPEND_CID=true" >> "$GITHUB_ENV"
+            echo "TRACE_SINSP_EVENTS=1" >> "$GITHUB_ENV"
 
             if [[ "${{ contains(github.event.pull_request.labels.*.name, 'address-sanitizer') }}" == "true" ]]; then
               echo "ADDRESS_SANITIZER=true" >> "$GITHUB_ENV"
             fi
           fi
 
-      - name: Create Ansible Vars (inc. secrets)
+      - name: Create ansible vars
         run: |
           {
             echo "---"
             echo "stackrox_io_username: ${{ secrets.QUAY_STACKROX_IO_RW_USERNAME }}"
             echo "stackrox_io_password: ${{ secrets.QUAY_STACKROX_IO_RW_PASSWORD }}"
             echo "rhacs_eng_username: ${{ secrets.QUAY_RHACS_ENG_RW_USERNAME }}"
             echo "rhacs_eng_password: ${{ secrets.QUAY_RHACS_ENG_RW_PASSWORD }}"
+            echo "collector_git_ref: ${{ github.ref }}"
+            echo "collector_git_sha: ${{ github.sha }}"
+            echo "collector_builder_tag: ${{ env.COLLECTOR_BUILDER_TAG }}"
+            echo "disable_profiling: ${{ matrix.arch != 'amd64' && matrix.arch != 'arm64' }}"
+            echo "rhacs_eng_image: ${{ env.RHACS_ENG_IMAGE }}"
+            echo "collector_image: ${{ inputs.collector-image }}"
+            echo "collector_tag: ${{ inputs.collector-tag }}"
           } > ${{ github.workspace }}/ansible/secrets.yml
 
-          if [[ "${RUNNER_DEBUG}" == "1" ]]; then
-            echo "ANSIBLE_STDOUT_CALLBACK=debug" >> "${GITHUB_ENV}"
-          fi
+      - name: Authenticate with GCP
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}'
+
+      - name: Setup GCP
+        uses: 'google-github-actions/setup-gcloud@v1'
 
       - name: Build images
         if: |
           github.event_name == 'push' ||
           matrix.arch == 'amd64' ||
-          contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds')
+          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch != 's390x')
         timeout-minutes: 480
         run: |
-          ansible-galaxy install -r ansible/requirements.yml
           ansible-playbook \
             --connection local \
             -i localhost, \
             --limit localhost \
-            -e collector_image='${{ inputs.collector-image }}' \
             -e arch='${{ matrix.arch }}' \
-            -e disable_profiling="${{ matrix.arch != 'amd64' && matrix.arch != 'arm64' }}" \
+            -e @'${{ github.workspace }}/ansible/secrets.yml' \
+            ansible/ci-build-collector.yml
+
+      - name: Build s390x image
+        if: |
+          github.event_name == 'push' ||
+          (contains(github.event.pull_request.labels.*.name, 'run-multiarch-builds') && matrix.arch == 's390x')
+        timeout-minutes: 480
+        run: |
+          ansible-playbook \
+            -i ansible/ci \
+            -e arch='${{ matrix.arch }}' \
+            -e build_hosts='job_id_${{ env.JOB_ID }}' \
             -e @'${{ github.workspace }}/ansible/secrets.yml' \
             ansible/ci-build-collector.yml
         env:
-          ANSIBLE_CONFIG: ansible/ansible.cfg
+          VM_TYPE: rhel-s390x
+
+      - name: Destroy Build VMs
+        if: always() && matrix.arch == 's390x'
+        run: |
+          make -C ansible destroy-vms
 
   create-multiarch-manifest:
     needs: