Merge downstream and upstream drivers for online download (#1306)

stackrox · Aug 31, 2023 · cf60cb1 · cf60cb1
1 parent 21a618f
commit cf60cb1
Show file tree

Hide file tree

Showing 5 changed files with 84 additions and 1 deletion.
diff --git a/.github/workflows/cpaas-sync-drivers.yml b/.github/workflows/cpaas-sync-drivers.yml
@@ -23,6 +23,16 @@ on:
         required: true
         description: |
           Bucket where CPaaS drivers will be pushed to.
+      merged-drivers-bucket:
+        type: string
+        required: true
+        description: |
+          Bucket where all drivers will be pushed to.
+      all-archs-drivers-bucket:
+        type: string
+        required: true
+        description: |
+          Bucket where CPaaS drivers for all archs will be pushed to.
       branch-name:
         type: string
         required: true
@@ -93,11 +103,21 @@ jobs:
           shopt -s nullglob
           shopt -s dotglob
 
+          source ${{ github.workspace }}/kernel-modules/support-packages/utils.sh
+
           for driver_version_dir in "${DRIVER_TMP_DIR}"/*; do
             files=("${driver_version_dir}"/*.{gz,unavail})
+            driver_version="$(basename "${driver_version_dir}")"
             [[ "${#files[@]}" -gt 0 ]] || continue
             printf '%s\n' "${files[@]}" | \
-              gsutil -m cp -n -I "gs://${{ inputs.drivers-bucket }}/${{ matrix.platform }}/$(basename "${driver_version_dir}")/"
+              gsutil -m cp -n -I "gs://${{ inputs.drivers-bucket }}/${{ matrix.platform }}/${driver_version}/"
+
+            # Starting with version 2.6.0, we allow for direct download of all
+            # downstream built drivers.
+            if use_downstream "${driver_version}"; then
+              printf '%s\n' "${files[@]}" | \
+                gsutil -m cp -n -I "gs://${{ inputs.all-archs-drivers-bucket }}/${driver_version}/"
+            fi
           done
 
       # x86 support packages will not be uploaded to GCP.
@@ -168,3 +188,21 @@ jobs:
           path: /tmp/support-packages/output/index.html
           parent: false
           destination: ${{ inputs.public-support-packages-bucket }}
+
+  copy-to-merged-bucket:
+    runs-on: ubuntu-latest
+    needs:
+    - sync-drivers
+    steps:
+      - name: Authenticate with GCP
+        uses: google-github-actions/auth@v1
+        with:
+          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}'
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v1
+
+      - name: Copy files to merged bucket
+        run: |
+          gsutil -m cp -r 'gs://${{ inputs.all-archs-drivers-bucket }}/*' \
+            gs://${{ inputs.merged-drivers-bucket }}/
diff --git a/.github/workflows/cpaas.yml b/.github/workflows/cpaas.yml
@@ -37,6 +37,8 @@ jobs:
       support-packages-index-bucket: ${{ needs.init.outputs.support-packages-index-bucket }}
       public-support-packages-bucket: ${{ needs.init.outputs.public-support-packages-bucket }}
       drivers-bucket: ${{ needs.init.outputs.cpaas-drivers-bucket }}
+      merged-drivers-bucket: ${{ needs.init.outputs.merged-drivers-bucket }}
+      all-archs-drivers-bucket: ${{ needs.init.outputs.cpaas-all-archs-drivers-bucket }}
       branch-name: ${{ needs.init.outputs.branch-name }}
 
   check-drivers-failures:

diff --git a/.github/workflows/init.yml b/.github/workflows/init.yml
@@ -17,6 +17,9 @@ on:
       drivers-bucket:
         description: Bucket used to pull collector drivers from
         value: ${{ jobs.common-variables.outputs.drivers-bucket }}
+      merged-drivers-bucket:
+        description: Bucket used to push collector drivers into
+        value: ${{ jobs.common-variables.outputs.merged-drivers-bucket }}
       push-drivers-bucket:
         description: Bucket used to push collector drivers into
         value: ${{ jobs.common-variables.outputs.push-drivers-bucket }}
@@ -40,6 +43,10 @@ on:
         description: |
           Bucket to push CPaaS built drivers into
         value: ${{ jobs.common-variables.outputs.cpaas-drivers-bucket }}
+      cpaas-all-archs-drivers-bucket:
+        description: |
+          Bucket to push CPaaS built drivers for all archs into
+        value: ${{ jobs.common-variables.outputs.cpaas-all-archs-drivers-bucket }}
       support-packages-index-bucket:
         description: |
           Bucket to push the generated support-packages index file into
@@ -53,11 +60,13 @@ jobs:
       collector-image: ${{ steps.collector-env.outputs.collector-image }}
       branch-name: ${{ steps.collector-env.outputs.branch-name }}
       drivers-bucket: ${{ steps.gcp-buckets.outputs.drivers-bucket }}
+      merged-drivers-bucket: ${{ steps.gcp-buckets.outputs.merged-drivers-bucket }}
       push-drivers-bucket: ${{ steps.gcp-buckets.outputs.push-drivers-bucket }}
       bundles-bucket: ${{ steps.gcp-buckets.outputs.bundles-bucket }}
       support-packages-bucket: ${{ steps.gcp-buckets.outputs.support-packages-bucket }}
       public-support-packages-bucket: ${{ steps.gcp-buckets.outputs.public-support-packages-bucket }}
       cpaas-drivers-bucket: ${{ steps.gcp-buckets.outputs.cpaas-drivers-bucket }}
+      cpaas-all-archs-drivers-bucket: ${{ steps.gcp-buckets.outputs.cpaas-all-archs-drivers-bucket }}
       cpaas-support-packages-bucket: ${{ steps.gcp-buckets.outputs.cpaas-support-packages-bucket }}
       support-packages-index-bucket: ${{ steps.gcp-buckets.outputs.support-packages-index-bucket }}
 
@@ -91,14 +100,18 @@ jobs:
           STAGING_RELATIVE_PATH="${GITHUB_HEAD_REF}/${{ github.run_id }}"
 
           MAIN_DRIVER_BUCKET="collector-modules-osci"
+          MERGED_DRIVER_BUCKET="${MAIN_DRIVER_BUCKET}/merged-build"
           STAGING_DRIVER_BUCKET="stackrox-collector-modules-staging/pr-builds/${STAGING_RELATIVE_PATH}"
+          STAGING_MERGED_DRIVER_BUCKET="${STAGING_DRIVER_BUCKET}/merged-build"
           BUNDLES_BUCKET="collector-kernel-bundles-public"
           SUPPORT_PACKAGES_BUCKET="sr-roxc/collector/support-packages"
           STAGING_SUPPORT_PACKAGES_BUCKET="${SUPPORT_PACKAGES_BUCKET}/${STAGING_RELATIVE_PATH}"
           PUBLIC_SUPPORT_PACKAGES_BUCKET="collector-support-public/offline/v1/support-packages"
 
           CPAAS_DRIVERS_BUCKET="${MAIN_DRIVER_BUCKET}/cpaas"
+          CPAAS_ALL_ARCHS_DRIVERS_BUCKET="${CPAAS_DRIVERS_BUCKET}/all-archs"
           CPAAS_STAGING_DRIVERS_BUCKET="${STAGING_DRIVER_BUCKET}/cpaas"
+          CPAAS_STAGING_ALL_ARCHS_DRIVERS_BUCKET="${CPAAS_STAGING_DRIVERS_BUCKET}/all-archs"
           CPAAS_SUPPORT_PACKAGES_BUCKET="${SUPPORT_PACKAGES_BUCKET}"
           CPAAS_STAGING_SUPPORT_PACKAGES_BUCKET="${STAGING_SUPPORT_PACKAGES_BUCKET}"
 
@@ -111,22 +124,27 @@ jobs:
           if [[ ${{ github.event_name }} == "pull_request" ]]; then
             {
               echo "push-drivers-bucket=${STAGING_DRIVER_BUCKET}"
+              echo "merged-drivers-bucket=${STAGING_MERGED_DRIVER_BUCKET}"
               echo "support-packages-bucket=${STAGING_SUPPORT_PACKAGES_BUCKET}"
               if [[ ${{ contains(github.event.pull_request.labels.*.name, 'run-cpaas-steps') }} == "true" ]]; then
                   echo "cpaas-drivers-bucket=${CPAAS_STAGING_DRIVERS_BUCKET}"
+                  echo "cpaas-all-archs-drivers-bucket=${CPAAS_STAGING_ALL_ARCHS_DRIVERS_BUCKET}"
                   echo "cpaas-support-packages-bucket=${CPAAS_STAGING_SUPPORT_PACKAGES_BUCKET}"
               else
                   # When running on PRs withouth the 'run-cpaas-steps' label, use the main buckets
                   echo "cpaas-drivers-bucket=${CPAAS_DRIVERS_BUCKET}"
+                  echo "cpaas-all-archs-drivers-bucket=${CPAAS_ALL_ARCHS_DRIVERS_BUCKET}"
                   echo "cpaas-support-packages-bucket=${CPAAS_SUPPORT_PACKAGES_BUCKET}"
               fi
               echo "support-packages-index-bucket=${STAGING_SUPPORT_PACKAGES_BUCKET}"
             } >> "$GITHUB_OUTPUT"
           else
             {
               echo "push-drivers-bucket=${MAIN_DRIVER_BUCKET}"
+              echo "merged-drivers-bucket=${MERGED_DRIVER_BUCKET}"
               echo "support-packages-bucket=${SUPPORT_PACKAGES_BUCKET}"
               echo "cpaas-drivers-bucket=${CPAAS_DRIVERS_BUCKET}"
+              echo "cpaas-all-archs-drivers-bucket=${CPAAS_ALL_ARCHS_DRIVERS_BUCKET}"
               echo "cpaas-support-packages-bucket=${CPAAS_SUPPORT_PACKAGES_BUCKET}"
               echo "support-packages-index-bucket=${SUPPORT_PACKAGES_BUCKET}"
             } >> "$GITHUB_OUTPUT"

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -40,6 +40,7 @@ jobs:
     uses: ./.github/workflows/upload-drivers.yml
     with:
       gcp-bucket: ${{ needs.init.outputs.push-drivers-bucket }}
+      merged-drivers-bucket: ${{ needs.init.outputs.merged-drivers-bucket }}
     if: ${{ needs.build-drivers.outputs.parallel-jobs > 0 }}
     needs:
     - init

diff --git a/.github/workflows/upload-drivers.yml b/.github/workflows/upload-drivers.yml
@@ -7,6 +7,10 @@ on:
         type: string
         required: true
         description: GCP bucket to push drivers into
+      merged-drivers-bucket:
+        type: string
+        required: true
+        description: GCP bucket unifying upstream and downstream drivers
 
 jobs:
   upload-drivers:
@@ -24,9 +28,29 @@ jobs:
         with:
           credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}'
 
+      - name: 'Set up Cloud SDK'
+        uses: 'google-github-actions/setup-gcloud@v1'
+
       - name: Push drivers
         uses: 'google-github-actions/upload-cloud-storage@v1'
         with:
           path: /tmp/output/
           parent: false
           destination: ${{ inputs.gcp-bucket }}
+
+      - name: Push to merged bucket
+        run: |
+         shopt -s nullglob
+         shopt -s dotglob
+
+          # We use gsutil here because we need to ignore files that already
+          # exist in the bucket, if they are there it means downstream
+          # built it and has prevalence over this set of drivers.
+          for driver_version_dir in /tmp/output/*; do
+            files=("${driver_version_dir}"/*.{gz,unavail})
+            driver_version="$(basename "${driver_version_dir}")"
+            [[ "${#files[@]}" -gt 0 ]] || continue
+
+            printf '%s\n' "${files[@]}" | \
+              gsutil -m cp -n -I "gs://${{ inputs.merged-drivers-bucket }}/${driver_version}/"
+          done