diff --git a/.github/workflows/cpaas-sync-drivers.yml b/.github/workflows/cpaas-sync-drivers.yml index c1bd572810..3187b74405 100644 --- a/.github/workflows/cpaas-sync-drivers.yml +++ b/.github/workflows/cpaas-sync-drivers.yml @@ -23,6 +23,16 @@ on: required: true description: | Bucket where CPaaS drivers will be pushed to. + merged-drivers-bucket: + type: string + required: true + description: | + Bucket where all drivers will be pushed to. + all-archs-drivers-bucket: + type: string + required: true + description: | + Bucket where CPaaS drivers for all archs will be pushed to. branch-name: type: string required: true @@ -93,11 +103,21 @@ jobs: shopt -s nullglob shopt -s dotglob + source ${{ github.workspace }}/kernel-modules/support-packages/utils.sh + for driver_version_dir in "${DRIVER_TMP_DIR}"/*; do files=("${driver_version_dir}"/*.{gz,unavail}) + driver_version="$(basename "${driver_version_dir}")" [[ "${#files[@]}" -gt 0 ]] || continue printf '%s\n' "${files[@]}" | \ - gsutil -m cp -n -I "gs://${{ inputs.drivers-bucket }}/${{ matrix.platform }}/$(basename "${driver_version_dir}")/" + gsutil -m cp -n -I "gs://${{ inputs.drivers-bucket }}/${{ matrix.platform }}/${driver_version}/" + + # Starting with version 2.6.0, we allow for direct download of all + # downstream built drivers. + if use_downstream "${driver_version}"; then + printf '%s\n' "${files[@]}" | \ + gsutil -m cp -n -I "gs://${{ inputs.all-archs-drivers-bucket }}/${driver_version}/" + fi done # x86 support packages will not be uploaded to GCP. @@ -168,3 +188,21 @@ jobs: path: /tmp/support-packages/output/index.html parent: false destination: ${{ inputs.public-support-packages-bucket }} + + copy-to-merged-bucket: + runs-on: ubuntu-latest + needs: + - sync-drivers + steps: + - name: Authenticate with GCP + uses: google-github-actions/auth@v1 + with: + credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}' + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v1 + + - name: Copy files to merged bucket + run: | + gsutil -m cp -r 'gs://${{ inputs.all-archs-drivers-bucket }}/*' \ + gs://${{ inputs.merged-drivers-bucket }}/ diff --git a/.github/workflows/cpaas.yml b/.github/workflows/cpaas.yml index 11e75bb860..f8118d026c 100644 --- a/.github/workflows/cpaas.yml +++ b/.github/workflows/cpaas.yml @@ -37,6 +37,8 @@ jobs: support-packages-index-bucket: ${{ needs.init.outputs.support-packages-index-bucket }} public-support-packages-bucket: ${{ needs.init.outputs.public-support-packages-bucket }} drivers-bucket: ${{ needs.init.outputs.cpaas-drivers-bucket }} + merged-drivers-bucket: ${{ needs.init.outputs.merged-drivers-bucket }} + all-archs-drivers-bucket: ${{ needs.init.outputs.cpaas-all-archs-drivers-bucket }} branch-name: ${{ needs.init.outputs.branch-name }} check-drivers-failures: diff --git a/.github/workflows/init.yml b/.github/workflows/init.yml index 0e30fcc3dd..b8e66abb69 100644 --- a/.github/workflows/init.yml +++ b/.github/workflows/init.yml @@ -17,6 +17,9 @@ on: drivers-bucket: description: Bucket used to pull collector drivers from value: ${{ jobs.common-variables.outputs.drivers-bucket }} + merged-drivers-bucket: + description: Bucket used to push collector drivers into + value: ${{ jobs.common-variables.outputs.merged-drivers-bucket }} push-drivers-bucket: description: Bucket used to push collector drivers into value: ${{ jobs.common-variables.outputs.push-drivers-bucket }} @@ -40,6 +43,10 @@ on: description: | Bucket to push CPaaS built drivers into value: ${{ jobs.common-variables.outputs.cpaas-drivers-bucket }} + cpaas-all-archs-drivers-bucket: + description: | + Bucket to push CPaaS built drivers for all archs into + value: ${{ jobs.common-variables.outputs.cpaas-all-archs-drivers-bucket }} support-packages-index-bucket: description: | Bucket to push the generated support-packages index file into @@ -53,11 +60,13 @@ jobs: collector-image: ${{ steps.collector-env.outputs.collector-image }} branch-name: ${{ steps.collector-env.outputs.branch-name }} drivers-bucket: ${{ steps.gcp-buckets.outputs.drivers-bucket }} + merged-drivers-bucket: ${{ steps.gcp-buckets.outputs.merged-drivers-bucket }} push-drivers-bucket: ${{ steps.gcp-buckets.outputs.push-drivers-bucket }} bundles-bucket: ${{ steps.gcp-buckets.outputs.bundles-bucket }} support-packages-bucket: ${{ steps.gcp-buckets.outputs.support-packages-bucket }} public-support-packages-bucket: ${{ steps.gcp-buckets.outputs.public-support-packages-bucket }} cpaas-drivers-bucket: ${{ steps.gcp-buckets.outputs.cpaas-drivers-bucket }} + cpaas-all-archs-drivers-bucket: ${{ steps.gcp-buckets.outputs.cpaas-all-archs-drivers-bucket }} cpaas-support-packages-bucket: ${{ steps.gcp-buckets.outputs.cpaas-support-packages-bucket }} support-packages-index-bucket: ${{ steps.gcp-buckets.outputs.support-packages-index-bucket }} @@ -91,14 +100,18 @@ jobs: STAGING_RELATIVE_PATH="${GITHUB_HEAD_REF}/${{ github.run_id }}" MAIN_DRIVER_BUCKET="collector-modules-osci" + MERGED_DRIVER_BUCKET="${MAIN_DRIVER_BUCKET}/merged-build" STAGING_DRIVER_BUCKET="stackrox-collector-modules-staging/pr-builds/${STAGING_RELATIVE_PATH}" + STAGING_MERGED_DRIVER_BUCKET="${STAGING_DRIVER_BUCKET}/merged-build" BUNDLES_BUCKET="collector-kernel-bundles-public" SUPPORT_PACKAGES_BUCKET="sr-roxc/collector/support-packages" STAGING_SUPPORT_PACKAGES_BUCKET="${SUPPORT_PACKAGES_BUCKET}/${STAGING_RELATIVE_PATH}" PUBLIC_SUPPORT_PACKAGES_BUCKET="collector-support-public/offline/v1/support-packages" CPAAS_DRIVERS_BUCKET="${MAIN_DRIVER_BUCKET}/cpaas" + CPAAS_ALL_ARCHS_DRIVERS_BUCKET="${CPAAS_DRIVERS_BUCKET}/all-archs" CPAAS_STAGING_DRIVERS_BUCKET="${STAGING_DRIVER_BUCKET}/cpaas" + CPAAS_STAGING_ALL_ARCHS_DRIVERS_BUCKET="${CPAAS_STAGING_DRIVERS_BUCKET}/all-archs" CPAAS_SUPPORT_PACKAGES_BUCKET="${SUPPORT_PACKAGES_BUCKET}" CPAAS_STAGING_SUPPORT_PACKAGES_BUCKET="${STAGING_SUPPORT_PACKAGES_BUCKET}" @@ -111,13 +124,16 @@ jobs: if [[ ${{ github.event_name }} == "pull_request" ]]; then { echo "push-drivers-bucket=${STAGING_DRIVER_BUCKET}" + echo "merged-drivers-bucket=${STAGING_MERGED_DRIVER_BUCKET}" echo "support-packages-bucket=${STAGING_SUPPORT_PACKAGES_BUCKET}" if [[ ${{ contains(github.event.pull_request.labels.*.name, 'run-cpaas-steps') }} == "true" ]]; then echo "cpaas-drivers-bucket=${CPAAS_STAGING_DRIVERS_BUCKET}" + echo "cpaas-all-archs-drivers-bucket=${CPAAS_STAGING_ALL_ARCHS_DRIVERS_BUCKET}" echo "cpaas-support-packages-bucket=${CPAAS_STAGING_SUPPORT_PACKAGES_BUCKET}" else # When running on PRs withouth the 'run-cpaas-steps' label, use the main buckets echo "cpaas-drivers-bucket=${CPAAS_DRIVERS_BUCKET}" + echo "cpaas-all-archs-drivers-bucket=${CPAAS_ALL_ARCHS_DRIVERS_BUCKET}" echo "cpaas-support-packages-bucket=${CPAAS_SUPPORT_PACKAGES_BUCKET}" fi echo "support-packages-index-bucket=${STAGING_SUPPORT_PACKAGES_BUCKET}" @@ -125,8 +141,10 @@ jobs: else { echo "push-drivers-bucket=${MAIN_DRIVER_BUCKET}" + echo "merged-drivers-bucket=${MERGED_DRIVER_BUCKET}" echo "support-packages-bucket=${SUPPORT_PACKAGES_BUCKET}" echo "cpaas-drivers-bucket=${CPAAS_DRIVERS_BUCKET}" + echo "cpaas-all-archs-drivers-bucket=${CPAAS_ALL_ARCHS_DRIVERS_BUCKET}" echo "cpaas-support-packages-bucket=${CPAAS_SUPPORT_PACKAGES_BUCKET}" echo "support-packages-index-bucket=${SUPPORT_PACKAGES_BUCKET}" } >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 21e8d73488..989faae039 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,6 +40,7 @@ jobs: uses: ./.github/workflows/upload-drivers.yml with: gcp-bucket: ${{ needs.init.outputs.push-drivers-bucket }} + merged-drivers-bucket: ${{ needs.init.outputs.merged-drivers-bucket }} if: ${{ needs.build-drivers.outputs.parallel-jobs > 0 }} needs: - init diff --git a/.github/workflows/upload-drivers.yml b/.github/workflows/upload-drivers.yml index fea77d50b2..8ef2a998db 100644 --- a/.github/workflows/upload-drivers.yml +++ b/.github/workflows/upload-drivers.yml @@ -7,6 +7,10 @@ on: type: string required: true description: GCP bucket to push drivers into + merged-drivers-bucket: + type: string + required: true + description: GCP bucket unifying upstream and downstream drivers jobs: upload-drivers: @@ -24,9 +28,29 @@ jobs: with: credentials_json: '${{ secrets.GOOGLE_CREDENTIALS_COLLECTOR_SVC_ACCT }}' + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v1' + - name: Push drivers uses: 'google-github-actions/upload-cloud-storage@v1' with: path: /tmp/output/ parent: false destination: ${{ inputs.gcp-bucket }} + + - name: Push to merged bucket + run: | + shopt -s nullglob + shopt -s dotglob + + # We use gsutil here because we need to ignore files that already + # exist in the bucket, if they are there it means downstream + # built it and has prevalence over this set of drivers. + for driver_version_dir in /tmp/output/*; do + files=("${driver_version_dir}"/*.{gz,unavail}) + driver_version="$(basename "${driver_version_dir}")" + [[ "${#files[@]}" -gt 0 ]] || continue + + printf '%s\n' "${files[@]}" | \ + gsutil -m cp -n -I "gs://${{ inputs.merged-drivers-bucket }}/${driver_version}/" + done