Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove CUDA and OFED builds from CI #479

Merged
merged 4 commits into from
Nov 15, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 8 additions & 15 deletions .github/workflows/fatimage.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,36 +15,29 @@ jobs:
openstack:
name: openstack-imagebuild
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }}-${{ matrix.build }} # to branch/PR + OS + build
group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }} # to branch/PR + OS
cancel-in-progress: true
runs-on: ubuntu-22.04
strategy:
fail-fast: false # allow other matrix jobs to continue even if one fails
matrix: # build RL8+OFED, RL9+OFED, RL9+OFED+CUDA versions
matrix: # build RL8, RL9
os_version:
- RL8
- RL9
build:
- openstack.openhpc
- openstack.openhpc-cuda
exclude:
- os_version: RL8
build: openstack.openhpc-cuda
env:
ANSIBLE_FORCE_COLOR: True
OS_CLOUD: openstack
CI_CLOUD: ${{ github.event.inputs.ci_cloud }}
SOURCE_IMAGES_MAP: |
{
"RL8": {
"openstack.openhpc": "rocky-latest-RL8",
"openstack.openhpc-cuda": "rocky-latest-cuda-RL8"
"openstack.openhpc": "rocky-latest-RL8"
},
"RL9": {
"openstack.openhpc": "rocky-latest-RL9",
"openstack.openhpc-cuda": "rocky-latest-cuda-RL9"
"openstack.openhpc": "rocky-latest-RL9"
}
}
BUILD: openstack.openhpc

steps:
- uses: actions/checkout@v2
Expand Down Expand Up @@ -90,13 +83,13 @@ jobs:

PACKER_LOG=1 packer build \
-on-error=${{ vars.PACKER_ON_ERROR }} \
-only=${{ matrix.build }} \
-only=${{ env.BUILD }} \
-var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
-var "source_image_name=${{ env.SOURCE_IMAGE }}" \
openstack.pkr.hcl
env:
PKR_VAR_os_version: ${{ matrix.os_version }}
SOURCE_IMAGE: ${{ fromJSON(env.SOURCE_IMAGES_MAP)[matrix.os_version][matrix.build] }}
SOURCE_IMAGE: ${{ fromJSON(env.SOURCE_IMAGES_MAP)[matrix.os_version][env.BUILD] }}

- name: Get created image names from manifest
id: manifest
Expand All @@ -113,7 +106,7 @@ jobs:
- name: Upload manifest artifact
uses: actions/upload-artifact@v4
with:
name: image-details-${{ matrix.build }}-${{ matrix.os_version }}
name: image-details-${{ env.BUILD }}-${{ matrix.os_version }}
path: |
./image-id.txt
./image-name.txt
Expand Down
17 changes: 4 additions & 13 deletions .github/workflows/nightlybuild.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,22 +17,15 @@ jobs:
openstack:
name: openstack-imagebuild
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }}-${{ matrix.build }} # to branch/PR + OS + build
group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.os_version }} # to branch/PR + OS
cancel-in-progress: true
runs-on: ubuntu-22.04
strategy:
fail-fast: false # allow other matrix jobs to continue even if one fails
matrix: # build RL8, RL9, RL9+CUDA versions
matrix: # build RL8, RL9
os_version:
- RL8
- RL9
build:
- openstack.rocky-latest
- openstack.rocky-latest-cuda
exclude:
- os_version: RL8
build: openstack.rocky-latest-cuda

env:
ANSIBLE_FORCE_COLOR: True
OS_CLOUD: openstack
Expand All @@ -42,6 +35,7 @@ jobs:
"RL8": "Rocky-8-GenericCloud-Base-8.9-20231119.0.x86_64.qcow2",
"RL9": "Rocky-9-GenericCloud-Base-9.4-20240523.0.x86_64.qcow2"
}
BUILD: openstack.openhpc
sjpb marked this conversation as resolved.
Show resolved Hide resolved

steps:
- uses: actions/checkout@v2
Expand Down Expand Up @@ -87,7 +81,7 @@ jobs:

PACKER_LOG=1 packer build \
-on-error=${{ vars.PACKER_ON_ERROR }} \
-only=${{ matrix.build }} \
-only=${{ env.BUILD }} \
-var-file=$PKR_VAR_environment_root/${{ env.CI_CLOUD }}.pkrvars.hcl \
-var "source_image_name=${{ env.SOURCE_IMAGE }}" \
openstack.pkr.hcl
Expand Down Expand Up @@ -144,10 +138,7 @@ jobs:
- RL9
image:
- rocky-latest
- rocky-latest-cuda
exclude:
- os_version: RL8
image: rocky-latest-cuda
- target_cloud: LEAFCLOUD
env:
OS_CLOUD: openstack
Expand Down
2 changes: 0 additions & 2 deletions .github/workflows/s3-image-sync.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ jobs:
build:
- RL8
- RL9
- RL9-cuda
env:
ANSIBLE_FORCE_COLOR: True
OS_CLOUD: openstack
Expand Down Expand Up @@ -112,7 +111,6 @@ jobs:
build:
- RL8
- RL9
- RL9-cuda
exclude:
- cloud: ${{ needs.image_upload.outputs.ci_cloud }}

Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/trivyscan.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ on:
jobs:
scan:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build }} # to branch/PR + OS + build
group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.build }} # to branch/PR + build
cancel-in-progress: true
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
build: ["RL8", "RL9", "RL9-cuda"]
build: ["RL8", "RL9"]
env:
JSON_PATH: environments/.stackhpc/terraform/cluster_image.auto.tfvars.json
OS_CLOUD: openstack
Expand Down Expand Up @@ -100,7 +100,7 @@ jobs:
uses: github/codeql-action/upload-sarif@v3
with:
sarif_file: "${{ steps.manifest.outputs.image-name }}.sarif"
category: "${{ matrix.os_version }}-${{ matrix.build }}"
category: "${{ matrix.build }}"

- name: Fail if scan has CRITICAL vulnerabilities
uses: aquasecurity/[email protected]
Expand Down
22 changes: 3 additions & 19 deletions packer/openstack.pkr.hcl
Original file line number Diff line number Diff line change
Expand Up @@ -127,15 +127,13 @@ variable "volume_size" {
default = {
# fat image builds, GB:
rocky-latest = 15
rocky-latest-cuda = 30
openhpc = 15
openhpc-cuda = 30
}
}

variable "extra_build_volume_size" {
type = number
default = 15 # same as default non-CUDA build
default = 15
}

variable "image_disk_format" {
Expand All @@ -153,10 +151,8 @@ variable "groups" {
description = "Additional inventory groups (other than 'builder') to add build VM to, keyed by source name"
default = {
# fat image builds:
rocky-latest = ["update", "ofed"]
rocky-latest-cuda = ["update", "ofed", "cuda"]
rocky-latest = ["update"]
openhpc = ["control", "compute", "login"]
openhpc-cuda = ["control", "compute", "login"]
}
}

Expand Down Expand Up @@ -210,24 +206,12 @@ build {
image_name = "${source.name}-${var.os_version}"
}

# latest nightly cuda image:
source "source.openstack.openhpc" {
name = "rocky-latest-cuda"
image_name = "${source.name}-${var.os_version}"
}

# OFED fat image:
# fat image:
source "source.openstack.openhpc" {
name = "openhpc"
image_name = "${source.name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}"
}

# CUDA fat image:
source "source.openstack.openhpc" {
name = "openhpc-cuda"
image_name = "${source.name}-${var.os_version}-${local.timestamp}-${substr(local.git_commit, 0, 8)}"
}

# Extended site-specific image, built on fat image:
source "source.openstack.openhpc" {
name = "openhpc-extra"
Expand Down
Loading