From d1104220b702d46acca55e335b8738f0bd63c7e6 Mon Sep 17 00:00:00 2001 From: Matt Pryor Date: Tue, 10 Sep 2024 11:36:36 +0100 Subject: [PATCH 1/9] Move common tasks into a shared linux-common role --- ansible/jupyter-repo2docker.yml | 1 + ansible/kubernetes.yml | 2 +- ansible/linux-rdp-gateway.yml | 1 + ansible/linux-rstudio.yml | 1 + ansible/linux-webconsole.yml | 1 + .../roles/jupyter-repo2docker/tasks/main.yml | 22 --------------- ansible/roles/linux-common/tasks/main.yml | 28 +++++++++++++++++++ .../roles/linux-rdp-gateway/tasks/main.yml | 22 --------------- ansible/roles/linux-rstudio/tasks/main.yml | 22 --------------- ansible/roles/linux-webconsole/tasks/main.yml | 22 --------------- 10 files changed, 33 insertions(+), 89 deletions(-) create mode 100644 ansible/roles/linux-common/tasks/main.yml diff --git a/ansible/jupyter-repo2docker.yml b/ansible/jupyter-repo2docker.yml index ca4557b3..5977da9b 100644 --- a/ansible/jupyter-repo2docker.yml +++ b/ansible/jupyter-repo2docker.yml @@ -3,4 +3,5 @@ - hosts: all become: yes roles: + - linux-common - jupyter-repo2docker diff --git a/ansible/kubernetes.yml b/ansible/kubernetes.yml index 6753ef36..23eff556 100644 --- a/ansible/kubernetes.yml +++ b/ansible/kubernetes.yml @@ -3,6 +3,6 @@ - hosts: all become: yes roles: - - linux-volumes-by-tag + - linux-common - import_playbook: ../vendor/image-builder/images/capi/ansible/node.yml diff --git a/ansible/linux-rdp-gateway.yml b/ansible/linux-rdp-gateway.yml index f4724c07..004d86b2 100644 --- a/ansible/linux-rdp-gateway.yml +++ b/ansible/linux-rdp-gateway.yml @@ -3,4 +3,5 @@ - hosts: all become: yes roles: + - linux-common - linux-rdp-gateway diff --git a/ansible/linux-rstudio.yml b/ansible/linux-rstudio.yml index dd27e734..db51fcba 100644 --- a/ansible/linux-rstudio.yml +++ b/ansible/linux-rstudio.yml @@ -3,4 +3,5 @@ - hosts: all become: yes roles: + - linux-common - linux-rstudio diff --git a/ansible/linux-webconsole.yml b/ansible/linux-webconsole.yml index e5e89284..2986f98b 100644 --- a/ansible/linux-webconsole.yml +++ b/ansible/linux-webconsole.yml @@ -3,4 +3,5 @@ - hosts: all become: yes roles: + - linux-common - linux-webconsole diff --git a/ansible/roles/jupyter-repo2docker/tasks/main.yml b/ansible/roles/jupyter-repo2docker/tasks/main.yml index ffa24c13..5b1199a0 100644 --- a/ansible/roles/jupyter-repo2docker/tasks/main.yml +++ b/ansible/roles/jupyter-repo2docker/tasks/main.yml @@ -1,27 +1,5 @@ --- -- name: Update apt cache - apt: - update_cache: yes - when: ansible_os_family == "Debian" - -- name: Ensure up-to-date CA certificates - package: - name: ca-certificates - state: latest - -# Required for become to an unprivileged user to work -# Using the apt module seems to work more reliably than package :-/ -- name: Install ACL package - apt: - update_cache: yes - name: acl - state: present - when: ansible_os_family == "Debian" - -- include_role: - name: azimuth_cloud.image_utils.linux_ansible_init - - include_role: name: linux-podman tasks_from: install.yml diff --git a/ansible/roles/linux-common/tasks/main.yml b/ansible/roles/linux-common/tasks/main.yml new file mode 100644 index 00000000..6190450d --- /dev/null +++ b/ansible/roles/linux-common/tasks/main.yml @@ -0,0 +1,28 @@ +--- + +- name: Update apt cache + apt: + update_cache: true + when: ansible_os_family == "Debian" + +- name: Ensure up-to-date CA certificates + package: + name: ca-certificates + state: latest + +# Required for become to an unprivileged user to work +# Using the apt module seems to work more reliably than package :-/ +- name: Install ACL package + apt: + update_cache: true + name: acl + state: present + when: ansible_os_family == "Debian" + +# Enables the ansible-init system, but does not install any default playbooks +- include_role: + name: azimuth_cloud.image_utils.linux_ansible_init + +# Enables volumes to be referred to by tag for setting up mounts in cloud-config +- include_role: + name: linux-volumes-by-tag diff --git a/ansible/roles/linux-rdp-gateway/tasks/main.yml b/ansible/roles/linux-rdp-gateway/tasks/main.yml index 91d8b2fe..f0c7cf20 100644 --- a/ansible/roles/linux-rdp-gateway/tasks/main.yml +++ b/ansible/roles/linux-rdp-gateway/tasks/main.yml @@ -1,27 +1,5 @@ --- -- name: Update apt cache - apt: - update_cache: yes - when: ansible_os_family == "Debian" - -- name: Ensure up-to-date CA certificates - package: - name: ca-certificates - state: latest - -# Required for become to an unprivileged user to work -# Using the apt module seems to work more reliably than package :-/ -- name: Install ACL package - apt: - update_cache: yes - name: acl - state: present - when: ansible_os_family == "Debian" - -- include_role: - name: azimuth_cloud.image_utils.linux_ansible_init - - include_role: name: linux-podman tasks_from: install.yml diff --git a/ansible/roles/linux-rstudio/tasks/main.yml b/ansible/roles/linux-rstudio/tasks/main.yml index 69c205dc..adec231a 100644 --- a/ansible/roles/linux-rstudio/tasks/main.yml +++ b/ansible/roles/linux-rstudio/tasks/main.yml @@ -1,27 +1,5 @@ --- -- name: Update apt cache - apt: - update_cache: yes - when: ansible_os_family == "Debian" - -- name: Ensure up-to-date CA certificates - package: - name: ca-certificates - state: latest - -# Required for become to an unprivileged user to work -# Using the apt module seems to work more reliably than package :-/ -- name: Install ACL package - apt: - update_cache: yes - name: acl - state: present - when: ansible_os_family == "Debian" - -- include_role: - name: azimuth_cloud.image_utils.linux_ansible_init - - include_role: name: linux-podman tasks_from: install.yml diff --git a/ansible/roles/linux-webconsole/tasks/main.yml b/ansible/roles/linux-webconsole/tasks/main.yml index f78bdf52..5b517be7 100644 --- a/ansible/roles/linux-webconsole/tasks/main.yml +++ b/ansible/roles/linux-webconsole/tasks/main.yml @@ -17,25 +17,6 @@ - "{{ role_path }}/vars/{{ ansible_distribution }}.yml" - "{{ role_path }}/vars/{{ ansible_distribution }}-{{ ansible_distribution_version }}.yml" -- name: Update apt cache - apt: - update_cache: true - when: ansible_os_family == "Debian" - -- name: Ensure up-to-date CA certificates - package: - name: ca-certificates - state: latest - -# Required for become to an unprivileged user to work -# Using the apt module seems to work more reliably than package :-/ -- name: Install ACL package - apt: - update_cache: true - name: acl - state: present - when: ansible_os_family == "Debian" - - name: Add apptainer repository apt_repository: repo: 'ppa:apptainer/ppa' @@ -51,9 +32,6 @@ - include_tasks: vnc_server.yml when: desktop_enabled is defined and desktop_enabled -- include_role: - name: azimuth_cloud.image_utils.linux_ansible_init - - include_role: name: linux-podman tasks_from: install.yml From 68b1ca982894ad4b2c4e7b82149b314f660c7fe8 Mon Sep 17 00:00:00 2001 From: Matt Pryor Date: Tue, 10 Sep 2024 13:39:00 +0100 Subject: [PATCH 2/9] Update Trivy action version --- .github/workflows/build_test_images.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_test_images.yaml b/.github/workflows/build_test_images.yaml index 8bd5568c..b9b9e9ee 100644 --- a/.github/workflows/build_test_images.yaml +++ b/.github/workflows/build_test_images.yaml @@ -88,7 +88,7 @@ jobs: run: sudo guestmount -a ${{ steps.publish-image.outputs.image-name }}.qcow2 -i --ro -o allow_other './${{ steps.publish-image.outputs.image-name }}' - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.17.0 + uses: aquasecurity/trivy-action@0.24.0 with: scan-type: fs scan-ref: "./${{ steps.publish-image.outputs.image-name }}" @@ -104,7 +104,7 @@ jobs: category: "${{ matrix.name }}" - name: Fail if scan has CRITICAL vulnerabilities - uses: aquasecurity/trivy-action@0.16.1 + uses: aquasecurity/trivy-action@0.24.0 with: scan-type: fs scan-ref: "./${{ steps.publish-image.outputs.image-name }}" From 0d88a82669c02be70fab4315f20f32df264c9510 Mon Sep 17 00:00:00 2001 From: Sean McConkey Date: Wed, 11 Sep 2024 15:36:46 +0100 Subject: [PATCH 3/9] New role to add the broadcom driver to Linx builds for Azimuth --- ansible/roles/linux-broadcom/tasks/broadcom.yml | 6 ++++++ ansible/roles/linux-common/tasks/main.yml | 3 +++ 2 files changed, 9 insertions(+) create mode 100644 ansible/roles/linux-broadcom/tasks/broadcom.yml diff --git a/ansible/roles/linux-broadcom/tasks/broadcom.yml b/ansible/roles/linux-broadcom/tasks/broadcom.yml new file mode 100644 index 00000000..bc38b0a0 --- /dev/null +++ b/ansible/roles/linux-broadcom/tasks/broadcom.yml @@ -0,0 +1,6 @@ +--- + + - name: Install the broadcom driver for NetXtreme-E Ethernet + ansible.builtin.apt: + deb: https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_7ac3c0a502cd46c783b2128116165566/bcom-nxe-drivers/netxtreme-peer-mem-dkms_229.2.49.0_all.deb + state: present \ No newline at end of file diff --git a/ansible/roles/linux-common/tasks/main.yml b/ansible/roles/linux-common/tasks/main.yml index 6190450d..ce163679 100644 --- a/ansible/roles/linux-common/tasks/main.yml +++ b/ansible/roles/linux-common/tasks/main.yml @@ -26,3 +26,6 @@ # Enables volumes to be referred to by tag for setting up mounts in cloud-config - include_role: name: linux-volumes-by-tag + +# Install the Broadcom network driver for NetXtreme-E Ethernet +- include_role: linux-broadcom \ No newline at end of file From 7319bb9f3191e19733b07d48cdcb563a10fff775 Mon Sep 17 00:00:00 2001 From: Paul Browne Date: Mon, 30 Sep 2024 23:13:01 +0100 Subject: [PATCH 4/9] Modify Arcus cloud variables to run in dev Keyston project rcp-pfb29-dev --- vars/arcus/common.json | 4 ++-- vars/arcus/linux.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/vars/arcus/common.json b/vars/arcus/common.json index fdfad6e7..e847f92a 100644 --- a/vars/arcus/common.json +++ b/vars/arcus/common.json @@ -1,5 +1,5 @@ { - "network": "97ed78b5-156c-441c-9fcb-5a26c8a4ed26", - "flavor": "vm.azimuth.ci.ec1.medium", + "network": "5e8d514a-1943-42e4-a757-6c533c343ece", + "flavor": "vm.v1.small", "volume_type": "arcus-ceph01-rbd" } diff --git a/vars/arcus/linux.json b/vars/arcus/linux.json index af641b56..2a0db531 100644 --- a/vars/arcus/linux.json +++ b/vars/arcus/linux.json @@ -1,3 +1,3 @@ { - "security_groups": ["default", "ssh-anywhere"] + "security_groups": ["default", "external"] } From 6f031cfe8b4dbcd746169c6c5769750cb87c5dc0 Mon Sep 17 00:00:00 2001 From: Paul Browne Date: Mon, 30 Sep 2024 23:23:09 +0100 Subject: [PATCH 5/9] Change pr.yaml workflow fail_on_remote repo specification --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1ce234e9..c40f681d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,8 +15,8 @@ jobs: fail_on_remote: runs-on: ubuntu-latest steps: - - name: PR must be from a branch in the azimuth-cloud/azimuth-images repo - run: exit ${{ github.repository == 'azimuth-cloud/azimuth-images' && '0' || '1' }} + - name: PR must be from a branch in the RSE-Cambridge/azimuth-images repo + run: exit ${{ github.repository == 'RSE-Cambridge/azimuth-images' && '0' || '1' }} # Build, publish and test the images build_test_images: From d760411cf7817e5eee135669c312fff8a5646106 Mon Sep 17 00:00:00 2001 From: Paul Browne Date: Mon, 30 Sep 2024 23:36:33 +0100 Subject: [PATCH 6/9] Minor syntax fix in include_role --- ansible/roles/linux-common/tasks/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/roles/linux-common/tasks/main.yml b/ansible/roles/linux-common/tasks/main.yml index ce163679..10a1aa19 100644 --- a/ansible/roles/linux-common/tasks/main.yml +++ b/ansible/roles/linux-common/tasks/main.yml @@ -28,4 +28,5 @@ name: linux-volumes-by-tag # Install the Broadcom network driver for NetXtreme-E Ethernet -- include_role: linux-broadcom \ No newline at end of file +- include_role: + name: linux-broadcom From 40f7c76f139a7250fb816e9a552ee7e3052686aa Mon Sep 17 00:00:00 2001 From: Paul Browne Date: Tue, 1 Oct 2024 00:00:23 +0100 Subject: [PATCH 7/9] Modify publish-image certificate verification regexp to reflect changed repo location --- bin/publish-image | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/publish-image b/bin/publish-image index 1e0f3540..8cc51ab7 100755 --- a/bin/publish-image +++ b/bin/publish-image @@ -21,7 +21,7 @@ cosign sign-blob "${IMAGE_NAME}.qcow2" --bundle "${IMAGE_NAME}.cosign.bundle" -y # test the bundle works validates cosign verify-blob "${IMAGE_NAME}.qcow2" --bundle "${IMAGE_NAME}.cosign.bundle" \ --certificate-oidc-issuer="https://token.actions.githubusercontent.com" \ - --certificate-identity-regexp="https://github.com/azimuth-cloud/azimuth-images/.github/.*" + --certificate-identity-regexp="https://github.com/RSE-Cambridge/azimuth-images/.github/.*" # Upload the compressed image to S3 cat < ~/.s3cfg From 19d558916af3ee3b7c500590c80392f93cfd2b54 Mon Sep 17 00:00:00 2001 From: Paul Browne Date: Tue, 1 Oct 2024 02:02:03 +0100 Subject: [PATCH 8/9] Ensure pre-reqs are installed for Broadcom NIC dpkg DEB install --- ansible/roles/linux-broadcom/tasks/broadcom.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ansible/roles/linux-broadcom/tasks/broadcom.yml b/ansible/roles/linux-broadcom/tasks/broadcom.yml index bc38b0a0..e9329455 100644 --- a/ansible/roles/linux-broadcom/tasks/broadcom.yml +++ b/ansible/roles/linux-broadcom/tasks/broadcom.yml @@ -1,6 +1,16 @@ --- + - name: Install pre-requisites + ansible.builtin.apt: + - dctrl-tools + - gcc-12 + - cpp-12 + - libgcc-12-dev + - libasan8 + - libtsan2 + - dkms + state: latest - name: Install the broadcom driver for NetXtreme-E Ethernet ansible.builtin.apt: deb: https://object.arcus.openstack.hpc.cam.ac.uk/swift/v1/AUTH_7ac3c0a502cd46c783b2128116165566/bcom-nxe-drivers/netxtreme-peer-mem-dkms_229.2.49.0_all.deb - state: present \ No newline at end of file + state: present From 1bb69faa771cb7991c551eaa335a7b6fcbbeae7e Mon Sep 17 00:00:00 2001 From: Paul Browne Date: Tue, 1 Oct 2024 09:57:25 +0100 Subject: [PATCH 9/9] Landing playbook in a role should be called main.yml where include_role is used --- .../tasks/{broadcom.yml => main.yml} | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) rename ansible/roles/linux-broadcom/tasks/{broadcom.yml => main.yml} (66%) diff --git a/ansible/roles/linux-broadcom/tasks/broadcom.yml b/ansible/roles/linux-broadcom/tasks/main.yml similarity index 66% rename from ansible/roles/linux-broadcom/tasks/broadcom.yml rename to ansible/roles/linux-broadcom/tasks/main.yml index e9329455..5e06dbec 100644 --- a/ansible/roles/linux-broadcom/tasks/broadcom.yml +++ b/ansible/roles/linux-broadcom/tasks/main.yml @@ -1,14 +1,15 @@ --- - name: Install pre-requisites ansible.builtin.apt: - - dctrl-tools - - gcc-12 - - cpp-12 - - libgcc-12-dev - - libasan8 - - libtsan2 - - dkms - state: latest + pkg: + - dctrl-tools + - gcc-12 + - cpp-12 + - libgcc-12-dev + - libasan8 + - libtsan2 + - dkms + state: latest - name: Install the broadcom driver for NetXtreme-E Ethernet ansible.builtin.apt: