hpc-amd-slurm.yaml

# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

---
blueprint_name: hpc-amd-slurm-v6

vars:
  project_id:  ## Set GCP Project ID Here ##
  deployment_name: amd-v6
  region: us-east4
  zone: us-east4-c

deployment_groups:
- group: primary
  modules:
  - id: network
    source: modules/network/vpc

  - id: homefs
    source: modules/file-system/filestore
    use: [network]
    settings:
      local_mount: /home

  - id: swfs
    source: modules/file-system/filestore
    use: [network]
    settings:
      local_mount: /sw

  - id: spack-setup
    source: community/modules/scripts/spack-setup
    settings:
      install_dir: /sw/spack

  - id: spack-execute
    source: community/modules/scripts/spack-execute
    use: [spack-setup]
    settings:
      log_file: /var/log/spack.log
      data_files:
      - destination: /tmp/projections-config.yaml
        content: |
          modules:
            default:
              tcl:
                hash_length: 0
                all:
                  conflict:
                    - '{name}'
                projections:
                  all: '{name}/{version}-{compiler.name}-{compiler.version}'
      - destination: /tmp/slurm-external-config.yaml
        content: |
          packages:
            slurm:
              externals:
                - spec: slurm@23-11-3
                  prefix: /usr/local
              buildable: False
      - destination: /sw/spack/openfoam_env.yaml
        content: |
          spack:
            definitions:
            - compilers:
              - gcc@13.1.0
            - mpis:
              - openmpi@4.1.3+legacylaunchers+pmi fabrics=none schedulers=slurm
            - packages:
              - flex@2.6.4
            - mpi_packages:
              - openfoam-org@8 ^flex@2.6.4 target=zen3
            specs:
            - matrix:
              - - $mpis
              - - $%compilers
            - matrix:
              - - $packages
              - - $%compilers
            - matrix:
              - - $mpi_packages
              - - $%compilers
              - - $^mpis
            concretizer:
              unify: when_possible
      commands: |
        spack config --scope defaults add config:build_stage:/opt/spack_build_stage
        spack config --scope defaults add -f /tmp/projections-config.yaml
        spack config --scope site add -f /tmp/slurm-external-config.yaml
        spack config --scope site add concretizer:targets:host_compatible:false

        # gcc 12.1.0 is known to have runtime failures with OpenFOAM 8
        spack install gcc@13.1.0 %gcc@8.5.0 target=x86_64
        spack load gcc@13.1.0 %gcc@8.5.0 target=x86_64
        spack compiler find --scope site

        if ! spack env list | grep -q openfoam; then
          spack env create openfoam /sw/spack/openfoam_env.yaml
          spack env activate openfoam
          spack concretize
          spack install
        fi

  - id: spack-startup
    source: modules/scripts/startup-script
    settings:
      runners:
      - $(spack-execute.spack_runner)
      - type: shell
        destination: shutdown.sh
        content: |
          #!/bin/bash
          if [ ! -f /etc/block_auto_shutdown ]; then
                  touch /etc/block_auto_shutdown
                  shutdown -h +1
          fi

  - id: slurm_startup
    source: modules/scripts/startup-script
    settings:
      runners:
      - $(spack-setup.spack_runner)
      # the following installation of AOCC may be automated in the future
      # with a clear direction to the user to read the EULA at
      # https://developer.amd.com/aocc-compiler-eula/
      - type: data
        destination: /var/tmp/install_aocc.sh
        content: |
          #!/bin/bash
          source /sw/spack/share/spack/setup-env.sh
          spack install aocc@3.2.0 +license-agreed
          spack load aocc@3.2.0
          spack compiler find --scope site
          spack -d install -v openmpi@4.1.3 %aocc@3.2.0 +legacylaunchers +pmi schedulers=slurm
      - type: data
        destination: /var/tmp/openfoam_test.sh
        content: |
          #!/bin/bash
          # the following line works around a problem activating environments
          # before directory is accessed
          ls -lha /sw/spack/var/spack/environments/openfoam/ &>/dev/null
          spack env activate openfoam
          DIR=$HOME/openfoam_test
          mkdir -p $DIR
          cd $DIR
          cp -fr $WM_PROJECT_DIR/tutorials/incompressible/simpleFoam/motorBike .
          cd motorBike
          ./Allrun

  - id: spack_builder
    source: modules/compute/vm-instance
    use: [network, swfs, spack-startup]
    settings:
      name_prefix: spack-builder
      machine_type: c2d-standard-16
      instance_image:
        # these images must match the images used by Slurm modules below because
        # we are building OpenMPI with PMI support in libraries contained in
        # Slurm installation
        family: slurm-gcp-6-5-hpc-rocky-linux-8
        project: schedmd-slurm-public

  - id: low_cost_nodeset
    source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset
    use: [network]
    settings:
      machine_type: c2d-standard-4
      node_count_dynamic_max: 10
      bandwidth_tier: gvnic_enabled
      enable_placement: false

  - id: low_cost_partition
    source: community/modules/compute/schedmd-slurm-gcp-v6-partition
    use: [low_cost_nodeset]
    settings:
      partition_name: lowcost

  - id: compute_nodeset
    source: community/modules/compute/schedmd-slurm-gcp-v6-nodeset
    use: [network]
    settings:
      machine_type: c2d-standard-112
      node_count_dynamic_max: 50
      bandwidth_tier: gvnic_enabled
      enable_placement: true

  # Because is_default is set to true, jobs will run on this partition unless an
  # alternative partition is specified using, for example, "srun -p lowcost"
  - id: compute_partition
    source: community/modules/compute/schedmd-slurm-gcp-v6-partition
    use: [compute_nodeset]
    settings:
      partition_name: compute
      is_default: true

  - id: slurm_login
    source: community/modules/scheduler/schedmd-slurm-gcp-v6-login
    use: [network]
    settings:
      # need at least 8 physical cores to run OpenFOAM test
      machine_type: c2d-standard-16

  - id: slurm_controller
    source: community/modules/scheduler/schedmd-slurm-gcp-v6-controller
    use:
    - network
    - homefs
    - swfs
    - low_cost_partition
    - compute_partition
    - slurm_login
    settings:
      machine_type: c2d-standard-4
      login_startup_script: $(slurm_startup.startup_script)
      login_startup_scripts_timeout: 21600