Gitlab CI CD for LC mirror

.gitignore

@@ -1,3 +1,4 @@
+builds/*
 build-*
 install-*
 .cproject

.gitlab-ci.yml

@@ -0,0 +1,180 @@
+stages:
+  - test
+  - deploy
+
+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+  GEOS_DIR: "geos"
+  GEOS_BRANCH: "feature/wrtobin/tpl-gitlab-ci-cd" # Default branch, can be overridden when CI is run manually
+  SPACK_DIR: "spack"
+  SPACK_BRANCH: "develop"
+  DEPLOY: "false" # default, override when launching manually to actually deploy
+  GEOS_TPL_BASE: ${GEOS_DIR}/src/coreComponents/LvArray/host-configs/LLNL/llnl-tpls-base.cmake
+  TIOGA_TPL_BASE: ${GEOS_DIR}/src/coreComponents/LvArray/host-configs/LLNL/tioga-base.cmake
+  TPL_ROOT_DIR: /usr/gapps/GEOSX/thirdPartyLibs
+  DATE: $(date +'%Y-%m-%d')
+  HOST_CONFIG_INSTALL_SCHEME: install-\$\{CONFIG_NAME\}-release
+  SPACK_INSTALL_SCHEME: install-\$\{CLUSTER\}-\$\{SYS_TYPE\}-release
+
+.clone_geos_no_integrated:
+  script:
+    - git clone -b ${GEOS_BRANCH} [email protected]:GEOS-DEV/GEOS.git ${GEOS_DIR}
+    - cd ${GEOS_DIR}
+    - git submodule init src/cmake/blt
+    - git submodule init src/coreComponents/LvArray
+    - git submodule init src/coreComponents/constitutive/PVTPackage
+    - git submodule init src/coreComponents/fileIO/coupling/hdf5_interface
+    - git submodule update
+    - cd ..
+
+.update_geos:
+  script:
+    - cd ${GEOS_DIR}/src/coreComponents/LvArray
+    - git checkout -b ${BRANCH}
+    - git add -A
+    - git commit -m "Update lc tpl installation."
+    - git push origin ${BRANCH}
+    - cd ${GEOS_DIR}
+    - git checkout -b ${BRANCH}
+    - git add -A
+    - git commit -m "Update lc tpl installation."
+    - git push origin ${BRANCH}
+
+.test_spack:
+  script:
+    - git clone [email protected]:spack/spack.git
+    - srun -N 1 -t 90 -n 1 scripts/spack-install.sh ${CLUSTER} ${SYS_TYPE} 64
+    # - source spack/share/spack/setup-env.sh
+    # - spack env create ${CLUSTER}-${SYS_TYPE} spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml
+    # - spack env activate ${CLUSTER}-${SYS_TYPE}
+    # - spack install
+    # - spack env deactivate
+
+.install_spack:
+  script:
+    - git clone [email protected]:spack/spack.git
+    - source spack/share/spack/setup-env.sh
+    - modifyYaml.py -i spack-environment/${CLUSTER-${SYSTEM}/spack.yaml -o spack-environments/${CLUSTER}-${SYSTEM}/spack.yaml -y "spack:config:install_tree:root:/usr/WS1/GEOS/GEOSX/TPLs_$(date +'%Y-%m-%d')"
+    - srun -N 1 -t 90 -n 1 scripts/spack-install.sh ${CLUSTER} ${SYS_TYPE} 64
+    # - spack env create ${CLUSTER}-${SYS_TYPE} spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml
+    # - spack env activate ${CLUSTER}-${SYS_TYPE}
+    # - spack install
+    # - INSTALL_SCHEME=${SPACK_INSTALL_SCHEME}
+    # - spack env deactivate
+
+test_spack_tioga:
+  stage: test
+  when: manual
+  tags:
+    - tioga
+    - shell
+  variables:
+    CLUSTER: tioga
+    SYS_TYPE: toss_4_x86_64_ib_cray
+  script:
+    - git clone -b $GEOS_BRANCH [email protected]:GEOS-DEV/GEOS.git ${GEOS_DIR}
+    - cd ${GEOS_DIR}
+    - git submodule init src/cmake/blt
+    - git submodule init src/coreComponents/LvArray
+    - git submodule init src/coreComponents/constitutive/PVTPackage
+    - git submodule init src/coreComponents/fileIO/coupling/hdf5_interface
+    - git submodule update
+    - cd ..
+    - git clone [email protected]:spack/spack.git ${SPACK_DIR}
+    - SPACK_DIR_ABS=$(realpath "${SPACK_DIR}")
+    - srun -N 1 -t 90 -n 1 scripts/spack-install.sh "${SPACK_DIR_ABS}" "${CLUSTER}" "${SYS_TYPE}" 64
+    - sed -i "7s|.*|set(GEOSX_TPL_DIR \"${SPACK_DIR_ABS}/opt/spack/install-rhel8-x86_64-cce-15.0.0a/\" CACHE PATH \"\")|" "${TIOGA_TPL_BASE}"
+    - cd ${GEOS_DIR}
+    - ./scripts/config-build.py -hc host-configs/LLNL/tioga-cce-15.cmake -bt Release
+    - cd build-tioga-cce-15-release
+    - srun -N 1 -t 90 -n 1 make -j 64 geosx
+
+test_uberenv_tioga:
+  stage: test
+  when: manual
+  tags:
+    - tioga
+    - shell
+  variables:
+    CLUSTER: tioga
+    SYS_TYPE: toss_4_x86_64_ib_cray
+  script:
+    - TPL_ROOT=${PWD}
+    - git clone -b $GEOS_BRANCH [email protected]:GEOS-DEV/GEOS.git ${GEOS_DIR}
+    - cd ${GEOS_DIR}
+    - git submodule init src/cmake/blt
+    - git submodule init src/coreComponents/LvArray
+    - git submodule init src/coreComponents/constitutive/PVTPackage
+    - git submodule init src/coreComponents/fileIO/coupling/hdf5_interface
+    - git submodule update
+    - cd scripts/uberenv
+    - srun -N 1 -t 90 -n 1 uberenv.py --spack-env-file ${TPL_ROOT}/spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml --spec "%[email protected] ++rocm amdgpu_targets==gfx90a +hypre lai=hypre"
+    # - cd ../..
+    # - cp spack-build-*/tioga-rhel8-x85_64_ib.cmake host-configs/LLNL/
+    # - cp spack-build-*/tioga-rhel8-x85_64_ib.cmake src/coreComponents/LvArray/host-configs/LLNL/
+
+
+test_lc_tpls:
+  stage: test
+  when: manual
+  tags:
+    - quartz
+    - shell
+  variables:
+    INSTALL_DIR: builds
+  script:
+    - git clone -b $GEOS_BRANCH [email protected]:GEOS-DEV/GEOS.git ${GEOS_DIR}
+    - cd ${GEOS_DIR}
+    - git submodule init src/cmake/blt
+    - git submodule init src/coreComponents/LvArray
+    - git submodule init src/coreComponents/constitutive/PVTPackage
+    - git submodule init src/coreComponents/fileIO/coupling/hdf5_interface
+    - git submodule update
+    - cd ..
+    - mkdir -p ${INSTALL_DIR}
+    - ./scripts/setupLC-TPL.bash ${GEOS_DIR} ${INSTALL_DIR}
+    - echo "set(GEOS_TPL_ROOT_DIR ${TPL_ROOT_DIR} CACHE PATH \"\")" > ${GEOS_TPL_BASE}
+    - echo "set(GEOS_TPL_DIR \$\{GEOS_TPL_ROOT_DIR\}/${TPL_DIR}/install-\$\{CONFIG_NAME\}-release CACHE PATH \"\")" >> ${GEOS_TPL_BASE}
+    - cd ${GEOS_DIR}
+    - scripts/setupLC.bash
+
+deploy_lc_tpls:
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master" && $DEPLOY == "true"'
+  stage: deploy
+  when: manual
+  tags:
+    - quartz
+    - shell
+  dependencies:
+    - test_lc_tpls
+  variables:
+    XSU: xsu geosadmn
+    INSTALL_DIR: ${TPL_ROOT_DIR}/${DATE}
+    BRANCH: feature/update-tpls-${DATE}
+  script:
+    - git clone -b $GEOS_BRANCH [email protected]:GEOS-DEV/GEOS.git ${GEOS_DIR}
+    - cd ${GEOS_DIR}
+    - git submodule init src/cmake/blt
+    - git submodule init src/coreComponents/LvArray
+    - git submodule init src/coreComponents/constitutive/PVTPackage
+    - git submodule init src/coreComponents/fileIO/coupling/hdf5_interface
+    - git submodule update
+    - cd ..
+    - ${XSU} mkdir -p ${INSTALL_DIR}
+    - ${XSU} ./scripts/setupLC-TPL.bash ${GEOS_DIR} ${INSTALL_DIR}
+    - echo "set(GEOS_TPL_ROOT_DIR ${TPL_ROOT_DIR} CACHE PATH \"\")" > ${GEOS_TPL_BASE}
+    - echo "set(GEOS_TPL_DIR \$\{GEOS_TPL_ROOT_DIR\}/${TPL_DIR}/install-\$\{CONFIG_NAME\}-release CACHE PATH \"\")" >> ${GEOS_TPL_BASE}
+    - cd ${GEOS_DIR}
+    - scripts/setupLC.bash
+    - cd ${GEOS_DIR}/src/coreComponents/LvArray
+    - git checkout -b ${BRANCH}
+    - git add -A
+    - git commit -m "Update lc tpl install location."
+    - git push origin ${BRANCH}
+    - cd ${GEOS_DIR}
+    - git checkout -b ${BRANCH}
+    - git add -A
+    - git commit -m "Update lc tpl install location."
+    - git push origin ${BRANCH}
+

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "cmake/blt"]
 	path = cmake/blt
-	url = ../../LLNL/blt.git
+  url = https://github.com/LLNL/blt.git
+

scripts/setupLC-TPL.bash

@@ -27,11 +27,11 @@ rm -rf toBeDeleted &
 echo "Building all LC TPLs from $GEOSX_DIR to be installed at $INSTALL_DIR"
 chmod -R g+rx $INSTALL_DIR
 chgrp -R GEOS $INSTALL_DIR
-./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR quartz clang-14 "srun -N 1 -t 90 -n 1 -A geosecp" $@ &
-./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR quartz gcc-12 "srun -N 1 -t 90 -n 1 -A geosecp" $@ &
-./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR lassen gcc-8-cuda-11       "lalloc 1 -qpdebug" $@ &
-./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR lassen clang-13-cuda-11       "lalloc 1 -qpdebug" $@ &
-./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR lassen clang-10-cuda-11       "lalloc 1 -qpdebug" $@ &
+./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR quartz clang-14 "srun -N 1 -t 90 -n 1 -A geosecp -J quartz-clang-14-tpls" $@ &
+./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR quartz gcc-12 "srun -N 1 -t 90 -n 1 -A geosecp -J quartz-gcc-12-tpls" $@ &
+./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR lassen gcc-8-cuda-11 "bsub -J lassen-gcc-8-cuda-11-tpls -nnodes 1 -Is -W 60 -core_isolation 2 /usr/tce/packages/lalloc/lalloc-2.0/bin/lexec" $@ &
+./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR lassen clang-13-cuda-11 "bsub -J lassen-clang-13-cuda-11-tpls -nnodes 1 -Is -W 60 -core_isolation 2 /usr/tce/packages/lalloc/lalloc-2.0/bin/lexec" $@ &
+./scripts/setupLC-TPL-helper.bash $GEOSX_DIR $INSTALL_DIR lassen clang-10-cuda-11 "bsub -J lassen-clang-10-cuda-11-tpls -nnodes 1 -Is -W 60 -core_isolation 2 /usr/tce/packages/lalloc/lalloc-2.0/bin/lexec" $@ &
 
 wait
 echo "Complete"

scripts/spack-install.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+SPACK_DIR=$1
+CLUSTER=$2
+SYS_TYPE=$3
+NPROCS=$4
+
+source spack/share/spack/setup-env.sh
+spack env create ${CLUSTER}-${SYS_TYPE} spack-environments/${CLUSTER}-${SYS_TYPE}/spack.yaml
+spack env activate ${CLUSTER}-${SYS_TYPE}
+spack install -j ${NPROCS}
+spack env deactivate