[WIP] Spack docker containers

.github/workflows/docker_build_tpls.yml

@@ -64,7 +64,6 @@ jobs:
           - name: Pangea 3 (AlmaLinux 8.8, gcc 9.4.0, open-mpi 4.1.2, cuda 11.5.0, openblas 0.3.10)
             DOCKER_REPOSITORY: geosx/pangea3-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18
             TPL_DOCKERFILE: docker/TotalEnergies/Pangea3.Dockerfile
-            HOST_CONFIG: host-configs/TOTAL/pangea3-gcc8.4.1-openmpi-4.1.2.cmake
             DOCKER_ROOT_IMAGE: 7g8efcehpff/pangea-almalinux8-gcc9.4-openmpi4.1.2-cuda11.5.0-openblas0.3.18:4
             HOST_ARCH: ppc64le
             RUNS_ON: streak2
@@ -95,6 +94,7 @@ jobs:
         DOCKER_COMPILER_BUILD_ARG: ${{ matrix.DOCKER_COMPILER_BUILD_ARG }}
         INSTALL_DIR_ROOT: ${{ matrix.INSTALL_DIR_ROOT || '/opt/GEOS' }}
         HOST_CONFIG: ${{ matrix.HOST_CONFIG || 'host-configs/environment.cmake'}}
+        SPEC: ${{ matrix.SPEC || 'undefined' }}
         DOCKER_ROOT_IMAGE: ${{ matrix.DOCKER_ROOT_IMAGE || 'undefined' }}
         COMMIT: ${{ github.sha }}
         BUILD_DIR: ${{ github.workspace }}

.gitignore

@@ -2,4 +2,6 @@ build-*/
 install-*/
 .cproject
 .project
-.pydevproject
+.pydevproject
+spack-*.txt
+__pycache__

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "cmake/blt"]
 	path = cmake/blt
 	url = ../../LLNL/blt.git
+[submodule "scripts/uberenv"]
+	path = scripts/uberenv
+	url = ../../LLNL/uberenv.git

.uberenv_config.json

@@ -0,0 +1,11 @@
+{
+"package_name": "geosx",
+"package_version": "develop",
+"package_final_phase": "lvarray_hostconfig",
+"package_source_dir": "../..",
+"spack_url": "https://github.com/spack/spack",
+"spack_commit": "d66dce2d668a6234504594661506cdd1eaca4adc",
+"spack_configs_path": "scripts/spack_configs",
+"spack_packages_path": "scripts/spack_packages/packages",
+"spack_concretizer": "clingo"
+}

docker/Stanford/Dockerfile

@@ -15,7 +15,18 @@ RUN yum install -y \
     ca-certificates \
     curl \
     python3 \
-    zlib-devel
+    zlib-devel \
+# Additional spack dependencies
+    python3-pip \
+    pkgconfig \
+    # xz \
+    unzip \
+    bzip2 \
+    && pip3 install virtualenv
+
+# Install clingo for Spack
+RUN python3 -m pip install --upgrade pip && \
+    python3 -m pip install clingo
 
 RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh
 
@@ -29,20 +40,43 @@ RUN yum install -y \
     file \
     bison \
     flex \
-    patch
+    patch \
+    ca-certificates \
+    autoconf \
+    automake \
+    git
 
-ARG HOST_CONFIG
+ARG SPEC
 
-RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh
-WORKDIR $BLD_DIR
-RUN --mount=src=.,dst=$SRC_DIR make
+# Run uberenv
+# Have to create install directory first for uberenv
+# -k flag is to ignore SSL errors
+RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \
+     mkdir -p ${GEOSX_TPL_DIR} && \
+     ./scripts/uberenv/uberenv.py \
+       # spack spec defined per image
+       --spec "${SPEC}" \
+       --spack-env-file=${SRC_DIR}/docker/stanford-spack.yaml \
+       --project-json=.uberenv_config.json \
+       --prefix ${GEOSX_TPL_DIR} \
+       -k && \
+# Remove host-config generated for LvArray
+     rm lvarray* && \
+# Rename and copy spack-generated host-config to root directory
+     cp *.cmake /spack-generated.cmake && \
+# Remove extraneous spack files
+     cd ${GEOSX_TPL_DIR} && \
+     rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/
 
 FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain
 ARG SRC_DIR
 
 COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR
 
-RUN yum -y install \
+# Extract the generated host-config
+COPY --from=tpl_toolchain /spack-generated.cmake /
+
+RUN yum install -y \
     openssh-clients \
     ca-certificates \
     curl \

docker/TotalEnergies/Dockerfile

@@ -11,38 +11,72 @@ ARG SRC_DIR
 ARG INSTALL_DIR
 ENV GEOSX_TPL_DIR=$INSTALL_DIR
 
-RUN yum install -y \
+RUN yum install --disablerepo=intel-mkl-repo -y \
     ca-certificates \
     curl \
     python3 \
-    zlib-devel
+    zlib-devel \
+# Additional spack dependencies
+    python3-pip \
+    pkgconfig \
+    # xz \
+    unzip \
+    bzip2 \
+    && pip3 install virtualenv
+
+# Install clingo for Spack
+RUN python3 -m pip install --upgrade pip && \
+    python3 -m pip install clingo
 
 RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh
 
 FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain
 ARG SRC_DIR
 ARG BLD_DIR
 
-RUN yum install -y \
+RUN yum install --disablerepo=intel-mkl-repo -y \
     make \
     bc \
     file \
     bison \
     flex \
-    patch
+    patch \
+    ca-certificates \
+    autoconf \
+    automake \
+    git
 
-ARG HOST_CONFIG
+ARG SPEC
 
-RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh
-WORKDIR $BLD_DIR
-RUN --mount=src=.,dst=$SRC_DIR make
+# Run uberenv
+# Have to create install directory first for uberenv
+# -k flag is to ignore SSL errors
+RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \
+     mkdir -p ${GEOSX_TPL_DIR} && \
+     ./scripts/uberenv/uberenv.py \
+       # spack spec defined per image
+       --spec "${SPEC}" \
+       --spack-env-file=${SRC_DIR}/docker/total-spack.yaml \
+       --project-json=.uberenv_config.json \
+       --prefix ${GEOSX_TPL_DIR} \
+       -k && \
+# Remove host-config generated for LvArray
+     rm lvarray* && \
+# Rename and copy spack-generated host-config to root directory
+     cp *.cmake /spack-generated.cmake && \
+# Remove extraneous spack files
+     cd ${GEOSX_TPL_DIR} && \
+     rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/
 
 FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain
 ARG SRC_DIR
 
 COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR
 
-RUN yum -y install \
+# Extract the generated host-config
+COPY --from=tpl_toolchain /spack-generated.cmake /
+
+RUN yum install --disablerepo=intel-mkl-repo -y \
     openssh-clients \
     ca-certificates \
     curl \

docker/TotalEnergies/Pangea3.Dockerfile

@@ -10,6 +10,16 @@ FROM $DOCKER_ROOT_IMAGE as tpl_toolchain_intersect_geosx_toolchain
 ARG SRC_DIR
 ARG BLD_DIR
 
+# Install additional required packages for spack
+RUN dnf clean all && \
+    dnf -y update && \
+    dnf -y install \
+        autoconf \
+        automake \
+        libtool \
+        bzip2 \
+        unzip
+
 # All the environment variables defined in this Dockerfile
 # (GEOSX_TPL_DIR but also compiler information like CC, CXX...)
 # are part of the image contract (otherwise ARG is used).
@@ -21,18 +31,65 @@ ARG INSTALL_DIR
 ENV GEOSX_TPL_DIR=$INSTALL_DIR
 
 # Get host config file from docker build arguments
-ARG HOST_CONFIG
+#ARG HOST_CONFIG
 
 FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain
+
 # We now configure the build...
-RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh
+#RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh
 # ... before we compile the TPLs!
-WORKDIR $BLD_DIR
-RUN --mount=src=.,dst=$SRC_DIR make
+#WORKDIR $BLD_DIR
+#RUN --mount=src=.,dst=$SRC_DIR make
+
+# Run uberenv
+# Have to create install directory first for uberenv
+# -k flag is to ignore SSL errors
+RUN --mount=src=.,dst=$SRC_DIR,readwrite cd ${SRC_DIR} && \
+     mkdir -p ${GEOSX_TPL_DIR} && \
+# Create symlink to existing libraries sp spack can find
+     ln -s /usr/lib64/libhwloc.so.15 /usr/lib64/libhwloc.so && \
+     ln -s /usr/lib64/libibverbs.so.1.14.48.0 /usr/lib64/libibverbs.so && \
+     ln -s /usr/lib64/libnsl.so.2.0.0  /usr/lib64/libnsl.so && \
+     ln -s /usr/lib64/librdmacm.so.1.3.48.0  /usr/lib64/librdmacm.so && \
+     ln -s /usr/lib64/liblustreapi.so.1 /usr/lib64/liblustreapi.so && \
+# Run uberenv
+     ./scripts/uberenv/uberenv.py \
+       --spec "%[email protected]+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^[email protected]+allow-unsupported-compilers ^[email protected]~gotcha~sampler~libunwind~libdw~papi" \
+       --spack-env-file=${SRC_DIR}/docker/pangea-spack.yaml \
+       --project-json=.uberenv_config.json \
+       --prefix ${GEOSX_TPL_DIR} \
+       -k && \
+# Remove host-config generated for LvArray
+     rm lvarray* && \
+# Rename and copy spack-generated host-config to root directory
+     cp *.cmake /spack-generated-wave-solver-only.cmake && \
+# Remove extraneous spack files
+     cd ${GEOSX_TPL_DIR} && \
+     rm -rf bin/ build_stage/ misc_cache/ spack/ spack_env/ .spack-db/
+
+# Build only the wave solver for Pangea 3
+RUN echo 'set ( GEOS_ENABLE_CONTACT OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \
+    echo 'set ( GEOS_ENABLE_FLUIDFLOW OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \
+    echo 'set ( GEOS_ENABLE_INDUCEDSEISMICITY OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \
+    echo 'set ( GEOS_ENABLE_MULTIPHYSICS OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \
+    echo 'set ( GEOS_ENABLE_SIMPLEPDE OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \
+    echo 'set ( GEOS_ENABLE_SOLIDMECHANICS OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake && \
+    echo 'set ( GEOS_ENABLE_SURFACEGENERATION OFF CACHE BOOL "" FORCE )' >> /spack-generated-wave-solver-only.cmake
+
 
 # Extract only TPL's from previous stage
 FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain
 
 COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR
 
+# Extract the generated host-config
+COPY --from=tpl_toolchain /spack-generated-wave-solver-only.cmake /
+
+# Regenerate symlinks for existing libraries
+RUN  ln -s /usr/lib64/libhwloc.so.15 /usr/lib64/libhwloc.so && \
+     ln -s /usr/lib64/libibverbs.so.1.14.48.0 /usr/lib64/libibverbs.so && \
+     ln -s /usr/lib64/libnsl.so.2.0.0  /usr/lib64/libnsl.so && \
+     ln -s /usr/lib64/librdmacm.so.1.3.48.0  /usr/lib64/librdmacm.so && \
+     ln -s /usr/lib64/liblustreapi.so.1 /usr/lib64/liblustreapi.so
+
 ENV SCCACHE=/opt/sccache/bin/sccache

docker/install-cmake.sh

@@ -5,5 +5,5 @@ set -xe
 VERSION=${1-3.28.3}
 PREFIX=${2-/usr/local}
 
-curl -s https://cmake.org/files/v${VERSION%.[0-9]*}/cmake-$VERSION-linux-x86_64.tar.gz | \
+curl -S https://cmake.org/files/v${VERSION%.[0-9]*}/cmake-$VERSION-linux-x86_64.tar.gz | \
 tar --directory=$PREFIX --strip-components=1 -xzf -