Try clang@10+cuda

.github/workflows/docker_build_tpls.yml

@@ -74,9 +74,9 @@ jobs:
             DOCKER_REPOSITORY: geosx/ubuntu22.04-clang15
             DOCKERFILE: docker/tpl-ubuntu-clang.Dockerfile
             DOCKER_COMPILER_BUILD_ARG: "--build-arg CLANG_MAJOR_VERSION=15 --build-arg GCC_MAJOR_VERSION=11"
-          # - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89)
-          #   DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89
-          #   DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile
+          - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89)
+            DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89
+            DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile
           # - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.89)
           #   DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89
           #   DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile

docker/spack.yaml

@@ -26,6 +26,19 @@ spack:
         fc: /usr/bin/gfortran-11
       spec: clang@15
       target: x86_64
+  - compiler:
+      environment: {}
+      extra_rpaths: []
+      flags: {}
+      modules: []
+      operating_system: ubuntu20.04
+      paths:
+        cc: /usr/bin/clang
+        cxx: /usr/bin/clang++
+        f77: /usr/bin/gfortran
+        fc: /usr/bin/gfortran
+      spec: clang@10
+      target: x86_64
   - compiler:
       environment: {}
       extra_rpaths: []
@@ -145,6 +158,11 @@ spack:
       externals:
       - spec: [email protected]
         prefix: /usr/local
+    cuda:
+      buildable: False
+      externals:
+      - spec: [email protected] +allow-unsupported-compilers
+        prefix: /usr/local/cuda
     flex:
       version: [2.6.4]
       buildable: false

docker/tpl-ubuntu-clang-cuda.Dockerfile

@@ -24,63 +24,77 @@ RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime && \
         openmpi-bin \
         libopenmpi-dev \
         python3 \
-        clang
+        clang \
+# Additional spack dependencies
+        python3-pip \
+        pkg-config \
+        xz-utils \
+        unzip \
+        libmpfr-dev \
+        lbzip2 \
+        virtualenv
+
+# Install clingo for Spack
+RUN python3 -m pip install --upgrade pip && \
+    python3 -m pip install clingo
 
 RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh
 
-ENV CC=/usr/bin/clang \
-    CXX=/usr/bin/clang++ \
-    MPICC=/usr/bin/mpicc \
-    MPICXX=/usr/bin/mpicxx \
-    MPIEXEC=/usr/bin/mpirun
-ENV OMPI_CC=$CC \
-    OMPI_CXX=$CXX 
-ENV ENABLE_CUDA=ON \
-    CMAKE_CUDA_FLAGS="-restrict -arch sm_70 --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations"
-
 # Installing TPL's
 FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain
 ARG SRC_DIR
 ARG BLD_DIR
 
-ENV FC=/usr/bin/gfortran \
-    MPIFC=/usr/bin/mpifort
-ENV OMPI_FC=$FC
-
 RUN apt-get install -y --no-install-recommends \
     libtbb-dev \
     bc \
     file \
     bison \
     flex \
-    patch
-
-ARG HOST_CONFIG
-
-ARG CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
-ARG CUDA_ARCH=sm_70
-ARG CMAKE_CUDA_COMPILER=$CUDA_TOOLKIT_ROOT_DIR/bin/nvcc
-ARG CMAKE_CUDA_ARCHITECTURES=70
-
-ENV HYPRE_CUDA_SM=70
-ENV CUDA_HOME=$CUDA_TOOLKIT_ROOT_DIR
+    patch \
+# `ca-certificates`  needed by `git` to download GEOS repo.
+    ca-certificates \
+    git
+
+# Clone branch with spack configs
+# TODO decide landing place of spack recipes
+RUN git clone --branch feature/han12/wip_docker_spack_recipes \
+              --depth 1 \
+          --single-branch \
+          https://github.com/GEOS-DEV/GEOS.git
+
+# Run uberenv
+# Have to create install directory first for uberenv
+# -k flag is to ignore SSL errors
+# Remove generated host-config for lvarray
+ RUN --mount=src=.,dst=$SRC_DIR cd GEOS && \
+     mkdir -p ${GEOSX_TPL_DIR} && \
+     git submodule init scripts/uberenv && \
+     git submodule update && \
+     ./scripts/uberenv/uberenv.py \
+       --spec "%clang@10+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^[email protected]+allow-unsupported-compilers ^[email protected]~gotcha~sampler~libunwind~libdw~papi" \
+       --spack-env-file=${SRC_DIR}/docker/spack.yaml \
+       --project-json=.uberenv_config.json \
+       --prefix ${GEOSX_TPL_DIR} \
+       -k && \
+# Remove host-config generated for LvArray
+     rm lvarray* && \
+# Rename and copy spack-generated host-config to root directory
+     cp *.cmake /spack-generated.cmake && \
+# Remove extraneous spack files
+     cd ${GEOSX_TPL_DIR} && \
+     rm -rf bin/ build_stage/ misc_cache/ spack/ spack-env/ .spack-db/
 
-RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh \
-    -DENABLE_CUDA=$ENABLE_CUDA \
-    -DENABLE_HYPRE_DEVICE="CUDA" \
-    -DCUDA_TOOLKIT_ROOT_DIR=$CUDA_TOOLKIT_ROOT_DIR \
-    -DCUDA_ARCH=$CUDA_ARCH \
-    -DCMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES \
-    -DCMAKE_CUDA_COMPILER=$CMAKE_CUDA_COMPILER
-WORKDIR $BLD_DIR
-RUN --mount=src=.,dst=$SRC_DIR make
 
 # Extract only TPL's from previous stage
 FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain
 ARG SRC_DIR
 
 COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR
 
+# Extract the generated host-config
+COPY --from=tpl_toolchain /spack-generated.cmake /
+
 RUN apt-get install -y --no-install-recommends \
     openssh-client \
     ca-certificates \