-
Notifications
You must be signed in to change notification settings - Fork 89
Skybridge Large Problem
Glen Hansen edited this page Feb 19, 2016
·
4 revisions
This build is experimental, focused on supporting problems over 2^31 degrees of freedom. It deliberately disables all 32 bit global ordinals in the Tpetra software stack, and eliminates Epetra to reduce build complexity. This build will not support general, generic applications.
module purge
module load gnu/4.9.2
module load intel/15.0
module load intel-mpi/5.1
module load mkl/15.0
module load hdf5/1.8
Note: One needs to load a C++11 - capable g++ compiler module or the Intel compiler will not compile Trilinos properly.
tar xzvf boost_1_60_0.tar.gz
cd boost_1_60_0
./bootstrap.sh \
--with-libraries=signals,regex,filesystem,system,mpi,serialization,thread,program_options,exception \
--prefix=/home/gahanse/intel-5.1/intel-15.0 --with-toolset=intel-linux
Edit the project-config.jam text file to add the MPI line below:
if ! intel-linux in [ feature.values <toolset> ]
{
using intel-linux ;
using mpi : /opt/intel-mpi-5.1/bin/mpiicpc ;
}
Back in terminal, build and install with b2.
./b2 -j 8
./b2 -j 8 install
CC=mpiicc CFLAGS="-axAVX -O3 -L/opt/intel-mpi-5.1/lib" ./configure --64 --prefix=/home/gahanse/intel-5.1/intel-15.0
make -j 8
make install
./configure --prefix=/home/gahanse/intel-5.1/intel-15.0 --enable-parallel CC=mpiicc \
CFLAGS="-fPIC -O3 -axAVX -L/opt/intel-mpi-5.1/lib" CXX=mpiicpc CXXFLAGS="-fPIC -O3 -axAVX -L/opt/intel-mpi-5.1/lib" \
FC=mpiifort FCFLAGS="-fPIC -O3 -axAVX -L/opt/intel-mpi-5.1/lib" --with-zlib=/home/gahanse/intel-5.1/intel-15.0
make -j 8
make install
Edit include/netcdf.h to change the following lines (near line 265):
#define NC_MAX_DIMS 65536 /* max dimensions per file */
#define NC_MAX_ATTRS 8192
#define NC_MAX_VARS 524288 /* max variables per file */
#define NC_MAX_NAME 256
#define NC_MAX_VAR_DIMS 8 /* max per variable dimensions */
./configure --prefix=/home/gahanse/intel-5.1/intel-15.0 --enable-netcdf-4 --disable-dap CC=mpiicc CXX=mpiicpc FC=mpiifort \
CPPFLAGS="-fPIC -axAVX -O3 -L/opt/intel-mpi-5.1/lib -I/home/gahanse/intel-5.1/intel-15.0/include" \
CFLAGS="-fPIC -axAVX -O3 -L/opt/intel-mpi-5.1/lib -I/home/gahanse/intel-5.1/intel-15.0/include" \
LDFLAGS="-fPIC -L/opt/intel-mpi-5.1/lib -L/home/gahanse/intel-5.1/intel-15.0/lib" \
CXXFLAGS="-fPIC -axAVX -O3 -L/opt/intel-mpi-5.1/lib -I/home/gahanse/intel-5.1/intel-15.0/include" \
FCFLAGS="-fPIC -axAVX -O3 -L/opt/intel-mpi-5.1/lib -I/home/gahanse/intel-5.1/intel-15.0/include"
make -j 8
make install
Edit metis.h to use an IDXTYPEWIDTH of 64
cd metis/include
vi metis.h
#define IDXTYPEWIDTH 64
#define REALTYPEWIDTH 64
cd ../../build
export metis=/home/gahanse/src/parmetis-4.0.3/metis
cmake \
-DCMAKE_INSTALL_PREFIX=/home/gahanse/intel-5.1/intel-15.0 \
-DMETIS_PATH=$metis \
-DGKLIB_PATH=$metis/GKlib \
-DCMAKE_C_COMPILER=mpiicc \
-DCMAKE_CXX_COMPILER=mpiicpc \
-DCMAKE_C_FLAGS="-fPIC -axAVX -O3 -L/opt/intel-mpi-5.1/lib" \
-DCMAKE_CXX_FLAGS="-fPIC -axAVX -O3 -L/opt/intel-mpi-5.1/lib" \
..
make -j 8
make -j 8 install
cp libmetis/libmetis.a /home/gahanse/intel-5.1/intel-15.0/lib
cp $metis/include/metis.h /home/gahanse/intel-5.1/intel-15.0/include
SuperLU is usually optional in a Trilinos build. In this Epetra-free build however, we currently disable Amesos-KLU in the Amesos2 package that is used for a coarse solver / smoother in MueLu. Thus, we must build an alternative, in this case SuperLU; and configure Trilinos to use it.
-
Version 5.1.1
mkdir /home/gahanse/intel-5.1/intel-15.0/SuperLU_4.3 mkdir /home/gahanse/intel-5.1/intel-15.0/SuperLU_4.3/include mkdir /home/gahanse/intel-5.1/intel-15.0/SuperLU_4.3/lib -
Edit
make.incfor your machine and environmentSuperLUroot = /home/gahanse/intel-5.1/intel-15.0/SuperLU_5.1.1 SUPERLULIB = $(SuperLUroot)/lib/libsuperlu_5.1.1.a TMGLIB = libtmglib.a BLASLIB = -L/opt/intel-15.0/mkl/lib/intel64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core LIBS = $(SUPERLULIB) $(BLASLIB) ARCH = /usr/bin/ar ARCHFLAGS = cr RANLIB = /usr/bin/ranlib CC = mpiicc -L/opt/intel-mpi-5.1/lib CFLAGS = -fPIC -axAVX -O3 -DPRNTlevel=0 -DAdd_ NOOPTS = -O0 FORTRAN = mpiifort -L/opt/intel-mpi-5.1/lib FFLAGS = -fPIC -axAVX -O3 LOADER = $(CC) LOADOPTS = -
Build SuperLU
make -j 4 cd SRC cp *.h /home/gahanse/intel-5.1/intel-15.0/SuperLU_5.1.1/include
\#!/bin/sh
\#
export TRILINOS_HOME=/ascldap/users/gahanse/src/Trilinos
BUILD_DIR=`pwd`
INSTALL_DIR=/ascldap/users/gahanse/trilinos/MPI_REL
BOOST_DIR=/home/gahanse/intel-5.1/intel-15.0
LIB_DIR=/home/gahanse/intel-5.1/intel-15.0
MPI_BASE_DIR=/opt/intel-mpi-5.1
NETCDF=/home/gahanse/intel-5.1/intel-15.0
HDFDIR=/home/gahanse/intel-5.1/intel-15.0
ZLIB_DIR=/home/gahanse/intel-5.1/intel-15.0
PARMETISDIR=/home/gahanse/intel-5.1/intel-15.0
SUPERLU_DIR=/home/gahanse/intel-5.1/intel-15.0/SuperLU_5.1.1
MKL_PATH=/opt/intel-15.0/mkl/lib/intel64
LABLAS_LIBRARIES="-L$MKL_PATH -Wl,--start-group $MKL_PATH/libmkl_intel_lp64.a $MKL_PATH/libmkl_core.a $MKL_PATH/libmkl_sequential.a -Wl,--end-group"
cmake \
-Wno-dev \
-D Trilinos_ENABLE_ALL_PACKAGES:BOOL=OFF \
-D Trilinos_DISABLE_ENABLED_FORWARD_DEP_PACKAGES=OFF \
-D Trilinos_ENABLE_SCOREC:BOOL=ON \
-D PCU_COMPRESS:BOOL=ON \
-D SCOREC_DISABLE_STRONG_WARNINGS:BOOL=ON \
-D BUILD_SHARED_LIBS:BOOL=ON \
-D CMAKE_BUILD_TYPE:STRING=NONE \
-D Trilinos_SHOW_DEPRECATED_WARNINGS:BOOL=OFF \
-D CMAKE_CXX_COMPILER:STRING=mpiicpc \
-D CMAKE_C_COMPILER:STRING=mpiicc \
-D CMAKE_Fortran_COMPILER:STRING=mpiifort \
-D CMAKE_CXX_FLAGS:STRING="-axAVX -O3 -L/opt/intel-mpi-5.1/lib" \
-D CMAKE_C_FLAGS:STRING="-axAVX -O3 -L/opt/intel-mpi-5.1/lib" \
-D CMAKE_Fortran_FLAGS:STRING="-axAVX -O3 -L/opt/intel-mpi-5.1/lib" \
-D Rythmos_ENABLE_DEBUG:BOOL=ON \
-D Trilinos_ENABLE_EXPLICIT_INSTANTIATION:BOOL=ON \
-D Tpetra_INST_INT_LONG_LONG:BOOL=ON \
-D Tpetra_INST_INT_INT:BOOL=OFF \
-D Tpetra_INST_DOUBLE:BOOL=ON \
-D Tpetra_INST_FLOAT:BOOL=OFF \
-D Tpetra_INST_COMPLEX_FLOAT:BOOL=OFF \
-D Tpetra_INST_COMPLEX_DOUBLE:BOOL=OFF \
-D Tpetra_INST_INT_LONG:BOOL=OFF \
-D Tpetra_INST_INT_UNSIGNED:BOOL=OFF \
-D Zoltan_ENABLE_ULONG_IDS:BOOL=ON \
-D MDS_ID_TYPE:STRING="long long int" \
-D Teuchos_ENABLE_LONG_LONG_INT:BOOL=ON \
-D Trilinos_ENABLE_Kokkos:BOOL=ON \
-D Trilinos_ENABLE_KokkosCore:BOOL=ON \
-D Phalanx_KOKKOS_DEVICE_TYPE:STRING="SERIAL" \
-D Phalanx_INDEX_SIZE_TYPE:STRING="LONGINT" \
-D Kokkos_ENABLE_Serial:BOOL=ON \
-D Kokkos_ENABLE_OpenMP:BOOL=OFF \
-D Kokkos_ENABLE_Pthread:BOOL=OFF \
\
-D TPL_ENABLE_MPI:BOOL=ON \
-D MPI_BASE_DIR:PATH=$MPI_BASE_DIR \
\
-D TPL_ENABLE_Pthread:BOOL=OFF \
\
-D TPL_ENABLE_Boost:BOOL=ON \
-D TPL_ENABLE_BoostLib:BOOL=ON \
-D TPL_ENABLE_BoostAlbLib:BOOL=ON \
-D Boost_INCLUDE_DIRS:PATH=$BOOST_DIR/include \
-D Boost_LIBRARY_DIRS:PATH=$BOOST_DIR/lib \
-D BoostLib_INCLUDE_DIRS:PATH=$BOOST_DIR/include \
-D BoostLib_LIBRARY_DIRS:PATH=$BOOST_DIR/lib \
-D BoostAlbLib_INCLUDE_DIRS:PATH=$BOOST_DIR/include \
-D BoostAlbLib_LIBRARY_DIRS:PATH=$BOOST_DIR/lib \
\
-D TPL_ENABLE_Netcdf:STRING=ON \
-D Netcdf_INCLUDE_DIRS:PATH=$NETCDF/include \
-D TPL_Netcdf_LIBRARIES:FILEPATH="${NETCDF}/lib/libnetcdf.a;${HDFDIR}/lib/libhdf5_hl.a;${HDFDIR}/lib/libhdf5.a;${ZLIB_DIR}/lib/libz.a" \
\
-D TPL_ENABLE_HDF5:STRING=ON \
-D HDF5_INCLUDE_DIRS:PATH=$HDFDIR/include \
-D HDF5_LIBRARY_DIRS:PATH=$HDFDIR/lib \
\
-D TPL_ENABLE_Zlib:STRING=ON \
-D Zlib_INCLUDE_DIRS:PATH=$HDFDIR/include \
-D Zlib_LIBRARY_DIRS:PATH=$HDFDIR/lib \
\
-D TPL_ENABLE_SuperLU:STRING=ON \
-D TPL_SuperLU_INCLUDE_DIRS:PATH="${SUPERLU_DIR}/include" \
-D TPL_SuperLU_LIBRARIES:PATH="${SUPERLU_DIR}/lib/libsuperlu_5.2.2.a" \
\
-D TPL_ENABLE_BLAS:STRING=ON \
-D TPL_ENABLE_LAPACK:STRING=ON \
-D TPL_BLAS_LIBRARIES:STRING="$LABLAS_LIBRARIES" \
-D TPL_LAPACK_LIBRARIES:STRING="$LABLAS_LIBRARIES" \
\
-D TPL_ENABLE_ParMETIS:STRING=ON \
-D ParMETIS_INCLUDE_DIRS:PATH="${PARMETISDIR}/include" \
-D ParMETIS_LIBRARY_DIRS:PATH="${PARMETISDIR}/lib" \
\
-D CMAKE_INSTALL_PREFIX:PATH=$INSTALL_DIR \
\
-D Trilinos_ENABLE_TriKota:BOOL=OFF \
-D Trilinos_ENABLE_SECONDARY_TESTED_CODE=OFF \
\
-D Trilinos_ENABLE_Teuchos:BOOL=ON \
-D Trilinos_ENABLE_Shards:BOOL=ON \
-D Trilinos_ENABLE_Sacado:BOOL=ON \
-D Trilinos_ENABLE_Belos:BOOL=ON \
-D Trilinos_ENABLE_Phalanx:BOOL=ON \
-D Trilinos_ENABLE_Intrepid:BOOL=ON \
-D Trilinos_ENABLE_Intrepid2:BOOL=ON \
-D Trilinos_ENABLE_NOX:BOOL=ON \
-D Trilinos_ENABLE_Stratimikos:BOOL=ON \
-D Stratimikos_ENABLE_ThyraTpetraAdapters:BOOL=ON \
-D Stratimikos_ENABLE_ThyraEpetraAdapters:BOOL=OFF \
-D Trilinos_ENABLE_Thyra:BOOL=ON \
-D Trilinos_ENABLE_Rythmos:BOOL=ON \
-D Trilinos_ENABLE_Stokhos:BOOL=OFF \
-D Trilinos_ENABLE_Piro:BOOL=ON \
-D Piro_ENABLE_Epetra:BOOL=OFF \
-D Trilinos_ENABLE_Teko:BOOL=OFF \
\
-D Trilinos_ENABLE_STKIO:BOOL=ON \
-D Trilinos_ENABLE_STKMesh:BOOL=ON \
-D Trilinos_ENABLE_SEACASIoss:BOOL=ON \
-D Trilinos_ENABLE_SEACASExodus:BOOL=ON \
-D Trilinos_ENABLE_Tpetra:BOOL=ON \
-D Trilinos_ENABLE_Kokkos:BOOL=ON \
-D Trilinos_ENABLE_Ifpack2:BOOL=ON \
-D Trilinos_ENABLE_Amesos2:BOOL=ON \
-D Amesos2_ENABLE_KLU2:BOOL=OFF \
-D Xpetra_ENABLE_Epetra:BOOL=OFF \
-D Trilinos_ENABLE_Zoltan2:BOOL=ON \
-D Trilinos_ENABLE_MueLu:BOOL=ON \
-D MueLu_ENABLE_Epetra:BOOL=OFF \
\
-D Trilinos_ENABLE_GlobiPack:BOOL=OFF \
-D Trilinos_ENABLE_OptiPack:BOOL=OFF \
-D Trilinos_ENABLE_Isorropia:BOOL=OFF \
-D Trilinos_ENABLE_Galeri:BOOL=OFF \
-D Trilinos_ENABLE_Pamgen:BOOL=OFF \
-D Trilinos_ENABLE_ML:BOOL=OFF \
-D Trilinos_ENABLE_Epetra:BOOL=OFF \
-D Trilinos_ENABLE_EpetraExt:BOOL=OFF \
-D Trilinos_ENABLE_Ifpack:BOOL=OFF \
-D Trilinos_ENABLE_AztecOO:BOOL=OFF \
-D Trilinos_ENABLE_Amesos:BOOL=OFF \
-D Trilinos_ENABLE_Anasazi:BOOL=ON \
\
$EXTRA_ARGS \
${TRILINOS_HOME}
exit 0
make -j 16 make -j 16 install
cd $HOME/Albany
mkdir build
cd build
Modify the following buildAlbany.sh script to reflect how you will use Albany.
\#!/bin/sh
export ALB_64BIT_INT=ON
export ALB_EPETRA=OFF
export ALB_ENABLE_SCOREC=ON
export ALB_ENABLE_LCM=ON
export ALB_ENABLE_LAME=OFF
export ALB_ENABLE_HYDRIDE=ON
export ALB_ENABLE_QCAD=OFF
export ALB_ENABLE_MOR=OFF
export ALB_ENABLE_ASCR=OFF
export ALB_ENABLE_AERAS=ON
export ALB_ENABLE_FELIX=ON
export ALB_ENABLE_AlbanyCI=OFF
export ALB_DEBUG=OFF
export ALB_ENABLE_CHECK_FPE=OFF
export ALB_ENABLE_SPECULATIVE=OFF
export ALB_ENABLE_SG_MP=OFF
export ALB_ENABLE_STOKHOS=OFF
export ALB_ENABLE_PERF_TESTS=ON
export TRILINOS_INSTALL_DIR=/home/gahanse/trilinos/MPI_REL
export ALB_LAME_INC_DIR=/projects/albany/src/lame-4.24.1/include
export ALB_LAME_LIB_DIR=/projects/albany/src/lame-4.24.1/build
export ALB_CROSS_COMPILE=ON
export ALB_VERBOSE=OFF
cmake \
-D ENABLE_CROSS_COMPILE:BOOL=${ALB_CROSS_COMPILE} \
-D ALBANY_TRILINOS_DIR:FILEPATH="$TRILINOS_INSTALL_DIR" \
-D ENABLE_LCM:BOOL=${ALB_ENABLE_LCM} \
-D ENABLE_AERAS:BOOL=${ALB_ENABLE_AERAS} \
-D ENABLE_QCAD:BOOL=${ALB_ENABLE_QCAD} \
-D ENABLE_HYDRIDE:BOOL=${ALB_ENABLE_HYDRIDE} \
-D ENABLE_LCM_SPECULATIVE:BOOL=${ALB_ENABLE_SPECULATIVE} \
-D ENABLE_LAME:BOOL=${ALB_ENABLE_LAME} \
-D CMAKE_VERBOSE_MAKEFILE:BOOL=${ALB_VERBOSE} \
-D ENABLE_DEBUGGING:BOOL=${ALB_DEBUG} \
-D ENABLE_CHECK_FPE:BOOL=${ALB_ENABLE_CHECK_FPE} \
-D ENABLE_SCOREC:BOOL=${ALB_ENABLE_SCOREC} \
-D ENABLE_FELIX:BOOL=${ALB_ENABLE_FELIX} \
-D ENABLE_MOR:BOOL=${ALB_ENABLE_MOR} \
-D ENABLE_ALBANY_CI:BOOL=${ALB_ENABLE_AlbanyCI} \
-D ENABLE_ASCR:BOOL=${ALB_ENABLE_ASCR} \
-D ENABLE_SG_MP:BOOL=${ALB_ENABLE_SG_MP} \
-D ENABLE_STOKHOS:BOOL=${ALB_ENABLE_STOKHOS} \
-D ENABLE_PERFORMANCE_TESTS:BOOL=${ALB_ENABLE_PERF_TESTS} \
-D LAME_INCLUDE_DIR:FILEPATH="$ALB_LAME_INC_DIR" \
-D LAME_LIBRARY_DIR:FILEPATH="$ALB_LAME_LIB_DIR" \
-D ALBANY_CTEST_TIMEOUT:INTEGER=70 \
-D ENABLE_64BIT_INT:BOOL=${ALB_64BIT_INT} \
-D ENABLE_ALBANY_EPETRA_EXE:BOOL=${ALB_EPETRA} \
-D ALBANY_MPI_OPTIONS:BOOL=ON \
-D ALBANY_MPI_EXEC:STRING=srun \
-D ALBANY_MPI_EXEC_NUMPROCS_FLAG:STRING=-n \
-D ALBANY_MPI_EXEC_MAX_NUMPROCS:STRING=4 \
-D ALBANY_MPI_TRAILING_OPTIONS:STRING="" \
\
..
make -j 16
Might need to add various shared libraried to LD_LIBRARY_PATH
export LD_LIBRARY_PATH=/ascldap/users/gahanse/trilinos/MPI_REL/lib:/ascldap/users/gahanse/intel-5.1/intel-15.0/lib:${LD_LIBRARY_PATH}
Setup an allocation to test in
salloc -N1 --time=1:00:00 --account=xxxxxxxx -p xxx
Run tests
ctest