Use caliper

.gitlab-ci.yml

@@ -24,6 +24,7 @@
 
 # We define the following GitLab pipeline variables:
 variables:
+# Required information about GitHub repository
   GITHUB_PROJECT_NAME: "RAJAPerf"
   GITHUB_PROJECT_ORG: "LLNL"
 # Use the umdev service user to run CI. This prevents from running pipelines as

.gitlab/corona-build-and-test-extra.yml

@@ -29,5 +29,12 @@ variables:
 # INFO: This job is activated in RAJA CI, but we don't use desul atomics here
 #rocmcc_5_1_1_hip_desul_atomics:
 #  variables:
-#    SPEC: "+rocm~openmp +desul amdgpu_target=gfx906 %[email protected] ^blt@develop ^[email protected]"
+#    SPEC: "~openmp +rocm +desul amdgpu_target=gfx906 %[email protected] ^[email protected] ^blt@develop"
 #  extends: .build_and_test_on_corona
+
+# EXTRA
+rocmcc_5_1_1_hip_caliper:
+  variables:
+    SPEC: "~openmp +caliper +rocm amdgpu_target=gfx906 %[email protected] ^[email protected] ^blt@develop"
+  extends: .build_and_test_on_corona
+

.gitlab/custom-jobs-and-variables.yml

@@ -16,9 +16,9 @@ variables:
 
 # Ruby
 # Arguments for top level allocation
-  RUBY_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --partition=pdebug --time=45 --nodes=1"
+  RUBY_BUILD_AND_TEST_SHARED_ALLOC: "--exclusive --reservation=ci --qos=ci_ruby --time=45 --nodes=1"
 # Arguments for job level allocation
-  RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=30 --nodes=1"
+  RUBY_BUILD_AND_TEST_JOB_ALLOC: "--reservation=ci --qos=ci_ruby --time=30 --nodes=1"
 # Project specific variants for ruby
   PROJECT_RUBY_VARIANTS: "+openmp "
 # Project specific deps for ruby

.gitlab/lassen-build-and-test-extra.yml

@@ -23,7 +23,15 @@ ibm_clang_9_0_0:
 # Overriding shared spec: Allow failures
 ibm_clang_9_0_0_gcc_8_3_1:
   variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} %[email protected] cxxflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ${PROJECT_LASSEN_DEPS}"
+    SPEC: " ${PROJECT_LASSEN_VARIANTS} %[email protected] cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ${PROJECT_LASSEN_DEPS}"
+  extends: .build_and_test_on_lassen
+  allow_failure: true
+
+# Overriding shared spec: Longer allocation + Allow failures
+ibm_clang_9_0_0_gcc_8_3_1_cuda_10_1_168:
+  variables:
+    SPEC: " ${PROJECT_LASSEN_VARIANTS} +cuda %[email protected] cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cuda_arch=70 ^[email protected] ${PROJECT_LASSEN_DEPS}"
+    LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 60"
   extends: .build_and_test_on_lassen
   allow_failure: true
 
@@ -38,28 +46,28 @@ ibm_clang_9_0_0_gcc_8_3_1_cuda_10_1_168:
 # Overriding shared spec: Extra flags
 gcc_8_3_1:
   variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} %[email protected] cxxflags=\"-finline-functions -finline-limit=20000\" cflags=\"-finline-functions -finline-limit=20000\" ${PROJECT_LASSEN_DEPS}"
+    SPEC: " ${PROJECT_LASSEN_VARIANTS} %[email protected] cxxflags==\"-finline-functions -finline-limit=20000\" cflags==\"-finline-functions -finline-limit=20000\" ${PROJECT_LASSEN_DEPS}"
   extends: .build_and_test_on_lassen
 
 # Overriding shared spec: Longer allocation + Allow failures
 pgi_20_4_gcc_8_3_1:
-  extends: .build_and_test_on_lassen
   variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} %[email protected] cxxflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ${PROJECT_LASSEN_DEPS}"
+    SPEC: " ${PROJECT_LASSEN_VARIANTS} %[email protected] cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ${PROJECT_LASSEN_DEPS}"
     LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 60"
+  extends: .build_and_test_on_lassen
   allow_failure: true
 
 # Overriding shared spec: Longer allocation + Extra flags
 xl_16_1_1_12:
   variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} %[email protected] cxxflags=\"-qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qsmp=omp -qnoeh -qsuppress=1500-029 -qsuppress=1500-036 ${PROJECT_LASSEN_DEPS}\""
+    SPEC: " ${PROJECT_LASSEN_VARIANTS} %[email protected] cxxflags==\"-qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qsmp=omp -qnoeh -qsuppress=1500-029 -qsuppress=1500-036 ${PROJECT_LASSEN_DEPS}\""
     LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 50"
   extends: .build_and_test_on_lassen
 
 # Overriding shared spec: Longer allocation + Extra flags
 xl_16_1_1_12_gcc_8_3_1:
   variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} %[email protected] cxxflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 -qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qsmp=omp -qnoeh -qsuppress=1500-029 -qsuppress=1500-036\" cflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ${PROJECT_LASSEN_DEPS}"
+    SPEC: " ${PROJECT_LASSEN_VARIANTS} %[email protected] cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 -qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qsmp=omp -qnoeh -qsuppress=1500-029 -qsuppress=1500-036\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ${PROJECT_LASSEN_DEPS}"
     LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 50"
   extends: .build_and_test_on_lassen
 
@@ -80,14 +88,14 @@ xl_16_1_1_12_gcc_7_3_1_cuda_10_1_168:
 # Overriding shared spec: Extra flags + Longer allocation + Allow failure
 xl_16_1_1_12_cuda_11_1_0:
   variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} +cuda %[email protected] cxxflags=\"-qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qpic -qsmp=omp -qsuppress=1500-029 -qsuppress=1500-036\" cuda_arch=70 ^[email protected] ${PROJECT_LASSEN_DEPS}"
+    SPEC: " ${PROJECT_LASSEN_VARIANTS} +cuda %[email protected] cxxflags==\"-qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qpic -qsmp=omp -qsuppress=1500-029 -qsuppress=1500-036\" cuda_arch=70 ^[email protected] ${PROJECT_LASSEN_DEPS}"
     LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 60"
   extends: .build_and_test_on_lassen
 
 # Overriding shared spec: Extra flags + Longer allocation + Allow failure
 xl_16_1_1_12_gcc_8_3_1_cuda_11_1_0:
   variables:
-    SPEC: " ${PROJECT_LASSEN_VARIANTS} +cuda %[email protected] cxxflags\"=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 -qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qpic -qsmp=omp -qsuppress=1500-029 -qsuppress=1500-036\" cflags=--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 cuda_arch=70 ^[email protected] ${PROJECT_LASSEN_DEPS}"
+    SPEC: " ${PROJECT_LASSEN_VARIANTS} +cuda %[email protected] cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1 -qthreaded -std=c++14 -O3 -qstrict -qxlcompatmacros -qlanglvl=extended0x -qalias=noansi -qhot -qpic -qsmp=omp -qsuppress=1500-029 -qsuppress=1500-036\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cuda_arch=70 ^[email protected] ${PROJECT_LASSEN_DEPS}"
     LASSEN_BUILD_AND_TEST_JOB_ALLOC: "1 -W 60"
   extends: .build_and_test_on_lassen
 
@@ -114,7 +122,7 @@ clang_14_0_5:
 
 clang_12_0_1_cuda_11_5_0:
   variables:
-    SPEC: " +openmp +cuda cuda_arch=70 %[email protected] cxxflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags=\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ^[email protected]"
+    SPEC: " +openmp +cuda cuda_arch=70 %[email protected] cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ^[email protected]"
   extends: .build_and_test_on_lassen
 
 gcc_8_3_1_cuda_11_1_0:
@@ -132,14 +140,19 @@ gcc_8_3_1_cuda_11_5_0_ats_disabled:
 # EXTRAS
 ##########
 
+clang_12_0_1_cuda_11_5_0_caliper:
+  variables:
+    SPEC: " +openmp +caliper +cuda cuda_arch=70 %[email protected] cxxflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" cflags==\"--gcc-toolchain=/usr/tce/packages/gcc/gcc-8.3.1\" ^[email protected]"
+  extends: .build_and_test_on_lassen
+
 clang_13_0_1_libcpp:
   variables:
     SPEC: " +openmp %[email protected]+libcpp"
   extends: .build_and_test_on_lassen
 
 clang_14_0_5_asan:
   variables:
-    SPEC: " +openmp %[email protected] cxxflags=-fsanitize=address"
+    SPEC: " +openmp %[email protected] cxxflags==\"-fsanitize=address\""
     ASAN_OPTIONS: "detect_leaks=1"
   extends: .build_and_test_on_lassen
 

.gitlab/ruby-build-and-test-extra.yml

@@ -23,7 +23,7 @@ gcc_8_1_0:
 # Overriding shared spec: Allow failures
 pgi_20_1_gcc_local_8_3_1:
   variables:
-    SPEC: " ${PROJECT_RUBY_VARIANTS} %[email protected] cxxflags\"=-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" cflags\"=-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" fflags=\"-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" ${PROJECT_RUBY_DEPS}"
+    SPEC: " ${PROJECT_RUBY_VARIANTS} %[email protected] cxxflags==\"-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" cflags==\"-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" fflags==\"-rc=/usr/workspace/umpire/pgi/x86_64/local-gcc-8.3.1-rc\" ${PROJECT_RUBY_DEPS}"
   extends: .build_and_test_on_ruby
   allow_failure: true
 
@@ -35,8 +35,16 @@ pgi_20_1_gcc_local_8_3_1:
 # ${PROJECT_<MACHINE>_DEPS} in the extra jobs. There is no reason not to fully
 # describe the spec here.
 
+gcc_8_3_1_caliper:
+  variables:
+    SPEC: " +openmp +caliper %[email protected]"
+    RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=60 --nodes=1"
+  extends: .build_and_test_on_ruby
+
 icpc_19_1_0:
   variables:
     SPEC: " +openmp %[email protected]"
     RUBY_BUILD_AND_TEST_JOB_ALLOC: "--time=40 --nodes=1"
+  before_script:
+    - export USE_DEV_SHM=False
   extends: .build_and_test_on_ruby

CMakeLists.txt

@@ -7,7 +7,9 @@
 ###############################################################################
 
 # C is required for googletest to find Threads
-project(RAJAPerfSuite LANGUAGES CXX C)
+cmake_policy(SET CMP0048 NEW)
+
+project(RAJAPerfSuite VERSION 2022.10.0 LANGUAGES CXX C)
 
 if (ENABLE_HIP)
   cmake_minimum_required(VERSION 3.23)
@@ -95,9 +97,10 @@ if (ENABLE_OPENMP)
   add_definitions(-DRUN_OPENMP)
 endif ()
 
-set(RAJA_PERFSUITE_VERSION_MAJOR 2022)
-set(RAJA_PERFSUITE_VERSION_MINOR 10)
-set(RAJA_PERFSUITE_VERSION_PATCHLEVEL 0)
+set(RAJA_PERFSUITE_VERSION_MAJOR ${RAJAPerfSuite_VERSION_MAJOR})
+set(RAJA_PERFSUITE_VERSION_MINOR ${RAJAPerfSuite_VERSION_MINOR})
+set(RAJA_PERFSUITE_VERSION_PATCHLEVEL ${RAJAPerfSuite_VERSION_PATCH})
+message(STATUS "CMAKE_PROJECT_VERSION:" ${CMAKE_PROJECT_VERSION})
 
 set(RAJA_PERFSUITE_DEPENDS RAJA)
 
@@ -124,6 +127,33 @@ if ((ENABLE_HIP) AND (NOT ENABLE_KOKKOS))
   list(APPEND RAJA_PERFSUITE_DEPENDS blt::hip_runtime)
 endif()
 
+#
+# Are we using Caliper
+#
+set(RAJA_PERFSUITE_USE_CALIPER off CACHE BOOL "")
+if (RAJA_PERFSUITE_USE_CALIPER)
+   find_package(caliper REQUIRED)
+   list(APPEND RAJA_PERFSUITE_DEPENDS caliper)
+   add_definitions(-DRAJA_PERFSUITE_USE_CALIPER)
+   message(STATUS "Using Caliper")
+   find_package(adiak REQUIRED)
+   # use ${adiak_LIBRARIES} since version could have adiak vs adiak::adiak export
+   list(APPEND RAJA_PERFSUITE_DEPENDS ${adiak_LIBRARIES})
+   if (ENABLE_CUDA)
+      # Adiak will propagate -pthread from spectrum mpi from a spack install of Caliper with +mpi; and needs to be handled even if RAJAPerf is non MPI program
+      # We should delegate to BLT to handle unguarded -pthread from any dependencies, but currently BLT doesn't
+      set_target_properties(${adiak_LIBRARIES} PROPERTIES INTERFACE_COMPILE_OPTIONS "$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:-pthread>;$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-pthread>")
+      # the following for adiak-0.2.2
+      if (TARGET adiak::mpi)
+        set_target_properties(adiak::mpi PROPERTIES INTERFACE_COMPILE_OPTIONS "$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:-pthread>;$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-pthread>")
+      endif ()
+   endif ()
+   message(STATUS "Caliper includes : ${caliper_INCLUDE_DIR}")
+   message(STATUS "Adiak includes : ${adiak_INCLUDE_DIRS}")
+   include_directories(${caliper_INCLUDE_DIR})
+   include_directories(${adiak_INCLUDE_DIRS})
+endif ()
+
 set(RAJAPERF_BUILD_SYSTYPE $ENV{SYS_TYPE})
 set(RAJAPERF_BUILD_HOST $ENV{HOSTNAME})
 
@@ -151,7 +181,7 @@ else()
 endif()
 
 configure_file(${CMAKE_SOURCE_DIR}/src/rajaperf_config.hpp.in
-  ${CMAKE_CURRENT_BINARY_DIR}/include/rajaperf_config.hpp)
+               ${CMAKE_CURRENT_BINARY_DIR}/include/rajaperf_config.hpp)
 
 include_directories($<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/include>)
 

docs/sphinx/user_guide/build.rst

@@ -210,3 +210,44 @@ sizes. The CMake option for this is
 
 will build versions of GPU kernels that use 64, 128, 256, 512, and 1024 threads
 per GPU thread-block.
+
+Building with Caliper
+---------------------
+
+RAJAPerf Suite may also use Caliper instrumentation, with per variant output into 
+Spot/Hatchet .cali files. Original timing is nested within Caliper annotations and 
+so is not impacted when Caliper support is turned on. While Caliper is low-overhead
+it is not zero, so it will add a small amount of timing skew in its data as 
+compared to the original. For much more on Caliper, read it's documentation here:
+[Caliper] (http://software.llnl.gov/Caliper/)
+
+Caliper *annotation* is in the following tree structure
+
+Variant
+   Group
+     Kernel
+       Kernel.Tuning
+
+| Build against these Caliper versions
+|
+|   **[email protected]** (preferred target)
+|   **caliper@master** (if using older Spack version)
+
+In Cmake scripts add
+  **-DRAJA_PERFSUITE_USE_CALIPER=On** 
+
+Add to **-DCMAKE_PREFIX_PATH**
+  ;${CALIPER_PREFIX}/share/cmake/caliper;${ADIAK_PREFIX}/lib/cmake/adiak
+
+or use 
+  -Dcaliper_DIR -Dadiak_DIR package prefixes
+
+For Spack : raja_perf +caliper ^[email protected]
+
+For Uberenv: python3 scripts/uberenv/uberenv.py --spec +caliper ^[email protected]
+
+If you intend on passing nvtx or roctx annotation to Nvidia or AMD profiling tools, 
+build Caliper with +cuda cuda_arch=XX or +rocm respectively. Then you can specify
+an additional Caliper service for nvtx or roctx like so: roctx example:
+
+CALI_SERVICES_ENABLE=roctx rocprof --roctx-trace --hip-trace raja-perf.exe 

docs/sphinx/user_guide/output.rst

@@ -159,3 +159,61 @@ storing the result in matrix A (N_i X N_j). Problem size could be chosen to be
 the maximum number of entries in matrix B or C. We choose the size of matrix 
 A (N_i * N_j), which is more closely aligned with the number of independent 
 operations (i.e., the amount of parallel work) in the kernels.
+
+
+===========================
+Caliper output files
+===========================
+
+If you've built RAJAPerf with Caliper support turned on, then in addition to the
+outputs mentioned above, we also save a .cali file for each variant run, such as:
+Base_OpenMP.cali, Lambda_OpenMP.cali, RAJA_OpenMP.cali, etc.
+
+There are several techniques to display the Caliper trees (Timing Hierarchy)
+
+| 1: Caliper's cali-query tool.
+| The first technique is with Caliper's own tool cali-query, we run it with 
+| **-T** to display tree, or you can specify **--tree**. 
+|
+| cali-query -T $HOME/data/default_problem_size/gcc/RAJA_Seq.cali
+
+2: Caliper's Python module *caliperreader*::
+
+  import os
+  import caliperreader as cr
+  DATA_DIR = os.getenv('HOME')+"/data/default_problem_size/gcc"
+  os.chdir(DATA_DIR)
+  r = cr.CaliperReader()
+  r.read("RAJA_Seq.cali")
+  metric = 'avg#inclusive#sum#time.duration'
+  for rec in r.records:
+    path = rec['path'] if 'path' in rec else 'UNKNOWN'
+    time = rec[metric] if metric in rec else '0'
+    if not 'UNKNOWN' in path:
+        if (isinstance(path, list)):
+            path = "/".join(path)
+        print("{0}: {1}".format(path, time))
+
+You can add a couple of lines to view the metadata keys captured by Caliper/Adiak::
+
+  for g in r.globals:  
+    print(g)  
+
+You can also add a line to display metadata value in the dictionary **r.globals**
+
+For example print out the OpenMP Max Threads value recorded at runtime:: 
+
+  print('OMP Max Threads: ' + r.globals['omp_max_threads'])`  
+
+or the variant represented in this file::  
+
+  print('Variant: ' + r.globals['variant'])
+
+3: Using the *Hatchet* Python module::
+
+  import hatchet as ht
+  DATA_DIR = os.getenv('HOME')+"/data/default_problem_size/gcc"
+  os.chdir(DATA_DIR)
+  gf1 = ht.GraphFrame.from_caliperreader("RAJA_Seq.cali")
+  print(gf1.tree())
+