WIP Caliper integration and instrumentation

.github/workflows/github-actions.yml

@@ -9,38 +9,50 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        config: [boilerplate, release, debug_cuda_only, debug_mpi_only, debug_threads_only]
+        config: [boilerplate, release, debug_cuda_only, debug_mpi_only, debug_threads_only, boilerplate_caliper]
 
         include:
           - config: boilerplate
             PERFFLOWASPECT_WITH_CUDA: OFF
             PERFFLOWASPECT_WITH_MPI: ON
             PERFFLOWASPECT_WITH_MULTITHREADS: ON
             CMAKE_BUILD_TYPE: Debug
+            PERFFLOWASPECT_WITH_CALIPER: OFF
 
           - config: release
             PERFFLOWASPECT_WITH_CUDA: OFF
             PERFFLOWASPECT_WITH_MPI: ON
             PERFFLOWASPECT_WITH_MULTITHREADS: ON
             CMAKE_BUILD_TYPE: Release
+            PERFFLOWASPECT_WITH_CALIPER: OFF
 
           - config: debug_cuda_only
             PERFFLOWASPECT_WITH_CUDA: OFF
             PERFFLOWASPECT_WITH_MPI: OFF
             PERFFLOWASPECT_WITH_MULTITHREADS: OFF
             CMAKE_BUILD_TYPE: Debug
+            PERFFLOWASPECT_WITH_CALIPER: OFF
 
           - config: debug_mpi_only
             PERFFLOWASPECT_WITH_CUDA: OFF
             PERFFLOWASPECT_WITH_MPI: ON
             PERFFLOWASPECT_WITH_MULTITHREADS: OFF
             CMAKE_BUILD_TYPE: Debug
+            PERFFLOWASPECT_WITH_CALIPER: OFF
 
           - config: debug_threads_only
             PERFFLOWASPECT_WITH_CUDA: OFF
             PERFFLOWASPECT_WITH_MPI: OFF
             PERFFLOWASPECT_WITH_MULTITHREADS: ON
             CMAKE_BUILD_TYPE: Debug
+            PERFFLOWASPECT_WITH_CALIPER: OFF
+
+          - config: boilerplate_caliper
+            PERFFLOWASPECT_WITH_CUDA: OFF
+            PERFFLOWASPECT_WITH_MPI: OFF
+            PERFFLOWASPECT_WITH_MULTITHREADS: OFF
+            CMAKE_BUILD_TYPE: Debug
+            PERFFLOWASPECT_WITH_CALIPER: ON
 
     steps:
       # Checkout PerfFlowAspect repository under $GITHUB_WORKSPACE
@@ -52,6 +64,21 @@ jobs:
           sudo apt install clang llvm-dev libjansson-dev libssl-dev bison flex make cmake mpich
           clang++ --version
 
+      - name: Clone Caliper
+        uses: actions/checkout@v2
+        with:
+          repository: LLNL/Caliper
+          path: Caliper
+
+      - name: Build Caliper
+        working-directory: Caliper
+        run: |
+          mkdir build && mkdir install
+          cd build
+          cmake -DCMAKE_INSTALL_PREFIX=../install ../
+          make VERBOSE=1
+          make install
+
       - name: Compile check
         run: |
           cd src/c
@@ -62,6 +89,7 @@ jobs:
           export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_CUDA=${{matrix.PERFFLOWASPECT_WITH_CUDA}}"
           export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_MPI=${{matrix.PERFFLOWASPECT_WITH_MPI}}"
           export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_MULTITHREADS=${{matrix.PERFFLOWASPECT_WITH_MULTITHREADS}}"
+          export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_CALIPER=${{matrix.PERFFLOWASPECT_WITH_CALIPER}} -Dcaliper_DIR=/home/runner/work/PerfFlowAspect/PerfFlowAspect/Caliper/install"
           echo -e ${CMAKE_OPTS}
           cmake ${CMAKE_OPTS} ..
           # build

docs/CaliperIntegration.rst

@@ -0,0 +1,77 @@
+..
+   # Copyright 2021 Lawrence Livermore National Security, LLC and other
+   # PerfFlowAspect Project Developers. See the top-level LICENSE file for
+   # details.
+   #
+   # SPDX-License-Identifier: LGPL-3.0
+
+#####################
+ Caliper Integration
+#####################
+
+PerfFlowAspect can be built with Caliper to leverage collection of additional
+performance data, such as hardware performance counters on CPUs and GPU measurements on
+NVIDIA GPUs. `Caliper <https://github.com/llnl/caliper>`_ is an instrumentation and
+performance annotation library. A Caliper install is required before building
+PerfFlowAspect:
+
+.. code:: bash
+
+    cmake -Dcaliper_DIR=<path-to-install>/share/lib/caliper ../
+
+Caliper can be configured at runtime, for example:
+
+.. code:: bash
+
+    CALI_CONFIG=runtime-report ./smoketest
+    CALI_CONFIG=runtime-report,output=test.cali ./smoketest
+
+.. code:: bash
+
+    Path        Min time/rank Max time/rank Avg time/rank Time %    
+    _Z3fooRKSs       0.004527      0.004527      0.004527 45.778137 
+      _Z3barv        0.004511      0.004511      0.004511 45.616341 
+        _Z3basv      0.000079      0.000079      0.000079  0.798867 
+
+.. code:: bash
+
+    CALI_PAPI_COUNTERS=PAPI_TOT_CYC,PAPI_L2_DCM CALI_SERVICES_ENABLE=event,trace,papi,report ./smoketest
+
+.. code:: bash
+
+    event.begin#annotation papi.PAPI_TOT_CYC papi.PAPI_L2_DCM annotation                 region.count event.end#annotation 
+    _Z3fooRKSs                        118289              679 
+    _Z3barv                           200050              765 _Z3fooRKSs                 
+    _Z3basv                           115098              352 _Z3fooRKSs/_Z3barv         
+                                       66564              242 _Z3fooRKSs/_Z3barv/_Z3basv            1 _Z3basv              
+                                      117061              385 _Z3fooRKSs/_Z3barv                    1 _Z3barv              
+                                       93592              206 _Z3fooRKSs                            1 _Z3fooRKSs           
+    _Z3fooRKSs                        146308              332                                                              
+    _Z3barv                            87811              255 _Z3fooRKSs                                                   
+    _Z3basv                            84904              244 _Z3fooRKSs/_Z3barv                                           
+                                       34547               66 _Z3fooRKSs/_Z3barv/_Z3basv            1 _Z3basv              
+                                       82540              168 _Z3fooRKSs/_Z3barv                    1 _Z3barv              
+                                       80711              144 _Z3fooRKSs                            1 _Z3fooRKSs           
+    _Z3fooRKSs                        127765              183                                                              
+    _Z3barv                            85440              241 _Z3fooRKSs                                                   
+    _Z3basv                            82100              250 _Z3fooRKSs/_Z3barv                                           
+                                       33969               67 _Z3fooRKSs/_Z3barv/_Z3basv            1 _Z3basv              
+                                       81511              161 _Z3fooRKSs/_Z3barv                    1 _Z3barv              
+                                       77498              128 _Z3fooRKSs                            1 _Z3fooRKSs           
+    _Z3fooRKSs                        119853              164                                                              
+    _Z3barv                            83285              227 _Z3fooRKSs                                                   
+    _Z3basv                            82702              297 _Z3fooRKSs/_Z3barv                                           
+                                       34170               78 _Z3fooRKSs/_Z3barv/_Z3basv            1 _Z3basv              
+                                       81589              149 _Z3fooRKSs/_Z3barv                    1 _Z3barv              
+                                       78920              119 _Z3fooRKSs                            1 _Z3fooRKSs
+
+.. code:: bash
+
+    PERFFLOW_OPTIONS='cpu-mem-usage=True:log-event=compact' CALI_CONFIG="load(time_exclusive.json),spot" ./smoketest
+
+.. code:: bash
+
+    Path        Min time/rank Max time/rank Avg time/rank Total time spot.channel  
+    _Z3fooRKSs       0.018068      0.018068      0.018068   0.018068 regionprofile 
+      _Z3barv        0.009124      0.009124      0.009124   0.009124 regionprofile 
+        _Z3basv      0.000074      0.000074      0.000074   0.000074 regionprofile 

docs/index.rst

@@ -85,6 +85,7 @@ uniformity as to how performance is measured and controlled.
 
    BuildingPerfFlowAspect
    Annotations
+   CaliperIntegration
    UpcomingFeatures
 
 .. toctree::

src/c/CMakeLists.txt

@@ -2,6 +2,12 @@ cmake_minimum_required(VERSION 3.12)
 
 project(PerfFlowAspect VERSION "0.1.0")
 
+# Build Options
+option(PERFFLOWASPECT_WITH_CUDA "Build CUDA smoketest" ON)
+option(PERFFLOWASPECT_WITH_MPI "Build MPI smoketest" ON)
+option(PERFFLOWASPECT_WITH_MULTITHREADS "Build multi-threaded smoketest" ON)
+option(PERFFLOWASPECT_WITH_CALIPER "Build with Caliper support" ON)
+
 # Fail if using Clang < 9.0
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
     # require at least Clang 9.0

src/c/cmake/Setup3rdParty.cmake

@@ -19,3 +19,19 @@ include(cmake/thirdparty/FindOpenSSL.cmake)
 if(PERFFLOWASPECT_WITH_MULTITHREADS)
     include(cmake/thirdparty/FindThreads.cmake)
 endif()
+
+if(PERFFLOWASPECT_WITH_CALIPER)
+    # first Check for CALIPER_DIR
+    if(NOT caliper_DIR)
+        MESSAGE(FATAL_ERROR "Caliper support needs explicit caliper_DIR")
+    endif()
+
+    if(caliper_DIR)
+        message(STATUS "PPP ${caliper_DIR}")
+        include(cmake/thirdparty/FindCaliper.cmake)
+    endif()
+
+    if(CALIPER_FOUND)
+        add_definitions(-DPERFFLOWASPECT_WITH_CALIPER)
+    endif()
+endif()

src/c/cmake/thirdparty/FindCaliper.cmake

@@ -0,0 +1,31 @@
+## most common case: caliper is built with adiak support
+## and caliper needs us to find adiak, or else find_pacakge caliper
+## will fail
+#
+## Check for ADIAK_DIR
+#
+#if(NOT ADIAK_DIR)
+#    MESSAGE(FATAL_ERROR "Caliper support needs explicit ADIAK_DIR")
+#endif()
+#
+#message(STATUS "Looking for Adiak in: ${ADIAK_DIR}")
+#
+#find_package(adiak REQUIRED
+#             NO_DEFAULT_PATH
+#             PATHS ${ADIAK_DIR}/lib/cmake/adiak)
+
+message(STATUS "Looking for Caliper in: ${caliper_DIR}")
+
+find_package(caliper REQUIRED
+             PATHS ${caliper_DIR}/share/cmake/caliper
+             NO_DEFAULT_PATH
+             NO_CMAKE_ENVIRONMENT_PATH
+             NO_CMAKE_PATH
+             NO_SYSTEM_ENVIRONMENT_PATH
+             NO_CMAKE_SYSTEM_PATH)
+
+message(STATUS "FOUND Caliper: ${caliper_INSTALL_PREFIX}")
+
+#set(ADIAK_FOUND TRUE)
+set(CALIPER_FOUND TRUE)
+set(PERFFLOWASPECT_CALIPER_ENABLED TRUE)

src/c/config/CMakeLists.txt

@@ -42,6 +42,8 @@ configure_package_config_file(
 install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/perfflowaspect-config.cmake
     ${CMAKE_CURRENT_BINARY_DIR}/perfflowaspect-config-version.cmake
+    perfflowaspect_setup_deps.cmake
+    perfflowaspect_setup_targets.cmake
     DESTINATION ${PERFFLOWASPECT_INSTALL_CMAKE_MODULE_DIR})
 
 # Create pkg-config .pc file

src/c/config/perfflowaspect-config.cmake.in

@@ -3,7 +3,10 @@ if (NOT PERFFLOWASPECT_CONFIG_LOADED)
   set(PERFFLOWASPECT_DIR     "@CMAKE_INSTALL_PREFIX@")
   set(PERFFLOWASPECT_LIB_DIR "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@")
 
-  include(CMakeFindDependencyMacro)
+  set(PERFFLOWASPECT_CALIPER_ENABLED "@PERFFLOWASPECT_CALIPER_ENABLED@")
+  set(PERFFLOWASPECT_CALIPER_DIR  "@caliper_DIR@")
+
+  include(${PERFFLOWASPECT_DIR}/share/perfflowaspect_setup_deps.cmake)
 
   find_dependency(OpenSSL REQUIRED)
 

src/c/config/perfflowaspect_setup_deps.cmake

@@ -0,0 +1,17 @@
+include(CMakeFindDependencyMacro)
+
+if (NOT caliper_DIR)
+    set(caliper_DIR ${PERFFLOWASPECT_CALIPER_DIR})
+endif()
+
+if(caliper_DIR)
+    if(NOT PerfFlowAspect_FIND_QUIETLY)
+        message(STATUS "PerfFlowAspect was built with Caliper Support")
+        message(STATUS "Looking for Caliper at: ${caliper_DIR}/share/cmake/caliper")
+    endif()
+
+    # find caliper
+    find_package(caliper REQUIRED
+                 NO_DEFAULT_PATH
+                 PATHS ${caliper_DIR}/share/cmake/caliper)
+endif()

src/c/config/perfflowaspect_setup_targets.cmake

@@ -0,0 +1,6 @@
+# create convenience target that bundles all reg perfflowaspect deps
+add_library(perfflowaspect::perfflowaspect INTERFACE IMPORTED)
+
+set_property(TARGET perfflowaspect::perfflowaspect
+             PROPERTY INTERFACE_LINK_LIBRARIES
+             perfflowaspect)

src/c/examples/using-with-cmake/CMakeLists.txt

@@ -23,6 +23,21 @@ cmake_minimum_required(VERSION 3.0)
 
 project(using_with_cmake)
 
+# Fail if using Clang < 9.0
+if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+    # require at least Clang 9.0
+    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
+        message(FATAL_ERROR "Clang++ version must be at least 9.0!")
+    elseif (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 11)
+        set(PERFFLOWASPECT_CLANG_11_NEWER TRUE CACHE BOOL "using >=clang11")
+        add_definitions(-DPERFFLOWASPECT_CLANG_11_NEWER)
+    elseif (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 11)
+        set(PERFFLOWASPECT_CLANG_11_NEWER FALSE CACHE BOOL "using >=clang11")
+    endif()
+else()
+    message(WARNING "Unsupported CXX compiler: please use Clang >= 9.0")
+endif()
+
 #
 # Provide default for PERFFLOWASPECT_DIR that works for an PerfFlowAspect install
 #
@@ -39,3 +54,16 @@ if(NOT EXISTS ${PERFFLOWASPECT_DIR}/share/perfflowaspect-config.cmake)
 endif()
 
 #
+# Use CMake's find_package to import PerfFlowAspect's targets
+#
+find_package(PerfFlowAspect REQUIRED
+             NO_DEFAULT_PATH
+             PATHS ${PERFFLOWASPECT_DIR}/share)
+
+# create our example
+add_executable(smoketest smoketest.cpp)
+
+# link to PerfFlowAspect
+target_link_libraries(smoketest perfflowaspect::perfflowaspect)
+
+#

src/c/runtime/CMakeLists.txt

@@ -14,12 +14,20 @@ set(perfflow_runtime_sources
 
 include_directories(${JANSSON_INCLUDE_DIRS})
 
+if(PERFFLOWASPECT_WITH_CALIPER)
+    include_directories(${caliper_INCLUDE_DIR})
+endif()
+
 add_library(perfflow_runtime SHARED
             ${perfflow_runtime_sources}
             ${perfflow_runtime_headers}
 )
 
-target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} OpenSSL::SSL OpenSSL::Crypto)
+if(PERFFLOWASPECT_WITH_CALIPER)
+    target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} OpenSSL::SSL OpenSSL::Crypto caliper)
+else()
+    target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} OpenSSL::SSL OpenSSL::Crypto)
+endif()
 
 install(TARGETS perfflow_runtime
         EXPORT perfflow_export

src/c/test/CMakeLists.txt

@@ -12,7 +12,11 @@ foreach(TEST ${SMOKETESTS})
     message(STATUS " [*] Adding test: ${TEST}")
     add_executable(${TEST} ${TEST}.cpp)
     set_source_files_properties(${TEST}.cpp COMPILE_FLAGS "-Xclang -load -Xclang ../weaver/weave/libWeavePass.so -fPIC")
-    target_link_libraries(${TEST} ${perfflow_deps})
+    if(PERFFLOWASPECT_WITH_CALIPER)
+        target_link_libraries(${TEST} ${perfflow_deps} caliper)
+    else()
+        target_link_libraries(${TEST} ${perfflow_deps})
+    endif()
 endforeach()
 
 # Build Options
@@ -35,22 +39,34 @@ if(PERFFLOWASPECT_WITH_MULTITHREADS)
     add_executable(smoketest_MT smoketest_MT.cpp)
     set_source_files_properties(smoketest_MT.cpp COMPILE_FLAGS "-Xclang -load -Xclang ../weaver/weave/libWeavePass.so -fPIC")
     set(THREADS_PREFER_PTHREAD_FLAG ON)
-    target_link_libraries(smoketest_MT ${perfflow_deps} pthread)
+    if(PERFFLOWASPECT_WITH_CALIPER)
+        target_link_libraries(smoketest_MT ${perfflow_deps} pthread caliper)
+    else()
+        target_link_libraries(smoketest_MT ${perfflow_deps} pthread)
+    endif()
 endif()
 
 if(PERFFLOWASPECT_WITH_MPI)
     message(STATUS " [*] Adding test: smoketest_MPI")
     add_executable(smoketest_MPI smoketest_MPI.cpp)
     set_source_files_properties(smoketest_MPI.cpp COMPILE_FLAGS "-Xclang -load -Xclang ../weaver/weave/libWeavePass.so -fPIC")
     include_directories(${MPI_INCLUDE_PATH})
-    target_link_libraries(smoketest_MPI ${perfflow_deps} ${MPI_LIBRARIES})
+    if(PERFFLOWASPECT_WITH_CALIPER)
+        target_link_libraries(smoketest_MPI ${perfflow_deps} ${MPI_LIBRARIES} caliper)
+    else()
+        target_link_libraries(smoketest_MPI ${perfflow_deps} ${MPI_LIBRARIES})
+    endif()
 endif()
 
 if(PERFFLOWASPECT_WITH_CUDA)
     message(STATUS " [*] Adding test: smoketest_cuda")
     set(CUDA_NVCC_FLAGS  "-ccbin ${CMAKE_CXX_COMPILER} -Xcompiler=-Xclang -Xcompiler=-load -Xcompiler=-Xclang -Xcompiler=../../../weaver/weave/libWeavePass.so")
     cuda_add_executable(smoketest_cuda smoketest_cuda_wrapper.cpp smoketest_cuda_kernel.cu)
-    target_link_libraries(smoketest_cuda ${perfflow_deps} ${CUDA_LIBRARIES})
+    if(PERFFLOWASPECT_WITH_CALIPER)
+        target_link_libraries(smoketest_cuda ${perfflow_deps} ${CUDA_LIBRARIES} caliper)
+    else()
+        target_link_libraries(smoketest_cuda ${perfflow_deps} ${CUDA_LIBRARIES})
+    endif()
 endif()
 
 configure_file(t0001-cbinding-basic.t.in