Run benchmarks weekly in CI

.gitlab-ci.yml

@@ -19,11 +19,15 @@ variables:
 
 .src_workflow:
   rules:
-    - if: '$FULL_BUILD != "ON"'
+    - if: '$FULL_BUILD != "ON" && $BENCHMARKS_BUILD != "ON"'
 
 .full_workflow:
   rules:
-    - if: '$FULL_BUILD == "ON"'
+    - if: '$FULL_BUILD == "ON" && $BENCHMARKS_BUILD != "ON"'
+
+.benchmarks_workflow:
+  rules:
+    - if: '$FULL_BUILD != "ON" && $BENCHMARKS_BUILD == "ON"'
 
 ####
 # Templates
@@ -66,6 +70,22 @@ variables:
     reports:
       junit: ${FULL_BUILD_ROOT}/${SYS_TYPE}/*/_serac_build_and_test_*/build-*/junit.xml
 
+
+.benchmarks_build_script:
+  script:
+    # Builds src, runs benchmarks, and stores Caliper files in shared location
+    - echo -e "section_start:$(date +%s):benchmarks_build\r\e[0K
+      Benchmarks Build ${CI_PROJECT_NAME}"
+    - ${ALLOC_COMMAND} python3 scripts/llnl/run_benchmarks.py
+    - echo -e "section_end:$(date +%s):benchmarks_build\r\e[0K"
+  artifacts:
+    expire_in: 2 weeks
+    when: always
+    paths:
+      - _serac_build_and_test_*/output.log*.txt
+      - _serac_build_and_test_*/build-*/output.log*.txt
+      - _serac_build_and_test_*/build-*/*.cali
+
 # This is where jobs are included for each system
 include:
   - local: .gitlab/build_blueos.yml

.gitlab/build_blueos.yml

@@ -37,6 +37,10 @@
   extends: [.full_build_script, .on_blueos, .full_workflow]
   needs: []
 
+.benchmarks_build_on_blueos:
+  extends: [.benchmarks_build_script, .on_blueos, .benchmarks_workflow]
+  needs: []
+
 ####
 # Build jobs
 blueos-clang_10_0_1-src:
@@ -58,3 +62,11 @@ blueos-clang_10_0_1-full:
     ALLOC_NODES: "1"
     ALLOC_TIME: "55"
   extends: [.full_build_on_blueos, .with_cuda]
+
+blueos-clang_10_0_1-benchmarks:
+  variables:
+    COMPILER: "[email protected]"
+    HOST_CONFIG: "lassen-blueos_3_ppc64le_ib_p9-${COMPILER}_cuda.cmake"
+    ALLOC_NODES: "1"
+    ALLOC_TIME: "30"
+  extends: [.benchmarks_build_on_blueos, .with_cuda]

.gitlab/build_toss4.yml

@@ -28,6 +28,10 @@
     # LC version of pip is ancient
     - if [[ $(python3 -c 'import pip; print(pip.__version__ < "19.3")') == "True" ]]; then python3 -m pip install --user --upgrade pip; fi
 
+.benchmarks_build_on_toss4:
+  extends: [.benchmarks_build_script, .on_toss4, .benchmarks_workflow]
+  needs: []
+
 
 ####
 # Build jobs
@@ -76,7 +80,6 @@ toss4-clang_14_0_6-full:
     SPEC: "--spec=%${COMPILER}"
     ALLOC_NODES: "1"
     ALLOC_TIME: "45"
-    EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON"
   extends: .full_build_on_toss4
 
 toss4-gcc_10_3_1-full:
@@ -85,5 +88,20 @@ toss4-gcc_10_3_1-full:
     SPEC: "--spec=%${COMPILER}"
     ALLOC_NODES: "1"
     ALLOC_TIME: "45"
-    EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON"
   extends: .full_build_on_toss4
+
+toss4-clang_14_0_6-benchmarks:
+  variables:
+    COMPILER: "[email protected]"
+    HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake"
+    ALLOC_NODES: "1"
+    ALLOC_TIME: "30"
+  extends: .benchmarks_build_on_toss4
+
+toss4-gcc_10_3_1-benchmarks:
+  variables:
+    COMPILER: "[email protected]"
+    HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake"
+    ALLOC_NODES: "1"
+    ALLOC_TIME: "30"
+  extends: .benchmarks_build_on_toss4

.gitlab/build_toss4_cray.yml

@@ -25,6 +25,10 @@
     # LC version of pip is ancient
     - if [[ $(python3 -c 'import pip; print(pip.__version__ < "19.3")') == "True" ]]; then python3 -m pip install --user --upgrade pip; fi
 
+.benchmarks_build_on_toss4_cray:
+  extends: [.benchmarks_build_script, .on_toss4_cray, .benchmarks_workflow]
+  needs: []
+
 
 ####
 # Build jobs
@@ -41,9 +45,14 @@ toss4_cray-clang_17_0_0-src:
 
 toss4_cray-clang_17_0_0-full:
   variables:
-    COMPILER: "[email protected]"
-    SPEC: "--spec=%${COMPILER}"
     ALLOC_NODES: "1"
     ALLOC_TIME: "45"
-    EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON"
   extends: .full_build_on_toss4_cray
+
+toss4_cray-clang_17_0_0-benchmarks:
+  variables:
+    COMPILER: "[email protected]"
+    HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake"
+    ALLOC_NODES: "1"
+    ALLOC_TIME: "30"
+  extends: .benchmarks_build_on_toss4_cray

scripts/llnl/run_benchmarks.py

@@ -0,0 +1,109 @@
+#!/bin/sh
+"exec" "python3" "-u" "-B" "$0" "$@"
+
+# Copyright (c) 2019-2024, Lawrence Livermore National Security, LLC and
+# other Serac Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+
+"""
+ file: run_benchmarks.py
+
+ description: 
+  Run benchmarks and update shared (or any desired) location with new Caliper files
+
+"""
+
+from common_build_functions import *
+
+from argparse import ArgumentParser
+
+import os
+
+
+def parse_args():
+    "Parses args from command line"
+    parser = ArgumentParser()
+    parser.add_argument("-e", "--extra-cmake-options",
+                      dest="extra_cmake_options",
+                      default=os.environ.get("EXTRA_CMAKE_OPTIONS", ""),
+                      help="Extra cmake options to add to the cmake configure line. Note '-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF' is always appended.")
+    parser.add_argument("-hc", "--host-config",
+                    dest="host_config",
+                    default=os.environ.get("HOST_CONFIG", None),
+                    help="Specific host-config file to build (defaults to HOST_CONFIG environment variable)")
+    parser.add_argument("-sd", "--spot-directory",
+                      dest="spot_dir",
+                      default=get_shared_spot_dir(),
+                      help="Where to put all resulting caliper files to use for SPOT analysis (defaults to a shared location)")
+    parser.add_argument("-t", "--timestamp",
+                    dest="timestamp",
+                    default=get_timestamp(),
+                    help="Set timestamp manually for debugging")
+
+    # Parse args
+    args, extra_args = parser.parse_known_args()
+    args = vars(args)
+
+    # Verify args
+    if args["host_config"] is None:
+        print("[ERROR: Both host_config argument and HOST_CONFIG environment variable unset!]")
+        sys.exit(1)
+
+    if not os.path.exists(args["host_config"]):
+        print("[ERROR: Host config path does not exist: %s]" % args["host_config"])
+        sys.exit(1)
+
+    return args
+
+
+def main():
+    # Args
+    args = parse_args()
+    cmake_options = args["extra_cmake_options"] + " -DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF"
+    host_config = args["host_config"]
+    spot_dir = args["spot_dir"]
+    timestamp = args["timestamp"]
+
+    # Vars
+    repo_dir = get_repo_dir()
+    test_root = get_build_and_test_root(repo_dir, timestamp)
+    host_config_root = get_host_config_root(host_config)
+    benchmarks_output_file = os.path.join(test_root, "output.log.%s.benchmarks.txt" % host_config_root)
+
+    # Build Serac
+    os.chdir(repo_dir)
+    os.makedirs(test_root, exist_ok=True)
+    build_and_test_host_config(test_root=test_root, host_config=host_config,
+                               report_to_stdout=True, extra_cmake_options=cmake_options,
+                               skip_install=True, skip_tests=True)
+
+    # Go to build location
+    build_dir=""
+    dirs = glob.glob(pjoin(test_root, "*"))
+    for dir in dirs:
+        if os.path.exists(dir) and "build-" in dir:
+            build_dir=dir
+    os.chdir(build_dir)
+
+    # Run benchmarks
+    shell_exec("make run_benchmarks", echo=True, print_output=True, output_file=benchmarks_output_file)
+
+    # Move resulting .cali files to specified directory
+    os.makedirs(spot_dir, exist_ok=True)
+    cali_files = glob.glob(pjoin(build_dir, "*.cali"))
+    for cali_file in cali_files:
+        if os.path.exists(cali_file):
+            shutil.copy2(cali_file, spot_dir)
+
+    # Print SPOT url
+    if on_rz():
+        print("[View SPOT directory here: https://rzlc.llnl.gov/spot2/?sf={0}]".format(spot_dir))
+    else:
+        print("[View SPOT directory here: https://lc.llnl.gov/spot2/?sf={0}]".format(spot_dir))
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())