diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 70a23d529..2f4fec344 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,11 +19,15 @@ variables: .src_workflow: rules: - - if: '$FULL_BUILD != "ON"' + - if: '$FULL_BUILD != "ON" && $BENCHMARKS_BUILD != "ON"' .full_workflow: rules: - - if: '$FULL_BUILD == "ON"' + - if: '$FULL_BUILD == "ON" && $BENCHMARKS_BUILD != "ON"' + +.benchmarks_workflow: + rules: + - if: '$FULL_BUILD != "ON" && $BENCHMARKS_BUILD == "ON"' #### # Templates @@ -66,10 +70,27 @@ variables: reports: junit: ${FULL_BUILD_ROOT}/${SYS_TYPE}/*/_serac_build_and_test_*/build-*/junit.xml + +.benchmarks_build_script: + script: + # Builds src, runs benchmarks, and stores Caliper files in shared location + - echo -e "section_start:$(date +%s):benchmarks_build\r\e[0K + Benchmarks Build ${CI_PROJECT_NAME}" + - ${ALLOC_COMMAND} python3 scripts/llnl/run_benchmarks.py + - echo -e "section_end:$(date +%s):benchmarks_build\r\e[0K" + artifacts: + expire_in: 2 weeks + when: always + paths: + - _serac_build_and_test_*/output.log*.txt + - _serac_build_and_test_*/build-*/output.log*.txt + - _serac_build_and_test_*/build-*/*.cali + # This is where jobs are included for each system include: - local: .gitlab/build_blueos.yml - local: .gitlab/build_toss4.yml - - local: .gitlab/build_toss4_cray.yml + # Disabling cray until all developers have access to the tioga machine + # - local: .gitlab/build_toss4_cray.yml - project: 'lc-templates/id_tokens' file: 'id_tokens.yml' diff --git a/.gitlab/build_blueos.yml b/.gitlab/build_blueos.yml index eff30dd57..bed3aa909 100644 --- a/.gitlab/build_blueos.yml +++ b/.gitlab/build_blueos.yml @@ -37,6 +37,10 @@ extends: [.full_build_script, .on_blueos, .full_workflow] needs: [] +.benchmarks_build_on_blueos: + extends: [.benchmarks_build_script, .on_blueos, .benchmarks_workflow] + needs: [] + #### # Build jobs blueos-clang_10_0_1-src: @@ -58,3 +62,11 @@ blueos-clang_10_0_1-full: ALLOC_NODES: "1" ALLOC_TIME: "55" extends: [.full_build_on_blueos, .with_cuda] + +blueos-clang_10_0_1-benchmarks: + variables: + COMPILER: "clang@10.0.1" + HOST_CONFIG: "lassen-blueos_3_ppc64le_ib_p9-${COMPILER}_cuda.cmake" + ALLOC_NODES: "1" + ALLOC_TIME: "120" + extends: [.benchmarks_build_on_blueos, .with_cuda] diff --git a/.gitlab/build_toss4.yml b/.gitlab/build_toss4.yml index c4f83eabc..5ceeda03a 100644 --- a/.gitlab/build_toss4.yml +++ b/.gitlab/build_toss4.yml @@ -2,7 +2,7 @@ # This is the shared configuration of jobs for toss4 .on_toss4: variables: - SCHEDULER_PARAMETERS: "--res=ci --exclusive=user --deadline=now+1hour -N ${ALLOC_NODES} -t ${ALLOC_TIME} -A ${ALLOC_BANK}" + SCHEDULER_PARAMETERS: "--res=ci --exclusive=user --deadline=now+${ALLOC_DEADLINE}minutes -N ${ALLOC_NODES} -t ${ALLOC_TIME} -A ${ALLOC_BANK}" tags: - batch - ruby @@ -28,6 +28,10 @@ # LC version of pip is ancient - if [[ $(python3 -c 'import pip; print(pip.__version__ < "19.3")') == "True" ]]; then python3 -m pip install --user --upgrade pip; fi +.benchmarks_build_on_toss4: + extends: [.benchmarks_build_script, .on_toss4, .benchmarks_workflow] + needs: [] + #### # Build jobs @@ -41,6 +45,7 @@ toss4-clang_14_0_6-src: DO_INTEGRATION_TESTS: "yes" ALLOC_NODES: "2" ALLOC_TIME: "30" + ALLOC_DEADLINE: "60" extends: .src_build_on_toss4 toss4-gcc_10_3_1-src: @@ -50,6 +55,7 @@ toss4-gcc_10_3_1-src: EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON" ALLOC_NODES: "1" ALLOC_TIME: "30" + ALLOC_DEADLINE: "60" extends: .src_build_on_toss4 toss4-gcc_10_3_1-src-no-tribol: @@ -59,6 +65,7 @@ toss4-gcc_10_3_1-src-no-tribol: EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -UTRIBOL_DIR" ALLOC_NODES: "1" ALLOC_TIME: "30" + ALLOC_DEADLINE: "60" extends: .src_build_on_toss4 toss4-gcc_10_3_1-src-no-optional-solvers: @@ -68,6 +75,7 @@ toss4-gcc_10_3_1-src-no-optional-solvers: EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -USUNDIALS_DIR -UPETSC_DIR" ALLOC_NODES: "1" ALLOC_TIME: "20" + ALLOC_DEADLINE: "60" extends: .src_build_on_toss4 toss4-clang_14_0_6-full: @@ -76,7 +84,7 @@ toss4-clang_14_0_6-full: SPEC: "--spec=%${COMPILER}" ALLOC_NODES: "1" ALLOC_TIME: "45" - EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON" + ALLOC_DEADLINE: "60" extends: .full_build_on_toss4 toss4-gcc_10_3_1-full: @@ -85,5 +93,23 @@ toss4-gcc_10_3_1-full: SPEC: "--spec=%${COMPILER}" ALLOC_NODES: "1" ALLOC_TIME: "45" - EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON" + ALLOC_DEADLINE: "60" extends: .full_build_on_toss4 + +toss4-clang_14_0_6-benchmarks: + variables: + COMPILER: "clang@14.0.6" + HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake" + ALLOC_NODES: "1" + ALLOC_TIME: "120" + ALLOC_DEADLINE: "180" + extends: .benchmarks_build_on_toss4 + +toss4-gcc_10_3_1-benchmarks: + variables: + COMPILER: "gcc@10.3.1" + HOST_CONFIG: "ruby-toss_4_x86_64_ib-${COMPILER}.cmake" + ALLOC_NODES: "1" + ALLOC_TIME: "120" + ALLOC_DEADLINE: "180" + extends: .benchmarks_build_on_toss4 diff --git a/.gitlab/build_toss4_cray.yml b/.gitlab/build_toss4_cray.yml index bf4a45fdb..56787c3b8 100644 --- a/.gitlab/build_toss4_cray.yml +++ b/.gitlab/build_toss4_cray.yml @@ -25,25 +25,34 @@ # LC version of pip is ancient - if [[ $(python3 -c 'import pip; print(pip.__version__ < "19.3")') == "True" ]]; then python3 -m pip install --user --upgrade pip; fi +.benchmarks_build_on_toss4_cray: + extends: [.benchmarks_build_script, .on_toss4_cray, .benchmarks_workflow] + needs: [] + #### # Build jobs # Only run integration tests on one spec -#toss4_cray-clang_17_0_0-src: -# variables: -# COMPILER: "clang@17.0.0" -# HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake" -# EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON" -# ALLOC_NODES: "1" -# ALLOC_TIME: "30" -# extends: .src_build_on_toss4_cray +toss4_cray-clang_17_0_0-src: + variables: + COMPILER: "clang@17.0.0" + HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake" + EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON" + ALLOC_NODES: "1" + ALLOC_TIME: "30" + extends: .src_build_on_toss4_cray toss4_cray-clang_17_0_0-full: variables: - COMPILER: "clang@17.0.0" - SPEC: "--spec=%${COMPILER}" ALLOC_NODES: "1" ALLOC_TIME: "45" - EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON" extends: .full_build_on_toss4_cray + +toss4_cray-clang_17_0_0-benchmarks: + variables: + COMPILER: "clang@17.0.0" + HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake" + ALLOC_NODES: "1" + ALLOC_TIME: "120" + extends: .benchmarks_build_on_toss4_cray diff --git a/scripts/llnl/build_src.py b/scripts/llnl/build_src.py index 158dd601c..6588cbcca 100755 --- a/scripts/llnl/build_src.py +++ b/scripts/llnl/build_src.py @@ -132,25 +132,7 @@ def main(): compiler = compiler.rsplit('-', 1)[0] hostconfig = "%s-%s-%s.cmake" % (hostname, sys_type, compiler) - # First try with where uberenv generates host-configs. - hostconfig_path = os.path.join(repo_dir, hostconfig) - if not os.path.isfile(hostconfig_path): - print("[INFO: Looking for hostconfig at %s]" % hostconfig_path) - print("[WARNING: Spack generated host-config not found, trying with predefined]") - - # Then look into project predefined host-configs. - hostconfig_path = os.path.join(repo_dir, "host-configs", hostconfig) - if not os.path.isfile(hostconfig_path): - print("[INFO: Looking for hostconfig at %s]" % hostconfig_path) - print("[WARNING: Predefined host-config not found, trying with Docker]") - - # Otherwise look into project predefined Docker host-configs. - hostconfig_path = os.path.join(repo_dir, "host-configs", "docker", hostconfig) - if not os.path.isfile(hostconfig_path): - print("[INFO: Looking for hostconfig at %s]" % hostconfig_path) - print("[WARNING: Predefined Docker host-config not found]") - print("[ERROR: Could not find any host-configs in any known path. Try giving fully qualified path.]") - return 1 + hostconfig_path = get_host_config_path(repo_dir, hostconfig) test_root = get_build_and_test_root(repo_dir, timestamp) os.mkdir(test_root) diff --git a/scripts/llnl/common_build_functions.py b/scripts/llnl/common_build_functions.py index 927aacf17..35971b902 100755 --- a/scripts/llnl/common_build_functions.py +++ b/scripts/llnl/common_build_functions.py @@ -713,3 +713,27 @@ def convertSecondsToReadableTime(seconds): m, s = divmod(seconds, 60) h, m = divmod(m, 60) return "%d:%02d:%02d" % (h, m, s) + + +def get_host_config_path(repo_dir, host_config): + # First try with where uberenv generates host-configs. + host_config_path = os.path.join(repo_dir, host_config) + if not os.path.isfile(host_config_path): + print("[INFO: Looking for host_config at %s]" % host_config_path) + print("[WARNING: Spack generated host-config not found, trying with predefined]") + + # Then look into project predefined host-configs. + host_config_path = os.path.join(repo_dir, "host-configs", host_config) + if not os.path.isfile(host_config_path): + print("[INFO: Looking for host_config at %s]" % host_config_path) + print("[WARNING: Predefined host-config not found, trying with Docker]") + + # Otherwise look into project predefined Docker host-configs. + host_config_path = os.path.join(repo_dir, "host-configs", "docker", host_config) + if not os.path.isfile(host_config_path): + print("[INFO: Looking for host_config at %s]" % host_config_path) + print("[WARNING: Predefined Docker host-config not found]") + print("[ERROR: Could not find any host-configs in any known path. Try giving fully qualified path.]") + sys.exit(1) + + return host_config_path diff --git a/scripts/llnl/run_benchmarks.py b/scripts/llnl/run_benchmarks.py new file mode 100755 index 000000000..9ac79d580 --- /dev/null +++ b/scripts/llnl/run_benchmarks.py @@ -0,0 +1,107 @@ +#!/bin/sh +"exec" "python3" "-u" "-B" "$0" "$@" + +# Copyright (c) 2019-2024, Lawrence Livermore National Security, LLC and +# other Serac Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: (BSD-3-Clause) + +""" + file: run_benchmarks.py + + description: + Run benchmarks and update shared (or any desired) location with new Caliper files + +""" + +from common_build_functions import * + +from argparse import ArgumentParser + +import os + + +def parse_args(): + "Parses args from command line" + parser = ArgumentParser() + parser.add_argument("-e", "--extra-cmake-options", + dest="extra_cmake_options", + default=os.environ.get("EXTRA_CMAKE_OPTIONS", ""), + help="Extra cmake options to add to the cmake configure line. Note that options to enable benchmarks, " + + "disable docs, and build as Release are always appended.") + parser.add_argument("-hc", "--host-config", + dest="host_config", + default=os.environ.get("HOST_CONFIG", None), + help="Specific host-config filename to build (defaults to HOST_CONFIG environment variable)") + parser.add_argument("-sd", "--spot-directory", + dest="spot_dir", + default=get_shared_spot_dir(), + help="Where to put all resulting caliper files to use for SPOT analysis (defaults to a shared location)") + parser.add_argument("-t", "--timestamp", + dest="timestamp", + default=get_timestamp(), + help="Set timestamp manually for debugging") + + # Parse args + args, extra_args = parser.parse_known_args() + args = vars(args) + + # Verify args + if args["host_config"] is None: + print("[ERROR: Both host_config argument and HOST_CONFIG environment variable unset!]") + sys.exit(1) + + return args + + +def main(): + # Args + args = parse_args() + cmake_options = args["extra_cmake_options"] + " -DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Release" + host_config = args["host_config"] + spot_dir = args["spot_dir"] + timestamp = args["timestamp"] + + # Vars + repo_dir = get_repo_dir() + test_root = get_build_and_test_root(repo_dir, timestamp) + host_config_path = get_host_config_path(repo_dir, host_config) + host_config_root = get_host_config_root(host_config) + benchmarks_output_file = os.path.join(test_root, "output.log.%s.benchmarks.txt" % host_config_root) + + # Build Serac + os.chdir(repo_dir) + os.makedirs(test_root, exist_ok=True) + build_and_test_host_config(test_root=test_root, host_config=host_config_path, + report_to_stdout=True, extra_cmake_options=cmake_options, + skip_install=True, skip_tests=True) + + # Go to build location + build_dir="" + dirs = glob.glob(pjoin(test_root, "*")) + for dir in dirs: + if os.path.exists(dir) and "build-" in dir: + build_dir=dir + os.chdir(build_dir) + + # Run benchmarks + result = shell_exec("make run_benchmarks", echo=True, print_output=True, output_file=benchmarks_output_file) + + # Move resulting .cali files to specified directory + os.makedirs(spot_dir, exist_ok=True) + cali_files = glob.glob(pjoin(build_dir, "*.cali")) + for cali_file in cali_files: + if os.path.exists(cali_file): + shutil.copy2(cali_file, spot_dir) + + # Print SPOT url + if on_rz(): + print("[View SPOT directory here: https://rzlc.llnl.gov/spot2/?sf={0}]".format(spot_dir)) + else: + print("[View SPOT directory here: https://lc.llnl.gov/spot2/?sf={0}]".format(spot_dir)) + + return result; + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/docs/sphinx/dev_guide/profiling.rst b/src/docs/sphinx/dev_guide/profiling.rst index 48df8a818..274693971 100644 --- a/src/docs/sphinx/dev_guide/profiling.rst +++ b/src/docs/sphinx/dev_guide/profiling.rst @@ -136,6 +136,8 @@ files: - `SPOT CZ `_ - `SPOT RZ `_ +The shared Caliper files for Serac are located here: https://lc.llnl.gov/spot2/?sf=/usr/WS2/smithdev/califiles + .. note:: There is a bug in SPOT where if you remove Caliper files from a directory, they still show up on SPOT - if you've visualized them previously. The current workaround is by removing the ``llnl.gov`` site cache manually. diff --git a/src/serac/physics/benchmarks/CMakeLists.txt b/src/serac/physics/benchmarks/CMakeLists.txt index 137a89978..9c9c94982 100644 --- a/src/serac/physics/benchmarks/CMakeLists.txt +++ b/src/serac/physics/benchmarks/CMakeLists.txt @@ -13,9 +13,9 @@ set(physics_benchmark_targets ) # Create executable for each benchmark -foreach(physics_benchmark ${physics_benchmark_targets}) - blt_add_executable(NAME ${physics_benchmark} - SOURCES ${physics_benchmark}.cpp +foreach(physics_benchmark_exec ${physics_benchmark_targets}) + blt_add_executable(NAME ${physics_benchmark_exec} + SOURCES ${physics_benchmark_exec}.cpp DEPENDS_ON ${physics_benchmark_depends} OUTPUT_DIR ${PROJECT_BINARY_DIR}/benchmarks FOLDER serac/benchmarks @@ -23,10 +23,14 @@ foreach(physics_benchmark ${physics_benchmark_targets}) # Add benchmarks with various task counts foreach(task_count 1 4 16) - blt_add_benchmark(NAME ${physics_benchmark}_${task_count}_task_count - COMMAND ${physics_benchmark} + set(physics_benchmark ${physics_benchmark_exec}_${task_count}_task_count) + blt_add_benchmark(NAME ${physics_benchmark} + COMMAND ${physics_benchmark_exec} NUM_MPI_TASKS ${task_count} WORKING_DIRECTORY ${PROJECT_BINARY_DIR} ) + + # Increase benchmark time limit + set_tests_properties(${physics_benchmark} PROPERTIES TIMEOUT 4800) endforeach() endforeach() diff --git a/src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp b/src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp index 192066803..e6dc8b792 100644 --- a/src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp +++ b/src/serac/physics/benchmarks/physics_benchmark_solid_nonlinear_solve.cpp @@ -382,9 +382,11 @@ int main(int argc, char* argv[]) functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::MULTIGRID, problemSize); SERAC_MARK_END("Multigrid Preconditioner"); +#ifdef SERAC_USE_PETSC SERAC_MARK_BEGIN("Petsc Multigrid Preconditioner"); functional_solid_test_nonlinear_buckle(NonlinSolve::NEWTON, Prec::PETSC_MULTIGRID, problemSize); SERAC_MARK_END("Petsc Multigrid Preconditioner"); +#endif } else { SERAC_SET_METADATA("nonlinear solver", nonlinSolveToString(nonlinSolve)); SERAC_SET_METADATA("preconditioner", precToString(prec));