diff --git a/.gitignore b/.gitignore index f5111f5..e6e4da9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .cache/ generated-docs/ build/ +builds/ +CMakeUserPresets.json diff --git a/CMakeLists.txt b/CMakeLists.txt index 2111e2c..55fa03b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,9 +1,11 @@ cmake_minimum_required(VERSION 3.21) -project(ctbench VERSION 0.0.1) +project(ctbench VERSION 0.1.0) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(CMAKE_CXX_STANDARD 20) +set(ENABLE_DOCS ON CACHE BOOL "Enable documentation target") + if(${ENABLE_CLANG_TIDY}) set(CMAKE_CXX_CLANG_TIDY clang-tidy -checks=-*,readability-*) endif() diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000..0d607e6 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,57 @@ +{ + "version": 4, + "cmakeMinimumRequired": { + "major": 3, + "minor": 23, + "patch": 0 + }, + "configurePresets": [ + { + "name": "dev", + "displayName": "Development", + "description": "Development build, provides compile_commands.json", + "generator": "Ninja", + "binaryDir": "${sourceDir}/build", + "cacheVariables": { + "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_CXX_COMPILER": "clang++", + "CMAKE_C_COMPILER": "clang", + "CMAKE_CXX_CLANG_TIDY": "clang-tidy;-checks=-*,readability-*" + } + }, + { + "name": "release", + "displayName": "Release", + "description": "Release build", + "binaryDir": "${sourceDir}/build/release", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "debug", + "inherits": "release", + "displayName": "Debug", + "description": "Debug build", + "binaryDir": "${sourceDir}/build/debug", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + } + ], + "buildPresets": [ + { + "name": "dev", + "configurePreset": "dev" + }, + { + "name": "release", + "configurePreset": "release" + }, + { + "name": "debug", + "configurePreset": "debug" + } + ] +} diff --git a/docs/llvm-timescopes.md b/docs/llvm-timescopes.md deleted file mode 100644 index 46adf24..0000000 --- a/docs/llvm-timescopes.md +++ /dev/null @@ -1,55 +0,0 @@ -# LLVM Timescopes - -Quick summary of what different Clang/LLVM's TimeScopes measure. - -- ExecuteCompiler - -Complete compiler execution time. - -- Frontend - -Files: `clang/lib/CodeGen/CodeGenAction.cpp`, `clang/lib/Parse/ParseAST.cpp` - -- Source - -File: `clang/lib/Sema/Sema.cpp` - -The Source timer measures pre-processing time. It can be used to ensure that -benchmark generation using macros has a negligible impact on final benchmark -results. - -- ParseClass - -File: `clang/lib/Parse/ParseDeclCXX.cpp` - -- InstantiateClass - -File: `clang/lib/Sema/SemaTemplateInstantiate.cpp` - -- Backend - -File: `clang/lib/CodeGen/BackendUtil.cpp` - -- ParseTemplate - -File: `clang/lib/Parse/ParseTemplate.cpp` - -- OptModule - -File: `llvm/lib/IR/LegacyPassManager.cpp` - -- CodeGenPasses - -File: `clang/lib/CodeGen/BackendUtil.cpp` - -- PerModulePasses - -File: `clang/lib/CodeGen/BackendUtil.cpp` - -- PerFunctionPasses - -File: `clang/lib/CodeGen/BackendUtil.cpp` - -- PerformPendingInstantiations - -File: `clang/lib/Sema/Sema.cpp` diff --git a/grapher/CMakeLists.txt b/grapher/CMakeLists.txt index 5f97847..5e91888 100644 --- a/grapher/CMakeLists.txt +++ b/grapher/CMakeLists.txt @@ -1,17 +1,19 @@ find_package(nlohmann_json 3.9.1 REQUIRED) find_package(sciplot REQUIRED) find_package(LLVM REQUIRED CONFIG) +find_package(fmt) # Declaring the grapher library file(GLOB_RECURSE GRAPHER_SOURCES lib/*.cpp) add_library(grapher STATIC ${GRAPHER_SOURCES}) + target_link_libraries(grapher PUBLIC ctbench-compile-opts) target_include_directories(grapher PUBLIC include) llvm_map_components_to_libnames(llvm_libs core support) target_link_libraries(grapher PUBLIC nlohmann_json::nlohmann_json - sciplot::sciplot stdc++fs tbb ${llvm_libs}) + sciplot::sciplot stdc++fs tbb fmt::fmt ${llvm_libs}) target_compile_options(grapher PUBLIC -DJSON_NOEXCEPTION) diff --git a/grapher/configs/readme.md b/grapher/configs/readme.md deleted file mode 100644 index dcecc63..0000000 --- a/grapher/configs/readme.md +++ /dev/null @@ -1,39 +0,0 @@ -# Useful configs - -Useful configs to break down time-trace data. - -## `feature_comparison.json` - -Generates comparison graphs for every major compile pass, namely: - -* `ExecuteCompiler` -* `Frontend` -* `Backend` -* `ModuleToFunctionPassAdaptor` -* `ModuleInlinerWrapperPass` -* `OptModule` -* `Source` -* `ParseClass` -* `ParseTemplate` -* `InstantiateClass` -* `InstantiateFunction` -* `CodeGen Function` -* `PerformPendingInstantiations` - -## `ExecuteCompiler-stacked.json` - * `Frontend` - * `Backend` - -## `Backend-stacked.json` - * `ModuleToFunctionPassAdaptor` - * `ModuleInlinerWrapperPass` - * `OptModule` - -## `Frontend-stacked.json` - * `Source` - * `ParseClass` - * `ParseTemplate` - * `InstantiateClass` - * `InstantiateFunction` - * `CodeGen Function` - * `PerformPendingInstantiations` diff --git a/grapher/grapher-plot.cpp b/grapher/grapher-plot.cpp index 411122f..7781ae3 100644 --- a/grapher/grapher-plot.cpp +++ b/grapher/grapher-plot.cpp @@ -28,7 +28,7 @@ int main(int argc, char const *argv[]) { llvm::cl::ParseCommandLineOptions(argc, argv); // Get configed - nlohmann::json config; + grapher::json_t config; { std::ifstream config_file(cli::config_opt.getValue()); if (!config_file) { @@ -47,7 +47,8 @@ int main(int argc, char const *argv[]) { // Acquire plotter grapher::plotter_t plotter = grapher::plotter_type_to_plotter(grapher::string_to_plotter_type( - grapher::json_value(config, "plotter"))); + grapher::json_at_ref(config, + "plotter"))); // Build cats grapher::benchmark_set_t bset = diff --git a/grapher/grapher-utils.cpp b/grapher/grapher-utils.cpp index 4aa4e83..98d28a1 100644 --- a/grapher/grapher-utils.cpp +++ b/grapher/grapher-utils.cpp @@ -1,7 +1,6 @@ #include #include -#include "grapher/plotters/plotter_i.hpp" #include "grapher/plotters/plotters.hpp" namespace cli { @@ -14,16 +13,13 @@ lc::opt plotter_opt("plotter", lc::Required, enum command_enum_t { generate_config_v, - help_v, }; -lc::opt command_opt( - "command", lc::Required, lc::desc("Command:"), - lc::values(lc::OptionEnumValue{"get-help", command_enum_t::help_v, - "Get plotter help."}, - lc::OptionEnumValue{"get-default-config", - command_enum_t::generate_config_v, - "Output plotter default config."})); +lc::opt + command_opt("command", lc::Required, lc::desc("Command:"), + lc::values(lc::OptionEnumValue{ + "get-default-config", command_enum_t::generate_config_v, + "Output plotter default config."})); } // namespace cli @@ -38,9 +34,6 @@ int main(int argc, char const *argv[]) { case cli::generate_config_v: llvm::outs() << plotter->get_default_config().dump(2) << '\n'; break; - case cli::help_v: - llvm::outs() << plotter->get_help() << '\n'; - break; } return 0; diff --git a/grapher/include/grapher/core.hpp b/grapher/include/grapher/core.hpp index 4350ced..64a14f9 100644 --- a/grapher/include/grapher/core.hpp +++ b/grapher/include/grapher/core.hpp @@ -5,12 +5,33 @@ #include #include -#include #include +#include + namespace grapher { +// Basic container types + +template > +using map_t = boost::container::flat_map; + +template > +using multimap_t = + boost::container::flat_multimap; + +/// Alias type for JSON objects. +using json_t = nlohmann::basic_json; + +// `time cmake --build --preset bench` results using different containers +// (poacher/brainfuck project, pre-built benchmark targets): +// - boost::container::flat_map -> 78.05 secs +// - std::map -> 87.08 secs +// - boost::container::map -> 80.16 secs + /// Set of results for a benchmark case iteration of a given size. struct benchmark_iteration_t { /// Size of the benchmark instance diff --git a/grapher/include/grapher/plotters/compare.hpp b/grapher/include/grapher/plotters/compare.hpp index 125f690..2978f72 100644 --- a/grapher/include/grapher/plotters/compare.hpp +++ b/grapher/include/grapher/plotters/compare.hpp @@ -1,18 +1,57 @@ #pragma once -#include "grapher/plotters/plotter_i.hpp" +#include "grapher/plotters/plotter_base.hpp" namespace grapher::plotters { -/// \ingroup plotters +/// For each group descriptor, generates a graph comparing all benchmark cases +/// in the set. +/// +/// Plotter-specific JSON parameters: +/// - `value_json_pointer` (`string`): pointer to JSON value to measure +/// - `draw_average` (`bool`): enable average curve drawing +/// - `draw_points` (`bool`): enable point value drawing +/// - `group_descriptors` (group descriptors): see group_descriptor_t +/// documentation +/// +/// \copydetails base_default_config +/// +/// Example config: +/// \code{.json} +/// { +/// "draw_average": true, +/// "draw_points": true, +/// "group_descriptors": [ +/// { +/// "name": "All", +/// "predicates": [ +/// { +/// "pointer": "/name", +/// "regex": "*", +/// "type": "regex" +/// } +/// ] +/// } +/// ], +/// "height": 500, +/// "legend_title": "Timings", +/// "plot_file_extensions": [ +/// ".svg", +/// ".png" +/// ], +/// "plotter": "compare", +/// "value_json_pointer": "/dur", +/// "width": 1500, +/// "x_label": "Benchmark size factor", +/// "y_label": "Time (µs)" +/// } +/// \endcode -struct plotter_compare_t : plotter_i { +struct plotter_compare_t : plotter_base_t { void plot(benchmark_set_t const &bset, std::filesystem::path const &dest, - nlohmann::json const &config) const override; + grapher::json_t const &config) const override; - std::string_view get_help() const override; - - nlohmann::json get_default_config() const override; + grapher::json_t get_default_config() const override; }; } // namespace grapher::plotters diff --git a/grapher/include/grapher/plotters/compare_by.hpp b/grapher/include/grapher/plotters/compare_by.hpp new file mode 100644 index 0000000..fcde533 --- /dev/null +++ b/grapher/include/grapher/plotters/compare_by.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include "grapher/plotters/plotter_base.hpp" + +namespace grapher::plotters { + +/// Generates a benchmark comparison graph for every matching key built from the +/// list of JSON pointers. +/// +/// JSON config parameters: +/// - `key_ptrs` (string array): pointers to JSON values to use as a key +/// - `value_ptr` (`string`): pointer to the JSON value to measure +/// - `draw_average` (`bool`): enable average curve drawing +/// - `draw_points` (`bool`): enable value point drawing +/// - `demangle` (`bool`): demangle C++ symbol names +/// +/// \copydoc base_default_config +/// +/// Example config: +/// \code{.json} +/// { +/// "draw_average": true, +/// "draw_points": true, +/// "height": 500, +/// "key_ptrs": [ +/// "/name", +/// "/args/detail" +/// ], +/// "legend_title": "Timings", +/// "plot_file_extensions": [ +/// ".svg", +/// ".png" +/// ], +/// "plotter": "compare_by", +/// "value_ptr": "/dur", +/// "width": 1500, +/// "x_label": "Benchmark size factor", +/// "y_label": "Time (µs)" +/// } +/// \endcode + +struct plotter_compare_by_t : plotter_base_t { + void plot(benchmark_set_t const &bset, std::filesystem::path const &dest, + grapher::json_t const &config) const override; + + grapher::json_t get_default_config() const override; +}; + +} // namespace grapher::plotters diff --git a/grapher/include/grapher/plotters/debug.hpp b/grapher/include/grapher/plotters/debug.hpp index f8c02d1..0dadb08 100644 --- a/grapher/include/grapher/plotters/debug.hpp +++ b/grapher/include/grapher/plotters/debug.hpp @@ -1,18 +1,24 @@ #pragma once -#include "grapher/plotters/plotter_i.hpp" +#include "grapher/plotters/plotter_base.hpp" namespace grapher::plotters { -/// \ingroup plotters -/// Debug plotter, outputs statistics on benchmark categories -struct plotter_debug_t : public plotter_i { - void plot(benchmark_set_t const &bset, std::filesystem::path const &dest, - nlohmann::json const &config) const override; +/// Debug plotter. Outputs various statistics on benchmark categories to debug +/// category building or traversal issues. +/// +/// Example config: +/// \code{.json} +/// { +/// "plotter": "debug" +/// } +/// \endcode - std::string_view get_help() const override; +struct plotter_debug_t : public plotter_base_t { + void plot(benchmark_set_t const &bset, std::filesystem::path const &dest, + grapher::json_t const &config) const override; - nlohmann::json get_default_config() const override; + grapher::json_t get_default_config() const override; }; } // namespace grapher::plotters diff --git a/grapher/include/grapher/plotters/grouped_histogram.hpp b/grapher/include/grapher/plotters/grouped_histogram.hpp index b7451d0..4f75659 100644 --- a/grapher/include/grapher/plotters/grouped_histogram.hpp +++ b/grapher/include/grapher/plotters/grouped_histogram.hpp @@ -1,18 +1,14 @@ #pragma once -#include "grapher/plotters/plotter_i.hpp" +#include "grapher/plotters/plotter_base.hpp" namespace grapher::plotters { -/// \ingroup plotters -/// Display bars -struct plotter_grouped_histogram_t : public plotter_i { +struct plotter_grouped_histogram_t : public plotter_base_t { void plot(benchmark_set_t const &bset, std::filesystem::path const &dest, - nlohmann::json const &config) const override; + grapher::json_t const &config) const override; - std::string_view get_help() const override; - - nlohmann::json get_default_config() const override; + grapher::json_t get_default_config() const override; }; } // namespace grapher::plotters diff --git a/grapher/include/grapher/plotters/plotter_base.hpp b/grapher/include/grapher/plotters/plotter_base.hpp new file mode 100644 index 0000000..d71b8c4 --- /dev/null +++ b/grapher/include/grapher/plotters/plotter_base.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +#include "grapher/core.hpp" + +namespace grapher { + +/// Plotters are objects that inherit the plotter_base_t virtual interface, and +/// thus override the plotter_base_t::plot, and +/// plotter_base_t::get_default_config methods. +/// +/// They're used to generate plots from a grapher::category_t object and a +/// grapher::json_t object for configuration. +/// +/// The plotter interface can also be used to implement other exportation modes +/// such as CSV, plain text, debug, or even HTML export if you want. +/// +/// Interface for plotters. Plotters should be able to: +/// - Plot a ctbench::category_t with a grapher::json_t configuration object, +/// - Output a default config as a grapher::json_t object. +struct plotter_base_t { + virtual ~plotter_base_t() = default; + + /// Plots a given ctbench::category_t at the given destination. + /// It receives a grapher::json_t object as a config. + virtual void plot(benchmark_set_t const &bset, + std::filesystem::path const &dest, + grapher::json_t const &config) const = 0; + + /// Returns a default config for end-users. + virtual grapher::json_t get_default_config() const = 0; +}; + +/// Polymorphic representation of a plotter. +using plotter_t = std::unique_ptr; + +} // namespace grapher diff --git a/grapher/include/grapher/plotters/plotter_i.hpp b/grapher/include/grapher/plotters/plotter_i.hpp deleted file mode 100644 index 8a018a2..0000000 --- a/grapher/include/grapher/plotters/plotter_i.hpp +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "grapher/core.hpp" - -namespace grapher { - -/// \defgroup plotters Plot generators - -/// \addtogroup plotters -/// -/// Plotters are objects that inherit the plotter_i virtual interface, and thus -/// override the plotter_i::plot, plotter_i::get_help, and -/// plotter_i::get_default_config methods. -/// -/// They're used to generate plots from a grapher::category_t object and a -/// nlohmann::json object for configuration. -/// -/// The plotter interface can also be used to implement other exportation modes -/// such as CSV, plain text, debug, or even HTML export if you want. - -/// \ingroup plotters - -/// Interface for plotters. Plotters should be able to: -/// - Plot a ctbench::category_t with a nlohmann::json configuration object, -/// - Output help as a std::string_view, -/// - Output a default config as a nlohmann::json object. -struct plotter_i { - virtual ~plotter_i() = default; - - /// Plots a given ctbench::category_t at the given destination. - /// It receives a nlohmann::json object as a config. - virtual void plot(benchmark_set_t const &bset, - std::filesystem::path const &dest, - nlohmann::json const &config) const = 0; - - /// Returns a help message for end-users. - virtual std::string_view get_help() const = 0; - - /// Returns a default config for end-users. - virtual nlohmann::json get_default_config() const = 0; -}; - -/// Polymorphic representation of a plotter. -using plotter_t = std::unique_ptr; - -} // namespace grapher diff --git a/grapher/include/grapher/plotters/plotters.hpp b/grapher/include/grapher/plotters/plotters.hpp index fa3967c..bc861b8 100644 --- a/grapher/include/grapher/plotters/plotters.hpp +++ b/grapher/include/grapher/plotters/plotters.hpp @@ -5,25 +5,38 @@ #include #include "grapher/plotters/compare.hpp" +#include "grapher/plotters/compare_by.hpp" #include "grapher/plotters/debug.hpp" -#include "grapher/plotters/grouped_histogram.hpp" +//#include "grapher/plotters/grouped_histogram.hpp" #include "grapher/plotters/stack.hpp" namespace grapher { +/// \page plotter_config Plotter configuration documentation +/// # compare +/// \copydoc grapher::plotters::plotter_compare_t +/// # compare_by +/// \copydoc grapher::plotters::plotter_compare_by_t +/// # debug +/// \copydoc grapher::plotters::plotter_debug_t +/// # stack +/// \copydoc grapher::plotters::plotter_stack_t + /// Plotter type enumeration. One per plotter. enum plotter_type_t { compare_v, + compare_by_v, debug_v, - grouped_histogram_v, + // grouped_histogram_v, stack_v, }; /// String to plotter type map. inline const std::map plotter_name_map = { {"compare", compare_v}, + {"compare_by", compare_by_v}, {"debug", debug_v}, - {"grouped_histogram", grouped_histogram_v}, + // {"grouped_histogram", grouped_histogram_v}, {"stack", stack_v}, }; @@ -31,27 +44,32 @@ inline const std::map plotter_name_map = { inline const llvm::cl::ValuesClass plotter_cl_values{ llvm::cl::OptionEnumValue{"compare", compare_v, "Compare benchmarks feature by feature."}, + llvm::cl::OptionEnumValue{"compare_by", compare_by_v, + "Stack features for each benchmark."}, llvm::cl::OptionEnumValue{"debug", debug_v, "Output category stats for debug."}, - llvm::cl::OptionEnumValue{"grouped_histogram", grouped_histogram_v, - "Stack features for groups of symbols."}, + // llvm::cl::OptionEnumValue{"grouped_histogram", grouped_histogram_v, + // "Stack features for groups of symbols."}, llvm::cl::OptionEnumValue{"stack", stack_v, - "Stack features for each benchmark."}}; + "Stack features for each benchmark."}, +}; /// Converts a plotter type to a plotter. inline plotter_t plotter_type_to_plotter(plotter_type_t type) { switch (type) { case compare_v: - return std::make_unique(); + return std::make_unique(); + case compare_by_v: + return std::make_unique(); case debug_v: - return std::make_unique(); - case grouped_histogram_v: - return std::make_unique(); + return std::make_unique(); + // case grouped_histogram_v: + // return std::make_unique(); case stack_v: - return std::make_unique(); + return std::make_unique(); } - return std::make_unique(); + return std::make_unique(); } /// Converts a string to a plotter type. diff --git a/grapher/include/grapher/plotters/stack.hpp b/grapher/include/grapher/plotters/stack.hpp index ebc246b..bef1304 100644 --- a/grapher/include/grapher/plotters/stack.hpp +++ b/grapher/include/grapher/plotters/stack.hpp @@ -1,20 +1,54 @@ #pragma once -#include "grapher/plotters/plotter_i.hpp" +#include "grapher/plotters/plotter_base.hpp" namespace grapher::plotters { -/// \ingroup plotters +/// For each benchmark in the category, generates a stakcked curve graph +/// where each curve corresponds to a matcher in the matchers JSON field. +/// +/// Plotter-specific JSON parameters: +/// - `value_json_pointer` (`string`): pointer to JSON value to measure +/// - `group_descriptors` (group descriptors): see group_descriptor_t +/// documentation +/// +/// \copydetails base_default_config +/// +/// Example config: +/// \code{.json} +/// { +/// "group_descriptors": [ +/// { +/// "name": "All", +/// "predicates": [ +/// { +/// "pointer": "/name", +/// "regex": "*", +/// "type": "regex" +/// } +/// ] +/// } +/// ], +/// "height": 500, +/// "legend_title": "Timings", +/// "name_json_pointer": "/name", +/// "plot_file_extensions": [ +/// ".svg", +/// ".png" +/// ], +/// "plotter": "stack", +/// "value_json_pointer": "/dur", +/// "width": 1500, +/// "x_label": "Benchmark size factor", +/// "y_label": "Time (µs)" +/// } +/// \endcode -/// Generates one plot per benchmark where all the targeted features are -/// visualized as stacked curves. -struct plotter_stack_t : plotter_i { +struct plotter_stack_t : plotter_base_t { void plot(benchmark_set_t const &bset, std::filesystem::path const &dest, - nlohmann::json const &config) const override; + grapher::json_t const &config) const override; - std::string_view get_help() const override; - - nlohmann::json get_default_config() const override; + grapher::json_t get_default_config() const override; }; } // namespace grapher::plotters diff --git a/grapher/include/grapher/predicates.hpp b/grapher/include/grapher/predicates.hpp index 9b56ec5..6c0f01b 100644 --- a/grapher/include/grapher/predicates.hpp +++ b/grapher/include/grapher/predicates.hpp @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -11,10 +13,10 @@ namespace grapher { /// \ingroup predicates /// Dynamic representation of a predicate. -using predicate_t = std::function; +using predicate_t = std::function; /// \ingroup predicates /// Builds predicate and stores it in an std::function object. -predicate_t get_predicate(nlohmann::json const &constraint); +predicate_t get_predicate(grapher::json_t const &constraint); } // namespace grapher diff --git a/grapher/include/grapher/utils/config.hpp b/grapher/include/grapher/utils/config.hpp deleted file mode 100644 index 42d5fe4..0000000 --- a/grapher/include/grapher/utils/config.hpp +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -#include - -#include - -#include - -/// \file -/// Common structure and function definitions that can be used for plotter -/// configuration. - -namespace grapher { - -/// Named set of constraint. -struct group_descriptor_t { - std::string name; - std::vector predicates; -}; - -group_descriptor_t get_default_group_descriptor(); - -std::vector get_predicates(group_descriptor_t const &descriptor); - -std::vector -extract_group(group_descriptor_t const &descriptor, - std::vector const &events); - -group_descriptor_t json_to_group_descriptor(nlohmann::json const &j); - -nlohmann::json group_descriptor_json(group_descriptor_t const &descriptor); - -std::vector -write_descriptors(std::vector const &descriptors); - -std::vector -read_descriptors(std::vector const &list); - -} // namespace grapher diff --git a/grapher/include/grapher/utils/error.hpp b/grapher/include/grapher/utils/error.hpp new file mode 100644 index 0000000..812c922 --- /dev/null +++ b/grapher/include/grapher/utils/error.hpp @@ -0,0 +1,57 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +#include + +namespace grapher { + +enum error_level_t : std::uint8_t { error_v, warning_v, info_v, log_v }; + +inline std::string to_string(error_level_t error_level) { + switch (error_level) { + case error_v: + return "ERROR"; + case warning_v: + return "WARNING"; + case info_v: + return "INFO"; + case log_v: + return "LOG"; + } + return ""; +} + +/// Prints a warning +inline void warn(std::string_view explain, + error_level_t error_level = warning_v, + std::experimental::source_location loc = + std::experimental::source_location::current()) { + llvm::errs() << fmt::format( + "[{}] {}:{}:{} ({}) - {}\n", to_string(error_level), loc.file_name(), + loc.line(), loc.column(), loc.function_name(), explain); +} + +/// Error management: if the condition is false, it will print a warning or +/// error message and termineate the program if error_level is error_v. +/// Condition value is then returned (useful for warnings). +inline bool check(bool condition, std::string_view explain, + error_level_t error_level = error_v, int err_code = -1, + std::experimental::source_location loc = + std::experimental::source_location::current()) { + if (!condition) { + warn(explain, error_level, loc); + if (error_level == error_v) { + std::exit(err_code); + } + } + return condition; +} + +} // namespace grapher diff --git a/grapher/include/grapher/utils/json.hpp b/grapher/include/grapher/utils/json.hpp index 3f432cb..dbf1a61 100644 --- a/grapher/include/grapher/utils/json.hpp +++ b/grapher/include/grapher/utils/json.hpp @@ -7,10 +7,13 @@ #include -#include +#include + +#include #include "grapher/core.hpp" -#include "grapher/utils/config.hpp" +#include "grapher/predicates.hpp" +#include "grapher/utils/error.hpp" namespace grapher { @@ -18,97 +21,176 @@ namespace grapher { /// value_jptr in the events matching the descriptor's predicates. std::vector get_values(benchmark_iteration_t const &iteration, std::vector const &predicates, - nlohmann::json::json_pointer value_jptr); + grapher::json_t::json_pointer value_jptr); /// Merges the contents of b into a, with items of a being overwritten if items /// present in b share the same key. -nlohmann::json merge_into(nlohmann::json a, nlohmann::json const &b); - -/// Wrapper for strict JSON type error management. -/// Exits program if value at given field location is empty or isn't of the -/// right type. -template -inline ValueType json_value(nlohmann::json const &object, - LocType const &field_location, - const std::experimental::source_location loc = - std::experimental::source_location::current()) { - // nlohmann::json - if constexpr (std::is_same::value) { - if (!object.contains(field_location)) { - llvm::errs() << loc.function_name() << " - Empty field " << field_location - << ":\n" - << object.dump(2) << '\n'; - std::exit(1); - } else { - return object[field_location]; - } +grapher::json_t merge_into(grapher::json_t a, grapher::json_t const &b); + +/// Generic conversion of a JSON field location to a std::string +template +inline std::string field_loc_to_string(T &&field_location) { + return std::string{std::forward(field_location)}; +} + +/// Specialized conversion of a JSON field location to a std::string +inline std::string +field_loc_to_string(grapher::json_t::json_pointer const &json_ptr) { + return json_ptr.to_string(); +} + +/// Wraps json_t object access with error management. +template +inline grapher::json_t::const_reference +json_at(grapher::json_t const &object, LocType const &field_location, + const std::experimental::source_location loc = + std::experimental::source_location::current()) { + std::string const field_location_str = field_loc_to_string(field_location); + + check(object.contains(field_location), + fmt::format("Empty field {}:\n{}", field_location_str, object.dump(2)), + error_v, -1, loc); + + return object[field_location]; +} + +/// Wraps reference access with type checking and error management. +template +inline ReferenceType +json_at_ref(grapher::json_t const &object, LocType const &field_location, + const std::experimental::source_location loc = + std::experimental::source_location::current()) { + + using ValueType = std::decay_t; + + std::string const field_location_str = field_loc_to_string(field_location); + + check(object.contains(field_location), + fmt::format("Empty field {}:\n{}", field_location_str, object.dump(2)), + error_v, -1, loc); + + // number_unsigned_t + if constexpr (std::is_same::value) { + check(object[field_location].type() == json_t::value_t::number_unsigned, + fmt::format("Invalid field {}, expected unsigned number:\n{}", + field_location_str, object.dump(2)), + error_v, -1, loc); } - // std::string - if constexpr (std::is_same::value) { - if (!object.contains(field_location) || - !object[field_location].is_string()) { - llvm::errs() << loc.function_name() << " - Invalid field " - << field_location << ", expected string:\n" - << object.dump(2) << '\n'; - std::exit(1); - } else { - return object[field_location]; - } + // number_integer_t + else if constexpr (std::is_same::value) { + check(object[field_location].type() == json_t::value_t::number_integer, + fmt::format("Invalid field {}, expected integer number:\n{}", + field_location_str, object.dump(2)), + error_v, -1, loc); } - // std::vector - if constexpr (std::is_same, ValueType>::value) { - if (!object.contains(field_location) || - !object[field_location].is_array()) { - llvm::errs() << loc.function_name() << " - Invalid field " - << field_location - << ", expected std::vector:\n" - << object.dump(2) << '\n'; - std::exit(1); - } else { - return object[field_location]; - } + // number_float_t + else if constexpr (std::is_same::value) { + check(object[field_location].type() == json_t::value_t::number_float, + fmt::format("Invalid field {}, expected float number:\n{}", + field_location_str, object.dump(2)), + error_v, -1, loc); } - // bool - if constexpr (std::is_same::value) { - if (!object.contains(field_location) || - !object[field_location].is_boolean()) { - llvm::errs() << loc.function_name() << " - Invalid field " - << field_location << ", expected bool:\n" - << object.dump(2) << '\n'; - std::exit(1); - } else { - return object[field_location]; - } + // array_t + else if constexpr (std::is_same::value) { + check(object[field_location].type() == json_t::value_t::array, + fmt::format("Invalid field {}, expected array:\n{}", + field_location_str, object.dump(2)), + error_v, -1, loc); } - // int - if constexpr (std::is_same::value) { - if (!object.contains(field_location) || - !object[field_location].is_number()) { - llvm::errs() << loc.function_name() << " - Invalid field " - << field_location << ", expected number:\n" - << object.dump(2) << '\n'; - std::exit(1); - } else { - return object[field_location]; - } + // boolean_t + else if constexpr (std::is_same::value) { + check(object[field_location].type() == json_t::value_t::boolean, + fmt::format("Invalid field {}, expected bool:\n{}", + field_location_str, object.dump(2)), + error_v, -1, loc); } - // double - if constexpr (std::is_same::value) { - if (!object.contains(field_location) || - !object[field_location].is_number()) { - llvm::errs() << loc.function_name() << " - Invalid field " - << field_location << ", expected number:\n" - << object.dump(2) << '\n'; - std::exit(1); - } else { - return object[field_location]; - } + // object_t + else if constexpr (std::is_same::value) { + check(object[field_location].type() == json_t::value_t::object, + fmt::format("Invalid field {}, expected JSON object:\n{}", + field_location_str, object.dump(2)), + error_v, -1, loc); } + + return object[field_location].template get_ref(); } +// ============================================================================= +// Group descriptors + +/// Describes a group of events with a name and a set of constraints. Group +/// descriptors are used by some plotters to target and aggregate trace events +/// for data collection. +/// +/// For example, this could be a group descriptor for collecting data about +/// Source trace events that are related to Boost include files: +/// +/// \code{.json} +/// { +/// "name": "Boost sources events", +/// "predicates": [ +/// { +/// "type": "match", +/// "regex_match": true, +/// "matcher": { +/// "name": "Source", +/// "args": { +/// "details": "/usr/include/boost/*" +/// } +/// } +/// } +/// ] +/// } +/// \endcode +struct group_descriptor_t { + std::string name; + grapher::json_t::array_t predicates; +}; + +/// Returns a default group descriptor as an example. +group_descriptor_t get_default_group_descriptor(); + +/// Extracts predicates from a group descriptor. +std::vector get_predicates(group_descriptor_t const &descriptor); + +grapher::json_t::array_t extract_group(group_descriptor_t const &descriptor, + grapher::json_t::array_t const &events); + +/// Exports a single group descriptor into a JSON object. +grapher::json_t group_descriptor_json(group_descriptor_t const &descriptor); + +/// Exports a vector of group descriptors into a JSON object. +grapher::json_t::array_t +write_descriptors(std::vector const &descriptors); + +/// Reads a single descriptor. +group_descriptor_t read_descriptor(grapher::json_t const &j); + +/// Reads descriptors from a predicate list. +std::vector +read_descriptors(grapher::json_t::array_t const &list); + +// ============================================================================= +// Plotter configuration + +/// Plot saving helper function. +void save_plot(sciplot::Plot2D const &plot, std::string const &dest, + grapher::json_t const &config); + +/// Returns the default configuration for apply_config. +grapher::json_t base_default_config(); + +/// Apply config to plot. +sciplot::Plot &apply_config(sciplot::Plot &plot, grapher::json_t config); + } // namespace grapher diff --git a/grapher/include/grapher/utils/plot.hpp b/grapher/include/grapher/utils/plot.hpp deleted file mode 100644 index 2ef87a1..0000000 --- a/grapher/include/grapher/utils/plot.hpp +++ /dev/null @@ -1,15 +0,0 @@ -#pragma once - -#include - -#include - -namespace grapher { - -/// Apply config to plot. -sciplot::Plot &apply_config(sciplot::Plot &plot, nlohmann::json config); - -/// Returns the default configuration for apply_config. -nlohmann::json base_default_config(); - -} // namespace grapher diff --git a/grapher/lib/grapher/plotters/compare.cpp b/grapher/lib/grapher/plotters/compare.cpp index 0bb11bf..77d99c0 100644 --- a/grapher/lib/grapher/plotters/compare.cpp +++ b/grapher/lib/grapher/plotters/compare.cpp @@ -5,6 +5,8 @@ #include +#include + #include #include @@ -12,19 +14,13 @@ #include "grapher/core.hpp" #include "grapher/plotters/compare.hpp" #include "grapher/predicates.hpp" -#include "grapher/utils/config.hpp" +#include "grapher/utils/error.hpp" #include "grapher/utils/json.hpp" -#include "grapher/utils/plot.hpp" namespace grapher::plotters { -std::string_view plotter_compare_t::get_help() const { - return "For each group descriptor, generates a graph comparing all " - "benchmark cases in the set."; -} - -nlohmann::json plotter_compare_t::get_default_config() const { - nlohmann::json res = grapher::base_default_config(); +grapher::json_t plotter_compare_t::get_default_config() const { + grapher::json_t res = grapher::base_default_config(); res["plotter"] = "compare"; @@ -32,8 +28,6 @@ nlohmann::json plotter_compare_t::get_default_config() const { res["draw_average"] = true; res["draw_points"] = true; - res["plot_file_extension"] = ".svg"; - res["group_descriptors"] = write_descriptors({get_default_group_descriptor()}); @@ -42,25 +36,23 @@ nlohmann::json plotter_compare_t::get_default_config() const { void plotter_compare_t::plot(benchmark_set_t const &bset, std::filesystem::path const &dest, - nlohmann::json const &config) const { + grapher::json_t const &config) const { // Config - nlohmann::json::json_pointer value_json_pointer( - json_value(config, "value_json_pointer")); + grapher::json_t::json_pointer value_json_pointer( + json_at_ref(config, "value_json_pointer")); bool draw_average = config.value("draw_average", true); bool draw_points = config.value("draw_points", true); - std::string plot_file_extension = config.value("plot_file_extension", ".svg"); - std::vector group_descriptors = read_descriptors( - json_value>(config, "group_descriptors")); + json_at_ref(config, "group_descriptors")); // Drawing for (group_descriptor_t const &descriptor : group_descriptors) { // Plot init - sciplot::Plot plot; + sciplot::Plot2D plot; apply_config(plot, config); // Generating predicates @@ -74,22 +66,19 @@ void plotter_compare_t::plot(benchmark_set_t const &bset, std::vector y_average; for (benchmark_iteration_t const &iteration : bench.iterations) { - if (iteration.samples.empty()) { - llvm::errs() << "[WARNING] No data in benchmark " << bench.name - << " for iteration size " << iteration.size << "\n"; - continue; - } + check(!iteration.samples.empty(), + fmt::format("No data in benchmark {} for iteration size {}.", + bench.name, iteration.size), + error_level_t::warning_v); std::vector const values = get_values(iteration, predicates, value_json_pointer); - if (values.empty()) { - llvm::errs() << "[WARNING] No event in benchmark " << bench.name - << " at size " << iteration.size - << " matched by group descriptor " << descriptor.name - << ".\n"; - continue; - } + check(!values.empty(), + fmt::format("No event in benchmark {} at size {} matched by " + "group descriptor {}.\n", + bench.name, iteration.size, descriptor.name), + error_level_t::warning_v); // Drawing points if (draw_points) { @@ -118,7 +107,7 @@ void plotter_compare_t::plot(benchmark_set_t const &bset, // Saving plot std::filesystem::create_directories(dest); - plot.save(dest / (std::move(descriptor.name) + plot_file_extension)); + save_plot(plot, dest / std::move(descriptor.name), config); } } diff --git a/grapher/lib/grapher/plotters/compare_by.cpp b/grapher/lib/grapher/plotters/compare_by.cpp new file mode 100644 index 0000000..bcb0908 --- /dev/null +++ b/grapher/lib/grapher/plotters/compare_by.cpp @@ -0,0 +1,221 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include + +#include + +#include "grapher/core.hpp" +#include "grapher/plotters/compare_by.hpp" +#include "grapher/utils/error.hpp" +#include "grapher/utils/json.hpp" + +namespace grapher::plotters { + +// ============================================================================= +// AUXILIARY + +// Plot-friendly data structures + +/// Point aggregate (multiple Y coordinates) +using point_data_t = std::vector; + +/// Curve: X -> vec +using benchmark_curve_t = map_t; + +/// Benchmark name -> Curve +using curve_aggregate_t = map_t; + +/// Value key type. Contains multiple values to group by a tuple of parameters +using key_t = boost::container::small_vector; + +/// Feature -> Benchmark aggregate +using curve_aggregate_map_t = map_t; + +/// Wrangles data into a structure that's easier to work with for plotting. +curve_aggregate_map_t +get_bench_curves(benchmark_set_t const &bset, + std::vector const &key_ptrs, + json_t::json_pointer const &val_ptr) { + namespace fs = std::filesystem; + + curve_aggregate_map_t res; + + for (benchmark_case_t const &bench_case : bset) { + for (benchmark_iteration_t const &iteration : bench_case.iterations) { + for (fs::path const &sample : iteration.samples) { + grapher::json_t sample_json; + { + std::ifstream sample_ifstream(sample); + sample_ifstream >> sample_json; + } + + for (grapher::json_t const &event : + json_at_ref(sample_json, "traceEvents")) { + + // Building key from JSON pointers + key_t key; + for (json_t::json_pointer const &key_ptr : key_ptrs) { + if (event.contains(key_ptr) && event[key_ptr].is_string()) { + key.push_back(event[key_ptr]); + } + } + + // Key/value presence and type checks + if (check(event.contains(val_ptr), + fmt::format("No value at {}: {}", val_ptr.to_string(), + event.dump()), + info_v) && + check(event[val_ptr].is_number(), + fmt::format("Value at {} is not an integer: {}", + val_ptr.to_string(), event.dump()), + info_v)) { + // Adding value + res[key][bench_case.name][iteration.size].push_back(event[val_ptr]); + } + } + } + } + } + + return res; +} + +/// Transforms a key into a string that's usable as a path. +std::string to_string(key_t const &key, bool demangle = true) { + if (key.empty()) { + return "empty"; + } + + std::string res = demangle ? llvm::demangle(key[0]) : key[0]; + std::for_each(key.begin() + 1, key.end(), [&](std::string const &part) { + res += '/'; + for (char const c : demangle ? llvm::demangle(part) : part) { + res += c == '/' ? '_' : c; + } + }); + + return res; +} + +// ============================================================================= +// OVERRIDES + +grapher::json_t plotter_compare_by_t::get_default_config() const { + grapher::json_t res = grapher::base_default_config(); + + // Plotter name + res["plotter"] = "compare_by"; + + res["key_ptrs"] = json_t::array({"/name", "/args/detail"}); + res["value_ptr"] = "/dur"; + res["draw_average"] = true; + res["draw_points"] = true; + res["demangle"] = true; + + return res; +} + +void plotter_compare_by_t::plot(benchmark_set_t const &bset, + std::filesystem::path const &dest, + grapher::json_t const &config) const { + namespace fs = std::filesystem; + + // Config reading + + std::vector key_strs = + config.value("key_ptrs", json_t::array({"/name", "/args/detail"})); + + json_t::json_pointer value_ptr(config.value("value_ptr", "/dur")); + + bool draw_average = config.value("draw_average", true); + bool draw_points = config.value("draw_points", true); + bool demangle = config.value("demangle", true); + + std::vector key_ptrs; + std::transform(key_strs.begin(), key_strs.end(), std::back_inserter(key_ptrs), + [](std::string const &s) -> json_t::json_pointer { + return json_t::json_pointer{s}; + }); + + // Wrangling + curve_aggregate_map_t curve_aggregate_map = + get_bench_curves(bset, key_ptrs, value_ptr); + + // Ensure the destination folder exists + fs::create_directories(dest); + + // Drawing, ie. unwrapping the nested maps and drawing curves + saving plots + + std::vector thread_pool; + + for (auto const &[key, curve_aggregate] : curve_aggregate_map) { + // Plot init + sciplot::Plot2D plot; + apply_config(plot, config); + + for (auto const &[bench_name, benchmark_curve] : curve_aggregate) { + // Average curve coord vectors + std::vector x_curve; + std::vector y_curve; + + // Point coord vectors + std::vector x_points; + std::vector y_points; + + // Build point & curve vectors + for (auto const &[x, y_vec] : benchmark_curve) { + // Building average curve vector + if (draw_average && !y_vec.empty()) { + double const sum = std::reduce(y_vec.begin(), y_vec.end()); + std::size_t const n = y_vec.size(); + + double const y = sum / n; + + x_curve.push_back(x); + y_curve.push_back(y); + } + + // Building point vector + if (draw_points) { + for (double y : y_vec) { + x_points.push_back(x); + y_points.push_back(y); + } + } + } + + // Plot drawing + + if (draw_average && !x_curve.empty()) { + // Draw average curve + plot.drawCurve(x_curve, y_curve).label(bench_name + " average"); + } + + if (draw_points && !x_points.empty()) { + // Draw points + plot.drawPoints(x_points, y_points).label(bench_name + " points"); + } + } + + // Writing in parallel + thread_pool.emplace_back(&save_plot, std::move(plot), + dest / to_string(key, demangle), config); + } + + // Joining before returning + for (std::thread &t : thread_pool) { + t.join(); + } +} + +} // namespace grapher::plotters diff --git a/grapher/lib/grapher/plotters/debug.cpp b/grapher/lib/grapher/plotters/debug.cpp index 83d8273..f973a50 100644 --- a/grapher/lib/grapher/plotters/debug.cpp +++ b/grapher/lib/grapher/plotters/debug.cpp @@ -1,26 +1,20 @@ #include "grapher/plotters/debug.hpp" #include -#include namespace grapher::plotters { -std::string_view plotter_debug_t::get_help() const { - return "Debug plotter. Outputs various statistics on benchmark categories to " - "debug category building or traversal issues."; -} - -nlohmann::json plotter_debug_t::get_default_config() const { - nlohmann::json res; +grapher::json_t plotter_debug_t::get_default_config() const { + grapher::json_t res; res["plotter"] = "debug"; return res; } void plotter_debug_t::plot(const benchmark_set_t &bset, const std::filesystem::path &dest, - const nlohmann::json & /* config */) const { - llvm::outs() << "Output path: " << dest << '\n'; - llvm::outs() << "Category size: " << bset.size() << '\n'; + const grapher::json_t & /* config */) const { + llvm::outs() << "Output path: " << dest << '\n' + << "Category size: " << bset.size() << '\n'; for (benchmark_case_t const &bench : bset) { llvm::outs() << "\tBenchmark case name: " << bench.name << '\n' diff --git a/grapher/lib/grapher/plotters/grouped_histogram.cpp b/grapher/lib/grapher/plotters/grouped_histogram.cpp index 9db7c1c..7b042fa 100644 --- a/grapher/lib/grapher/plotters/grouped_histogram.cpp +++ b/grapher/lib/grapher/plotters/grouped_histogram.cpp @@ -3,16 +3,10 @@ #include "grapher/plotters/grouped_histogram.hpp" #include "grapher/plotters/plotters.hpp" #include "grapher/utils/json.hpp" -#include "grapher/utils/plot.hpp" namespace grapher::plotters { -std::string_view plotter_grouped_histogram_t::get_help() const { - // TODO: Add doc - return "TODO"; -} - -nlohmann::json plotter_grouped_histogram_t::get_default_config() const { +grapher::json_t plotter_grouped_histogram_t::get_default_config() const { // TODO return grapher::base_default_config(); } @@ -20,7 +14,7 @@ nlohmann::json plotter_grouped_histogram_t::get_default_config() const { void plotter_grouped_histogram_t::plot( const benchmark_set_t & /* bset */, const std::filesystem::path & /* dest */, - const nlohmann::json & /* config */) const { + const grapher::json_t & /* config */) const { // TODO } diff --git a/grapher/lib/grapher/plotters/plotters.cpp b/grapher/lib/grapher/plotters/plotters.cpp index 6e8ab43..f160f23 100644 --- a/grapher/lib/grapher/plotters/plotters.cpp +++ b/grapher/lib/grapher/plotters/plotters.cpp @@ -1,5 +1,5 @@ #include "grapher/plotters/plotters.hpp" -#include "grapher/plotters/plotter_i.hpp" +#include "grapher/plotters/plotter_base.hpp" namespace grapher { diff --git a/grapher/lib/grapher/plotters/stack.cpp b/grapher/lib/grapher/plotters/stack.cpp index 9169a25..a88b4a8 100644 --- a/grapher/lib/grapher/plotters/stack.cpp +++ b/grapher/lib/grapher/plotters/stack.cpp @@ -1,39 +1,27 @@ #include #include +#include #include -#include +#include -#include +#include #include #include "grapher/core.hpp" #include "grapher/plotters/stack.hpp" #include "grapher/predicates.hpp" -#include "grapher/utils/config.hpp" +#include "grapher/utils/error.hpp" #include "grapher/utils/json.hpp" -#include "grapher/utils/plot.hpp" namespace grapher::plotters { -std::string_view plotter_stack_t::get_help() const { - return "For each benchmark in the category, generates a stakcked curve graph " - "where each curve corresponds to a matcher in the \'matchers\' JSON " - "field."; -} - -nlohmann::json plotter_stack_t::get_default_config() const { - nlohmann::json res = grapher::base_default_config(); +grapher::json_t plotter_stack_t::get_default_config() const { + grapher::json_t res = grapher::base_default_config(); res["plotter"] = "stack"; - - // Basic values, probably no need to change them res["value_json_pointer"] = "/dur"; - res["name_json_pointer"] = "/name"; - res["plot_file_extension"] = ".svg"; - - // Some matchers as an example... res["group_descriptors"] = write_descriptors({get_default_group_descriptor()}); @@ -42,32 +30,25 @@ nlohmann::json plotter_stack_t::get_default_config() const { void plotter_stack_t::plot(benchmark_set_t const &bset, std::filesystem::path const &dest, - nlohmann::json const &config) const { - // Config + grapher::json_t const &config) const { + // Config reading - nlohmann::json::json_pointer feature_value_jptr( + grapher::json_t::json_pointer feature_value_jptr( config.value("value_json_pointer", "/dur")); - nlohmann::json::json_pointer feature_name_jptr( - config.value("name_json_pointer", "/name")); - - std::string plot_file_extension = config.value("plot_file_extension", ".svg"); - std::vector descriptors = read_descriptors( - json_value>(config, "group_descriptors")); + json_at_ref(config, "group_descriptors")); // Drawing - std::vector plots; + std::vector plots; - // Saving max y value for normalization + // Storing max y value for normalization double max_y_val = 0.; /// Draws a stacked curve graph for a given benchmark - auto draw_plot = [&](benchmark_case_t const &bench) -> sciplot::Plot { - namespace sp = sciplot; - - sp::Plot plot; + auto draw_plot = [&](benchmark_case_t const &bench) -> sciplot::Plot2D { + sciplot::Plot2D plot; apply_config(plot, config); // x axis @@ -92,12 +73,10 @@ void plotter_stack_t::plot(benchmark_set_t const &bset, std::vector const values = get_values(iteration, predicates, feature_value_jptr); - if (values.empty()) { - llvm::errs() << "[ERROR] No event matching descriptor " - << descriptor.name << " in benchmark " << bench.name - << " with iteration size " << iteration.size << ".\n"; - std::exit(1); - } + check(values.empty(), + fmt::format("No event matching descriptor {} in benchmark {} " + "with iteration size {}.\n", + descriptor.name, bench.name, iteration.size)); // TODO: Get better stats (standard deviation, etc...) double const y_val = @@ -128,7 +107,7 @@ void plotter_stack_t::plot(benchmark_set_t const &bset, std::filesystem::create_directories(dest); for (std::size_t i = 0; i < bset.size(); i++) { plots[i].yrange(0., max_y_val); - plots[i].save(dest / (bset[i].name + plot_file_extension)); + save_plot(plots[i], dest / bset[i].name, config); } } diff --git a/grapher/lib/grapher/predicates.cpp b/grapher/lib/grapher/predicates.cpp index ea31369..4e99c0a 100644 --- a/grapher/lib/grapher/predicates.cpp +++ b/grapher/lib/grapher/predicates.cpp @@ -1,8 +1,4 @@ -#include "grapher/predicates.hpp" -#include "grapher/utils/json.hpp" - -#include -#include +#include #include #include @@ -10,6 +6,13 @@ #include +#include + +#include "grapher/core.hpp" +#include "grapher/predicates.hpp" +#include "grapher/utils/error.hpp" +#include "grapher/utils/json.hpp" + namespace grapher::predicates { /// \ingroup predicates @@ -23,17 +26,19 @@ namespace grapher::predicates { /// "regex": "Total*" /// } /// ``` -inline auto regex(nlohmann::json const &constraint) { +inline auto regex(grapher::json_t const &constraint) { // Validating pointer parameter - return [pointer = nlohmann::json::json_pointer{json_value( - constraint, "pointer")}, - regex = std::regex(json_value(constraint, "regex"))]( - nlohmann::json const &value) -> bool { + return [pointer = + grapher::json_t::json_pointer{ + json_at_ref(constraint, "pointer")}, + regex = std::regex(json_at_ref( + constraint, "regex"))](grapher::json_t const &value) -> bool { if (!value.contains(pointer) || !value[pointer].is_string()) { return false; } - return std::regex_match(json_value(value, pointer), regex); + return std::regex_match( + json_at_ref(value, pointer), regex); }; } @@ -77,17 +82,16 @@ inline auto regex(nlohmann::json const &constraint) { /// } /// } /// ``` -inline auto match(nlohmann::json const &constraint) { - return [matcher_flat = - json_value(constraint, "matcher").flatten(), +inline auto match(grapher::json_t const &constraint) { + return [matcher_flat = json_at(constraint, "matcher").flatten(), regex_match_opt = constraint.value("regex", false)]( - nlohmann::json const &value) -> bool { + grapher::json_t const &value) -> bool { auto items_iteration_proxy = matcher_flat.items(); return std::all_of( items_iteration_proxy.begin(), items_iteration_proxy.end(), [&](auto const &matcher_item_kv) -> bool { // Pointer to the value we should observe - nlohmann::json::json_pointer const ptr(matcher_item_kv.key()); + grapher::json_t::json_pointer const ptr(matcher_item_kv.key()); // Checking for existence of matching value if (!value.contains(ptr)) { @@ -117,16 +121,17 @@ inline auto match(nlohmann::json const &constraint) { /// "string": "Total Source" /// } /// ``` -inline auto streq(nlohmann::json const &constraint) { - return [pointer = nlohmann::json::json_pointer{json_value( - constraint, "pointer")}, - str = json_value(constraint, "string")]( - nlohmann::json const &value) -> bool { +inline auto streq(grapher::json_t const &constraint) { + return [pointer = + grapher::json_t::json_pointer{ + json_at_ref(constraint, "pointer")}, + str = json_at_ref(constraint, "string")]( + grapher::json_t const &value) -> bool { if (!value.contains(pointer) || !value[pointer].is_string()) { return false; } - return value[pointer].get_ref() == str; + return value[pointer].get_ref() == str; }; } @@ -152,13 +157,12 @@ inline auto streq(nlohmann::json const &constraint) { /// } /// } /// ``` -inline auto op_or(nlohmann::json const &constraint) { - return - [first = get_predicate(json_value(constraint, "first")), - second = get_predicate(json_value( - constraint, "second"))](nlohmann::json const &value) -> bool { - return first(value) || second(value); - }; +inline auto op_or(grapher::json_t const &constraint) { + return [first = get_predicate(json_at(constraint, "first")), + second = get_predicate(json_at(constraint, "second"))]( + grapher::json_t const &value) -> bool { + return first(value) || second(value); + }; } /// \ingroup predicates @@ -182,13 +186,12 @@ inline auto op_or(nlohmann::json const &constraint) { /// } /// } /// ``` -inline auto op_and(nlohmann::json const &constraint) { - return - [first = get_predicate(json_value(constraint, "first")), - second = get_predicate(json_value( - constraint, "second"))](nlohmann::json const &value) -> bool { - return first(value) && second(value); - }; +inline auto op_and(grapher::json_t const &constraint) { + return [first = get_predicate(json_at(constraint, "first")), + second = get_predicate(json_at(constraint, "second"))]( + grapher::json_t const &value) -> bool { + return first(value) && second(value); + }; } /// \ingroup predicates @@ -200,8 +203,8 @@ inline auto op_and(nlohmann::json const &constraint) { /// "type": "val_true", /// } /// ``` -inline auto val_true(nlohmann::json const & /* unused */) { - return [](nlohmann::json const &) -> bool { return true; }; +inline auto val_true(grapher::json_t const & /* unused */) { + return [](grapher::json_t const &) -> bool { return true; }; } /// \ingroup predicates @@ -213,8 +216,8 @@ inline auto val_true(nlohmann::json const & /* unused */) { /// "type": "val_false", /// } /// ``` -inline auto val_false(nlohmann::json const & /* unused */) { - return [](nlohmann::json const &) -> bool { return false; }; +inline auto val_false(grapher::json_t const & /* unused */) { + return [](grapher::json_t const &) -> bool { return false; }; } } // namespace grapher::predicates @@ -223,8 +226,9 @@ namespace grapher { /// \ingroup predicates /// Builds predicate and stores it in an std::function object. -predicate_t get_predicate(nlohmann::json const &constraint) { - std::string constraint_type = json_value(constraint, "type"); +predicate_t get_predicate(grapher::json_t const &constraint) { + std::string constraint_type = + json_at_ref(constraint, "type"); #define REGISTER_PREDICATE(name) \ if (constraint_type == #name) { \ @@ -241,9 +245,9 @@ predicate_t get_predicate(nlohmann::json const &constraint) { #undef REGISTER_PREDICATE - llvm::errs() << "[ERROR] Predicate error, invalid type:\n" - << constraint.dump(2) << '\n'; - std::exit(1); + check(false, + fmt::format("Predicate error, invalid type:\n{}", constraint.dump(2))); + return {}; } } // namespace grapher diff --git a/grapher/lib/grapher/utils/cli.cpp b/grapher/lib/grapher/utils/cli.cpp index 7bcdedb..33735a6 100644 --- a/grapher/lib/grapher/utils/cli.cpp +++ b/grapher/lib/grapher/utils/cli.cpp @@ -4,7 +4,10 @@ #include +#include + #include "grapher/utils/cli.hpp" +#include "grapher/utils/error.hpp" namespace grapher { @@ -18,8 +21,8 @@ build_category(llvm::cl::list const &benchmark_path_list) { fs::path bench_path(bench_path_str.data()); if (!fs::is_directory(bench_path)) { - llvm::errs() << "[WARNING] Not a directory: " << bench_path - << " (current path: " << fs::current_path() << ").\n"; + warn(fmt::format("Not a directory: {} (current path: {}).", + bench_path.string(), fs::current_path().string())); continue; } @@ -36,10 +39,9 @@ build_category(llvm::cl::list const &benchmark_path_list) { { std::istringstream iss(entry_dir.path().filename().stem()); if (!(iss >> iteration.size)) { - llvm::errs() << "[WARNING] Entry directory name is not a size: " - << entry_dir.path() - << " (current path: " << fs::current_path() << ").\n"; - + warn(fmt::format( + "Entry directory name is not a size: {} (current path: {}).", + entry_dir.path().string(), fs::current_path().string())); continue; } } @@ -50,11 +52,10 @@ build_category(llvm::cl::list const &benchmark_path_list) { // Basic property check if (!std::filesystem::is_regular_file(sample_path_entry)) { - llvm::errs() - << "[WARNING] Invalid repetition file (not a regular file): " - << sample_path_entry.path() - << " (current path: " << fs::current_path() << ").\n"; - + warn(fmt::format("Invalid repetition file (not a regular file): {} " + "(current path: {}).", + sample_path_entry.path().string(), + fs::current_path().string())); continue; } diff --git a/grapher/lib/grapher/utils/config.cpp b/grapher/lib/grapher/utils/config.cpp deleted file mode 100644 index c5af597..0000000 --- a/grapher/lib/grapher/utils/config.cpp +++ /dev/null @@ -1,84 +0,0 @@ -#include "grapher/utils/config.hpp" - -#include -#include -#include -#include - -#include - -#include -#include - -#include "grapher/predicates.hpp" -#include "grapher/utils/json.hpp" - -namespace grapher { - -group_descriptor_t get_default_group_descriptor() { - return {.name = "All", - .predicates = nlohmann::json::array({nlohmann::json{ - {"type", "regex"}, - {"pointer", "/name"}, - {"regex", "*"}, - }})}; -} - -std::vector get_predicates(group_descriptor_t const &descriptor) { - std::vector predicates; - - predicates.reserve(descriptor.predicates.size()); - std::ranges::transform(descriptor.predicates, std::back_inserter(predicates), - get_predicate); - return predicates; -} - -std::vector -extract_group(group_descriptor_t const &descriptor, - std::vector const &events) { - std::vector predicates = get_predicates(descriptor); - - std::vector res; - - std::ranges::copy_if( - events, std::back_inserter(res), [&](nlohmann::json const &event) { - return std::ranges::all_of( - predicates, [&](predicate_t const &p) { return p(event); }); - }); - - return res; -} - -group_descriptor_t json_to_group_descriptor(nlohmann::json const &j) { - return {.name = json_value(j, "name"), - .predicates = - json_value>(j, "predicates")}; -} -nlohmann::json group_descriptor_json(group_descriptor_t const &descriptor) { - return { - {"name", descriptor.name}, - {"predicates", descriptor.predicates}, - }; -} - -std::vector -write_descriptors(std::vector const &descriptors) { - std::vector res; - res.reserve(descriptors.size()); - - for (group_descriptor_t const &d : descriptors) { - res.push_back(group_descriptor_json(d)); - } - return res; -} - -std::vector -read_descriptors(std::vector const &list) { - std::vector res; - res.reserve(list.size()); - std::transform(list.begin(), list.end(), std::back_inserter(res), - &json_to_group_descriptor); - return res; -} - -} // namespace grapher diff --git a/grapher/lib/grapher/utils/json.cpp b/grapher/lib/grapher/utils/json.cpp index b774a31..e5643a8 100644 --- a/grapher/lib/grapher/utils/json.cpp +++ b/grapher/lib/grapher/utils/json.cpp @@ -1,42 +1,125 @@ #include "grapher/utils/json.hpp" +#include "grapher/core.hpp" #include "grapher/predicates.hpp" -#include "grapher/utils/config.hpp" #include #include #include #include +#include +#include namespace grapher { +/// Default group descriptor values: +/// +/// \code{.js} +/// { +/// "group_descriptors": [ +/// { +/// "name": "All", +/// "predicates": [ +/// { +/// "pointer": "/name", +/// "regex": "*", +/// "type": "regex" +/// } +/// ] +/// } +/// ] +/// } +/// \endcode + +group_descriptor_t get_default_group_descriptor() { + return {.name = "All", + .predicates = grapher::json_t::array({grapher::json_t{ + {"type", "regex"}, + {"pointer", "/name"}, + {"regex", "*"}, + }})}; +} + +std::vector get_predicates(group_descriptor_t const &descriptor) { + std::vector predicates; + + predicates.reserve(descriptor.predicates.size()); + std::ranges::transform(descriptor.predicates, std::back_inserter(predicates), + get_predicate); + return predicates; +} + +grapher::json_t::array_t extract_group(group_descriptor_t const &descriptor, + grapher::json_t::array_t const &events) { + std::vector predicates = get_predicates(descriptor); + + grapher::json_t::array_t res; + + std::ranges::copy_if( + events, std::back_inserter(res), [&](grapher::json_t const &event) { + return std::ranges::all_of( + predicates, [&](predicate_t const &p) { return p(event); }); + }); + + return res; +} + +group_descriptor_t read_descriptor(grapher::json_t const &j) { + return {.name = json_at_ref(j, "name"), + .predicates = + json_at_ref(j, "predicates")}; +} + +grapher::json_t group_descriptor_json(group_descriptor_t const &descriptor) { + return { + {"name", descriptor.name}, + {"predicates", descriptor.predicates}, + }; +} + +grapher::json_t::array_t +write_descriptors(std::vector const &descriptors) { + grapher::json_t::array_t res; + res.reserve(descriptors.size()); + + std::transform(descriptors.begin(), descriptors.end(), + std::back_inserter(res), &group_descriptor_json); + + return res; +} + +std::vector +read_descriptors(grapher::json_t::array_t const &list) { + std::vector res; + res.reserve(list.size()); + std::transform(list.begin(), list.end(), std::back_inserter(res), + &read_descriptor); + return res; +} + std::vector get_values(benchmark_iteration_t const &iteration, std::vector const &predicates, - nlohmann::json::json_pointer value_jptr) { + grapher::json_t::json_pointer value_jptr) { std::vector res(iteration.samples.size()); auto get_val = [&](std::filesystem::path const &repetition_path) -> double { // Extract events - nlohmann::json j; + grapher::json_t j; { std::ifstream repetition_ifstream(repetition_path); repetition_ifstream >> j; } - if (!j.contains("traceEvents") || !j["traceEvents"].is_array() || - j["traceEvents"].empty()) { - return 0.; - } - - nlohmann::json::array_t const &events = - j["traceEvents"].get_ref(); + grapher::json_t::array_t const &events = + json_at_ref(j, "traceEvents"); // Accumulate double val = 0.; - for (nlohmann::json const &event : events) { + for (grapher::json_t const &event : events) { if (std::all_of(predicates.begin(), predicates.end(), [&](predicate_t const &p) -> bool { return p(event); })) { - val += json_value(event, value_jptr); + val += + json_at_ref(event, value_jptr); } } return val; @@ -48,12 +131,76 @@ std::vector get_values(benchmark_iteration_t const &iteration, return res; } -nlohmann::json merge_into(nlohmann::json a, nlohmann::json const &b) { - for (nlohmann::json const b_flat = b.flatten(); +grapher::json_t merge_into(grapher::json_t a, grapher::json_t const &b) { + for (grapher::json_t const b_flat = b.flatten(); auto const &[k_ptr, v] : b_flat.items()) { - a[nlohmann::json::json_pointer(k_ptr)] = v; + a[grapher::json_t::json_pointer(k_ptr)] = v; } return a; } +void save_plot(sciplot::Plot2D const &plot, std::string const &dest, + grapher::json_t const &config) { + namespace fs = std::filesystem; + std::vector plot_file_extensions = config.value( + "plot_file_extensions", grapher::json_t::array({".svg", ".png"})); + + // Saving file for all extensions + for (std::string const &extension : plot_file_extensions) { + fs::path file_dest = dest + extension; + fs::create_directories(file_dest.parent_path()); + + constexpr std::size_t max_filename_size = 256; + + // Avoid filename hitting OS filename size limit (yes, this is bad) + if (file_dest.filename().string().size() > max_filename_size) { + std::string new_filename = file_dest.stem(); + new_filename.resize(max_filename_size - extension.size()); + file_dest.replace_filename(new_filename); + } + + sciplot::Canvas{{sciplot::Figure{{plot}}}}.save(dest + extension); + } +} + +sciplot::Plot &apply_config(sciplot::Plot &plot, grapher::json_t config) { + // Dimensions + if (config.contains("width") && config.contains("height")) { + plot.size(config["width"], config["height"]); + } + + // Labels + if (config.contains("legend_title")) { + plot.legend().atOutsideRightTop().title(config["legend_title"]); + } + + if (config.contains("x_label")) { + plot.xlabel(config["x_label"]); + } + + if (config.contains("y_label")) { + plot.ylabel(config["y_label"]); + } + + return plot; +} + +grapher::json_t const default_config = { + {"width", 1500}, + {"height", 500}, + {"legend_title", "Timings"}, + {"x_label", "Benchmark size factor"}, + {"y_label", "Time (µs)"}, + {"plot_file_extensions", grapher::json_t::array({".svg", ".png"})}, +}; + +/// Common plot JSON parameters: +/// - `width` (`int`): Graph width +/// - `height` (`int`): Graph height +/// - `legend_title` (`string`): Graph legend title +/// - `x_label` (`string`): X axis label +/// - `y_label` (`string`): Y axis label +/// - `plot_file_extensions` (string array): List of extensions for the export +grapher::json_t base_default_config() { return default_config; } + } // namespace grapher diff --git a/grapher/lib/grapher/utils/plot.cpp b/grapher/lib/grapher/utils/plot.cpp deleted file mode 100644 index 756cd12..0000000 --- a/grapher/lib/grapher/utils/plot.cpp +++ /dev/null @@ -1,43 +0,0 @@ -#include "grapher/utils/plot.hpp" - -#include -#include - -#include "grapher/plotters/plotter_i.hpp" - -namespace grapher { - -/// Default config file for plots -nlohmann::json const default_config = { - {"width", 1500}, - {"height", 500}, - {"legend_title", "Timings"}, - {"xlabel", "Benchmark size factor"}, - {"ylabel", "Time (µs)"}, -}; - -sciplot::Plot &apply_config(sciplot::Plot &plot, nlohmann::json config) { - // Dimensions - if (config.contains("width") && config.contains("height")) { - plot.size(config["width"], config["height"]); - } - - // Labels - if (config.contains("legend_title")) { - plot.legend().atOutsideRightTop().title(config["legend_title"]); - } - - if (config.contains("xlabel")) { - plot.xlabel(config["xlabel"]); - } - - if (config.contains("ylabel")) { - plot.ylabel(config["ylabel"]); - } - - return plot; -} - -nlohmann::json base_default_config() { return default_config; } - -} // namespace grapher diff --git a/grapher/tests/grapher/utils/json.cpp b/grapher/tests/grapher/utils/json.cpp index e852aa5..dc4b43a 100644 --- a/grapher/tests/grapher/utils/json.cpp +++ b/grapher/tests/grapher/utils/json.cpp @@ -6,17 +6,17 @@ /* TTS_CASE("find_matching - Basic") { - nlohmann::json json_a = {{"a", 42}}; - nlohmann::json json_b = {{"b", 42}}; - nlohmann::json json_c = {{"c", "42"}}; + grapher::json_t json_a = {{"a", 42}}; + grapher::json_t json_b = {{"b", 42}}; + grapher::json_t json_c = {{"c", "42"}}; - nlohmann::json json_ac = {{"a", 42}, {"c", "42"}}; - nlohmann::json json_bc = {{"b", 42}, {"c", "42"}}; + grapher::json_t json_ac = {{"a", 42}, {"c", "42"}}; + grapher::json_t json_bc = {{"b", 42}, {"c", "42"}}; - nlohmann::json json_d = {{"d", 42}}; - nlohmann::json json_ab = {{"a", 42}, {"b", 42}}; + grapher::json_t json_d = {{"d", 42}}; + grapher::json_t json_ab = {{"a", 42}, {"b", 42}}; - nlohmann::json json_ac_bc; + grapher::json_t json_ac_bc; json_ac_bc.push_back(json_ac); json_ac_bc.push_back(json_bc); @@ -55,18 +55,18 @@ TTS_CASE("find_matching - Basic") { }; TTS_CASE("find_matching - Imbricated JSON") { - nlohmann::json json_a = {{"a", 42}}; - nlohmann::json json_b = {{"b", 42}}; - nlohmann::json json_c = {{"c", "42"}}; + grapher::json_t json_a = {{"a", 42}}; + grapher::json_t json_b = {{"b", 42}}; + grapher::json_t json_c = {{"c", "42"}}; - nlohmann::json json_jsonc = {{"json_c", json_c}}; + grapher::json_t json_jsonc = {{"json_c", json_c}}; - nlohmann::json json_ab_jsonc = {{"a", 42}, {"b", 42}, {"json_c", json_c}}; - nlohmann::json json_a_jsonc = {{"a", 42}, {"json_c", json_c}}; - nlohmann::json json_b_jsonc = {{"b", 42}, {"json_c", json_c}}; + grapher::json_t json_ab_jsonc = {{"a", 42}, {"b", 42}, {"json_c", json_c}}; + grapher::json_t json_a_jsonc = {{"a", 42}, {"json_c", json_c}}; + grapher::json_t json_b_jsonc = {{"b", 42}, {"json_c", json_c}}; { - nlohmann::json list; + grapher::json_t list; list.push_back(json_a); list.push_back(json_b); @@ -85,7 +85,7 @@ TTS_CASE("find_matching - Imbricated JSON") { } { - nlohmann::json list; + grapher::json_t list; list.push_back(json_a); list.push_back(json_a_jsonc); @@ -103,7 +103,7 @@ TTS_CASE("find_matching - Imbricated JSON") { } { - nlohmann::json list; + grapher::json_t list; list.push_back(json_a); list.push_back(json_b); @@ -120,7 +120,7 @@ TTS_CASE("find_matching - Imbricated JSON") { }; TTS_CASE("merge_into - Basic") { - nlohmann::json a, b; + grapher::json_t a, b; a["a"] = 0; a["b"] = 0; diff --git a/readme.md b/readme.md index e02f8a1..0390b39 100644 --- a/readme.md +++ b/readme.md @@ -97,7 +97,8 @@ ctbench_add_benchmark(function_selection.requires # Benchmark case name Once you have several benchmark cases, you can start writing a graph config. -Example configs can be found [here](./grapher/configs/), or by running +Example configs can be found [here]( +https://github.com/JPenuchot/ctbench/tree/main/grapher/configs), or by running `ctbench-grapher-utils --plotter= --command=get-default-config`. ```json @@ -173,5 +174,8 @@ per benchmark case. In this case, you would then get 3 graphs ## Additional +- [ctbench: compile time benchmarking for Clang]( + https://www.youtube.com/watch?v=1RZY6skM0Rc) at [CPPP 2021]( + https://cppp.fr/schedule2021/) - [Pyperf - Tune the system for benchmarks]( https://pyperf.readthedocs.io/en/latest/system.html)