benchmark working

.cmake-format

@@ -2,13 +2,42 @@ format:
   disable: false
   line_width: 100
   tab_size: 2
-  max_subgroups_hwrap: 3
+  max_subgroups_hwrap: 2
   dangle_parens: true
 
 enable_markup: false
 
 parse:
   additional_commands:
+    cpmaddpackage:
+      pargs:
+        nargs: '*'
+        flags: []
+      spelling: CPMAddPackage
+      kwargs: &cpmaddpackagekwargs
+        NAME: 1
+        FORCE: 1
+        VERSION: 1
+        GIT_TAG: 1
+        DOWNLOAD_ONLY: 1
+        GITHUB_REPOSITORY: 1
+        GITLAB_REPOSITORY: 1
+        GIT_REPOSITORY: 1
+        SVN_REPOSITORY: 1
+        SVN_REVISION: 1
+        SOURCE_DIR: 1
+        DOWNLOAD_COMMAND: 1
+        FIND_PACKAGE_ARGUMENTS: 1
+        NO_CACHE: 1
+        GIT_SHALLOW: 1
+        URL: 1
+        URL_HASH: 1
+        URL_MD5: 1
+        DOWNLOAD_NAME: 1
+        DOWNLOAD_NO_EXTRACT: 1
+        HTTP_USERNAME: 1
+        HTTP_PASSWORD: 1
+        OPTIONS: +
     cpmfindpackage:
       pargs:
         nargs: '*'

CMakeLists.txt

@@ -1,12 +1,18 @@
 cmake_minimum_required(VERSION 3.15)
 cmake_policy(SET CMP0148 OLD)
 
-project(dmt VERSION 0.1.0 LANGUAGES CXX)
+project(
+  dmt
+  VERSION 0.1.0
+  LANGUAGES CXX
+)
 
 # Configuration options
 option(BUILD_DOCS "Build documentation" OFF)
 option(BUILD_PYTHON "Build Python bindings" ON)
 option(BUILD_TESTING "Build tests" OFF)
+option(ENABLE_COVERAGE "Enable coverage reporting" OFF)
+option(BUILD_BENCHMARKS "Build benchmarks" OFF)
 option(ENABLE_FAST_MATH "Enable fast math flags" ON)
 
 set(CMAKE_CXX_STANDARD 17)
@@ -30,7 +36,6 @@ endif()
 include(GNUInstallDirs)
 include(cmake/CPM.cmake)
 
-
 # Build the C++ library
 set(LIBRARY_NAME dmt)
 add_subdirectory(lib)
@@ -42,6 +47,10 @@ endif()
 if(BUILD_TESTING)
   add_subdirectory(tests/cpp)
 endif()
+# Build benchmarks
+if(BUILD_BENCHMARKS)
+  add_subdirectory(bench)
+endif()
 
 # Install targets and configuration
 install(
@@ -52,7 +61,11 @@ install(
   ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
 )
 
-install(EXPORT dmt_config NAMESPACE dmt:: DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/dmt)
+install(
+  EXPORT dmt_config
+  NAMESPACE dmt::
+  DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/dmt
+)
 install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 
 # This prints a summary of found dependencies

README.md

@@ -27,4 +27,11 @@ thefdmt = FDMT(f_min, f_max, nchans, nsamps, tsamp, dt_max=dt_max, dt_min=0, dt_
 dmt_transform = thefdmt.execute(frb.astype(np.float32))
 ```
 
+## Benchmarks
+
+```python
+f_min = 704.0, f_max = 1216.0, nchans = 4096, tsamp = 0.00008192, dt_max = 2048, nsamps = n;
+```
+![](bench/results/bench.png)
+
 

bench/CMakeLists.txt

@@ -0,0 +1,13 @@
+CPMAddPackage(
+  NAME benchmark
+  VERSION 1.8.0
+  GITHUB_REPOSITORY google/benchmark
+  OPTIONS "BENCHMARK_ENABLE_TESTING Off"
+)
+
+set(TARGET_BENCHMARKS ${PROJECT_NAME}_bench)
+file(GLOB TEST_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp)
+
+add_executable(${TARGET_BENCHMARKS} ${TEST_SOURCES})
+target_include_directories(${TARGET_BENCHMARKS} PUBLIC ${CMAKE_SOURCE_DIR}/include)
+target_link_libraries(${TARGET_BENCHMARKS} PRIVATE ${LIBRARY_NAME} benchmark::benchmark)

bench/bench_plots.py

@@ -0,0 +1,56 @@
+import json
+import sys
+from pathlib import Path
+
+import pandas as pd
+import seaborn as sns
+from matplotlib import pyplot as plt
+
+filename = sys.argv[1]
+basename = filename.split(".")[0]
+
+# load benchmarks data
+with Path(filename).open("r") as jfile:
+    jdata = json.load(jfile)
+bench_df = pd.DataFrame(jdata["benchmarks"])
+
+# only keep names with /, i.e. drop avg/rms/etc entries
+bench_df = bench_df.loc[bench_df.name.str.contains("/")]
+
+# create a column with complexity n
+bench_df[["benchmark_name", "benchmark_type", "n"]] = bench_df.name.str.split(
+    "/BM_|_|/",
+    expand=True,
+).apply(
+    lambda x: ["_".join([x[1], x[2]]), "_".join([x[3], x[4]]), x[5]],
+    axis=1,
+    result_type="expand",
+)
+bench_df["n"] = bench_df["n"].astype("uint32")
+bench_df = bench_df[["benchmark_name", "n", "cpu_time", "benchmark_type"]]
+benchmarks = bench_df.benchmark_name.unique()
+
+palette = sns.color_palette("husl", len(benchmarks))
+fig, axes = plt.subplots(2, 2, figsize=(10, 6), sharex=True)
+for i, benchmark in enumerate(benchmarks):
+    ax = axes[0, i] if i < 2 else axes[1, i - 2]
+    data = bench_df[bench_df["benchmark_name"] == benchmark]
+    sns.lineplot(
+        x="n",
+        y="cpu_time",
+        hue="benchmark_type",
+        data=data,
+        ax=ax,
+        color=palette[i],
+        marker="o",
+        markersize=8,
+    )
+    ax.set_title(benchmark)
+    ax.set_xscale("log")
+    ax.set_yscale("log")
+    ax.set_xlabel("n")
+    ax.set_ylabel("Time (s)")
+    ax.legend()
+
+fig.tight_layout()
+fig.savefig(f"{basename}.png", bbox_inches="tight", dpi=120)

bench/fdmt_b.cpp

@@ -0,0 +1,101 @@
+#include <benchmark/benchmark.h>
+
+#include <algorithm>
+#include <random>
+#include <vector>
+#include <dmt/fdmt.hpp>
+
+class FDMTFixture : public benchmark::Fixture {
+public:
+    void SetUp(const ::benchmark::State& state) {
+        f_min  = 704.0F;
+        f_max  = 1216.0F;
+        nchans = 4096;
+        tsamp  = 0.00008192F;
+        dt_max = 2048;
+        nsamps = state.range(0);
+    }
+
+    void TearDown(const ::benchmark::State&) {}
+
+    template <typename T>
+    std::vector<T> generate_vector(size_t size, std::mt19937& gen) {
+        std::vector<T> vec(size);
+        std::uniform_real_distribution<T> dis(0.0, 1.0);
+        std::generate(vec.begin(), vec.end(), [&]() { return dis(gen); });
+        return vec;
+    }
+
+    float f_min;
+    float f_max;
+    size_t nchans;
+    float tsamp;
+    size_t dt_max;
+    size_t nsamps;
+};
+
+BENCHMARK_DEFINE_F(FDMTFixture, BM_fdmt_plan_seq_cpu)(benchmark::State& state) {
+    for (auto _ : state) {
+        FDMT fdmt(f_min, f_max, nchans, nsamps, tsamp, dt_max);
+    }
+}
+
+BENCHMARK_DEFINE_F(FDMTFixture, BM_fdmt_initialise_seq_cpu)(benchmark::State& state) {
+    FDMT fdmt(f_min, f_max, nchans, nsamps, tsamp, dt_max);
+
+    std::random_device rd;
+    std::mt19937 gen(rd());
+    auto waterfall = generate_vector<float>(nchans * nsamps, gen);
+    std::vector<float> state_init(
+        nchans * fdmt.get_dt_grid_init().size() * nsamps, 0.0F);
+    for (auto _ : state) {
+        fdmt.initialise(waterfall.data(), state_init.data());
+    }
+}
+
+BENCHMARK_DEFINE_F(FDMTFixture, BM_fdmt_execute_seq_cpu)(benchmark::State& state) {
+    FDMT fdmt(f_min, f_max, nchans, nsamps, tsamp, dt_max);
+
+    std::random_device rd;
+    std::mt19937 gen(rd());
+    auto waterfall = generate_vector<float>(nchans * nsamps, gen);
+    std::vector<float> dmt(fdmt.get_dt_grid_final().size() * nsamps, 0.0F);
+    for (auto _ : state) {
+        fdmt.execute(waterfall.data(), waterfall.size(), dmt.data(),
+                     dmt.size());
+    }
+}
+
+BENCHMARK_DEFINE_F(FDMTFixture, BM_fdmt_overall_seq_cpu)(benchmark::State& state) {
+    std::random_device rd;
+    std::mt19937 gen(rd());
+    auto waterfall = generate_vector<float>(nchans * nsamps, gen);
+
+    for (auto _ : state) {
+        FDMT fdmt(f_min, f_max, nchans, nsamps, tsamp, dt_max);
+        state.PauseTiming();
+        std::vector<float> dmt(fdmt.get_dt_grid_final().size() * nsamps, 0.0F);
+        state.ResumeTiming();
+
+        fdmt.execute(waterfall.data(), waterfall.size(), dmt.data(),
+                     dmt.size());
+    }
+}
+
+constexpr size_t min_nsamps = 1 << 11;
+constexpr size_t max_nsamps = 1 << 16;
+
+BENCHMARK_REGISTER_F(FDMTFixture, BM_fdmt_plan_seq_cpu)
+    ->RangeMultiplier(2)
+    ->Range(min_nsamps, max_nsamps);
+BENCHMARK_REGISTER_F(FDMTFixture, BM_fdmt_initialise_seq_cpu)
+    ->RangeMultiplier(2)
+    ->Range(min_nsamps, max_nsamps);
+BENCHMARK_REGISTER_F(FDMTFixture, BM_fdmt_execute_seq_cpu)
+    ->RangeMultiplier(2)
+    ->Range(min_nsamps, max_nsamps);
+BENCHMARK_REGISTER_F(FDMTFixture, BM_fdmt_overall_seq_cpu)
+    ->RangeMultiplier(2)
+    ->Range(min_nsamps, max_nsamps);
+
+BENCHMARK_MAIN();