From 2576261e5d1904ea9e32dba84b6397971dc0da46 Mon Sep 17 00:00:00 2001 From: "C.A.P. Linssen" Date: Thu, 17 Oct 2024 11:44:16 +0200 Subject: [PATCH] add continuous benchmarking --- .github/workflows/continuous_benchmarking.yml | 91 +++++ .../test_nest_continuous_benchmarking.py | 311 ++++++++++++++++++ 2 files changed, 402 insertions(+) create mode 100644 .github/workflows/continuous_benchmarking.yml create mode 100644 tests/nest_continuous_benchmarking/test_nest_continuous_benchmarking.py diff --git a/.github/workflows/continuous_benchmarking.yml b/.github/workflows/continuous_benchmarking.yml new file mode 100644 index 000000000..980da3473 --- /dev/null +++ b/.github/workflows/continuous_benchmarking.yml @@ -0,0 +1,91 @@ +on: + pull_request_target: + types: [opened, reopened, edited, synchronize] + +jobs: + fork_pr_requires_review: + environment: ${{ (github.event.pull_request.head.repo.full_name == github.repository && 'internal') || 'external' }} + runs-on: ubuntu-latest + steps: + - run: true + + benchmark_fork_pr_branch: + needs: fork_pr_requires_review + name: Continuous Benchmarking Fork PRs with Bencher + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + repository: ${{ github.event.pull_request.head.repo.full_name }} + ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false + + - uses: bencherdev/bencher@main + + # Setup Python version + - name: Setup Python 3.8 + uses: actions/setup-python@v5 + with: + python-version: 3.8 + + # Install dependencies + - name: Install apt dependencies + run: | + sudo apt-get update + sudo apt-get install libltdl7-dev libgsl0-dev libncurses5-dev libreadline6-dev pkg-config + sudo apt-get install python3-all-dev python3-matplotlib python3-numpy python3-scipy ipython3 + + # Install Python dependencies + - name: Python dependencies + run: | + python -m pip install --upgrade pip pytest jupyterlab matplotlib pycodestyle scipy pandas pytest-benchmark + python -m pip install -r requirements.txt + + # Install NEST simulator + - name: NEST simulator + run: | + python -m pip install cython + echo "GITHUB_WORKSPACE = $GITHUB_WORKSPACE" + NEST_SIMULATOR=$(pwd)/nest-simulator + NEST_INSTALL=$(pwd)/nest_install + echo "NEST_SIMULATOR = $NEST_SIMULATOR" + echo "NEST_INSTALL = $NEST_INSTALL" + + git clone --depth=1 https://github.com/nest/nest-simulator + mkdir nest_install + echo "NEST_INSTALL=$NEST_INSTALL" >> $GITHUB_ENV + cd nest_install + cmake -DCMAKE_INSTALL_PREFIX=$NEST_INSTALL $NEST_SIMULATOR + make && make install + cd .. + + # Install NESTML (repeated) + - name: Install NESTML + run: | + export PYTHONPATH=${{ env.PYTHONPATH }}:${{ env.NEST_INSTALL }}/lib/python3.8/site-packages + #echo PYTHONPATH=`pwd` >> $GITHUB_ENV + echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV + python setup.py install + + - name: Track Fork PR Benchmarks with Bencher + env: + LD_LIBRARY_PATH: ${{ env.NEST_INSTALL }}/lib/nest + run: | + echo "NEST_INSTALL = $NEST_INSTALL" + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${{ env.NEST_INSTALL }}/lib/nest bencher run \ + --project nestml \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + --branch '${{ github.event.number }}/merge' \ + --branch-start-point '${{ github.base_ref }}' \ + --branch-start-point-hash '${{ github.event.pull_request.base.sha }}' \ + --branch-reset \ + --github-actions "${{ secrets.GITHUB_TOKEN }}" \ + --testbed ubuntu-latest \ + --adapter python_pytest \ + --file results.json \ + --err \ + 'LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${{ env.NEST_INSTALL }}/lib/nest python3 -m pytest --benchmark-json results.json -s $GITHUB_WORKSPACE/tests/nest_continuous_benchmarking/test_nest_continuous_benchmarking.py' + + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 diff --git a/tests/nest_continuous_benchmarking/test_nest_continuous_benchmarking.py b/tests/nest_continuous_benchmarking/test_nest_continuous_benchmarking.py new file mode 100644 index 000000000..70e94706b --- /dev/null +++ b/tests/nest_continuous_benchmarking/test_nest_continuous_benchmarking.py @@ -0,0 +1,311 @@ +# -*- coding: utf-8 -*- +# +# test_nest_continuous_benchmarking.py +# +# This file is part of NEST. +# +# Copyright (C) 2004 The NEST Initiative +# +# NEST is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# NEST is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with NEST. If not, see . + +import numpy as np +import os +import pytest + +import nest + +from pynestml.frontend.pynestml_frontend import generate_nest_target + +try: + import matplotlib + matplotlib.use('Agg') + import matplotlib.ticker + import matplotlib.pyplot as plt + TEST_PLOTS = True +except Exception: + TEST_PLOTS = False + +sim_mdl = True +sim_ref = True + + +class TestNESTContinuousBenchmarking: + + neuron_model_name = "iaf_psc_exp_neuron_nestml__with_stdp_nn_symm_synapse_nestml" + ref_neuron_model_name = "iaf_psc_exp_neuron_nestml_non_jit" + + synapse_model_name = "stdp_nn_symm_synapse_nestml__with_iaf_psc_exp_neuron_nestml" + ref_synapse_model_name = "stdp_nn_symm_synapse" + + @pytest.fixture(scope="module", autouse=True) + def setUp(self): + """Generate the neuron model code""" + + # generate the "jit" model (co-generated neuron and synapse), that does not rely on ArchivingNode + files = [os.path.join("models", "neurons", "iaf_psc_exp_neuron.nestml"), + os.path.join("models", "synapses", "stdp_nn_symm_synapse.nestml")] + input_path = [os.path.realpath(os.path.join(os.path.dirname(__file__), os.path.join( + os.pardir, os.pardir, s))) for s in files] + generate_nest_target(input_path=input_path, + target_path="/tmp/nestml-jit", + logging_level="INFO", + module_name="nestml_jit_module", + suffix="_nestml", + codegen_opts={"neuron_parent_class": "StructuralPlasticityNode", + "neuron_parent_class_include": "structural_plasticity_node.h", + "neuron_synapse_pairs": [{"neuron": "iaf_psc_exp_neuron", + "synapse": "stdp_nn_symm_synapse", + "post_ports": ["post_spikes"]}], + "delay_variable": {"stdp_nn_symm_synapse": "d"}, + "weight_variable": {"stdp_nn_symm_synapse": "w"}}) + + # generate the "non-jit" model, that relies on ArchivingNode + generate_nest_target(input_path=os.path.realpath(os.path.join(os.path.dirname(__file__), + os.path.join(os.pardir, os.pardir, "models", "neurons", "iaf_psc_exp_neuron.nestml"))), + target_path="/tmp/nestml-non-jit", + logging_level="INFO", + module_name="nestml_non_jit_module", + suffix="_nestml_non_jit", + codegen_opts={"neuron_parent_class": "ArchivingNode", + "neuron_parent_class_include": "archiving_node.h"}) + + @pytest.mark.benchmark + def test_stdp_nn_synapse(self, benchmark): + return benchmark(self._test_stdp_nn_synapse) + + def _test_stdp_nn_synapse(self): + + fname_snip = "" + + pre_spike_times = [1., 11., 21.] # [ms] + post_spike_times = [6., 16., 26.] # [ms] + + post_spike_times = np.sort(np.unique(1 + np.round(10 * np.sort(np.abs(np.random.randn(10)))))) # [ms] + pre_spike_times = np.sort(np.unique(1 + np.round(10 * np.sort(np.abs(np.random.randn(10)))))) # [ms] + + post_spike_times = np.sort(np.unique(1 + np.round(100 * np.sort(np.abs(np.random.randn(100)))))) # [ms] + pre_spike_times = np.sort(np.unique(1 + np.round(100 * np.sort(np.abs(np.random.randn(100)))))) # [ms] + + pre_spike_times = np.array([2., 4., 7., 8., 12., 13., 19., 23., 24., 28., 29., 30., 33., 34., + 35., 36., 38., 40., 42., 46., 51., 53., 54., 55., 56., 59., 63., 64., + 65., 66., 68., 72., 73., 76., 79., 80., 83., 84., 86., 87., 90., 95., + 99., 100., 103., 104., 105., 111., 112., 126., 131., 133., 134., 139., 147., 150., + 152., 155., 172., 175., 176., 181., 196., 197., 199., 202., 213., 215., 217., 265.]) + post_spike_times = np.array([4., 5., 6., 7., 10., 11., 12., 16., 17., 18., 19., 20., 22., 23., + 25., 27., 29., 30., 31., 32., 34., 36., 37., 38., 39., 42., 44., 46., + 48., 49., 50., 54., 56., 57., 59., 60., 61., 62., 67., 74., 76., 79., + 80., 81., 83., 88., 93., 94., 97., 99., 100., 105., 111., 113., 114., 115., + 116., 119., 123., 130., 132., 134., 135., 145., 152., 155., 158., 166., 172., 174., + 188., 194., 202., 245., 249., 289., 454.]) + + self.run_synapse_test(neuron_model_name=self.neuron_model_name, + ref_neuron_model_name=self.ref_neuron_model_name, + synapse_model_name=self.synapse_model_name, + ref_synapse_model_name=self.ref_synapse_model_name, + resolution=1., # [ms] + delay=1., # [ms] + pre_spike_times=pre_spike_times, + post_spike_times=post_spike_times, + fname_snip=fname_snip) + + def run_synapse_test(self, neuron_model_name, + ref_neuron_model_name, + synapse_model_name, + ref_synapse_model_name, + resolution=1., # [ms] + delay=1., # [ms] + sim_time=None, # if None, computed from pre and post spike times + pre_spike_times=None, + post_spike_times=None, + fname_snip=""): + + if pre_spike_times is None: + pre_spike_times = [] + + if post_spike_times is None: + post_spike_times = [] + + if sim_time is None: + sim_time = max(np.amax(pre_spike_times), np.amax(post_spike_times)) + 5 * delay + + nest.ResetKernel() + nest.set_verbosity("M_ALL") + nest.SetKernelStatus({'resolution': resolution}) + + if sim_mdl: + try: + nest.Install("nestml_jit_module") + except Exception: + # ResetKernel() does not unload modules for NEST Simulator < v3.7; ignore exception if module is already loaded on earlier versions + pass + + if sim_ref: + try: + nest.Install("nestml_non_jit_module") + except Exception: + # ResetKernel() does not unload modules for NEST Simulator < v3.7; ignore exception if module is already loaded on earlier versions + pass + + print("Pre spike times: " + str(pre_spike_times)) + print("Post spike times: " + str(post_spike_times)) + + wr = nest.Create('weight_recorder') + wr_ref = nest.Create('weight_recorder') + if sim_mdl: + nest.CopyModel(synapse_model_name, "stdp_nestml_rec", + {"weight_recorder": wr[0], "w": 1., "d": 1., "receptor_type": 0}) + if sim_ref: + nest.CopyModel(ref_synapse_model_name, "stdp_ref_rec", + {"weight_recorder": wr_ref[0], "weight": 1., "delay": 1., "receptor_type": 0}) + + # create spike_generators with these times + pre_sg = nest.Create("spike_generator", + params={"spike_times": pre_spike_times}) + post_sg = nest.Create("spike_generator", + params={"spike_times": post_spike_times, + 'allow_offgrid_times': True}) + + # create parrot neurons and connect spike_generators + if sim_mdl: + pre_neuron = nest.Create("parrot_neuron") + post_neuron = nest.Create(neuron_model_name) + + if sim_ref: + pre_neuron_ref = nest.Create("parrot_neuron") + post_neuron_ref = nest.Create(ref_neuron_model_name) + + if sim_mdl: + spikedet_pre = nest.Create("spike_recorder") + spikedet_post = nest.Create("spike_recorder") + mm = nest.Create("multimeter", params={"record_from": ["V_m", "post_trace__for_stdp_nn_symm_synapse_nestml"]}) + if sim_ref: + spikedet_pre_ref = nest.Create("spike_recorder") + spikedet_post_ref = nest.Create("spike_recorder") + mm_ref = nest.Create("multimeter", params={"record_from": ["V_m"]}) + + if sim_mdl: + nest.Connect(pre_sg, pre_neuron, "one_to_one", syn_spec={"delay": 1.}) + nest.Connect(post_sg, post_neuron, "one_to_one", syn_spec={"delay": 1., "weight": 9999.}) + nest.Connect(pre_neuron, post_neuron, "all_to_all", syn_spec={'synapse_model': 'stdp_nestml_rec'}) + nest.Connect(mm, post_neuron) + nest.Connect(pre_neuron, spikedet_pre) + nest.Connect(post_neuron, spikedet_post) + if sim_ref: + nest.Connect(pre_sg, pre_neuron_ref, "one_to_one", syn_spec={"delay": 1.}) + nest.Connect(post_sg, post_neuron_ref, "one_to_one", syn_spec={"delay": 1., "weight": 9999.}) + nest.Connect(pre_neuron_ref, post_neuron_ref, "all_to_all", + syn_spec={'synapse_model': ref_synapse_model_name}) + nest.Connect(mm_ref, post_neuron_ref) + nest.Connect(pre_neuron_ref, spikedet_pre_ref) + nest.Connect(post_neuron_ref, spikedet_post_ref) + + # get STDP synapse and weight before protocol + if sim_mdl: + syn = nest.GetConnections(source=pre_neuron, synapse_model="stdp_nestml_rec") + if sim_ref: + syn_ref = nest.GetConnections(source=pre_neuron_ref, synapse_model=ref_synapse_model_name) + + n_steps = int(np.ceil(sim_time / resolution)) + 1 + t = 0. + t_hist = [] + if sim_mdl: + w_hist = [] + if sim_ref: + w_hist_ref = [] + while t <= sim_time: + nest.Simulate(resolution) + t += resolution + t_hist.append(t) + if sim_ref: + w_hist_ref.append(nest.GetStatus(syn_ref)[0]['weight']) + if sim_mdl: + w_hist.append(nest.GetStatus(syn)[0]['w']) + + # plot + if TEST_PLOTS: + fig, ax = plt.subplots(nrows=3) + ax1, ax2, ax3 = ax + + if sim_mdl: + pre_spike_times_ = nest.GetStatus(spikedet_pre, "events")[0]["times"] + print("Actual pre spike times: " + str(pre_spike_times_)) + if sim_ref: + pre_ref_spike_times_ = nest.GetStatus(spikedet_pre_ref, "events")[0]["times"] + print("Actual pre ref spike times: " + str(pre_ref_spike_times_)) + + if sim_mdl: + n_spikes = len(pre_spike_times_) + for i in range(n_spikes): + if i == 0: + _lbl = "nestml" + else: + _lbl = None + ax1.plot(2 * [pre_spike_times_[i] + delay], [0, 1], linewidth=2, color="blue", alpha=.4, label=_lbl) + + if sim_mdl: + post_spike_times_ = nest.GetStatus(spikedet_post, "events")[0]["times"] + print("Actual post spike times: " + str(post_spike_times_)) + if sim_ref: + post_ref_spike_times_ = nest.GetStatus(spikedet_post_ref, "events")[0]["times"] + print("Actual post ref spike times: " + str(post_ref_spike_times_)) + + if sim_ref: + n_spikes = len(pre_ref_spike_times_) + for i in range(n_spikes): + if i == 0: + _lbl = "nest ref" + else: + _lbl = None + ax1.plot(2 * [pre_ref_spike_times_[i] + delay], [0, 1], + linewidth=2, color="cyan", label=_lbl, alpha=.4) + ax1.set_ylabel("Pre spikes") + + if sim_mdl: + n_spikes = len(post_spike_times_) + for i in range(n_spikes): + if i == 0: + _lbl = "nestml" + else: + _lbl = None + ax2.plot(2 * [post_spike_times_[i]], [0, 1], linewidth=2, color="black", alpha=.4, label=_lbl) + if sim_ref: + n_spikes = len(post_ref_spike_times_) + for i in range(n_spikes): + if i == 0: + _lbl = "nest ref" + else: + _lbl = None + ax2.plot(2 * [post_ref_spike_times_[i]], [0, 1], linewidth=2, color="red", alpha=.4, label=_lbl) + if sim_mdl: + ax2.plot(nest.GetStatus(mm, "events")[0]["times"], nest.GetStatus(mm, "events")[ + 0]["post_trace__for_stdp_nn_symm_synapse_nestml"], label="nestml post tr") + ax2.set_ylabel("Post spikes") + + if sim_mdl: + ax3.plot(t_hist, w_hist, marker="o", label="nestml") + if sim_ref: + ax3.plot(t_hist, w_hist_ref, linestyle="--", marker="x", label="ref") + + ax3.set_xlabel("Time [ms]") + ax3.set_ylabel("w") + for _ax in ax: + _ax.grid(which="major", axis="both") + _ax.xaxis.set_major_locator(matplotlib.ticker.FixedLocator(np.arange(0, np.ceil(sim_time)))) + _ax.set_xlim(0., sim_time) + _ax.legend() + fig.savefig("/tmp/stdp_synapse_test" + fname_snip + ".png", dpi=300) + + # verify + MAX_ABS_ERROR = 1E-6 + assert np.all(np.abs(np.array(w_hist) - np.array(w_hist_ref)) < MAX_ABS_ERROR)