diff --git a/.gitattributes b/.gitattributes index 0461245..d865b62 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,6 @@ *.jpg filter=lfs diff=lfs merge=lfs -text *.jpeg filter=lfs diff=lfs merge=lfs -text *.root filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pb.txt filter=lfs diff=lfs merge=lfs -text +*.pbtxt filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index 0cf467f..64cb7a8 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,6 @@ *.hdf5 *.json *.yaml -*.pb *.out *.parquet .coverage diff --git a/README.md b/README.md index 868ec73..08617aa 100644 --- a/README.md +++ b/README.md @@ -14,30 +14,27 @@ source setup.sh This command is also used to activate your venv at the start of each new bash session. -To check the setup and create a basic plot with this tool on an already existing graph, you may run the task [PlotRuntimes](#plotruntimes) -with a simple call: +To check the setup and create a basic plot with this tool on an already existing graph, you may run the task [PlotRuntimes](#plotruntimes) with a simple call: -``` shell +```shell law run PlotRuntimes --version test_mlprof ``` ## Law introduction -As you can already see from the Quickstart section, this tool uses [law](https://github.com/riga/law) for -the orchestration. Therefore, a short introduction to the most essential functions of law you should be -aware of when using this tool are provided here. More informations are available for example in the "[Examples](https://github.com/riga/law#examples)" -section of this [Github repository](https://github.com/riga/law). +As you can already see from the Quickstart section, this tool uses [law](https://github.com/riga/law) for the orchestration. +Therefore, a short introduction to the most essential functions of law you should be aware of when using this tool are provided here. +More informations are available for example in the "[Examples](https://github.com/riga/law#examples)" section of this [Github repository](https://github.com/riga/law). This section can be ignored if you are already familiar with law. In [law](https://github.com/riga/law), tasks are defined and separated by purpose and may have dependencies to each other. -As an example, MLProf defines a task for the runtime measurement of a network for -several batch sizes and a different task to make a plot of these data. The plotting task requires the -runtime measurement task to have already run, in order to have data to plot. This is checked -by the presence or absence of the corresponding output file from the required task. If the required file is not present, -the required task will be automatically started with the corresponding parameters before the called task. +As an example, MLProf defines a task for the runtime measurement of a network for several batch sizes and a different task to make a plot of these data. +The plotting task requires the runtime measurement task to have already run, in order to have data to plot. +This is checked by the presence or absence of the corresponding output file from the required task. +If the required file is not present, the required task will be automatically started with the corresponding parameters before the called task. -The tree of runtime tasks in MLPROF is : +The tree of runtime tasks in MLPROF is: ```mermaid flowchart TD @@ -48,36 +45,30 @@ flowchart TD C --> F[PlotRuntimesMultipleNetworks] ``` -A task is run with the command ```law run``` followed by the name of the task. -A version, given by the argument ```--version```, followed by the name of the version, is required. -Each ```version``` has its own set of outputs for the different existing tasks. - -In law, the intermediate results (=the outputs to the different tasks) are saved locally in the corresponding directory -(default path in MLPROF is -```data/name_of_the_task/CMSSW_architecture/model_identifier_with_run_parameters/name_of_the_version/```). -Hence the name of the version should be selected to match your -purpose, for example ```--version convolutional_nn_for_b_tagging```. - -Tasks in law are organized as a graph with dependencies. Therefore a "depth" for the different required tasks exists, -depending on which task required which other task. In order to see the different required tasks for a single task, -you might use the argument ```--print-status -1```, which will show all required tasks and the existence or absence -of their output for the given input parameters up to depth "-1", hence the deepest one. The called task -with ```law run``` will have depth 0. You might check the output path of a task with the -argument ```--print-output```, followed by the depth of the task. If you want a finished task to be run anew without changing -the version (e.g. do a new runtime measurement with a new -training of the same original network), you might remove the previous outputs with the -```--remove-output``` argument, followed by the depth up to which to remove the outputs. There are three removal modes: -- ```a``` (all: remove all outputs of the different tasks up to the given depth), -- ```i``` (interactive: prompt a selection of the tasks to remove up to the given depth) -- ```d``` (dry: show which files might be deleted with the same selection options, but do not remove the outputs). - -The ```--remove-output``` argument does not allow the depth "-1", check the task -tree with ```--print-output``` before selecting the depth you want. The removal mode -can be already selected in the command, e.g. with ```--remove-output 1,a``` (remove all outputs up to depth 1). - -Once the output has been removed, it is possible to run the task again. It is also possible to rerun the task -in the same command as the removal by adding the ```y``` argument at the end. Therefore, removing all outputs -of a selected task (but not its dependencies) and running it again at once would correspond to the following command: +A task is run with the command `law run` followed by the name of the task. +A version, given by the argument `--version```, followed by the name of the version, is required. +Each `version` has its own set of outputs for the different existing tasks. + +In law, the intermediate results (=the outputs to the different tasks) are saved locally in the corresponding directory (default path in MLPROF is ```data/name_of_the_task/CMSSW_architecture/model_identifier_with_run_parameters/name_of_the_version/```). +Hence the name of the version should be selected to match your purpose, for example `--version convolutional_nn_for_b_tagging`. + +Tasks in law are organized as a graph with dependencies. +Therefore a "depth" for the different required tasks exists, depending on which task required which other task. +In order to see the different required tasks for a single task, you might use the argument `--print-status -1`, which will show all required tasks and the existence or absence of their output for the given input parameters up to depth "-1", hence the deepest one. +The called task with `law run` will have depth 0. +You might check the output path of a task with the argument `--print-output```, followed by the depth of the task. +If you want a finished task to be run anew without changing the version (e.g. do a new runtime measurement with a new training of the same original network), you might remove the previous outputs with the `--remove-output` argument, followed by the depth up to which to remove the outputs. +There are three removal modes: +- `a` (all: remove all outputs of the different tasks up to the given depth), +- `i` (interactive: prompt a selection of the tasks to remove up to the given depth) +- `d` (dry: show which files might be deleted with the same selection options, but do not remove the outputs). + +The `--remove-output` argument does not allow the depth "-1", check the task tree with `--print-output` before selecting the depth you want. +The removal mode can be already selected in the command, e.g. with `--remove-output 1,a` (remove all outputs up to depth 1). + +Once the output has been removed, it is possible to run the task again. +It is also possible to rerun the task in the same command as the removal by adding the `y` argument at the end. +Therefore, removing all outputs of a selected task (but not its dependencies) and running it again at once would correspond to the following command: ```shell law run name_of_the_task --version name_of_the_version --remove-output 0,a,y @@ -90,13 +81,12 @@ law run PlotRuntimes --version test_mlprof --print-output 0 ``` - ## Profiling -This tools uses the c++ `````` library for runtime measurements and (soon) [IgProf](https://igprof.org/) for the memory profiling. -It allows for the measurement of TensorFlow graphs with several input layers. The inputs can be up to 3 dimensional. As this tool is set -to work in CMSSW, it requires a frozen graph (it is recommended to use the cmsml [save_graph](https://cmsml.readthedocs.io/en/latest/api/tensorflow.html#cmsml.tensorflow.save_graph) -function with the argument "True" for variables_to_constant). +This tools uses the c++ `` library for runtime measurements and (soon) [IgProf](https://igprof.org/) for the memory profiling. +It allows for the measurement of TensorFlow graphs with several input layers. +The inputs can be up to 3 dimensional. +As this tool is set to work in CMSSW, it requires a frozen graph (it is recommended to use the cmsml [save_graph](https://cmsml.readthedocs.io/en/latest/api/tensorflow.html#cmsml.tensorflow.save_graph) function with the argument "True" for variables_to_constant). ## Runtime measurement @@ -167,80 +157,75 @@ There are already a few examples of these configswith working paths for the netw # CreateRuntimeConfig This task create the CMSSW config file to run the inference in the corresponding task, using the template file in -the ```cmssw/MLProf/RuntimeModule/test/``` directory. The parameters of the inference except the batch sizes are fixed by the created -configuration file, therefore this task will be run again for every change in the inference. -(e.g. the number of runs for the statistics, the path to the graph to check...). +the `cmssw/MLProf/RuntimeModule/test/` directory. +The parameters of the inference except the batch sizes are fixed by the created +configuration file, therefore this task will be run again for every change in the inference (e.g. the number of runs for the statistics, the path to the graph to check...). ## Parameters: -- model-file: str. The absolute path of the json file containing the informations of the model to be tested. - default: ```$MLP_BASE/examples/model1/model.json```. +- model-file: str. The absolute path of the json file containing the informations of the model to be tested. default: `$MLP_BASE/examples/model1/model.json`. -- model-name: str. When set, use this name for the path used when storing outputs instead of a hashed version of - ```--model-file```. default: empty. +- model-name: str. When set, use this name for the path used when storing outputs instead of a hashed version of `--model-file```. default: empty. -- input-files: str. comma-separated list of absolute paths of input files for the CMSSW analyzer (TODO: not implemented); - when empty, random input values will be used; default: empty +- input-files: str. Comma-separated list of absolute paths of input files for the CMSSW analyzer (TODO: not implemented); when empty, random input values will be used. default: empty. -- events: int. The number of events to read from each input file for averaging measurements. default: ```1``` +- events: int. The number of events to read from each input file for averaging measurements. default: `1`. -- repetitions: int. The number of repetitions to be performed per evaluation for averaging. default: ```100``` +- repetitions: int. The number of repetitions to be performed per evaluation for averaging. default: `100`. -- warmup: int. The number of evaluations to be performed before starting the actual measurement. - default: ```10``` +- warmup: int. The number of evaluations to be performed before starting the actual measurement. default: `10`. -- cmssw-version: str. The CMSSW version used for the inference. default: ```CMSSW_12_2_4``` +- cmssw-version: str. The CMSSW version used for the inference. default: `CMSSW_12_2_4`. -- scram-arch: str. The SCRAM architecture used for the inference. default: ```slc7_amd64_gcc10``` +- scram-arch: str. The SCRAM architecture used for the inference. default: `slc7_amd64_gcc10`. ## Output: -- ```cfg.py```: the config file for the ```RuntimeModule``` in ```cmssw/MLProf```. + +- `cfg.py`: The config file for the `RuntimeModule` in `cmssw/MLProf`. ## Example: ```shell -law run CreateRuntimeConfig --version test_simple_dnn \ - --model-file $MLP_BASE/examples/model1/model.json \ - --model-name dnn \ - --repetitions 500 \ - --cmssw-version CMSSW_12_2_4 +law run CreateRuntimeConfig \ + --version test_simple_dnn \ + --model-file "$MLP_BASE/examples/model1/model.json" \ + --model-name dnn \ + --repetitions 500 \ + --cmssw-version CMSSW_12_2_4 ``` # MeasureRuntime -Task to provide the time measurements of the inference of a network in CMSSW, given the input parameters -and a single batch size. The batch size and the (```repetitions * events```) measured values in -milliseconds are saved in csv format. +Task to provide the time measurements of the inference of a network in CMSSW, given the input parameters and a single batch size. +The batch size and the (```repetitions * events```) measured values in milliseconds are saved in csv format. ## Requires: -- The config file created by ```CreateRuntimeConfig```. + +- The config file created by `CreateRuntimeConfig`. ## Parameters: -- batch-size: int. the batch size to measure the runtime for; default: ```1```. -- model-file: str. The absolute path of the json file containing the informations of the model to be tested. - default: ```$MLP_BASE/examples/model1/model.json```. +- batch-size: int. the batch size to measure the runtime for. default: `1`. -- model-name: str. When set, use this name for the path used when storing outputs instead of a hashed version of - ```--model-file```. default: empty. +- model-file: str. The absolute path of the json file containing the informations of the model to be tested. default: `$MLP_BASE/examples/model1/model.json`. -- input-files: str. comma-separated list of absolute paths of input files for the CMSSW analyzer (TODO: not implemented); - when empty, random input values will be used; default: empty +- model-name: str. When set, use this name for the path used when storing outputs instead of a hashed version of `--model-file`. default: empty. -- events: int. The number of events to read from each input file for averaging measurements. default: ```1``` +- input-files: str. comma-separated list of absolute paths of input files for the CMSSW analyzer (TODO: not implemented). When empty, random input values will be used. default: empty. -- repetitions: int. The number of repetitions to be performed per evaluation for averaging. default: ```100``` +- events: int. The number of events to read from each input file for averaging measurements. default: `1`. -- warmup: int. The number of evaluations to be performed before starting the actual measurement. - default: ```10``` +- repetitions: int. The number of repetitions to be performed per evaluation for averaging. default: `100`. + +- warmup: int. The number of evaluations to be performed before starting the actual measurement. default: `10`. -- cmssw-version: str. The CMSSW version used for the inference. default: ```CMSSW_12_2_4``` +- cmssw-version: str. The CMSSW version used for the inference. default: `CMSSW_12_2_4`. -- scram-arch: str. The SCRAM architecture used for the inference. default: ```slc7_amd64_gcc10``` +- scram-arch: str. The SCRAM architecture used for the inference. default: `slc7_amd64_gcc10`. ## Output: -- ```runtime_bs_{batch-size}.csv```: The batch size and measured values of the runtime +- `runtime_bs_{batch-size}.csv```: The batch size and measured values of the runtime for each repetition and event. ## Example: @@ -258,37 +243,37 @@ law run MeasureRuntime --version test_simple_dnn \ # MergeRuntimes This task merges the .csv output files with the required multiple batch sizes -from the different occurences of the ```MeasureRuntime``` task to obtain a single .csv +from the different occurences of the `MeasureRuntime` task to obtain a single .csv file containing the informations to plot. ## Requires: -- The .csv files from the several occurences of ```MeasureRuntime``` (one for each batch size). +- The .csv files from the several occurences of `MeasureRuntime` (one for each batch size). ## Parameters: -- batch-sizes: int. The comma-separated list of batch sizes to be tested; default: ```1,2,4```. +- batch-sizes: int. The comma-separated list of batch sizes to be tested; default: `1,2,4```. - model-file: str. The absolute path of the json file containing the informations of the model to be tested. - default: ```$MLP_BASE/examples/model1/model.json```. + default: `$MLP_BASE/examples/model1/model.json```. - model-name: str. When set, use this name for the path used when storing outputs instead of a hashed version of - ```--model-file```. default: empty. + `--model-file```. default: empty. - input-files: str. comma-separated list of absolute paths of input files for the CMSSW analyzer (TODO: not implemented); when empty, random input values will be used; default: empty -- events: int. The number of events to read from each input file for averaging measurements. default: ```1``` +- events: int. The number of events to read from each input file for averaging measurements. default: `1``` -- repetitions: int. The number of repetitions to be performed per evaluation for averaging. default: ```100``` +- repetitions: int. The number of repetitions to be performed per evaluation for averaging. default: `100``` - warmup: int. The number of evaluations to be performed before starting the actual measurement. - default: ```10``` + default: `10``` -- cmssw-version: str. The CMSSW version used for the inference. default: ```CMSSW_12_2_4``` +- cmssw-version: str. The CMSSW version used for the inference. default: `CMSSW_12_2_4``` -- scram-arch: str. The SCRAM architecture used for the inference. default: ```slc7_amd64_gcc10``` +- scram-arch: str. The SCRAM architecture used for the inference. default: `slc7_amd64_gcc10``` ## Output: -- ```runtime_bs_{batch_size_1}_{batch_size_2}_{...}.csv```: The batch size and measured values of the runtime +- `runtime_bs_{batch_size_1}_{batch_size_2}_{...}.csv```: The batch size and measured values of the runtime for each repetition and event in the several measurements. ## Example: @@ -308,42 +293,42 @@ This task plots the results of the runtime measurement against the given batch s given by the median of the data series and the boundaries of the uncertainty bands are given by the 16 and 84 percentiles of the data series (Therefore the uncertainty band contains 68% of the data points, which corresponds to a $1\sigma$ uncertainty for gaussian uncertainties). The number of inferences behind one -plotted data point is given by ```events * repetitions```. +plotted data point is given by `events * repetitions```. ## Requires: -- The .csv file from the ```MergeRuntimes``` task. +- The .csv file from the `MergeRuntimes` task. ## Parameters: -- log-y: bool. Plot the y-axis values logarithmically; default: ```False```. +- log-y: bool. Plot the y-axis values logarithmically; default: `False```. -- bs-normalized: bool. Normalize the measured values with the batch size before plotting; default: ```True```. +- bs-normalized: bool. Normalize the measured values with the batch size before plotting; default: `True```. -- filling: bool. Plot the errors as error bands instead of error bars; default: ```True```. +- filling: bool. Plot the errors as error bands instead of error bars; default: `True```. -- batch-sizes: int. The comma-separated list of batch sizes to be tested; default: ```1,2,4```. +- batch-sizes: int. The comma-separated list of batch sizes to be tested; default: `1,2,4```. - model-file: str. The absolute path of the json file containing the informations of the model to be tested. - default: ```$MLP_BASE/examples/model1/model.json```. + default: `$MLP_BASE/examples/model1/model.json```. - model-name: str. When set, use this name for the path used when storing outputs instead of a hashed version of - ```--model-file```. default: empty. + `--model-file```. default: empty. - input-files: str. comma-separated list of absolute paths of input files for the CMSSW analyzer (TODO: not implemented); when empty, random input values will be used; default: empty -- events: int. The number of events to read from each input file for averaging measurements. default: ```1``` +- events: int. The number of events to read from each input file for averaging measurements. default: `1``` -- repetitions: int. The number of repetitions to be performed per evaluation for averaging. default: ```100``` +- repetitions: int. The number of repetitions to be performed per evaluation for averaging. default: `100``` - warmup: int. The number of evaluations to be performed before starting the actual measurement. - default: ```10``` + default: `10``` -- cmssw-version: str. The CMSSW version used for the inference. default: ```CMSSW_12_2_4``` +- cmssw-version: str. The CMSSW version used for the inference. default: `CMSSW_12_2_4``` -- scram-arch: str. The SCRAM architecture used for the inference. default: ```slc7_amd64_gcc10``` +- scram-arch: str. The SCRAM architecture used for the inference. default: `slc7_amd64_gcc10``` ## Output: -- ```runtime_plot_different_batch_sizes_{batch_size_1}_{batch_size_2}_{...}.pdf```: The plot of the runtime measurement +- `runtime_plot_different_batch_sizes_{batch_size_1}_{batch_size_2}_{...}.pdf```: The plot of the runtime measurement against the different batch sizes given. ## Example: @@ -366,42 +351,42 @@ The model-files argument is required and replaces the module-file argument. The given by the median of the data series and the boundaries of the uncertainty bands are given by the 16 and 84 percentiles of the data series (Therefore the uncertainty band contains 68% of the data points, which corresponds to a $1\sigma$ uncertainty for gaussian uncertainties). The number of inferences behind one -plotted data point is given by ```events * repetitions```. +plotted data point is given by `events * repetitions```. ## Requires: -- The .csv file from the ```MergeRuntimes``` task. +- The .csv file from the `MergeRuntimes` task. ## Parameters: - model-files: str. The comma-separated list of the absolute paths of the json files containing the informations of the model to be tested. No default value. -- log-y: bool. Plot the y-axis values logarithmically; default: ```False```. +- log-y: bool. Plot the y-axis values logarithmically; default: `False```. -- bs-normalized: bool. Normalize the measured values with the batch size before plotting; default: ```True```. +- bs-normalized: bool. Normalize the measured values with the batch size before plotting; default: `True```. -- filling: bool. Plot the errors as error bands instead of error bars; default: ```True```. +- filling: bool. Plot the errors as error bands instead of error bars; default: `True```. -- batch-sizes: int. The comma-separated list of batch sizes to be tested; default: ```1,2,4```. +- batch-sizes: int. The comma-separated list of batch sizes to be tested; default: `1,2,4```. - model-name: str. When set, use this name for the path used when storing outputs instead of a hashed version of - ```--model-file```. default: empty. + `--model-file```. default: empty. - input-files: str. comma-separated list of absolute paths of input files for the CMSSW analyzer (TODO: not implemented); when empty, random input values will be used; default: empty -- events: int. The number of events to read from each input file for averaging measurements. default: ```1``` +- events: int. The number of events to read from each input file for averaging measurements. default: `1``` -- repetitions: int. The number of repetitions to be performed per evaluation for averaging. default: ```100``` +- repetitions: int. The number of repetitions to be performed per evaluation for averaging. default: `100``` - warmup: int. The number of evaluations to be performed before starting the actual measurement. - default: ```10``` + default: `10``` -- cmssw-version: str. The CMSSW version used for the inference. default: ```CMSSW_12_2_4``` +- cmssw-version: str. The CMSSW version used for the inference. default: `CMSSW_12_2_4``` -- scram-arch: str. The SCRAM architecture used for the inference. default: ```slc7_amd64_gcc10``` +- scram-arch: str. The SCRAM architecture used for the inference. default: `slc7_amd64_gcc10``` ## Output: -- ```runtime_plot_networks_{network_name_1}_{network_name_2}_{...}_different_batch_sizes_{batch_size_1}_{batch_size_2}_{...}.pdf```: The plot of the runtime measurement +- `runtime_plot_networks_{network_name_1}_{network_name_2}_{...}_different_batch_sizes_{batch_size_1}_{batch_size_2}_{...}.pdf```: The plot of the runtime measurement against the different batch sizes given. ## Example: @@ -424,43 +409,43 @@ see below in the Example subsection. The points are given by the median of the data series and the boundaries of the uncertainty bands are given by the 16 and 84 percentiles of the data series (Therefore the uncertainty band contains 68% of the data points, which corresponds to a $1\sigma$ uncertainty for gaussian uncertainties). The number of inferences behind one -plotted data point is given by ```events * repetitions```. +plotted data point is given by `events * repetitions```. ## Requires: -- The .csv file from the ```MergeRuntimes``` task. +- The .csv file from the `MergeRuntimes` task. ## Parameters: - cmssw-versions: str. The comma separated list of CMSSW version used for the inference. - default: ```"CMSSW_12_2_4","CMSSW_12_2_2"``` + default: `"CMSSW_12_2_4","CMSSW_12_2_2"``` -- log-y: bool. Plot the y-axis values logarithmically; default: ```False```. +- log-y: bool. Plot the y-axis values logarithmically; default: `False```. -- bs-normalized: bool. Normalize the measured values with the batch size before plotting; default: ```True```. +- bs-normalized: bool. Normalize the measured values with the batch size before plotting; default: `True```. -- filling: bool. Plot the errors as error bands instead of error bars; default: ```True```. +- filling: bool. Plot the errors as error bands instead of error bars; default: `True```. -- batch-sizes: int. The comma-separated list of batch sizes to be tested; default: ```1,2,4```. +- batch-sizes: int. The comma-separated list of batch sizes to be tested; default: `1,2,4```. - model-file: str. The absolute path of the json file containing the informations of the model to be tested. - default: ```$MLP_BASE/examples/model1/model.json```. + default: `$MLP_BASE/examples/model1/model.json```. - model-name: str. When set, use this name for the path used when storing outputs instead of a hashed version of - ```--model-file```. default: empty. + `--model-file```. default: empty. - input-files: str. comma-separated list of absolute paths of input files for the CMSSW analyzer (TODO: not implemented); when empty, random input values will be used; default: empty -- events: int. The number of events to read from each input file for averaging measurements. default: ```1``` +- events: int. The number of events to read from each input file for averaging measurements. default: `1``` -- repetitions: int. The number of repetitions to be performed per evaluation for averaging. default: ```100``` +- repetitions: int. The number of repetitions to be performed per evaluation for averaging. default: `100``` - warmup: int. The number of evaluations to be performed before starting the actual measurement. - default: ```10``` + default: `10``` -- scram-arch: str. The SCRAM architecture used for the inference. default: ```slc7_amd64_gcc10``` +- scram-arch: str. The SCRAM architecture used for the inference. default: `slc7_amd64_gcc10``` ## Output: -- ```runtime_plot__multiple_cmssw_{cmssw_version_1}_{cmssw_version_2}_{...}_different_batch_sizes_{batch_size_1}_{batch_size_2}_{...}.pdf```: The plot of the runtime measurement +- `runtime_plot__multiple_cmssw_{cmssw_version_1}_{cmssw_version_2}_{...}_different_batch_sizes_{batch_size_1}_{batch_size_2}_{...}.pdf```: The plot of the runtime measurement against the different batch sizes given. ## Example: diff --git a/cmssw/MLProf/RuntimeModule/plugins/BuildFile.xml b/cmssw/MLProf/RuntimeMeasurement/plugins/BuildFile.xml similarity index 100% rename from cmssw/MLProf/RuntimeModule/plugins/BuildFile.xml rename to cmssw/MLProf/RuntimeMeasurement/plugins/BuildFile.xml diff --git a/cmssw/MLProf/RuntimeMeasurement/plugins/TFRuntime.cpp b/cmssw/MLProf/RuntimeMeasurement/plugins/TFRuntime.cpp new file mode 100644 index 0000000..6a44e53 --- /dev/null +++ b/cmssw/MLProf/RuntimeMeasurement/plugins/TFRuntime.cpp @@ -0,0 +1,213 @@ +/* + * Plugin to measure the runtime of a tensorflow graph. + */ + +#include +#include +#include +#include +#include +#include + +#include "FWCore/Framework/interface/Event.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/MakerMacros.h" +#include "FWCore/Framework/interface/stream/EDAnalyzer.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/Utilities/interface/Exception.h" +#include "PhysicsTools/TensorFlow/interface/TensorFlow.h" + +#include "MLProf/Utils/interface/utils.h" + +class TFRuntime : public edm::stream::EDAnalyzer> { +public: + explicit TFRuntime(const edm::ParameterSet&, const tensorflow::SessionCache*); + ~TFRuntime(){}; + + static void fillDescriptions(edm::ConfigurationDescriptions&); + + static std::unique_ptr initializeGlobalCache(const edm::ParameterSet&); + static void globalEndJob(const tensorflow::SessionCache*); + +private: + void beginJob(); + void analyze(const edm::Event&, const edm::EventSetup&); + void endJob(); + + inline float drawNormal() { return normalPdf_(rndGen_); } + tensorflow::Tensor createInputTensor(int rank, std::vector shape); + + // parameters + std::vector inputTensorNames_; + std::vector outputTensorNames_; + std::string outputFile_; + std::string inputTypeStr_; + std::vector inputRanks_; + std::vector flatInputSizes_; + std::vector batchSizes_; + int nCalls_; + + // other members + int nInputs_; + int nPreCalls_; + mlprof::InputType inputType_; + std::random_device rnd_; + std::default_random_engine rndGen_; + std::normal_distribution normalPdf_; + const tensorflow::Session* session_; +}; + +std::unique_ptr TFRuntime::initializeGlobalCache(const edm::ParameterSet& params) { + std::string graphPath = edm::FileInPath(params.getParameter("graphPath")).fullPath(); + // cpu-only for now + tensorflow::Options options{tensorflow::Backend::cpu}; + return std::make_unique(graphPath, options); +} + +void TFRuntime::globalEndJob(const tensorflow::SessionCache* cache) {} + +void TFRuntime::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { + edm::ParameterSetDescription desc; + + // the path to the file containing the graph + desc.add("graphPath"); + // the names of the input tensors + desc.add>("inputTensorNames"); + // the names of the output tensors + desc.add>("outputTensorNames"); + // the name of the output csv file + desc.add("outputFile"); + // the type of input values, either "incremental" or "random" + desc.add("inputType", "random"); + // the rank (number of dimensions) of each input tensor + desc.add>("inputRanks"); + // flat list of sizes of each dimension of each input tensor + // (for a graph with a 1D and a 2D input tensor, this would be a vector of three values) + desc.add>("flatInputSizes"); + // batch sizes to test + desc.add>("batchSizes"); + // the number of calls to the graph to measure the runtime + desc.add("nCalls"); + + descriptions.addWithDefaultLabel(desc); +} + +TFRuntime::TFRuntime(const edm::ParameterSet& config, const tensorflow::SessionCache* cache) + : inputTensorNames_(config.getParameter>("inputTensorNames")), + outputTensorNames_(config.getParameter>("outputTensorNames")), + outputFile_(config.getParameter("outputFile")), + inputTypeStr_(config.getParameter("inputType")), + inputRanks_(config.getParameter>("inputRanks")), + flatInputSizes_(config.getParameter>("flatInputSizes")), + batchSizes_(config.getParameter>("batchSizes")), + nCalls_(config.getParameter("nCalls")), + nInputs_(inputTensorNames_.size()), + nPreCalls_(10), + rndGen_(rnd_()), + normalPdf_(0.0, 1.0), + session_(cache->getSession()) { + // the number of input ranks must match the number of input tensors + if ((int)inputRanks_.size() != nInputs_) { + throw cms::Exception("InvalidInputRanks") << "number of input ranks must match number of input tensors"; + } + // input ranks below 1 and above 3 are not supported + for (auto rank : inputRanks_) { + if (rank < 1) { + throw cms::Exception("InvalidRank") << "only ranks above 0 are supported, got " << rank; + } + if (rank > 3) { + throw cms::Exception("InvalidRank") << "only ranks up to 3 are supported, got " << rank; + } + } + // the sum of ranks must match the number of flat input sizes + if (std::accumulate(inputRanks_.begin(), inputRanks_.end(), 0) != (int)flatInputSizes_.size()) { + throw cms::Exception("InvalidFlatInputSizes") + << "sum of input ranks must match number of flat input sizes, got " << flatInputSizes_.size(); + } + // batch size must be positive + for (auto batchSize : batchSizes_) { + if (batchSize < 1) { + throw cms::Exception("InvalidBatchSize") << "batch sizes must be positive, got " << batchSize; + } + } + // input sizes must be postitive + for (auto size : flatInputSizes_) { + if (size < 1) { + throw cms::Exception("InvalidInputSize") << "input sizes must be positive, got " << size; + } + } + // check the input type + if (inputTypeStr_ == "incremental") { + inputType_ = mlprof::InputType::Incremental; + } else if (inputTypeStr_ == "random") { + inputType_ = mlprof::InputType::Random; + } else { + throw cms::Exception("InvalidInputType") + << "input type must be either 'incremental' or 'random', got " << inputTypeStr_; + } +} + +void TFRuntime::beginJob() {} + +void TFRuntime::endJob() {} + +tensorflow::Tensor TFRuntime::createInputTensor(int rank, std::vector shape) { + // convert the shape to a tf shape + tensorflow::TensorShape tShape; + for (auto dim : shape) { + tShape.AddDim(dim); + } + + // create the tensor + tensorflow::Tensor tensor(tensorflow::DT_FLOAT, tShape); + + // fill it + float* data = tensor.flat().data(); + for (int i = 0; i < tensor.NumElements(); i++, data++) { + *data = inputType_ == mlprof::InputType::Incremental ? float(i) : drawNormal(); + } + + // print the tensor meta data + std::cout << "tensor: " << tensor.DebugString() << std::endl; + + return tensor; +} + +void TFRuntime::analyze(const edm::Event& event, const edm::EventSetup& setup) { + for (int batchSize : batchSizes_) { + // prepare inputs + std::vector> inputs; + int sizeOffset = 0; + for (int i = 0; i < nInputs_; i++) { + // build the shape + std::vector shape = {batchSize}; + for (int j = 0; j < inputRanks_[i]; j++, sizeOffset++) { + shape.push_back(flatInputSizes_[sizeOffset]); + } + // create and save it + inputs.push_back({inputTensorNames_[i], createInputTensor(inputRanks_[i], shape)}); + } + + // prepare output vectors + std::vector outputs; + + // pre calls to "warm up" + for (int r = 0; r < nPreCalls_; r++) { + tensorflow::run(session_, inputs, outputTensorNames_, &outputs); + } + + // actual calls to measure runtimes + std::vector runtimes; + for (int r = 0; r < nCalls_; r++) { + auto start = std::chrono::high_resolution_clock::now(); + tensorflow::run(session_, inputs, outputTensorNames_, &outputs); + auto end = std::chrono::high_resolution_clock::now(); + runtimes.push_back((end - start).count() * 1000); + } + + // save them + mlprof::writeRuntimes(outputFile_, batchSize, runtimes); + } +} + +DEFINE_FWK_MODULE(TFRuntime); diff --git a/cmssw/MLProf/RuntimeMeasurement/test/tf_runtime_template_cfg.py b/cmssw/MLProf/RuntimeMeasurement/test/tf_runtime_template_cfg.py new file mode 100644 index 0000000..def47f7 --- /dev/null +++ b/cmssw/MLProf/RuntimeMeasurement/test/tf_runtime_template_cfg.py @@ -0,0 +1,58 @@ +# coding: utf-8 + +import FWCore.ParameterSet.Config as cms +from FWCore.ParameterSet.VarParsing import VarParsing + +# setup minimal options +options = VarParsing("python") +options.register( + "batchSizes", + [1], + VarParsing.multiplicity.list, + VarParsing.varType.int, + "Batch sizes to be tested", +) +options.register( + "csvFile", + "results.csv", + VarParsing.multiplicity.singleton, + VarParsing.varType.string, + "The path of the csv file to save results", +) +options.parseArguments() + + +# define the process to run +process = cms.Process("MLPROF") + +# minimal configuration +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1 +process.maxEvents = cms.untracked.PSet( + input=cms.untracked.int32(__N_EVENTS__), # noqa +) +process.source = cms.Source( + "PoolSource", + fileNames=cms.untracked.vstring(*__INPUT_FILES__), # noqa +) + +# process options +process.options = cms.untracked.PSet( + allowUnscheduled=cms.untracked.bool(True), + wantSummary=cms.untracked.bool(False), +) + +# setup the plugin +process.load("MLProf.RuntimeMeasurement.tfRuntime_cfi") +process.tfRuntime.graphPath = cms.string("__GRAPH_PATH__") +process.tfRuntime.inputTensorNames = cms.vstring(__INPUT_TENSOR_NAMES__) # noqa +process.tfRuntime.outputTensorNames = cms.vstring(__OUTPUT_TENSOR_NAMES__) # noqa +process.tfRuntime.outputFile = cms.string(options.csvFile) +process.tfRuntime.inputType = cms.string("__INPUT_TYPE__") +process.tfRuntime.inputRanks = cms.vint32(__INPUT_RANKS__) # noqa +process.tfRuntime.flatInputSizes = cms.vint32(__FLAT_INPUT_SIZES__) # noqa +process.tfRuntime.batchSizes = cms.vint32(list(options.batchSizes)) +process.tfRuntime.nCalls = cms.int32(__N_CALLS__) # noqa + +# define what to run in the path +process.p = cms.Path(process.tfRuntime) diff --git a/cmssw/MLProf/RuntimeModule/plugins/MyPluginRuntime.cpp b/cmssw/MLProf/RuntimeModule/plugins/MyPluginRuntime.cpp deleted file mode 100644 index 66d178a..0000000 --- a/cmssw/MLProf/RuntimeModule/plugins/MyPluginRuntime.cpp +++ /dev/null @@ -1,353 +0,0 @@ -/* - * Example plugin to demonstrate the direct multi-threaded inference with - * TensorFlow 2. - */ - -#include -#include -#include -#include -#include -#include - -#include "FWCore/Framework/interface/Event.h" -#include "FWCore/Framework/interface/Frameworkfwd.h" -#include "FWCore/Framework/interface/MakerMacros.h" -#include "FWCore/Framework/interface/stream/EDAnalyzer.h" -#include "FWCore/ParameterSet/interface/ParameterSet.h" -#include "PhysicsTools/TensorFlow/interface/TensorFlow.h" - -// define the cache object -// it could handle graph loading and destruction on its own, -// but in this example, we define it as a logicless container -struct CacheData { - CacheData() : graphDef(nullptr) {} - std::atomic graphDef; -}; - -class MyPluginRuntime - : public edm::stream::EDAnalyzer> { - public: - explicit MyPluginRuntime(const edm::ParameterSet&, const CacheData*); - ~MyPluginRuntime(){}; - - static void fillDescriptions(edm::ConfigurationDescriptions&); - - // two additional static methods for handling the global cache - static std::unique_ptr initializeGlobalCache( - const edm::ParameterSet&); - static void globalEndJob(const CacheData*); - - private: - void beginJob(); - void analyze(const edm::Event&, const edm::EventSetup&); - void endJob(); - - std::vector inputTensorNames_; - std::vector outputTensorNames_; - std::string filenameOutputCsv_; - std::string inputType_; - std::vector inputLengths_; - std::vector inputSizes_; - // std:vector batchSize_; - int nRuns_; - int nWarmUps_; - std::vector batchSizes_; - - tensorflow::Session* session_; -}; - -std::unique_ptr MyPluginRuntime::initializeGlobalCache( - const edm::ParameterSet& config) { - // this method is supposed to create, initialize and return a CacheData - // instance - CacheData* cacheData = new CacheData(); - - // load the graph def and save it - std::string graphPath = config.getParameter("graphPath"); - cacheData->graphDef = tensorflow::loadGraphDef(graphPath); - - // set tensorflow log leven to warning - tensorflow::setLogging("2"); - - return std::unique_ptr(cacheData); -} - -void MyPluginRuntime::globalEndJob(const CacheData* cacheData) { - // reset the graphDef - if (cacheData->graphDef != nullptr) { - delete cacheData->graphDef; - } -} - -void MyPluginRuntime::fillDescriptions( - edm::ConfigurationDescriptions& descriptions) { - // defining this function will lead to a *_cfi file being generated when - // compiling - edm::ParameterSetDescription desc; - desc.add("graphPath"); - desc.add>("inputTensorNames"); - desc.add>("outputTensorNames"); - desc.add("filenameOutputCsv"); - desc.add("inputType"); - desc.add>("inputLengths"); - desc.add>("inputSizes"); - desc.add>("batchSizes"); - desc.add("numberRuns"); - desc.add("numberWarmUps"); - descriptions.addWithDefaultLabel(desc); -} - -MyPluginRuntime::MyPluginRuntime(const edm::ParameterSet& config, - const CacheData* cacheData) - : inputTensorNames_( - config.getParameter>("inputTensorNames")), - outputTensorNames_( - config.getParameter>("outputTensorNames")), - filenameOutputCsv_(config.getParameter("filenameOutputCsv")), - inputType_(config.getParameter("inputType")), - inputLengths_(config.getParameter>("inputLengths")), - inputSizes_(config.getParameter>("inputSizes")), - // batchSize_(config.getParameter("batchSize")), - nRuns_(config.getParameter("numberRuns")), - nWarmUps_(config.getParameter("numberWarmUps")), - batchSizes_(config.getParameter>("batchSizes")), - session_(tensorflow::createSession(cacheData->graphDef)) {} - -void MyPluginRuntime::beginJob() {} - -void MyPluginRuntime::endJob() { - // close the session - tensorflow::closeSession(session_); -} - -std::tuple mean_and_std(std::vector data_vector) { - float runs = data_vector.size(); - - // calculate mean runtime - float mean_runtime = 0; - for (float time : data_vector) { - mean_runtime += time; - } - - // calculate std of mean runtime - mean_runtime /= runs; - float std_runtime = 0; - - for (float time : data_vector) { - std_runtime += pow(time - mean_runtime, 2); - } - std_runtime /= runs; - std_runtime = pow(std_runtime, 0.5); - - // wrapp results in tuple - auto results = std::make_tuple(mean_runtime, std_runtime); - return results; -} - -void deletePreviousFileContent(std::string csv_file) { - std::ofstream file(csv_file, std::ios::out | std::ios::trunc); - file.close(); -} - -void writeFile(float batch_size, float mean, float std, std::string csv_file) { - std::ofstream file(csv_file, std::ios::out | std::ios::app); - file << batch_size << "," << mean << "," << std << std::endl; - file.close(); -} - -void writeFileWholeVector(float batch_size, std::vector runtimes, - std::string csv_file) { - std::cout << runtimes.size() << std::endl; - std::ofstream file(csv_file, std::ios::out | std::ios::app); - for (int i = 0; i < (int)runtimes.size(); i++) { - file << batch_size << "," << runtimes[i] << std::endl; - } - - file.close(); -} - -void print_list(std::list const& list) { - for (auto const& i : list) { - std::cout << i << std::endl; - } -} - -void print_vector(std::vector const& vector) { - for (auto const& i : vector) { - std::cout << i << std::endl; - } -} - -float choose_value(std::string type, int incremental_value, - std::default_random_engine generator, - std::normal_distribution distribution) { - float value; - if (type == "incremental") { - value = float(incremental_value); - } else if (type == "random") { - value = distribution(generator); - } - return value; -} - -std::vector get_tensor_shape(tensorflow::Tensor& tensor) { - std::vector shape; - int num_dimensions = tensor.shape().dims(); - - for (int ii_dim = 0; ii_dim < num_dimensions; ii_dim++) { - shape.push_back(tensor.shape().dim_size(ii_dim)); - } - return shape; -} - -void MyPluginRuntime::analyze(const edm::Event& event, - const edm::EventSetup& setup) { - std::list mean_runtimes; - std::list std_runtimes; - for (int batchSize : batchSizes_) { - std::random_device rd; - std::default_random_engine generator(rd()); - std::normal_distribution distribution(1.0, 1.0); - std::vector input_classes_vector; - int counter = 0; - tensorflow::Tensor input; - for (int input_class = 0; input_class < (int)inputLengths_.size(); - input_class++) { - // definition and filling of the input tensors for the differents input - // classes here with if statements -> 1D,2D,3D inputs (creating input - // tensor with std::vector does not work.) - if (inputLengths_[input_class] == 1) { - input = tensorflow::Tensor(tensorflow::DT_FLOAT, - {batchSize, inputSizes_[counter]}); - auto input_eigen_mapped = input.tensor(); - for (int b = 0; b < batchSize; b++) { - for (int i = 0; i < inputSizes_[counter]; i++) { - float value; - if (inputType_ == "incremental") { - value = float(i); - } else if (inputType_ == "random") { - value = distribution(generator); - } - // float value = choose_value(inputType_, i, generator, - // distribution); input.matrix()(b, i) = value; - input_eigen_mapped(b, i) = value; - } - } - counter++; - } else if (inputLengths_[input_class] == 2) { - input = tensorflow::Tensor( - tensorflow::DT_FLOAT, - {batchSize, inputSizes_[counter], inputSizes_[counter + 1]}); - auto input_eigen_mapped = input.tensor(); - std::cout << input.shape().dims() << std::endl; - for (int b = 0; b < batchSize; b++) { - for (int i = 0; i < inputSizes_[counter]; i++) { - for (int j = 0; j < inputSizes_[counter + 1]; j++) { - // float value = choose_value(inputType_, i+j, generator, - // distribution); - float value; - if (inputType_ == "incremental") { - value = float(i + j); - } else if (inputType_ == "random") { - value = distribution(generator); - } - // float value = choose_value(inputType_, i, generator, - // distribution); input.matrix()(b, i, j) = value; - input_eigen_mapped(b, i, j) = value; - } - } - } - counter++; - counter++; - } else if (inputLengths_[input_class] == 3) { - input = tensorflow::Tensor( - tensorflow::DT_FLOAT, - {batchSize, inputSizes_[counter], inputSizes_[counter + 1], - inputSizes_[counter + 2]}); - auto input_eigen_mapped = input.tensor(); - for (int b = 0; b < batchSize; b++) { - for (int i = 0; i < inputSizes_[counter]; i++) { - for (int j = 0; j < inputSizes_[counter + 1]; j++) { - for (int k = 0; k < inputSizes_[counter + 2]; k++) { - float value; - if (inputType_ == "incremental") { - value = float(i + j + k); - } else if (inputType_ == "random") { - value = distribution(generator); - } - // float value = choose_value(inputType_, i+j+k, generator, - // distribution); - // input.matrix()(b, i, j, k) = value; - input_eigen_mapped(b, i, j, k) = value; - } - } - } - } - counter++; - counter++; - counter++; - } else { - std::cout << "The inputs must be one-, two- or three dimensional, an " - "error will be thrown" - << std::endl; - throw std::invalid_argument( - "input " + std::to_string(input_class) + " with name " + - inputTensorNames_[input_class] + " has not the right dimension"); - } - // feeding all input tensors into an input vector - - input_classes_vector.push_back(input); - } - - // define the output vector - std::vector outputs; - - // from cmssw source code: inputs in run function is a vector of pairs - // consisting of a string and a tensor - std::vector> inputs; - for (int input_class = 0; input_class < (int)inputLengths_.size(); - input_class++) { - inputs.push_back( - {inputTensorNames_[input_class], input_classes_vector[input_class]}); - } - - // run and measure time - int nRuns = nRuns_; - std::vector runtimes; - int nWarmUps = nWarmUps_; - for (int r = 0; r < nRuns + nWarmUps; r++) { - auto start = std::chrono::high_resolution_clock::now(); - // run the graph with given inputs and outputs - tensorflow::run(session_, inputs, outputTensorNames_, &outputs); - - auto end = std::chrono::high_resolution_clock::now(); - std::chrono::duration runtime_in_seconds = (end - start); - - if (r >= nWarmUps) { - // std::cout << "Current epoch: " << r - nWarmUps + 1 << std::endl; - // conver runtimes to milli seconds - runtimes.push_back(runtime_in_seconds.count() * 1000); - // std::cout << "Corresponding runtime: " - // << runtime_in_seconds.count() * 1000 << " ms" << std::endl; - } else { - // std::cout << "Current warm-up epoch: " << r + 1 << std::endl; - continue; - } - } - // calculate metrics - float mean_runtime = 0; - float std_runtime = 0; - std::tie(mean_runtime, std_runtime) = mean_and_std(runtimes); - - mean_runtimes.push_back(mean_runtime); - std_runtimes.push_back(std_runtime); - - // save time performance not divided by batch size - std::cout << "measurement done, begin writing file" << std::endl; - writeFileWholeVector(batchSize, runtimes, filenameOutputCsv_); - std::cout << "file written" << std::endl; - } -} - -DEFINE_FWK_MODULE(MyPluginRuntime); diff --git a/cmssw/MLProf/RuntimeModule/test/runtime_template_cfg.py b/cmssw/MLProf/RuntimeModule/test/runtime_template_cfg.py deleted file mode 100644 index 176e7aa..0000000 --- a/cmssw/MLProf/RuntimeModule/test/runtime_template_cfg.py +++ /dev/null @@ -1,60 +0,0 @@ -# coding: utf-8 - -import FWCore.ParameterSet.Config as cms -from FWCore.ParameterSet.VarParsing import VarParsing - -# setup minimal options -options = VarParsing("python") -options.register( - "batchSizes", - [1], # default - VarParsing.multiplicity.list, - VarParsing.varType.int, - "Batch sizes to be tested", -) -options.register( - "csvFile", - "results.csv", # default - VarParsing.multiplicity.singleton, - VarParsing.varType.string, - "The path of the csv file to save results", -) -options.parseArguments() - - -# define the process to run -process = cms.Process("TEST") - -# minimal configuration -process.load("FWCore.MessageService.MessageLogger_cfi") -process.MessageLogger.cerr.FwkReport.reportEvery = 1 -process.maxEvents = cms.untracked.PSet( - input=cms.untracked.int32(NUMBER_EVENTS_TAKEN), -) -process.source = cms.Source( - "PoolSource", - fileNames=cms.untracked.vstring(*INPUT_FILES_PLACEHOLDER), -) - -# process options -process.options = cms.untracked.PSet( - allowUnscheduled=cms.untracked.bool(True), - wantSummary=cms.untracked.bool(False), -) - -# setup MyPluginRuntime by loading the auto-generated cfi (see MyPlugin.fillDescriptions) -process.load("MLProf.RuntimeModule.myPluginRuntime_cfi") -process.myPluginRuntime.graphPath = cms.string("GRAPH_PATH_PLACEHOLDER") -process.myPluginRuntime.inputTensorNames = cms.vstring(INPUT_TENSOR_NAME_PLACEHOLDER) -process.myPluginRuntime.outputTensorNames = cms.vstring(OUTPUT_TENSOR_NAME_PLACEHOLDER) -process.myPluginRuntime.filenameOutputCsv = cms.string(options.csvFile) -process.myPluginRuntime.inputType = cms.string("INPUT_TYPE_PLACEHOLDER") - -process.myPluginRuntime.inputSizes = cms.vint32(INPUT_SIZE_PLACEHOLDER) -process.myPluginRuntime.inputLengths = cms.vint32(INPUT_CLASS_DIMENSION_PLACEHOLDER) -process.myPluginRuntime.numberRuns = cms.int32(NUMBER_RUNS_PLACEHOLDER) -process.myPluginRuntime.numberWarmUps = cms.int32(NUMBER_WARM_UPS_PLACEHOLDER) -process.myPluginRuntime.batchSizes = cms.vint32(list(options.batchSizes)) - -# define what to run in the path -process.p = cms.Path(process.myPluginRuntime) diff --git a/cmssw/MLProf/Utils/interface/utils.h b/cmssw/MLProf/Utils/interface/utils.h new file mode 100644 index 0000000..db0dda0 --- /dev/null +++ b/cmssw/MLProf/Utils/interface/utils.h @@ -0,0 +1,26 @@ +/* + * Helper functions. + */ + +#include +#include +#include +#include +#include + +namespace mlprof { + +enum InputType { + Incremental, + Random, +}; + +void writeRuntimes(const std::string& path, float batchSize, std::vector runtimes) { + std::ofstream file(path, std::ios::out | std::ios::app); + for (int i = 0; i < (int)runtimes.size(); i++) { + file << batchSize << "," << runtimes[i] << std::endl; + } + file.close(); +} + +} // namespace mlprof diff --git a/cmssw/MLProf/utils/utils.cpp b/cmssw/MLProf/utils/utils.cpp deleted file mode 100644 index 251fed3..0000000 --- a/cmssw/MLProf/utils/utils.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include -#include -#include -#include -#include - -// #include ? - -// Protected header? - -std::tuple mean_and_std(std::vector data_vector) { - float runs = data_vector.size(); - - // calculate mean runtime - float mean_runtime = 0; - for (float time : data_vector) { - mean_runtime += time; - } - - // calculate std of mean runtime - mean_runtime /= runs; - float std_runtime = 0; - - for (float time : data_vector) { - std_runtime += pow(time - mean_runtime, 2); - } - std_runtime /= runs; - std_runtime = pow(std_runtime, 0.5); - - // wrapp results in tuple - auto results = std::make_tuple(mean_runtime, std_runtime); - return results; -} - -void deletePreviousFileContent(std::string csv_file) { - std::ofstream file(csv_file, std::ios::out | std::ios::trunc); - file.close(); -} - -void writeFile(float batch_size, float mean, float std, std::string csv_file) { - std::ofstream file(csv_file, std::ios::out | std::ios::app); - file << batch_size << "," << mean << "," << std << std::endl; - file.close(); -} - -void print_list(std::list const &list) { - for (auto const &i : list) { - std::cout << i << std::endl; - } -} - -void print_vector(std::vector const &vector) { - for (auto const &i : vector) { - std::cout << i << std::endl; - } -} diff --git a/cmssw/install_sandbox.sh b/cmssw/install_sandbox.sh index cd11eb9..e80b33b 100644 --- a/cmssw/install_sandbox.sh +++ b/cmssw/install_sandbox.sh @@ -2,3 +2,4 @@ rm -rf MLProf cp -r "${MLP_BASE}/cmssw/MLProf" . +rm -rf MLProf/*/test diff --git a/examples/2_inputs/model_2_inputs.json b/examples/2_inputs/model_2_inputs.json deleted file mode 100644 index bbba1e3..0000000 --- a/examples/2_inputs/model_2_inputs.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "file": "/afs/cern.ch/user/n/nprouvos/public/dnn_2_inputs.pb", - "inputs": [ - { - "name": "input_0", - "shape": [392] - }, - { - "name": "input_1", - "shape": [392] - } - ], - "outputs": [ - { - "name": "Identity" - } - ], - "network_name": "dnn_2_inputs" -} diff --git a/examples/cnn/conv_2d_inputs.pb b/examples/cnn/conv_2d_inputs.pb new file mode 100644 index 0000000..3deb7ec --- /dev/null +++ b/examples/cnn/conv_2d_inputs.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:154d11b45faab5e3b6c7b69cf8ca55c39ff42dd8abced7552d25fe523dbb7b9b +size 399875 diff --git a/examples/cnn/model_cnn.json b/examples/cnn/model_cnn.json deleted file mode 100644 index fcefc8d..0000000 --- a/examples/cnn/model_cnn.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "file": "/afs/cern.ch/user/n/nprouvos/public/conv_2d_inputs.pb", - "inputs": [ - { - "name": "input_0_input", - "shape": [28,28,1] - } - ], - "outputs": [ - { - "name": "Identity" - } - ], - "network_name": "cnn" -} diff --git a/examples/dnn_2_inputs/dnn_2_inputs.pb b/examples/dnn_2_inputs/dnn_2_inputs.pb new file mode 100644 index 0000000..48071b9 --- /dev/null +++ b/examples/dnn_2_inputs/dnn_2_inputs.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db40f419bf1e1726abd287eb2a0133dba829e83bbc635534f6e3d260f9c2108b +size 103855 diff --git a/examples/model1/model.json b/examples/model1/model.json deleted file mode 100644 index a02ecdf..0000000 --- a/examples/model1/model.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "file": "/afs/cern.ch/user/n/nprouvos/public/simple_dnn.pb", - "inputs": [ - { - "name": "input_0", - "shape": [784] - } - ], - "outputs": [ - { - "name": "Identity" - } - ], - "network_name": "dnn" -} diff --git a/examples/simple_dnn/simple_dnn.pb b/examples/simple_dnn/simple_dnn.pb new file mode 100644 index 0000000..24f079c --- /dev/null +++ b/examples/simple_dnn/simple_dnn.pb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8da8cdf45063f479ab3f7c0d09a64120994efd779cd8d2d42f236e14acd0fd44 +size 103821 diff --git a/mlprof/tasks/base.py b/mlprof/tasks/base.py index 3227a13..64e54e0 100644 --- a/mlprof/tasks/base.py +++ b/mlprof/tasks/base.py @@ -27,9 +27,9 @@ class BaseTask(law.SandboxTask): def store_parts(self): """ - Returns a :py:class:`law.util.InsertableDict` whose values are used to create a store path. - For instance, the parts ``{"keyA": "a", "keyB": "b", 2: "c"}`` lead to the path "a/b/c". The - keys can be used by subclassing tasks to overwrite values. + Returns a :py:class:`law.util.InsertableDict` whose values are used to create a store path. For instance, the + parts ``{"keyA": "a", "keyB": "b", 2: "c"}`` lead to the path "a/b/c". The keys can be used by subclassing tasks + to overwrite values. """ parts = law.util.InsertableDict() @@ -44,8 +44,8 @@ def store_parts(self): def local_path(self, *path, **kwargs): """ - Joins path fragments from *store* (defaulting to :py:attr:`default_store`), *store_parts()* - and *path* and returns the joined path. + Joins path fragments from *store* (defaulting to :py:attr:`default_store`), :py:meth:`store_parts` and *path*, + and returns the joined path. """ # determine the main store directory store = "$MLP_STORE_LOCAL" @@ -58,8 +58,8 @@ def local_path(self, *path, **kwargs): def local_target(self, *path, **kwargs): """ local_target(*path, dir=False, **kwargs) - Creates either a local file or directory target, depending on *dir*, forwarding all *path* - fragments and *store* to :py:meth:`local_path` and all *kwargs* the respective target class. + Creates either a local file or directory target, depending on *dir*, forwarding all *path* fragments and *store* + to :py:meth:`local_path` and all *kwargs* the respective target class. """ # select the target class cls = law.LocalDirectoryTarget if kwargs.pop("dir", False) else law.LocalFileTarget @@ -73,8 +73,8 @@ def local_target(self, *path, **kwargs): class CommandTask(BaseTask): """ - A task that provides convenience methods to work with shell commands, i.e., printing them on the - command line and executing them with error handling. + A task that provides convenience methods to work with shell commands, i.e., printing them on the command line and + executing them with error handling. """ print_command = law.CSVParameter( @@ -102,7 +102,7 @@ def _print_command(self, args): max_depth = int(args[0]) - print("print task commands with max_depth {}".format(max_depth)) + print(f"print task commands with max_depth {max_depth}") print("") # get the format chars @@ -140,7 +140,7 @@ def _print_command(self, args): task_offset = offset if depth > 0: task_offset += fmt["l" if is_last else "t"] + fmt["ind"] * fmt["-"] - task_prefix = "{} {} ".format(depth, fmt[">"]) + task_prefix = f"{depth} {fmt['>']} " # determine text offset and prefix text_offset = offset @@ -162,7 +162,7 @@ def _print_command(self, args): text = law.util.colored("command", style="bright") if isinstance(dep, law.BaseWorkflow) and dep.is_workflow(): dep = dep.as_branch(0) - text += " (from branch {})".format(law.util.colored("0", "red")) + text += f" (from branch {law.util.colored('0', 'red')})" text += ": " cmd = dep.build_command() @@ -211,10 +211,10 @@ def run_command(self, cmd, optional=False, **kwargs): tmp_dir = law.LocalDirectoryTarget(is_tmp=True) tmp_dir.touch() kwargs["cwd"] = tmp_dir.path - self.publish_message("cwd: {}".format(kwargs.get("cwd", os.getcwd()))) + self.publish_message(f"cwd: {kwargs.get('cwd', os.getcwd())}") # call it - with self.publish_step("running '{}' ...".format(law.util.colored(cmd, "cyan"))): + with self.publish_step(f"running '{law.util.colored(cmd, 'cyan')}' ..."): p, lines = law.util.readable_popen(cmd, shell=True, executable="/bin/bash", **kwargs) for line in lines: print(line) @@ -258,22 +258,22 @@ class PlotTask(BaseTask): ) plot_postfix = luigi.Parameter( default=law.NO_STR, - description="an arbitrary postfix that is added with two underscores to all paths of " - "produced plots; default: empty", + description="an arbitrary postfix that is added with two underscores to all paths of produced plots; " + "default: empty", ) view_cmd = luigi.Parameter( default=law.NO_STR, significant=False, - description="a command to execute after the task has run to visualize plots right in the " - "terminal; default: empty", + description="a command to execute after the task has run to visualize plots right in the terminal; " + "default: empty", ) def create_plot_names(self, parts): plot_file_types = ["pdf", "png", "root", "c", "eps"] if any(t not in plot_file_types for t in self.file_types): - raise Exception("plot names only allowed for file types {}, got {}".format( - ",".join(plot_file_types), ",".join(self.file_types), - )) + raise Exception( + f"plot names only allowed for file types {','.join(plot_file_types)}, got {','.join(self.file_types)}", + ) if self.plot_postfix and self.plot_postfix != law.NO_STR: parts.append((self.plot_postfix,)) @@ -314,7 +314,7 @@ def after_call(state): # loop through targets and view them for target in view_targets.values(): - task.publish_message("showing {}".format(target.path)) + task.publish_message(f"showing {target.path}") with target.localize("r") as tmp: law.util.interruptable_popen( view_cmd.format(tmp.path), diff --git a/mlprof/tasks/parameters.py b/mlprof/tasks/parameters.py index ab02a0e..2a0b15a 100644 --- a/mlprof/tasks/parameters.py +++ b/mlprof/tasks/parameters.py @@ -1,7 +1,7 @@ # coding: utf-8 """ -Collection of the recurrent luigi parameters for different tasks +Collection of the recurrent luigi parameters for different tasks. """ import os @@ -18,21 +18,18 @@ class CMSSWParameters(BaseTask): """ cmssw_version = luigi.Parameter( - default="CMSSW_12_2_4", - description="CMSSW version; default: CMSSW_12_2_4", + default="CMSSW_13_3_1", + description="CMSSW version; default: CMSSW_13_3_1", ) scram_arch = luigi.Parameter( - default="slc7_amd64_gcc10", - description="SCRAM architecture; default: slc7_amd64_gcc10", + default="slc7_amd64_gcc12", + description="SCRAM architecture; default: slc7_amd64_gcc12", ) def store_parts(self): parts = super().store_parts() - cmssw_repr = [ - self.cmssw_version, - self.scram_arch, - ] + cmssw_repr = [self.cmssw_version, self.scram_arch] parts.insert_before("version", "cmssw", "__".join(cmssw_repr)) return parts @@ -40,46 +37,53 @@ def store_parts(self): class RuntimeParameters(BaseTask): """ - General parameters for the model definition and the runtime measurement + General parameters for the model definition and the runtime measurement. """ model_file = luigi.Parameter( - default="$MLP_BASE/examples/model1/model.json", + default="$MLP_BASE/examples/simple_dnn/model.json", description="json file containing information of model to be tested; " - "default: $MLP_BASE/examples/model1/model.json", + "default: $MLP_BASE/examples/simple_dnn/model.json", ) model_name = luigi.Parameter( default=law.NO_STR, description="when set, use this name for storing outputs instead of a hashed version of " "--model-file; default: empty", ) - input_files = law.CSVParameter( - default=(), - description="comma-separated list of absolute paths of input files for the CMSSW analyzer; " - "when empty, random input values will be used; default: empty", + input_type = luigi.Parameter( + default="random", + description="either 'random', 'incremental', or a path to a root file; default: random", ) - events = luigi.IntParameter( + n_events = luigi.IntParameter( default=1, - description="number of events to read from each input file for averaging measurements; " - "default: 1", + description="number of events to be processed; default: 1", ) - repetitions = luigi.IntParameter( + n_calls = luigi.IntParameter( default=100, - description="number of repetitions to be performed per evaluation for averaging; " - "default: 100", - ) - warmup = luigi.IntParameter( - default=10, - significant=False, - description="number of evaluations to be performed before starting the actual measurement; default: 10", + description="number of evaluation calls for averaging; default: 100", ) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # verify the input type + self.input_file = None + if self.input_type not in ("random", "incremental"): + self.input_file = os.path.abspath(os.path.expandvars(os.path.expanduser(self.input_type))) + if not os.path.exists(self.input_file): + raise ValueError( + f"input type '{self.input_type}' is neither 'random' nor 'incremental' nor a path to an existing " + f"root file", + ) + + # cached model content + self._model_data = None + @property - def abs_input_files(self): - return [ - os.path.abspath(os.path.expandvars(os.path.expanduser(path))) - for path in self.input_files - ] + def model_data(self): + if self._model_data is None: + self._model_data = law.LocalFileTarget(self.model_file).load(formatter="json") + return self._model_data @property def full_model_name(self): @@ -97,9 +101,9 @@ def store_parts(self): # build a combined string that represents the significant parameters params = [ f"model_{self.full_model_name}", - f"inputs_{law.util.create_hash(sorted(self.abs_input_files)) if self.input_files else 'empty'}", - f"events_{self.events}", - f"repeat_{self.repetitions}", + f"input_{law.util.create_hash(self.input_file) if self.input_file else self.input_type}", + f"nevents_{self.n_events}", + f"ncalls_{self.n_calls}", ] parts.insert_before("version", "model_params", "__".join(params)) diff --git a/mlprof/tasks/runtime.py b/mlprof/tasks/runtime.py index 3f676c9..d2b71a5 100644 --- a/mlprof/tasks/runtime.py +++ b/mlprof/tasks/runtime.py @@ -4,6 +4,8 @@ Collection of test tasks. """ +import os + import luigi import law @@ -14,21 +16,17 @@ class CreateRuntimeConfig(RuntimeParameters, CMSSWParameters): - """ - TODO: - - rename template vars - """ - default_cmssw_files = { + default_input_files = { "CMSSW_*": ["/afs/cern.ch/user/n/nprouvos/public/testfile.root"], } - def find_default_cmssw_files(self): - for version_pattern, files in self.default_cmssw_files.items(): + def find_default_input_files(self): + for version_pattern, files in self.default_input_files.items(): if law.util.multi_match(self.cmssw_version, version_pattern): return files - raise Exception(f"no default cmssw files found for version '{self.cmssw_version}'") + raise Exception(f"no default input files found for '{self.cmssw_version}'") def output(self): return self.local_target("cfg.py") @@ -38,46 +36,50 @@ def run(self): output = self.output() output.parent.touch() - # open the model file - model_data = law.LocalFileTarget(self.model_file).load(formatter="json") + # get model data + model_data = self.model_data + + # resolve the graph path relative to the model file + graph_path = os.path.expandvars(os.path.expanduser(model_data["file"])) + graph_path = os.path.join(os.path.dirname(self.model_file), graph_path) # determine input files - input_files = self.abs_input_files - input_type = "file" - if not input_files: - input_files = self.find_default_cmssw_files() + if self.input_file: + input_files = [self.input_file] input_type = "random" + else: + input_files = self.find_default_input_files() + input_type = self.input_type # prepare template variables template_vars = { - "GRAPH_PATH_PLACEHOLDER": model_data["file"], - "INPUT_FILES_PLACEHOLDER": [ + "GRAPH_PATH": graph_path, + "INPUT_FILES": [ law.target.file.add_scheme(path, "file://") for path in input_files ], - "NUMBER_EVENTS_TAKEN": self.events, - "INPUT_SIZE_PLACEHOLDER": sum( - (inp["shape"] for inp in model_data["inputs"]), - [], - ), - "INPUT_CLASS_DIMENSION_PLACEHOLDER": [ + "N_EVENTS": self.n_events, + "INPUT_TYPE": input_type, + "INPUT_RANKS": [ len(inp["shape"]) for inp in model_data["inputs"] ], - "INPUT_TYPE_PLACEHOLDER": input_type, - "INPUT_TENSOR_NAME_PLACEHOLDER": [inp["name"] for inp in model_data["inputs"]], - "OUTPUT_TENSOR_NAME_PLACEHOLDER": [outp["name"] for outp in model_data["outputs"]], - "NUMBER_RUNS_PLACEHOLDER": self.repetitions, - "NUMBER_WARM_UPS_PLACEHOLDER": self.warmup, + "FLAT_INPUT_SIZES": sum( + (inp["shape"] for inp in model_data["inputs"]), + [], + ), + "INPUT_TENSOR_NAMES": [inp["name"] for inp in model_data["inputs"]], + "OUTPUT_TENSOR_NAMES": [outp["name"] for outp in model_data["outputs"]], + "N_CALLS": self.n_calls, } # load the template content - template = "$MLP_BASE/cmssw/MLProf/RuntimeModule/test/runtime_template_cfg.py" + template = "$MLP_BASE/cmssw/MLProf/RuntimeMeasurement/test/tf_runtime_template_cfg.py" content = law.LocalFileTarget(template).load(formatter="text") # replace variables for key, value in template_vars.items(): - content = content.replace(key, str(value)) + content = content.replace(f"__{key}__", str(value)) # write the output output.dump(content, formatter="text") @@ -160,23 +162,30 @@ def run(self): output.parent.touch() # get name network for legend - model_data = law.LocalFileTarget(self.model_file).load(formatter="json") + model_data = self.model_data network_name = model_data["network_name"] # create the plot - plot_batch_size_several_measurements(self.batch_sizes, [self.input().path], output.path, [network_name], - self.custom_plot_params) + plot_batch_size_several_measurements( + self.batch_sizes, + [self.input().path], + output.path, + [network_name], + self.custom_plot_params, + ) print("plot saved") -class PlotRuntimesMultipleNetworks(RuntimeParameters, - CMSSWParameters, - BatchSizesParameters, - PlotTask, - CustomPlotParameters): +class PlotRuntimesMultipleNetworks( + RuntimeParameters, + CMSSWParameters, + BatchSizesParameters, + PlotTask, + CustomPlotParameters, +): """ - Task to plot the results from the runtime measurements for several networks, depending on the batch sizes - given as parameters, default are 1, 2 and 4. + Task to plot the results from the runtime measurements for several networks, depending on the batch sizes given as + parameters, default are 1, 2 and 4. """ sandbox = "bash::$MLP_BASE/sandboxes/plotting.sh" @@ -186,15 +195,17 @@ class PlotRuntimesMultipleNetworks(RuntimeParameters, ) def requires(self): - return [MergeRuntimes.req(self, model_file=model_file) for model_file in self.model_files] + return [ + MergeRuntimes.req(self, model_file=model_file) + for model_file in self.model_files + ] def output(self): - network_names = [] - for model_file in self.model_files: - model_data = law.LocalFileTarget(model_file).load(formatter="json") - network_names += [model_data["network_name"]] + network_names = [req.model_data["network_name"] for req in self.requires()] network_names_repr = "_".join(network_names) - return self.local_target(f"runtime_plot_networks_{network_names_repr}_different_batch_sizes_{self.batch_sizes_repr}.pdf") # noqa + return self.local_target( + f"runtime_plot_networks_{network_names_repr}_different_batch_sizes_{self.batch_sizes_repr}.pdf", + ) @view_output_plots def run(self): @@ -203,23 +214,24 @@ def run(self): output.parent.touch() # create the plot - network_names = [] - input_paths = [] - for model_file in self.model_files: - model_data = law.LocalFileTarget(model_file).load(formatter="json") - network_names += [model_data["network_name"]] - for input_task in self.input(): - input_paths += [input_task.path] - plot_batch_size_several_measurements(self.batch_sizes, input_paths, - output.path, network_names, self.custom_plot_params) - print("plot saved") + network_names = [req.model_data["network_name"] for req in self.requires()] + input_paths = [inp.path for inp in self.input()] + plot_batch_size_several_measurements( + self.batch_sizes, + input_paths, + output.path, + network_names, + self.custom_plot_params, + ) -class PlotRuntimesMultipleCMSSW(RuntimeParameters, - CMSSWParameters, - BatchSizesParameters, - PlotTask, - CustomPlotParameters): +class PlotRuntimesMultipleCMSSW( + RuntimeParameters, + CMSSWParameters, + BatchSizesParameters, + PlotTask, + CustomPlotParameters, +): """ Task to plot the results from the runtime measurements for inferences performed in multiple cmssw versions, depending on the batch sizes given as parameters, default are 1, 2 and 4. @@ -230,16 +242,21 @@ class PlotRuntimesMultipleCMSSW(RuntimeParameters, cmssw_versions = law.CSVParameter( cls=luigi.Parameter, default=("CMSSW_12_2_4", "CMSSW_12_2_2"), - description="comma-separated list of CMSSW versions; default: ('CMSSW_12_2_4','CMSSW_12_2_2')", + description="comma-separated list of CMSSW versions; default: CMSSW_12_2_4,CMSSW_12_2_2", brace_expand=True, ) def requires(self): - return [MergeRuntimes.req(self, cmssw_version=cmssw_version) for cmssw_version in self.cmssw_versions] + return [ + MergeRuntimes.req(self, cmssw_version=cmssw_version) + for cmssw_version in self.cmssw_versions + ] def output(self): cmssw_versions_repr = "_".join(self.cmssw_versions) - return self.local_target(f"runtime_plot_multiple_cmssw_{cmssw_versions_repr}_different_batch_sizes_{self.batch_sizes_repr}.pdf") # noqa + return self.local_target( + f"runtime_plot_multiple_cmssw_{cmssw_versions_repr}_different_batch_sizes_{self.batch_sizes_repr}.pdf", + ) @view_output_plots def run(self): @@ -248,9 +265,11 @@ def run(self): output.parent.touch() # create the plot - input_paths = [] - for input_task in self.input(): - input_paths += [input_task.path] - plot_batch_size_several_measurements(self.batch_sizes, input_paths, - output.path, self.cmssw_versions, self.custom_plot_params) - print("plot saved") + input_paths = [inp.path for inp in self.input()] + plot_batch_size_several_measurements( + self.batch_sizes, + input_paths, + output.path, + self.cmssw_versions, + self.custom_plot_params, + ) diff --git a/mlprof/tasks/sandboxes.py b/mlprof/tasks/sandboxes.py index 137dc81..812670c 100644 --- a/mlprof/tasks/sandboxes.py +++ b/mlprof/tasks/sandboxes.py @@ -9,8 +9,7 @@ class CMSSWSandboxTask(CMSSWParameters): """ - TODO: - - move to base (or even better: a different name) + Base class for tasks in cmssw sandboxes. """ @property diff --git a/modules/law b/modules/law index 6fb41c1..5d0f2ed 160000 --- a/modules/law +++ b/modules/law @@ -1 +1 @@ -Subproject commit 6fb41c143a5a94b5a00d92431a0d9657673c4ece +Subproject commit 5d0f2ed887a261d0cf304341b6c7e6ecf00cd0ee