From 5aaea245a291b053d0389e1ca6e7344b6da5945e Mon Sep 17 00:00:00 2001 From: Oliver Schulz Date: Fri, 6 Nov 2015 15:14:04 +0100 Subject: [PATCH 1/8] Disabled ROOT on-screen graphics output during dbrx execution --- src/dbrx.cxx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/dbrx.cxx b/src/dbrx.cxx index 94172aa..206327b 100644 --- a/src/dbrx.cxx +++ b/src/dbrx.cxx @@ -182,6 +182,9 @@ int main(int argc, char *argv[], char *envp[]) { // (still true for ROOT-6?): g_rootApplication = unique_ptr(new TApplication("dbrx", 0, 0)); + // Disable ROOT on-screen graphics output: + gROOT->SetBatch(true); + // Set ROOT program name (necessary / useful ?): gSystem->SetProgname("dbrx"); From 32a588c7a1f1c7f37a5c051df556e48e071c48bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Achim=20G=C3=BCtlein?= Date: Mon, 23 Nov 2015 12:28:07 +0100 Subject: [PATCH 2/8] Added options for update mode of TFile and overwrite of TObject::Write to RootFileWriter --- src/rootiobrics.cxx | 18 +++++++++++++----- src/rootiobrics.h | 8 +++++++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/rootiobrics.cxx b/src/rootiobrics.cxx index 0c13a09..3d2bdd4 100644 --- a/src/rootiobrics.cxx +++ b/src/rootiobrics.cxx @@ -348,7 +348,7 @@ void RootFileWriter::ContentGroup::processInput() { // Need to clone inputObject to own it: dbrx_log_trace("Cloning object \"%s\" to content group \"%s\"", inputObject->GetName(), absolutePath()); TNamed *outputObject = (TNamed*) inputObject->Clone(); - writeObject(outputObject); + writeObject(outputObject, m_writer->overwrite.get()); } } } @@ -420,7 +420,7 @@ RootFileWriter::ContentGroup::ContentGroup(RootFileWriter *writer, Bric *parentB -void RootFileWriter::writeObject(TNamed *obj) { +void RootFileWriter::writeObject(TNamed *obj, bool overwrite) { if (string(obj->GetName()).empty()) throw invalid_argument("Refusing to add object with empty name to TDirectory"); @@ -431,10 +431,12 @@ void RootFileWriter::writeObject(TNamed *obj) { TDirectory::AddDirectoryStatus() && (obj->IsA()->GetDirectoryAutoAdd() != nullptr) ); - if (!autoAdded) obj->Write(); + if (!autoAdded) + { + if (overwrite) obj->Write(obj->GetName(), TObject::kOverwrite); + } } - void RootFileWriter::connectInputs() { dbrx_log_trace("Setting up content groups for bric \"%s\"", absolutePath()); inputs.addContent(content); @@ -466,7 +468,13 @@ void RootFileWriter::openOutputForWrite() { const char *outFileName = fileName->c_str(); const char *outFileTitle = title->c_str(); dbrx_log_debug("Creating TFile \"%s\" with title \"%s\" in bric \"%s\""_format(outFileName, outFileTitle, absolutePath())); - TFile *tfile = TFile::Open(outFileName, "RECREATE", outFileTitle); + + // Open tfile in update mode if chosen that way + TFile *tfile = nullptr; + if (recreateFile.get()) tfile = TFile::Open(outFileName, "RECREATE", outFileTitle); + else tfile = TFile::Open(outFileName, "UPDATE", outFileTitle); + + if (tfile == nullptr) throw runtime_error("Could not create TFile \"%s\""_format(outFileName)); outputFile.value() = unique_ptr(tfile); diff --git a/src/rootiobrics.h b/src/rootiobrics.h index 7e1103b..b160b7f 100644 --- a/src/rootiobrics.h +++ b/src/rootiobrics.h @@ -157,7 +157,8 @@ class RootFileWriter: public AsyncReducerBric { protected: static const PropKey s_thisDirName; - static void writeObject(TNamed *obj); + // Additional argument to choose if the object is overwritten + static void writeObject(TNamed *obj, bool overwrite = false); bool m_outputReadyForWrite = false; @@ -204,6 +205,11 @@ class RootFileWriter: public AsyncReducerBric { Param title{this, "title", "Title"}; Param content{this, "content", "Content"}; + // Is used to choose the mode for opening the root file + Param recreateFile{this, "recreate", "Recreate file", true}; + // Is used to choose if the contents of the root file are overwritten + Param overwrite{this, "overwrite", "Overwrite objects in file", false}; + Output output{this, "output", "Output File Name"}; Output outputFile{this, "outputFile", "Output TFile"}; From 1ec560d297c6b70b8c6270dadfefeb3d4cd0d2c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Achim=20G=C3=BCtlein?= Date: Mon, 23 Nov 2015 13:21:40 +0100 Subject: [PATCH 3/8] Added progress informations to RootTreeReader --- src/rootiobrics.cxx | 52 +++++++++++++++++++++++++++++++++++++++++++++ src/rootiobrics.h | 19 +++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/src/rootiobrics.cxx b/src/rootiobrics.cxx index 3d2bdd4..950193a 100644 --- a/src/rootiobrics.cxx +++ b/src/rootiobrics.cxx @@ -66,6 +66,16 @@ void RootTreeReader::processInput() { index = firstEntry - 1; size = m_chain->GetEntries() - firstEntry.get(); if (ssize_t(nEntries) > 0) size = std::min(ssize_t(nEntries), size.get()); + + if (printETA.get()) + { + gettimeofday(&m_StartTime, NULL); + + m_LastTime.tv_sec = m_StartTime.tv_sec; + m_LastTime.tv_usec = m_StartTime.tv_usec; + } + + m_previousProgress = 0.; } @@ -73,6 +83,48 @@ bool RootTreeReader::nextOutput() { if (index.get() + 1 < firstEntry.get() + size.get()) { ++index; m_chain->GetEntry(index); + + if (!printProgress.get()) return true; + + double progress = 100.*index/size.get(); + + if (progress >= m_previousProgress + progressPrecent.get()) + { + dbrx_log_info("%6.2d % processed (%i entries)!", progress, index.get()); + m_previousProgress = progress; + + if (printETA.get()) + { + gettimeofday(&m_CurrentTime, NULL); + + double timeDiff = m_CurrentTime.tv_sec - m_LastTime.tv_sec + 1.0e-6*(m_CurrentTime.tv_usec - m_LastTime.tv_usec); + double averageTimeDiff = m_CurrentTime.tv_sec - m_StartTime.tv_sec + 1.0e-6*(m_CurrentTime.tv_usec - m_StartTime.tv_usec); + + double rate = progressPrecent.get()*size.get()/timeDiff; + double averageRate = index/averageTimeDiff; + + dbrx_log_info("Rate: %5.1f Entries/s (average: %5.1f Entries/s)", rate, averageRate); + + double eta = (size.get() - index.get())/averageRate; + + int seconds = eta; + int usecs = (eta-seconds)*1.0e6; + + time_t finishTime; + finishTime = m_CurrentTime.tv_sec+seconds; + struct tm* finishtm; + finishtm = localtime(&finishTime); + + char tmbuf[64]; + strftime(tmbuf, sizeof tmbuf, "%a %b %d - %H:%M:%S", finishtm); + + dbrx_log_info("ETA: %s", tmbuf); + + m_LastTime.tv_sec = m_CurrentTime.tv_sec; + m_LastTime.tv_usec = m_CurrentTime.tv_usec; + } + } + return true; } else return false; } diff --git a/src/rootiobrics.h b/src/rootiobrics.h index b160b7f..1628a02 100644 --- a/src/rootiobrics.h +++ b/src/rootiobrics.h @@ -27,6 +27,9 @@ #include #include +#include +#include + namespace dbrx { @@ -34,6 +37,16 @@ class RootTreeReader: public MapperBric { protected: std::unique_ptr m_chain; + // Additional members for progress informations + //! Previously printed progress + double m_previousProgress; + //! CPU time at begin of reading + struct timeval m_StartTime; + //! CPU time from last event + struct timeval m_LastTime; + //! Current CPU time + struct timeval m_CurrentTime; + public: class Entry final: public DynOutputGroup { public: @@ -47,6 +60,12 @@ class RootTreeReader: public MapperBric { Param nEntries{this, "nEntries", "Number of entries to read (-1 for all)", -1}; Param firstEntry{this, "firstEntry", "First entry to read", 0}; + // Parmeters for printing the progress informations while looping over a tree/chain + Param printProgress{this, "printProgress", "Print progress", false}; + Param printETA{this, "printETA", "Print ETA", false}; + Param progressPrecent{this, "progressPrecent", "Precentage when progress info is printed", 1000 }; + + Entry entry{this, "entry"}; Output size{this, "size", "Number of entries"}; From 064b4ab2b55128f453452755b7c9dbe0ae28f627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Achim=20G=C3=BCtlein?= Date: Mon, 23 Nov 2015 14:29:46 +0100 Subject: [PATCH 4/8] Fix for the segmentation fault at the end of execution --- src/dbrx.cxx | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/dbrx.cxx b/src/dbrx.cxx index 94172aa..aa480a3 100644 --- a/src/dbrx.cxx +++ b/src/dbrx.cxx @@ -19,6 +19,7 @@ #include #include #include +#include #include @@ -178,9 +179,24 @@ void main_printUsage(const char* progName) { int main(int argc, char *argv[], char *envp[]) { try { + + // Set the command line options to -b (batch mode) for TApplication + // Fixes the segmentation fault at the end of the execution + // Does this allow the use of the HTTP server? + // -> At least for databricxx example. Do we need more tests? + int nArgs = 2; + char** args = new char*[nArgs]; + for (int i=0; i(new TApplication("dbrx", 0, 0)); + g_rootApplication = unique_ptr(new TApplication("dbrx", &nArgs, args)); // Set ROOT program name (necessary / useful ?): gSystem->SetProgname("dbrx"); @@ -218,4 +234,5 @@ int main(int argc, char *argv[], char *envp[]) { } dbrx_log_info("Done."); + return 0; } From 3a47c1d10746e60e555dd13fc668f0d96b45c7a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Achim=20G=C3=BCtlein?= Date: Fri, 27 Nov 2015 11:17:55 +0100 Subject: [PATCH 5/8] Changed bool parameters to int in RootTreeReader/FileWriter; Fixed bug in progress info of TreeReader --- src/rootiobrics.cxx | 28 ++++++++++++++++++++++------ src/rootiobrics.h | 8 ++++---- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/rootiobrics.cxx b/src/rootiobrics.cxx index 950193a..17bda30 100644 --- a/src/rootiobrics.cxx +++ b/src/rootiobrics.cxx @@ -86,11 +86,11 @@ bool RootTreeReader::nextOutput() { if (!printProgress.get()) return true; - double progress = 100.*index/size.get(); + double progress = 100.*index.get()/size.get(); if (progress >= m_previousProgress + progressPrecent.get()) { - dbrx_log_info("%6.2d % processed (%i entries)!", progress, index.get()); + dbrx_log_info("%6.2f%s processed (%li entries)!", progress, "%", index.get()); m_previousProgress = progress; if (printETA.get()) @@ -485,7 +485,14 @@ void RootFileWriter::writeObject(TNamed *obj, bool overwrite) { if (!autoAdded) { - if (overwrite) obj->Write(obj->GetName(), TObject::kOverwrite); + if (overwrite) + { + obj->Write(obj->GetName(), TObject::kOverwrite); + } + else + { + obj->Write(); + } } } @@ -523,8 +530,16 @@ void RootFileWriter::openOutputForWrite() { // Open tfile in update mode if chosen that way TFile *tfile = nullptr; - if (recreateFile.get()) tfile = TFile::Open(outFileName, "RECREATE", outFileTitle); - else tfile = TFile::Open(outFileName, "UPDATE", outFileTitle); + if (recreateFile.get()) + { + dbrx_log_debug("Recreate file %s", outFileName); + tfile = TFile::Open(outFileName, "RECREATE", outFileTitle); + } + else + { + dbrx_log_debug("Update file %s", outFileName); + tfile = TFile::Open(outFileName, "UPDATE", outFileTitle); + } if (tfile == nullptr) throw runtime_error("Could not create TFile \"%s\""_format(outFileName)); @@ -541,7 +556,8 @@ void RootFileWriter::finalizeOutput() { if (! m_outputReadyForWrite) return; dbrx_log_debug("Writing TFile \"%s\" in bric \"%s\""_format(outputFile->GetName(), absolutePath())); - outputFile->Write(); + if (overwrite.get()) outputFile->Write(0, TObject::kOverwrite); + else outputFile->Write(); outputFile->ReOpen("READ"); diff --git a/src/rootiobrics.h b/src/rootiobrics.h index 1628a02..569123b 100644 --- a/src/rootiobrics.h +++ b/src/rootiobrics.h @@ -61,8 +61,8 @@ class RootTreeReader: public MapperBric { Param firstEntry{this, "firstEntry", "First entry to read", 0}; // Parmeters for printing the progress informations while looping over a tree/chain - Param printProgress{this, "printProgress", "Print progress", false}; - Param printETA{this, "printETA", "Print ETA", false}; + Param printProgress{this, "printProgress", "Print progress", false}; + Param printETA{this, "printETA", "Print ETA", false}; Param progressPrecent{this, "progressPrecent", "Precentage when progress info is printed", 1000 }; @@ -225,9 +225,9 @@ class RootFileWriter: public AsyncReducerBric { Param content{this, "content", "Content"}; // Is used to choose the mode for opening the root file - Param recreateFile{this, "recreate", "Recreate file", true}; + Param recreateFile{this, "recreate", "Recreate file", true}; // Is used to choose if the contents of the root file are overwritten - Param overwrite{this, "overwrite", "Overwrite objects in file", false}; + Param overwrite{this, "overwrite", "Overwrite objects in file", false}; Output output{this, "output", "Output File Name"}; Output outputFile{this, "outputFile", "Output TFile"}; From e52272ccaf863ba15f75b05021e39d873515a6bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Achim=20G=C3=BCtlein?= Date: Fri, 5 Feb 2016 11:42:27 +0100 Subject: [PATCH 6/8] Removed -b option for TApplication --- src/dbrx.cxx | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/dbrx.cxx b/src/dbrx.cxx index 7511d55..cebe558 100644 --- a/src/dbrx.cxx +++ b/src/dbrx.cxx @@ -179,19 +179,19 @@ void main_printUsage(const char* progName) { int main(int argc, char *argv[], char *envp[]) { try { - // Set the command line options to -b (batch mode) for TApplication - // Fixes the segmentation fault at the end of the execution - // Does this allow the use of the HTTP server? - // -> At least for databricxx example. Do we need more tests? - int nArgs = 2; - char** args = new char*[nArgs]; - for (int i=0; i At least for databricxx example. Do we need more tests? +// int nArgs = 2; +// char** args = new char*[nArgs]; +// for (int i=0; i(new TApplication("dbrx", &nArgs, args)); +// g_rootApplication = unique_ptr(new TApplication("dbrx", &nArgs, args)); + g_rootApplication = unique_ptr(new TApplication("dbrx", 0, 0)); // Set ROOT program name (necessary / useful ?): gSystem->SetProgname("dbrx"); From bc7e833062f77d593d893495bffcca17880a98c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Achim=20G=C3=BCtlein?= Date: Fri, 5 Feb 2016 11:42:45 +0100 Subject: [PATCH 7/8] Removed -b option for TApplication --- src/dbrx.cxx | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/dbrx.cxx b/src/dbrx.cxx index cebe558..11e5b06 100644 --- a/src/dbrx.cxx +++ b/src/dbrx.cxx @@ -179,20 +179,6 @@ void main_printUsage(const char* progName) { int main(int argc, char *argv[], char *envp[]) { try { -// // Set the command line options to -b (batch mode) for TApplication -// // Fixes the segmentation fault at the end of the execution -// // Does this allow the use of the HTTP server? -// // -> At least for databricxx example. Do we need more tests? -// int nArgs = 2; -// char** args = new char*[nArgs]; -// for (int i=0; iSetBatch(true); From 3b45fbc9fa936a818e612dd60f635fc187037d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Achim=20G=C3=BCtlein?= Date: Mon, 1 Aug 2016 15:52:08 +0200 Subject: [PATCH 8/8] Changed progress info for RootTreeReader: the update intervall can be selected via progressUpdateTime (default: NaN for no progress info) --- doc/examples/mca-calib.json | 3 +- src/rootiobrics.cxx | 76 ++++++++++++++++++++----------------- src/rootiobrics.h | 10 ++--- 3 files changed, 47 insertions(+), 42 deletions(-) diff --git a/doc/examples/mca-calib.json b/doc/examples/mca-calib.json index 4aa26a9..a14b886 100644 --- a/doc/examples/mca-calib.json +++ b/doc/examples/mca-calib.json @@ -48,7 +48,8 @@ "type": "dbrx::RootTreeReader", "input": "&mcaFileReader.content.mcaEvents", "nEntries": -1, - "firstEntry": 0 + "firstEntry": 0, + "progressUpdateTime": 0.5 }, "calib": { diff --git a/src/rootiobrics.cxx b/src/rootiobrics.cxx index 17bda30..5ba6077 100644 --- a/src/rootiobrics.cxx +++ b/src/rootiobrics.cxx @@ -67,15 +67,12 @@ void RootTreeReader::processInput() { size = m_chain->GetEntries() - firstEntry.get(); if (ssize_t(nEntries) > 0) size = std::min(ssize_t(nEntries), size.get()); - if (printETA.get()) - { - gettimeofday(&m_StartTime, NULL); + gettimeofday(&m_StartTime, NULL); - m_LastTime.tv_sec = m_StartTime.tv_sec; - m_LastTime.tv_usec = m_StartTime.tv_usec; - } + m_LastTime.tv_sec = m_StartTime.tv_sec; + m_LastTime.tv_usec = m_StartTime.tv_usec; - m_previousProgress = 0.; + m_PreviousNEntries = index.get(); } @@ -84,46 +81,55 @@ bool RootTreeReader::nextOutput() { ++index; m_chain->GetEntry(index); - if (!printProgress.get()) return true; + // Update progress infos: - double progress = 100.*index.get()/size.get(); + if (pProgressUpdateTime.get() != pProgressUpdateTime.get()) return true; // No progress update - if (progress >= m_previousProgress + progressPrecent.get()) - { - dbrx_log_info("%6.2f%s processed (%li entries)!", progress, "%", index.get()); - m_previousProgress = progress; + gettimeofday(&m_CurrentTime, NULL); - if (printETA.get()) - { - gettimeofday(&m_CurrentTime, NULL); + double timeDiff = m_CurrentTime.tv_sec - m_LastTime.tv_sec + 1.0e-6*(m_CurrentTime.tv_usec - m_LastTime.tv_usec); - double timeDiff = m_CurrentTime.tv_sec - m_LastTime.tv_sec + 1.0e-6*(m_CurrentTime.tv_usec - m_LastTime.tv_usec); - double averageTimeDiff = m_CurrentTime.tv_sec - m_StartTime.tv_sec + 1.0e-6*(m_CurrentTime.tv_usec - m_StartTime.tv_usec); + if (timeDiff < pProgressUpdateTime.get()) return true; - double rate = progressPrecent.get()*size.get()/timeDiff; - double averageRate = index/averageTimeDiff; + double averageTimeDiff = m_CurrentTime.tv_sec - m_StartTime.tv_sec + 1.0e-6*(m_CurrentTime.tv_usec - m_StartTime.tv_usec); - dbrx_log_info("Rate: %5.1f Entries/s (average: %5.1f Entries/s)", rate, averageRate); + double percentage = 100. * (index.get()-firstEntry.get())/size.get(); - double eta = (size.get() - index.get())/averageRate; + int64_t nEntries = index.get() - m_PreviousNEntries; + m_PreviousNEntries = index.get()-firstEntry.get(); - int seconds = eta; - int usecs = (eta-seconds)*1.0e6; + dbrx_log_info("Reading from chain %s ...", m_chain->GetName()); + dbrx_log_info(" %5.1f %% done (%u/%u pulses read)", + percentage, + (unsigned int) index.get()-firstEntry.get(), + (unsigned int) size.get()); - time_t finishTime; - finishTime = m_CurrentTime.tv_sec+seconds; - struct tm* finishtm; - finishtm = localtime(&finishTime); + double rate = (nEntries)/timeDiff; + double averageRate = (index.get()-firstEntry.get())/averageTimeDiff; - char tmbuf[64]; - strftime(tmbuf, sizeof tmbuf, "%a %b %d - %H:%M:%S", finishtm); + dbrx_log_info(" Rate: %5.1f Entries/s (average rate: %5.1f Entries/s)", rate, averageRate); - dbrx_log_info("ETA: %s", tmbuf); + double eta = (size.get() - (index.get()-firstEntry.get()))/averageRate; - m_LastTime.tv_sec = m_CurrentTime.tv_sec; - m_LastTime.tv_usec = m_CurrentTime.tv_usec; - } - } + int fullSeconds = eta; + int usecs = (eta-fullSeconds)*1.0e6; + + int hours = eta/(3600.); + int minutes = (eta - hours*3600.)/60.; + double seconds = eta - hours*3600. - minutes*60.; + + time_t finishTime; + finishTime = m_CurrentTime.tv_sec+fullSeconds; + struct tm* finishtm; + finishtm = localtime(&finishTime); + + char tmbuf[64]; + strftime(tmbuf, sizeof tmbuf, "%a %b %d - %H:%M:%S", finishtm); + + dbrx_log_info("ETA: %i:%i:%5.2f (%s)", hours, minutes, seconds, tmbuf); + + m_LastTime.tv_sec = m_CurrentTime.tv_sec; + m_LastTime.tv_usec = m_CurrentTime.tv_usec; return true; } else return false; diff --git a/src/rootiobrics.h b/src/rootiobrics.h index 569123b..96583dc 100644 --- a/src/rootiobrics.h +++ b/src/rootiobrics.h @@ -19,6 +19,7 @@ #define DBRX_ROOTIOBRICS_H #include +#include #include "Bric.h" @@ -39,7 +40,7 @@ class RootTreeReader: public MapperBric { // Additional members for progress informations //! Previously printed progress - double m_previousProgress; + int64_t m_PreviousNEntries; //! CPU time at begin of reading struct timeval m_StartTime; //! CPU time from last event @@ -60,11 +61,8 @@ class RootTreeReader: public MapperBric { Param nEntries{this, "nEntries", "Number of entries to read (-1 for all)", -1}; Param firstEntry{this, "firstEntry", "First entry to read", 0}; - // Parmeters for printing the progress informations while looping over a tree/chain - Param printProgress{this, "printProgress", "Print progress", false}; - Param printETA{this, "printETA", "Print ETA", false}; - Param progressPrecent{this, "progressPrecent", "Precentage when progress info is printed", 1000 }; - + //! Parameter: Time to update the progress information + Param pProgressUpdateTime {this, "progressUpdateTime", "Time to update the progress information [s]", std::numeric_limits::quiet_NaN()}; Entry entry{this, "entry"};