7 #include <TDirectory.h>
8 #include <TEfficiency.h>
13 #include <TIterator.h>
36 #include "TTreeReader.h"
37 #include "TTreeReaderArray.h"
50 if (
level <= m_currLvl)
51 std::cout <<
msg << std::endl;
60 static dbgPrint s_dbg;
61 static bool s_checkEquality=
false;
63 std::optional<std::regex> checkRegEx(
const std::string&
re) {
70 std::string
test(
"Test String");
71 std::regex_match(
test, reNew);
73 std::cout <<
"ERROR: Invalid RegEx string \"" <<
re <<
"\"." << std::endl;
74 std::cout <<
"See http://www.boost.org/doc/libs/1_42_0/libs/regex/doc/html/boost_regex/syntax.html for allowed regular expression syntax" << std::endl;
85 typedef std::map<std::string, std::vector<std::string>> fileLBMap_t;
86 explicit histCollection(TFile*
out,
bool skipExisting =
false) : m_out{
out}, m_skipExisting(skipExisting) {};
91 void addDirectory(TDirectory*
dir,
const std::string&
dirName,
const std::string&
filename =
"");
96 void addDirExclusion(
const std::optional<std::regex>& dirEx);
97 void addHistExclusion(
const std::optional<std::regex>& histEx);
99 unsigned size()
const;
102 fileLBMap_t getFileLBMapAndClear() {
104 return std::move(m_fileLBMap);
112 histPerDir_t(
const std::string& nameIn, std::unique_ptr<TObject>&& objIn, TTree* md);
113 histPerDir_t(dqutils::histCollection::histPerDir_t&&
other)
117 std::unique_ptr<TObject>
obj;
118 std::array<std::string, 3>
metadata{
"unset",
"",
"<default>"};
119 std::clock_t cpuSum = 0;
120 void (*mergeMethod)(TObject*
a,
const TObject*
b) =
nullptr;
124 bool fillMD(TTree* mdTree);
128 std::unordered_map<std::string, histPerDir_t>
histos;
129 void writeMD(TDirectory* outDir)
const;
135 std::unordered_map<std::string, histDir_t>
m_data;
136 std::optional<std::regex> m_dirExclusion;
137 std::optional<std::regex> m_histExclusion;
138 fileLBMap_t m_fileLBMap;
146 void histCollection::printTiming() {
147 std::vector<std::pair<std::string, clock_t>> cpuPerHistVec;
153 auto ordering = [](std::pair<std::string, clock_t>
a, std::pair<std::string, clock_t>
b) {
return a.second <
b.second; };
154 std::sort(cpuPerHistVec.begin(), cpuPerHistVec.end(),
ordering);
155 for (
const auto& [
name, time] : cpuPerHistVec) {
156 const double tSec =
double(time) / CLOCKS_PER_SEC;
157 std::cout <<
std::format(
"{:<30} : {:10.3f}",
name, tSec) << std::endl;
162 bool histCollection::histPerDir_t::fillMD(TTree * md) {
164 TTreeReaderArray<char> i_name(
reader,
"Name");
165 TTreeReaderArray<char> i_interval(
reader,
"Interval");
166 TTreeReaderArray<char> i_chain(
reader,
"TriggerChain");
167 TTreeReaderArray<char> i_merge(
reader,
"MergeMethod");
171 const std::string nameStr(
static_cast<char*
>(i_name.GetAddress()));
172 if (
name == nameStr) {
173 metadata = {
static_cast<char*
>(i_interval.GetAddress()),
static_cast<char*
>(i_chain.GetAddress()),
static_cast<char*
>(i_merge.GetAddress())};
183 const std::clock_t cpuStart = std::clock();
184 this->mergeMethod(
obj.get(),
other);
185 cpuSum += std::clock() - cpuStart;
190 void histCollection::histDir_t::writeMD(TDirectory *
out)
const {
193 std::map<std::string, std::array<std::string, 3>> metadatamap;
194 std::unique_ptr<TTree> oldMD((TTree*)
out->Get(
"metadata"));
196 TTreeReader
reader(oldMD.get());
197 TTreeReaderArray<char> i_name(
reader,
"Name");
198 TTreeReaderArray<char> i_interval(
reader,
"Interval");
199 TTreeReaderArray<char> i_chain(
reader,
"TriggerChain");
200 TTreeReaderArray<char> i_merge(
reader,
"MergeMethod");
203 const std::string
name(
static_cast<char*
>(i_name.GetAddress()));
204 metadatamap[
name] = {
static_cast<char*
>(i_interval.GetAddress()),
static_cast<char*
>(i_chain.GetAddress()),
static_cast<char*
>(i_merge.GetAddress())};
209 if (
h.metadata[0]!=
"unset")
210 metadatamap[
key] =
h.metadata;
213 if (metadatamap.empty())
return;
216 std::unique_ptr<TTree> mdTree = std::make_unique<TTree>(
"metadata",
"Monitoring Metadata");
217 mdTree->SetDirectory(
out);
219 mdTree->Branch(
"Name", (
void*)
nullptr,
"Name/C");
220 mdTree->Branch(
"Interval",
interval.data(),
"Interval/C");
221 mdTree->Branch(
"TriggerChain",
chain.data(),
"TriggerChain/C");
222 mdTree->Branch(
"MergeMethod",
merge.data(),
"MergeMethod/C");
224 mdTree->SetBranchAddress(
"Name",
histname);
226 for (
auto& [
key,
h] : metadatamap) {
233 mdTree->Write(0, TObject::kOverwrite);
236 histCollection::~histCollection() {}
238 void histCollection::addDirExclusion(
const std::optional<std::regex>&
dir) {
239 m_dirExclusion =
dir;
243 void histCollection::addHistExclusion(
const std::optional<std::regex>&
dir) {
244 m_histExclusion =
dir;
251 s +=
it.second.histos.size();
258 const histDir_t& hd =
it.second;
259 std::cout <<
"Dir: " <<
it.first <<
" has " << hd.histos.size() <<
" histos" << std::endl;
260 for (
const auto&
it1 : hd.histos)
261 std::cout <<
"\t" <<
it1.second.name << std::endl;
266 template <
class HIST>
268 ((HIST*)
a)->Add((HIST*)
b);
273 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
274 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
276 std::cout <<
"ERROR in weightedAverageTH1: Object not of type TH1" << std::endl;
278 if (b1->GetEntries()==0)
return;
285 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
286 const TH1* b1 = (
dynamic_cast<const TH1*
>(
b));
288 std::cout <<
"ERROR in weightedEff: Object not of type TH1" << std::endl;
290 if (b1->GetEntries()==0)
return;
297 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
298 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
300 std::cout <<
"ERROR in mergeRMS: Object not of type TH1" << std::endl;
302 if (b1->GetEntries()==0)
return;
309 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
310 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
312 std::cout <<
"ERROR in RMSpercentDeviation: Object not of type TH1" << std::endl;
314 if (b1->GetEntries()==0)
return;
321 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
322 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
324 std::cout <<
"ERROR in getBinEffPerCent: Object not of type TH1" << std::endl;
326 if (b1->GetEntries()==0)
return;
333 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
334 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
336 std::cout <<
"ERROR in lowerLB: Object not of type TH1" << std::endl;
342 template <
class HIST>
344 if (!s_checkEquality)
346 HIST* a1 = (
dynamic_cast<HIST*
>(
a));
347 const HIST* b1 =
dynamic_cast<const HIST*
>(
b);
349 std::cout <<
"ERROR in identical: Object not of correct type" << std::endl;
356 void merge_rebinned(TObject *
a,
const TObject*
b) {
357 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
358 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
360 std::cout <<
"ERROR, in merge_rebinned: Object not of type TH1";
363 TH1* b2 =
const_cast<TH1*
>(b1);
368 void merge_eventSample(TObject *
a,
const TObject*
b) {
369 TH2* a1 = (
dynamic_cast<TH2*
>(
a));
370 const TH2* b1 =
dynamic_cast<const TH2*
>(
b);
372 std::cout <<
"ERROR in merge_eventSample: Object not of type TH2" << std::endl;
378 void merge_TEfficency(TObject *
a,
const TObject*
b) {
379 TEfficiency* a1 =
dynamic_cast<TEfficiency*
>(
a);
380 const TEfficiency* b1 =
dynamic_cast<const TEfficiency*
>(
b);
381 TEfficiency* b2 =
const_cast<TEfficiency*
>(b1);
383 std::cout <<
"ERROR in merge_TEfficiency: Object not of type TEfficiency" << std::endl;
392 void merge_TTree(TObject *
a,
const TObject*
b) {
393 TTree* a1 =
dynamic_cast<TTree*
>(
a);
394 const TTree* b1 =
dynamic_cast<const TTree*
>(
b);
396 std::cout <<
"ERROR in merge_TTree: Object not of type TTree" << std::endl;
399 TTree* b2 =
const_cast<TTree*
>(b1);
407 :
name(nameIn),
obj(std::move(objIn)), mergeMethod(
nullptr) {
410 std::cout <<
"ERROR while adding " << nameIn <<
": Histogram pointer is NULL" << std::endl;
418 s_dbg(
VERBOSE,
"No matadata found for " +
name +
", use defaults");
420 const std::string& howToMerge =
metadata[2];
421 s_dbg(
VERBOSE,
"Name: " +
name +
" mergeMethod=" + howToMerge);
423 TH1* th1 =
dynamic_cast<TH1*
>(
obj.get());
424 TH2* th2 =
dynamic_cast<TH2*
>(
obj.get());
425 TEfficiency* teff =
dynamic_cast<TEfficiency*
>(
obj.get());
427 th1->SetDirectory(
nullptr);
428 if (howToMerge ==
"<default>") {
430 mergeMethod = &defaultMerge<TH2>;
432 mergeMethod = &defaultMerge<TH1>;
435 else if (howToMerge ==
"weightedAverage" || howToMerge==
"weightedAverage2D")
436 mergeMethod = &weightedAverage;
437 else if (howToMerge ==
"weightedEff")
438 mergeMethod = &weightedEff;
439 else if (howToMerge ==
"mergeRMS")
440 mergeMethod = &mergeRMS;
441 else if (howToMerge ==
"RMSpercentDeviation")
442 mergeMethod = &RMSpercentDeviation;
443 else if (howToMerge ==
"perBinEffPerCent")
444 mergeMethod = &perBinEffPerCent;
445 else if (howToMerge ==
"lowerLB")
447 else if (howToMerge ==
"identical")
449 mergeMethod = &identical<TH2>;
451 mergeMethod = &identical<TH1>;
453 else if ((howToMerge ==
"mergeRebinned") || (howToMerge ==
"merge"))
454 mergeMethod = &merge_rebinned;
456 std::cout <<
"ERROR: Unknown merging method (" << howToMerge <<
") for object of type TH1 named " << nameIn << std::endl;
461 teff->SetDirectory(
nullptr);
462 if (howToMerge ==
"<default>")
463 mergeMethod = &merge_TEfficency;
465 std::cout <<
"ERROR: Unknown merging method (" << howToMerge <<
") for object of type TEfficiency named " << nameIn << std::endl;
467 else if (
nullptr !=
dynamic_cast<TTree*
>(
obj.get())) {
468 mergeMethod = &merge_TTree;
470 std::cout <<
"ERROR Object " <<
name <<
" has unkown type" << std::endl;
478 if (m_dirExclusion && !std::regex_search(
dirName, *m_dirExclusion)) {
483 for (TObject* oKey : *
dir->GetListOfKeys()) {
484 TKey*
key =
static_cast<TKey*
>(oKey);
485 const std::string
name =
key->GetName();
486 const std::string classname =
key->GetClassName();
487 if ((classname ==
"TTree") && (
name ==
"metadata")) {
491 s_dbg(
VERBOSE,
"Found name " +
name +
", classname=" + classname);
496 if (classname.starts_with(
"TH") || classname.starts_with(
"TProfile") || classname.starts_with(
"TEfficiency") || classname ==
"TTree") {
497 if (m_histExclusion && !std::regex_search(
name, *m_histExclusion)) {
498 s_dbg(
DEBUG,
"Histogram with name " +
name +
" is excluded");
503 if (m_skipExisting) {
505 std::unique_ptr<TObject> existingObj(m_out->Get(
newName.c_str()));
510 std::unique_ptr<TTree> md;
511 if (itDir ==
m_data.end()) {
518 auto itH = itDir->second.histos.find(
name);
519 if (itH == itDir->second.histos.end()) {
523 md.reset((TTree*)
dir->Get(
"metadata"));
526 std::unique_ptr<TObject>
obj{
key->ReadObj()};
527 TTree* treeObj =
dynamic_cast<TTree*
>(
obj.get());
529 TDirectory* outDir = m_out->GetDirectory(
dirName.c_str());
531 outDir = m_out->mkdir(
dirName.c_str());
535 TTree* cloneTree = treeObj->CloneTree();
537 obj.reset(cloneTree);
541 itH = itDir->second.histos.emplace(
name, std::move(
histo)).first;
545 std::unique_ptr<TObject>
other(
key->ReadObj());
547 std::cout <<
"ERROR, got NULL key";
549 itH->second.merge(
other.get());
553 }
else if (classname.starts_with(
"TDirectory")) {
554 std::unique_ptr<TObject>
obj(
key->ReadObj());
555 TDirectory*
subdir =
dynamic_cast<TDirectory*
>(
obj.get());
560 if (!
name.starts_with(
"lb_") && !
name.starts_with(
"lowStat_LB")) {
568 std::cout <<
"Ignored objects '" <<
name <<
"' of type " << classname << std::endl;
576 unsigned nIgnored = 0;
580 TDirectory* histDir = m_out->GetDirectory(
fulldir.c_str());
581 if (histDir ==
nullptr) {
582 histDir = m_out->mkdir(
fulldir.c_str());
583 if (histDir ==
nullptr) {
584 std::cout <<
"ERROR, failed to create directory " <<
fulldir << std::endl;
587 s_dbg(
VERBOSE,
"Created directory " +
fulldir +
" in file " + m_out->GetName());
597 std::cout <<
"NOT writing " <<
name <<
". Invalid." << std::endl;
601 it.second.writeMD(histDir);
603 std::cout <<
"Wrote " <<
nWritten <<
" histograms to " << nDirs <<
" directories in output file " << m_out->GetName() << std::endl;
605 std::cout <<
" Omitting " << nIgnored <<
" histograms." << std::endl;
613 makeBranch(
"Name",
"Name/C");
614 makeBranch(
"Interval",
"Interval/C");
615 makeBranch(
"TriggerChain",
"TriggerChain/C");
616 makeBranch(
"MergeMethod",
"MergeMethod/C");
621 m_metadata->Branch(branchName, (
void*)
nullptr, branchstr);
626 const std::string& theMerge) {
627 std::string
name = theName;
629 std::string
chain = theChain;
630 std::string
merge = theMerge;
631 m_metadata->SetBranchAddress(
"Name",
name.data());
632 m_metadata->SetBranchAddress(
"Interval",
interval.data());
633 m_metadata->SetBranchAddress(
"TriggerChain",
chain.data());
634 m_metadata->SetBranchAddress(
"MergeMethod",
merge.data());
681 dirmap.insert(dirmapVal);
684 TIter
next(
dir->GetListOfKeys());
686 while ((
key =
dynamic_cast<TKey*
>(
next())) != 0) {
688 TObject*
obj =
key->ReadObj();
689 TDirectory*
subdir =
dynamic_cast<TDirectory*
>(
obj);
691 std::string subdirName(
subdir->GetName());
692 std::string
fName(
"");
710 DirMap_t::const_iterator diter;
711 std::string::size_type
i =
path.find_first_of(
'/');
712 std::string
fName(
"");
718 if (
i != std::string::npos) {
719 std::string dName(
path, 0,
i);
720 std::string pName(
path,
i + 1, std::string::npos);
723 diter = dirmap.find(
fName);
724 if (diter != dirmap.end()) {
729 dirmap.insert(dirmapVal);
739 diter = dirmap.find(
fName);
740 if (diter != dirmap.end()) {
741 return diter->second;
746 dirmap.insert(dirmapVal);
756 std::string::size_type
i =
path.find_first_of(
'/');
757 if (
i != std::string::npos) {
758 std::string dName(
path, 0,
i);
759 std::string pName(
path,
i + 1, std::string::npos);
761 key =
dir->FindKey(dName.c_str());
763 TDirectory* subDir =
dynamic_cast<TDirectory*
>(
key->ReadObj());
773 return dir->FindKey(
path.c_str());
779 TTree* md =
dynamic_cast<TTree*
>(
dir->Get(
"metadata"));
784 TTreeReaderArray<char> i_name(
reader,
"Name");
785 TTreeReaderArray<char> i_interval(
reader,
"Interval");
786 TTreeReaderArray<char> i_chain(
reader,
"TriggerChain");
787 TTreeReaderArray<char> i_merge(
reader,
"MergeMethod");
790 const std::string nameStr(
static_cast<char*
>(i_name.GetAddress()));
791 if (mdMap.find(nameStr) == mdMap.end()) {
792 MetaData md(nameStr,
static_cast<char*
>(i_interval.GetAddress()),
static_cast<char*
>(i_chain.GetAddress()),
static_cast<char*
>(i_merge.GetAddress()));
793 std::map<std::string, MetaData>::value_type mdVal(nameStr, md);
802 std::cout <<
"Writing file: " <<
outFileName << std::endl;
803 std::cout <<
"Start merging [" <<
files.size() <<
"] histogram files" << std::endl;
805 TH1::AddDirectory(
false);
807 std::cout <<
" ========== Using regular expressions for selective merging ========== " << std::endl;
810 std::cout <<
"CPU time measurement activated " << std::endl;
819 std::cout <<
"Got exactly one input file. Will copy input -> output" << std::endl;
821 std::cout <<
"regular expressions for selective merging will have no effect!" << std::endl;
826 std::filesystem::copy_file(
inPath,
outPath, std::filesystem::copy_options::overwrite_existing);
832 std::cout <<
" ERROR, cound not open output file " <<
outFileName << std::endl;
835 std::cout <<
"Opened/created output file " <<
outFileName << std::endl;
842 std::unique_ptr<TFile> in1(TFile::Open(
files[0].c_str()));
844 std::cout <<
"ERROR, could not open input file " <<
files[0] << std::endl;
847 std::cout <<
"Working on file 1/" <<
nFiles <<
": " <<
files[0] << std::endl;
848 std::string runDir, runDirFwd;
850 TIter
next(in1->GetListOfKeys());
852 while ((
key = (TKey*)
next())) {
853 const char*
name =
key->GetName();
854 if (std::regex_match(
name, runDirPattern)) {
855 if (runDir.size() > 0) {
856 std::cout <<
"ERROR More than one run_XXX directory found! Ignoring " <<
name << std::endl;
861 if (runDir.empty()) {
862 std::cout <<
"No run-directory found, start with '/'" << std::endl;
867 std::cout <<
"Found run directory " << runDir << std::endl;
872 TDirectory*
dir(
dynamic_cast<TDirectory*
>(in1->GetDirectory(runDir.c_str())));
874 std::cout <<
"ERROR, can't access directory " << runDir;
885 for (
size_t i = 1;
i <
files.size(); ++
i) {
886 std::cout <<
"Working on file " << 1+
i <<
"/" <<
nFiles <<
": " <<
files[
i] << std::endl;
887 std::unique_ptr<TFile> in(TFile::Open(
files[
i].c_str()));
889 std::cout <<
"ERROR, could not open input file " <<
files[
i] << std::endl;
892 TDirectory*
dir(
dynamic_cast<TDirectory*
>(in->GetDirectory(runDir.c_str())));
894 std::cout <<
"ERROR, could not cast to directory" << std::endl;
902 std::cout <<
"Accumulated a total of " << hc.
size() <<
" histograms." << std::endl;
904 std::cout <<
"Start writing output ..." << std::endl;
908 std::cout <<
"CPU time for histogram merging: (regular histograms)" << std::endl;
911 const auto lbmap = hc.getFileLBMapAndClear();
912 if (!lbmap.empty()) {
913 std::cout <<
"Start merging lb_nnn and lowStat_LB directories (" << lbmap.
size() <<
" in total)" << std::endl;
919 std::cout <<
"Merging/copying directory " <<
dir << std::endl;
921 std::unique_ptr<TFile> in(TFile::Open(
fName.c_str()));
923 std::cout <<
"ERROR, could not open input file " <<
fName << std::endl;
926 TDirectory* tDir = (
dynamic_cast<TDirectory*
>(in->Get(
dir.c_str())));
928 std::cout <<
"ERROR, failed to get directory " <<
dir <<
" from file " <<
fName << std::endl;
937 std::cout <<
"CPU time for histogram merging: (lumiblock-histograms)" << std::endl;
948 typedef std::vector<std::string> FileList_t;
950 const unsigned int nFilesAtOnce = 50;
955 std::cout <<
"ERROR Failed ot read list of input files" << std::endl;
959 if (
allFiles.size() <= nFilesAtOnce) {
963 FileList_t procFiles, tmpIntermediateFiles;
965 FileList_t::const_iterator filesEnd =
allFiles.end();
966 FileList_t::const_iterator fi =
allFiles.begin();
969 std::string tmpInputFile(
"");
970 std::string tmpOutputFile(
"");
973 while (fi != filesEnd) {
975 procFiles.push_back(*fi);
978 if (
counter % nFilesAtOnce == 0 || fi == filesEnd) {
979 std::ostringstream nameStream;
980 nameStream <<
"tmp_merge_" <<
counter <<
".root";
981 tmpOutputFile = nameStream.str();
982 tmpIntermediateFiles.push_back(tmpOutputFile);
992 for (
const auto& tmpFile : tmpIntermediateFiles) {
1000 std::cerr <<
"MonitoringFile::printStatistics(): "
1001 <<
"No input file is open\n";
1009 DirMap_t::const_iterator idirend = indirmap.end();
1010 for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1011 std::string idirName = idir->first;
1013 GatherStatistics stat_shift(idirName);
1014 GatherStatistics stat_all(idirName);
1019 std::cout.setf(std::ios_base::left, std::ios_base::adjustfield);
1020 std::cout.width(80);
1021 std::cout << idirName <<
" ";
1023 std::cout.setf(std::ios_base::right, std::ios_base::adjustfield);
1024 std::cout <<
" shift: ";
1026 std::cout << stat_shift.m_nHist1D <<
" ";
1028 std::cout << stat_shift.m_nHist1DBins <<
" ";
1030 std::cout << stat_shift.m_nHist2D <<
" ";
1032 std::cout << stat_shift.m_nHist2DBins <<
" ";
1034 std::cout << stat_shift.m_nGraph <<
" ";
1036 std::cout << stat_shift.m_nGraphPoints <<
" ";
1038 std::cout <<
" all: ";
1039 std::cout << stat_all.m_nHist1D <<
" ";
1041 std::cout << stat_all.m_nHist1DBins <<
" ";
1043 std::cout << stat_all.m_nHist2D <<
" ";
1045 std::cout << stat_all.m_nHist2DBins <<
" ";
1047 std::cout << stat_all.m_nGraph <<
" ";
1049 std::cout << stat_all.m_nGraphPoints <<
"\n";
1061 std::cerr <<
"MonitoringFile::copyHistograms(): "
1062 <<
"No input file is open\n";
1073 std::cerr <<
"MonitoringFile::copyHistograms(): "
1074 <<
"Directory \'" <<
dirName <<
"\' not found in input file\n";
1078 TDirectory* fromDir =
dynamic_cast<TDirectory*
>(dkey->ReadObj());
1080 DirMap_t::value_type dirmapVal(
dirName, fromDir);
1081 indirmap.insert(dirmapVal);
1083 std::cout <<
"Building list of all TDirectories in file...\n" <<
std::flush;
1087 DirMap_t::const_iterator idirend = indirmap.end();
1088 for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1090 std::string idirName = idir->first;
1091 std::cout <<
"Checking " << idirName <<
"\n" <<
std::flush;
1103 reducedmap.insert(*idir);
1108 std::cerr <<
"MonitoringFile::copyHistograms(): "
1109 <<
"Output file not opened\n";
1113 idirend = reducedmap.end();
1114 for (DirMap_t::const_iterator idir = reducedmap.begin(); idir != idirend; ++idir) {
1116 std::string idirName = idir->first;
1117 std::cout <<
"Processing " << idirName <<
"\n" <<
std::flush;
1121 std::cerr <<
"MonitoringFile::copyHistograms(): "
1122 <<
"Directory \'" << idirName <<
"\' not created in output file\n";
1126 CopyHistogram copyFcn(toDir, idirName);
1139 const std::string& hcfg_lowStat,
const std::string& hcfg_medStat) {
1142 std::cout <<
"\nUsing han configurations:\n"
1143 <<
" entire run: " <<
hcfg <<
"\n"
1144 <<
" low stat interval: " << hcfg_lowStat <<
"\n"
1145 <<
" medium stat interval: " << hcfg_medStat <<
"\n\n"
1148 TFile*
infile = TFile::Open(input.c_str());
1150 std::cerr <<
"MonitoringFile::getHanResults(): "
1151 <<
"Cannot open input file \"" << input <<
"\"\n";
1155 std::vector<std::string> run_dirs;
1156 std::vector<std::string> lowStat_dirs;
1157 std::vector<std::string> medStat_dirs;
1159 TIter next_run(
infile->GetListOfKeys());
1161 while ((key_run =
dynamic_cast<TKey*
>(next_run())) != 0) {
1162 TObject* obj_run = key_run->ReadObj();
1163 TDirectory* tdir_run =
dynamic_cast<TDirectory*
>(obj_run);
1164 if (tdir_run != 0) {
1165 std::string tdir_run_name(tdir_run->GetName());
1166 if (tdir_run_name.find(
"run") != std::string::npos) {
1167 run_dirs.push_back(tdir_run_name);
1168 TIter next_minutes(tdir_run->GetListOfKeys());
1169 TKey* key_minutes(0);
1170 while ((key_minutes =
dynamic_cast<TKey*
>(next_minutes())) != 0) {
1171 TObject* obj_minutes = key_minutes->ReadObj();
1172 TDirectory* tdir_minutes =
dynamic_cast<TDirectory*
>(obj_minutes);
1173 if (tdir_minutes != 0) {
1174 std::string tdir_minutes_name(tdir_minutes->GetName());
1175 if (tdir_minutes_name.find(
"lowStat") != std::string::npos) {
1176 lowStat_dirs.push_back(tdir_run_name +
'/' + tdir_minutes_name);
1177 }
else if (tdir_minutes_name.find(
"medStat") != std::string::npos) {
1178 medStat_dirs.push_back(tdir_run_name +
'/' + tdir_minutes_name);
1193 std::vector<std::string>::const_iterator dirs_end;
1194 std::vector<std::string>::const_iterator
dir;
1196 dirs_end = run_dirs.end();
1197 for (
dir = run_dirs.begin();
dir != dirs_end; ++
dir) {
1198 const std::string& tdir_run_name = *
dir;
1199 std::string han_output_run =
hanResultsDir +
'/' + tdir_run_name +
"_han.root";
1200 std::cout <<
"Calling han( " <<
hcfg <<
", " << input <<
", " << tdir_run_name <<
", " << han_output_run <<
" ):\n" <<
std::flush;
1201 han.Analyze(
hcfg, input, han_output_run, tdir_run_name);
1203 fileList += han_output_run +
" " + tdir_run_name +
"\n";
1206 dirs_end = lowStat_dirs.end();
1207 for (
dir = lowStat_dirs.begin();
dir != dirs_end; ++
dir) {
1208 const std::string& tdir_minutes_path = *
dir;
1210 std::string tdir_minutes_underscore = tdir_minutes_path;
1211 std::string::size_type tdir_minutes_i = tdir_minutes_underscore.find(
'/');
1212 tdir_minutes_underscore.replace(tdir_minutes_i, 1,
"_");
1214 std::string han_output_lowStat =
hanResultsDir +
'/' + tdir_minutes_underscore +
"_han.root";
1215 std::cout <<
"Running han, writing to " << han_output_lowStat <<
":\n" <<
std::flush;
1216 han.Analyze(hcfg_lowStat, input, han_output_lowStat, tdir_minutes_path);
1218 std::string subdirname(tdir_minutes_path, tdir_minutes_i + 1, std::string::npos);
1219 std::string
dirname(tdir_minutes_path, 0, tdir_minutes_i);
1220 fileList += han_output_lowStat +
" " + subdirname +
" " +
dirname +
" " + subdirname +
"\n";
1223 dirs_end = medStat_dirs.end();
1224 for (
dir = medStat_dirs.begin();
dir != dirs_end; ++
dir) {
1225 const std::string& tdir_minutes_path = *
dir;
1227 std::string tdir_minutes_underscore = tdir_minutes_path;
1228 std::string::size_type tdir_minutes_i = tdir_minutes_underscore.find(
'/');
1229 tdir_minutes_underscore.replace(tdir_minutes_i, 1,
"_");
1231 std::string han_output_medStat =
hanResultsDir +
'/' + tdir_minutes_underscore +
"_han.root";
1232 std::cout <<
"Running han, writing to " << han_output_medStat <<
":\n" <<
std::flush;
1233 han.Analyze(hcfg_medStat, input, han_output_medStat, tdir_minutes_path);
1235 std::string subdirname(tdir_minutes_path, tdir_minutes_i + 1, std::string::npos);
1236 std::string
dirname(tdir_minutes_path, 0, tdir_minutes_i);
1237 fileList += han_output_medStat +
" " + subdirname +
" " +
dirname +
" " + subdirname +
"\n";
1245 std::cerr <<
"MonitoringFile::printHanConfig(): "
1246 <<
"No input file is open\n";
1254 std::string
indent, indent_p, indent_c;
1255 std::string idirName_p;
1256 DirMap_t::const_iterator idirend = indirmap.end();
1257 for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1258 std::string idirName = idir->first;
1259 std::string::size_type shortNameIndex = idirName.rfind(
'/');
1260 std::string
shortName = idirName.substr(shortNameIndex + 1, std::string::npos);
1262 std::string::size_type fsIndex = idirName.find(
'/');
1263 std::string shortPath;
1264 if (fsIndex != shortNameIndex)
1265 shortPath = idirName.substr(fsIndex + 1, shortNameIndex);
1267 shortPath = idirName.substr(fsIndex + 1, std::string::npos);
1269 std::cout << idirName <<
"\n";
1270 std::cout << shortPath <<
", " <<
shortName <<
"\n";
1309 std::string space = leadingSpace;
1310 std::string::size_type
i =
pathName.find_first_of(
'/');
1311 if (
i != std::string::npos) {
1312 std::string subPath(
pathName,
i + 1, std::string::npos);
1338 m_metadata =
new TTree(
"metadata",
"Monitoring Metadata");
1339 m_metadata->SetDirectory(0);
1340 m_metadata->Branch(
"Name", (
void*)
nullptr,
"Name/C");
1341 m_metadata->Branch(
"Interval", (
void*)
nullptr,
"Interval/C");
1342 m_metadata->Branch(
"TriggerChain", (
void*)
nullptr,
"TriggerChain/C");
1343 m_metadata->Branch(
"MergeMethod", (
void*)
nullptr,
"MergeMethod/C");
1348 m_metadata->SetDirectory(m_target);
1349 m_metadata->Write();
1355 hist->SetDirectory(m_target);
1375 std::string
name(md.name);
1377 std::string
chain(md.chain);
1378 std::string
merge(md.merge);
1379 m_metadata->SetBranchAddress(
"Name",
name.data());
1380 m_metadata->SetBranchAddress(
"Interval",
interval.data());
1381 m_metadata->SetBranchAddress(
"TriggerChain",
chain.data());
1382 m_metadata->SetBranchAddress(
"MergeMethod",
merge.data());
1388 hist->SetDirectory(m_target);
1413 : m_dirName(
dirName), m_nHist1D(0), m_nHist1DBins(0), m_nGraph(0), m_nGraphPoints(0), m_nHist2D(0), m_nHist2DBins(0) {}
1416 TH2* hist2d =
dynamic_cast<TH2*
>(
hist);
1419 m_nHist2DBins += (hist2d->GetNbinsX() * hist2d->GetNbinsY());
1423 m_nHist1DBins +=
hist->GetNbinsX();
1429 m_nGraphPoints += graph->GetMaxSize();
1436 TH1* h_total =
eff->GetCopyPassedHisto();
1437 TH2* h_total2D =
dynamic_cast<TH2*
>(h_total);
1439 if (h_total2D != 0) {
1440 m_nEfficiencyBins += (h_total2D->GetNbinsX() * h_total2D->GetNbinsY());
1443 m_nEfficiencyBins += h_total->GetNbinsX();
1451 m_names.push_back(std::string(
hist->GetName()));
1456 m_names.push_back(std::string(graph->GetName()));
1461 m_names.push_back(std::string(
eff->GetName()));
1477 TKey* mdKey =
dir->FindKey(
"metadata");
1482 TTree* md =
dynamic_cast<TTree*
>(mdKey->ReadObj());
1494 std::cerr <<
"Exception: \"" <<
e.what() <<
"\" in directory \"" <<
dir->GetName() <<
"\"\n" <<
std::flush;
1506 TIter
next(
dir->GetListOfKeys());
1508 while ((
key =
dynamic_cast<TKey*
>(
next())) != 0) {
1509 TObject*
obj =
key->ReadObj();
1513 if ((
h =
dynamic_cast<TH1*
>(
obj))) {
1515 }
else if ((
g =
dynamic_cast<TGraph*
>(
obj))) {
1517 }
else if ((
e =
dynamic_cast<TEfficiency*
>(
obj))) {
1526 TKey* mdKey =
dir->FindKey(
"metadata");
1531 TTree* md =
dynamic_cast<TTree*
>(mdKey->ReadObj());
1539 TTreeReaderArray<char> i_name(
reader,
"Name");
1540 TTreeReaderArray<char> i_interval(
reader,
"Interval");
1541 TTreeReaderArray<char> i_chain(
reader,
"TriggerChain");
1542 TTreeReaderArray<char> i_merge(
reader,
"MergeMethod");
1545 const std::string nameStr(
static_cast<char*
>(i_name.GetAddress()));
1547 i_key =
dir->FindKey(
static_cast<char*
>(i_name.GetAddress()));
1549 std::cerr <<
"MonitoringFile::loopOnHistogramsInMetadata(): "
1550 <<
"No \'" << nameStr <<
"\' object found\n";
1553 MetaData md(nameStr,
static_cast<char*
>(i_interval.GetAddress()),
static_cast<char*
>(i_chain.GetAddress()),
static_cast<char*
>(i_merge.GetAddress()));
1554 TObject*
obj = i_key->ReadObj();
1555 TH1*
h =
dynamic_cast<TH1*
>(
obj);
1557 fcn.executeMD(
h, md);
1559 TGraph*
g =
dynamic_cast<TGraph*
>(
obj);
1561 fcn.executeMD(
g, md);
1573 using namespace std;
1577 ifstream listfile(listFileName.c_str());
1579 cerr <<
"MonitoringFile::setListFromFile(): "
1580 <<
"cannot read from file: " << listFileName <<
"\n";
1587 while (getline(listfile,
line)) {
1588 istringstream linestream(
line);
1589 while (linestream.get(
c)) {
1596 linestream.putback(
c);
1599 cerr <<
"MonitoringFile::setListFromFile(): "
1600 <<
"badly formatted line: " <<
line <<
"\n";
1614 std::cout <<
"Running mergeLBintervals on " << inFilename << std::endl;
1616 std::unique_ptr<TFile>
f(TFile::Open(inFilename.c_str(),
"UPDATE"));
1618 std::cout <<
"ERROR, could not open file " << inFilename <<
" for update" << std::endl;
1621 std::string runDirName;
1622 const std::regex runDirPattern(
"run_[0-9]*");
1623 TIter
next(
f->GetListOfKeys());
1625 while ((
key = (TKey*)
next())) {
1626 const char*
name =
key->GetName();
1627 if (std::regex_match(
name, runDirPattern)) {
1628 if (runDirName.size() > 0) {
1629 std::cout <<
"ERROR More than one run_XXX directory found! Ignoring " <<
name << std::endl;
1636 TDirectory* runDir =
f->GetDirectory(runDirName.c_str());
1639 if (s_dbg.getLvl() ==
VERBOSE) {
1640 std::cout <<
"LB directory mapping:" << std::endl;
1641 for (
const auto& i1 :
mapping) {
1642 std::cout << i1.first;
1643 for (
const auto& i2 : i1.second) {
1644 std::cout <<
"\t" << i2 << std::endl;
1649 for (
const auto& [outDir, inDIrs] :
mapping) {
1655 f.reset(TFile::Open(inFilename.c_str(),
"UPDATE"));
1656 runDir =
f->GetDirectory(runDirName.c_str());
1658 std::cout <<
"merging lowStat_LB dirs into run-dir" << std::endl;
1659 std::vector<std::string> lowStatDirs;
1660 for (TObject* oKey : *runDir->GetListOfKeys()) {
1661 TKey*
key =
static_cast<TKey*
>(oKey);
1662 const std::string
name =
key->GetName();
1663 const std::string classname =
key->GetClassName();
1664 if (classname.starts_with(
"TDirectory") and
name.starts_with(
"lowStat_LB")) {
1665 lowStatDirs.push_back(runDirName +
"/" +
name);
1666 s_dbg(
VERBOSE,
"Found input: " + runDirName +
"/" +
name);
1678 std::map<std::string, std::vector<std::string>>
ranges;
1681 const std::string runDirName = runDir->GetName();
1682 for (TObject* oKey : *runDir->GetListOfKeys()) {
1683 TKey*
key =
static_cast<TKey*
>(oKey);
1684 const std::string
name =
key->GetName();
1685 const std::string classname =
key->GetClassName();
1686 if (!classname.starts_with(
"TDirectory"))
1688 if (
name.starts_with(
"lb_")) {
1692 }
catch (std::invalid_argument&
e) {
1693 std::cout <<
"ERROR, unexpected directory name " <<
name <<
". Can't parse lb number" << std::endl;
1694 std::cout <<
e.what() << std::endl;
1700 ranges[lbString].push_back(runDirName +
"/" +
name);
1708 TDirectory* outDir =
file->GetDirectory(outputDirName.c_str());
1710 outDir =
file->mkdir(outputDirName.c_str());
1713 std::cout <<
"ERROR, can't obtain nor create directory " << outputDirName <<
" in file " <<
file->GetName() << std::endl;
1721 for (
const std::string& inDirName : inputDirNames) {
1722 TDirectory* inDir =
file->GetDirectory(inDirName.c_str());
1725 if (hc.
size() == 0) {
1726 std::cout <<
"mergeLB_processLBinterval: No new objects found for " << outputDirName << std::endl;
1734 std::unique_ptr<TObject>
obj(
f->Get(HistoName));
1743 return s_dbg.getLvl();
1760 std::string
path =
dir->GetPath();
1761 if (
path.find(
':') != std::string::npos)