50 if (
level <= m_currLvl)
51 std::cout <<
msg << std::endl;
60 static dbgPrint s_dbg;
61 static bool s_checkEquality=
false;
63 std::optional<std::regex> checkRegEx(
const std::string&
re) {
70 std::string
test(
"Test String");
71 std::regex_match(
test, reNew);
73 std::cout <<
"ERROR: Invalid RegEx string \"" <<
re <<
"\"." << std::endl;
74 std::cout <<
"See http://www.boost.org/doc/libs/1_42_0/libs/regex/doc/html/boost_regex/syntax.html for allowed regular expression syntax" << std::endl;
84 explicit histCollection(TFile*
out,
bool skipExisting =
false) : m_out{
out}, m_skipExisting(skipExisting) {};
94 void addDirExclusion(
const std::optional<std::regex>& dirEx);
95 void addHistExclusion(
const std::optional<std::regex>& histEx);
97 unsigned size()
const;
102 return std::move(m_fileLBMap);
110 histPerDir_t(
const std::string& nameIn, std::unique_ptr<TObject>&& objIn, TTree* md);
115 std::unique_ptr<TObject>
obj;
116 std::array<std::string, 3>
metadata{
"unset",
"",
"<default>"};
117 std::clock_t cpuSum = 0;
122 bool fillMD(TTree* mdTree);
126 std::unordered_map<std::string, histPerDir_t>
histos;
127 void writeMD(TDirectory* outDir)
const;
133 std::unordered_map<std::string, histDir_t>
m_data;
134 std::optional<std::regex> m_dirExclusion;
135 std::optional<std::regex> m_histExclusion;
144 void histCollection::printTiming() {
145 std::vector<std::pair<std::string, clock_t>> cpuPerHistVec;
151 auto ordering = [](std::pair<std::string, clock_t>
a, std::pair<std::string, clock_t>
b) {
return a.second <
b.second; };
153 for (
const auto& [
name,
time] : cpuPerHistVec) {
154 const double tSec =
double(
time) / CLOCKS_PER_SEC;
155 std::cout <<
std::format(
"{:<30} : {:10.3f}",
name, tSec) << std::endl;
160 bool histCollection::histPerDir_t::fillMD(TTree * md) {
162 TTreeReaderArray<char> i_name(
reader,
"Name");
163 TTreeReaderArray<char> i_interval(
reader,
"Interval");
164 TTreeReaderArray<char> i_chain(
reader,
"TriggerChain");
165 TTreeReaderArray<char> i_merge(
reader,
"MergeMethod");
169 const std::string nameStr(
static_cast<char*
>(i_name.GetAddress()));
170 if (
name == nameStr) {
171 metadata = {
static_cast<char*
>(i_interval.GetAddress()),
static_cast<char*
>(i_chain.GetAddress()),
static_cast<char*
>(i_merge.GetAddress())};
181 const std::clock_t cpuStart = std::clock();
182 this->mergeMethod(
obj.get(),
other);
183 cpuSum += std::clock() - cpuStart;
188 void histCollection::histDir_t::writeMD(TDirectory *
out)
const {
191 std::map<std::string, std::array<std::string, 3>> metadatamap;
192 std::unique_ptr<TTree> oldMD((TTree*)
out->Get(
"metadata"));
194 TTreeReader
reader(oldMD.get());
195 TTreeReaderArray<char> i_name(
reader,
"Name");
196 TTreeReaderArray<char> i_interval(
reader,
"Interval");
197 TTreeReaderArray<char> i_chain(
reader,
"TriggerChain");
198 TTreeReaderArray<char> i_merge(
reader,
"MergeMethod");
201 const std::string
name(
static_cast<char*
>(i_name.GetAddress()));
202 metadatamap[
name] = {
static_cast<char*
>(i_interval.GetAddress()),
static_cast<char*
>(i_chain.GetAddress()),
static_cast<char*
>(i_merge.GetAddress())};
207 if (
h.metadata[0]!=
"unset")
208 metadatamap[
key] =
h.metadata;
211 if (metadatamap.empty())
return;
214 std::unique_ptr<TTree> mdTree = std::make_unique<TTree>(
"metadata",
"Monitoring Metadata");
215 mdTree->SetDirectory(
out);
217 mdTree->Branch(
"Name", (
void*)
nullptr,
"Name/C");
218 mdTree->Branch(
"Interval",
interval.data(),
"Interval/C");
219 mdTree->Branch(
"TriggerChain",
chain.data(),
"TriggerChain/C");
220 mdTree->Branch(
"MergeMethod",
merge.data(),
"MergeMethod/C");
222 mdTree->SetBranchAddress(
"Name",
histname);
224 for (
auto& [
key,
h] : metadatamap) {
231 mdTree->Write(0, TObject::kOverwrite);
234 histCollection::~histCollection() {}
236 void histCollection::addDirExclusion(
const std::optional<std::regex>&
dir) {
237 m_dirExclusion =
dir;
241 void histCollection::addHistExclusion(
const std::optional<std::regex>&
dir) {
242 m_histExclusion =
dir;
249 s +=
it.second.histos.size();
256 const histDir_t& hd =
it.second;
257 std::cout <<
"Dir: " <<
it.first <<
" has " << hd.histos.size() <<
" histos" << std::endl;
258 for (
const auto& it1 : hd.histos)
259 std::cout <<
"\t" << it1.second.name << std::endl;
264 template <
class HIST>
266 static_cast<HIST*
>(
a)->Add(
static_cast<const HIST*
>(
b));
271 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
272 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
274 std::cout <<
"ERROR in weightedAverageTH1: Object not of type TH1" << std::endl;
276 if (b1->GetEntries()==0)
return;
283 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
284 const TH1* b1 = (
dynamic_cast<const TH1*
>(
b));
286 std::cout <<
"ERROR in weightedEff: Object not of type TH1" << std::endl;
288 if (b1->GetEntries()==0)
return;
295 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
296 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
298 std::cout <<
"ERROR in mergeRMS: Object not of type TH1" << std::endl;
300 if (b1->GetEntries()==0)
return;
307 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
308 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
310 std::cout <<
"ERROR in RMSpercentDeviation: Object not of type TH1" << std::endl;
312 if (b1->GetEntries()==0)
return;
319 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
320 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
322 std::cout <<
"ERROR in getBinEffPerCent: Object not of type TH1" << std::endl;
324 if (b1->GetEntries()==0)
return;
331 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
332 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
334 std::cout <<
"ERROR in lowerLB: Object not of type TH1" << std::endl;
340 template <
class HIST>
342 if (!s_checkEquality)
344 HIST* a1 = (
dynamic_cast<HIST*
>(
a));
345 const HIST* b1 =
dynamic_cast<const HIST*
>(
b);
347 std::cout <<
"ERROR in identical: Object not of correct type" << std::endl;
354 void merge_rebinned(TObject *
a,
const TObject*
b) {
355 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
356 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
358 std::cout <<
"ERROR, in merge_rebinned: Object not of type TH1";
361 TH1* b2 =
const_cast<TH1*
>(b1);
366 void merge_eventSample(TObject *
a,
const TObject*
b) {
367 TH2* a1 = (
dynamic_cast<TH2*
>(
a));
368 const TH2* b1 =
dynamic_cast<const TH2*
>(
b);
370 std::cout <<
"ERROR in merge_eventSample: Object not of type TH2" << std::endl;
376 void merge_TEfficency(TObject *
a,
const TObject*
b) {
377 TEfficiency* a1 =
dynamic_cast<TEfficiency*
>(
a);
378 const TEfficiency* b1 =
dynamic_cast<const TEfficiency*
>(
b);
379 TEfficiency* b2 =
const_cast<TEfficiency*
>(b1);
381 std::cout <<
"ERROR in merge_TEfficiency: Object not of type TEfficiency" << std::endl;
390 void merge_TTree(TObject *
a,
const TObject*
b) {
391 TTree* a1 =
dynamic_cast<TTree*
>(
a);
392 const TTree* b1 =
dynamic_cast<const TTree*
>(
b);
394 std::cout <<
"ERROR in merge_TTree: Object not of type TTree" << std::endl;
397 TTree* b2 =
const_cast<TTree*
>(b1);
405 :
name(nameIn),
obj(std::move(objIn)), mergeMethod(nullptr) {
408 std::cout <<
"ERROR while adding " << nameIn <<
": Histogram pointer is NULL" << std::endl;
416 s_dbg(
VERBOSE,
"No matadata found for " +
name +
", use defaults");
418 const std::string& howToMerge =
metadata[2];
419 s_dbg(
VERBOSE,
"Name: " +
name +
" mergeMethod=" + howToMerge);
421 TH1* th1 =
dynamic_cast<TH1*
>(
obj.get());
422 TH2* th2 =
dynamic_cast<TH2*
>(
obj.get());
423 TEfficiency* teff =
dynamic_cast<TEfficiency*
>(
obj.get());
425 th1->SetDirectory(
nullptr);
426 if (howToMerge ==
"<default>") {
428 mergeMethod = &defaultMerge<TH2>;
430 mergeMethod = &defaultMerge<TH1>;
433 else if (howToMerge ==
"weightedAverage" || howToMerge==
"weightedAverage2D")
434 mergeMethod = &weightedAverage;
435 else if (howToMerge ==
"weightedEff")
436 mergeMethod = &weightedEff;
437 else if (howToMerge ==
"mergeRMS")
438 mergeMethod = &mergeRMS;
439 else if (howToMerge ==
"RMSpercentDeviation")
440 mergeMethod = &RMSpercentDeviation;
441 else if (howToMerge ==
"perBinEffPerCent")
442 mergeMethod = &perBinEffPerCent;
443 else if (howToMerge ==
"lowerLB")
445 else if (howToMerge ==
"identical")
447 mergeMethod = &identical<TH2>;
449 mergeMethod = &identical<TH1>;
451 else if ((howToMerge ==
"mergeRebinned") || (howToMerge ==
"merge"))
452 mergeMethod = &merge_rebinned;
454 std::cout <<
"ERROR: Unknown merging method (" << howToMerge <<
") for object of type TH1 named " << nameIn << std::endl;
459 teff->SetDirectory(
nullptr);
460 if (howToMerge ==
"<default>")
461 mergeMethod = &merge_TEfficency;
463 std::cout <<
"ERROR: Unknown merging method (" << howToMerge <<
") for object of type TEfficiency named " << nameIn << std::endl;
465 else if (
nullptr !=
dynamic_cast<TTree*
>(
obj.get())) {
466 mergeMethod = &merge_TTree;
468 std::cout <<
"ERROR Object " <<
name <<
" has unkown type" << std::endl;
476 if (m_dirExclusion && !std::regex_search(
dirName, *m_dirExclusion)) {
481 for (TObject* oKey : *
dir->GetListOfKeys()) {
482 TKey*
key =
static_cast<TKey*
>(oKey);
483 const std::string
name =
key->GetName();
484 const std::string classname =
key->GetClassName();
485 if ((classname ==
"TTree") && (
name ==
"metadata")) {
489 s_dbg(
VERBOSE,
"Found name " +
name +
", classname=" + classname);
494 if (classname.starts_with(
"TH") || classname.starts_with(
"TProfile") || classname.starts_with(
"TEfficiency") || classname ==
"TTree") {
495 if (m_histExclusion && !std::regex_search(
name, *m_histExclusion)) {
496 s_dbg(
DEBUG,
"Histogram with name " +
name +
" is excluded");
501 if (m_skipExisting) {
503 std::unique_ptr<TObject> existingObj(m_out->Get(
newName.c_str()));
508 std::unique_ptr<TTree> md;
509 if (itDir ==
m_data.end()) {
516 auto itH = itDir->second.histos.find(
name);
517 if (itH == itDir->second.histos.end()) {
521 md.reset((TTree*)
dir->Get(
"metadata"));
524 std::unique_ptr<TObject>
obj{
key->ReadObj()};
525 TTree* treeObj =
dynamic_cast<TTree*
>(
obj.get());
527 TDirectory* outDir = m_out->GetDirectory(
dirName.c_str());
529 outDir = m_out->mkdir(
dirName.c_str());
533 TTree* cloneTree = treeObj->CloneTree();
535 obj.reset(cloneTree);
539 itH = itDir->second.histos.emplace(
name, std::move(
histo)).first;
543 std::unique_ptr<TObject>
other(
key->ReadObj());
545 std::cout <<
"ERROR, got NULL key";
547 itH->second.merge(
other.get());
551 }
else if (classname.starts_with(
"TDirectory")) {
552 std::unique_ptr<TObject>
obj(
key->ReadObj());
553 TDirectory*
subdir =
dynamic_cast<TDirectory*
>(
obj.get());
558 if (!
name.starts_with(
"lb_") && !
name.starts_with(
"lowStat_LB")) {
566 std::cout <<
"Ignored objects '" <<
name <<
"' of type " << classname << std::endl;
574 unsigned nIgnored = 0;
578 TDirectory* histDir = m_out->GetDirectory(
fulldir.c_str());
579 if (histDir ==
nullptr) {
580 histDir = m_out->mkdir(
fulldir.c_str());
581 if (histDir ==
nullptr) {
582 std::cout <<
"ERROR, failed to create directory " <<
fulldir << std::endl;
585 s_dbg(
VERBOSE,
"Created directory " +
fulldir +
" in file " + m_out->GetName());
595 std::cout <<
"NOT writing " <<
name <<
". Invalid." << std::endl;
599 it.second.writeMD(histDir);
601 std::cout <<
"Wrote " <<
nWritten <<
" histograms to " << nDirs <<
" directories in output file " << m_out->GetName() << std::endl;
603 std::cout <<
" Omitting " << nIgnored <<
" histograms." << std::endl;
610 MonitoringFile::OutputMetadata::OutputMetadata(TTree *
metadata) : m_metadata(
metadata) {
611 makeBranch(
"Name",
"Name/C");
612 makeBranch(
"Interval",
"Interval/C");
613 makeBranch(
"TriggerChain",
"TriggerChain/C");
614 makeBranch(
"MergeMethod",
"MergeMethod/C");
617 void MonitoringFile::OutputMetadata::makeBranch(
const char* branchName,
const char* branchstr) {
618 if (!m_metadata->GetBranch(branchName)) {
619 m_metadata->Branch(branchName, (
void*)
nullptr, branchstr);
624 const std::string& theMerge) {
625 std::string
name = theName;
627 std::string
chain = theChain;
628 std::string
merge = theMerge;
629 m_metadata->SetBranchAddress(
"Name",
name.data());
630 m_metadata->SetBranchAddress(
"Interval",
interval.data());
631 m_metadata->SetBranchAddress(
"TriggerChain",
chain.data());
632 m_metadata->SetBranchAddress(
"MergeMethod",
merge.data());
636 MonitoringFile::MonitoringFile() :
m_file(0) {
637 m_fileCompressionLevel = 1;
639 MonitoringFile::clearData();
642 bool MonitoringFile::setFile(
const std::string&
fileName) {
650 MonitoringFile::MonitoringFile(
const std::string&
fileName) :
m_file(0) {
651 m_fileCompressionLevel = 1;
653 MonitoringFile::clearData();
657 MonitoringFile::~MonitoringFile() {
663 bool MonitoringFile::setHistogramRegEx(
const std::string&
re) {
664 m_mergeMatchHistoRE = checkRegEx(
re);
665 return m_mergeMatchHistoRE.has_value();
668 bool MonitoringFile::setDirectoryRegEx(
const std::string&
re) {
669 m_mergeMatchDirRE = checkRegEx(
re);
670 return m_mergeMatchDirRE.has_value();
673 void MonitoringFile::getAllDirs(DirMap_t & dirmap, TDirectory *
dir,
const std::string&
dirName) {
679 dirmap.insert(dirmapVal);
682 TIter
next(
dir->GetListOfKeys());
684 while ((
key =
dynamic_cast<TKey*
>(
next())) != 0) {
686 TObject*
obj =
key->ReadObj();
687 TDirectory*
subdir =
dynamic_cast<TDirectory*
>(
obj);
689 std::string subdirName(
subdir->GetName());
690 std::string
fName(
"");
703 TDirectory* MonitoringFile::createDir(DirMap_t & dirmap, TDirectory *
dir,
const std::string&
parent,
const std::string&
path) {
708 DirMap_t::const_iterator diter;
709 std::string::size_type
i =
path.find_first_of(
'/');
710 std::string
fName(
"");
716 if (
i != std::string::npos) {
717 std::string dName(
path, 0,
i);
718 std::string pName(
path,
i + 1, std::string::npos);
721 diter = dirmap.find(
fName);
722 if (diter != dirmap.end()) {
727 dirmap.insert(dirmapVal);
737 diter = dirmap.find(
fName);
738 if (diter != dirmap.end()) {
739 return diter->second;
744 dirmap.insert(dirmapVal);
754 std::string::size_type
i =
path.find_first_of(
'/');
755 if (
i != std::string::npos) {
756 std::string dName(
path, 0,
i);
757 std::string pName(
path,
i + 1, std::string::npos);
759 key =
dir->FindKey(dName.c_str());
761 TDirectory* subDir =
dynamic_cast<TDirectory*
>(
key->ReadObj());
771 return dir->FindKey(
path.c_str());
774 void MonitoringFile::fillMetaDataMap(std::map<std::string, dqutils::MonitoringFile::MetaData> & mdMap, TDirectory *
dir) {
777 TTree* md =
dynamic_cast<TTree*
>(
dir->Get(
"metadata"));
782 TTreeReaderArray<char> i_name(
reader,
"Name");
783 TTreeReaderArray<char> i_interval(
reader,
"Interval");
784 TTreeReaderArray<char> i_chain(
reader,
"TriggerChain");
785 TTreeReaderArray<char> i_merge(
reader,
"MergeMethod");
788 const std::string nameStr(
static_cast<char*
>(i_name.GetAddress()));
789 if (mdMap.find(nameStr) == mdMap.end()) {
790 MetaData md(nameStr,
static_cast<char*
>(i_interval.GetAddress()),
static_cast<char*
>(i_chain.GetAddress()),
static_cast<char*
>(i_merge.GetAddress()));
791 std::map<std::string, MetaData>::value_type mdVal(nameStr, md);
800 std::cout <<
"Writing file: " <<
outFileName << std::endl;
801 std::cout <<
"Start merging [" <<
files.size() <<
"] histogram files" << std::endl;
803 TH1::AddDirectory(
false);
804 if (m_mergeMatchDirRE.has_value() || m_mergeMatchHistoRE.has_value()) {
805 std::cout <<
" ========== Using regular expressions for selective merging ========== " << std::endl;
808 std::cout <<
"CPU time measurement activated " << std::endl;
817 std::cout <<
"Got exactly one input file. Will copy input -> output" << std::endl;
818 if (m_mergeMatchDirRE.has_value() || m_mergeMatchHistoRE.has_value()) {
819 std::cout <<
"regular expressions for selective merging will have no effect!" << std::endl;
824 std::filesystem::copy_file(
inPath,
outPath, std::filesystem::copy_options::overwrite_existing);
830 std::cout <<
" ERROR, cound not open output file " <<
outFileName << std::endl;
833 std::cout <<
"Opened/created output file " <<
outFileName << std::endl;
836 hc.addDirExclusion(m_mergeMatchDirRE);
837 hc.addHistExclusion(m_mergeMatchHistoRE);
840 std::unique_ptr<TFile> in1(TFile::Open(
files[0].c_str()));
842 std::cout <<
"ERROR, could not open input file " <<
files[0] << std::endl;
845 std::cout <<
"Working on file 1/" <<
nFiles <<
": " <<
files[0] << std::endl;
846 std::string runDir, runDirFwd;
848 TIter
next(in1->GetListOfKeys());
850 while ((
key = (TKey*)
next())) {
851 const char*
name =
key->GetName();
852 if (std::regex_match(
name, runDirPattern)) {
853 if (runDir.size() > 0) {
854 std::cout <<
"ERROR More than one run_XXX directory found! Ignoring " <<
name << std::endl;
859 if (runDir.empty()) {
860 std::cout <<
"No run-directory found, start with '/'" << std::endl;
865 std::cout <<
"Found run directory " << runDir << std::endl;
870 TDirectory*
dir(
dynamic_cast<TDirectory*
>(in1->GetDirectory(runDir.c_str())));
872 std::cout <<
"ERROR, can't access directory " << runDir;
876 hc.addDirectory(
dir, runDirFwd,
files[0]);
881 for (
size_t i = 1;
i <
files.size(); ++
i) {
882 std::cout <<
"Working on file " << 1+
i <<
"/" <<
nFiles <<
": " <<
files[
i] << std::endl;
883 std::unique_ptr<TFile> in(TFile::Open(
files[
i].c_str()));
885 std::cout <<
"ERROR, could not open input file " <<
files[
i] << std::endl;
888 TDirectory*
dir(
dynamic_cast<TDirectory*
>(in->GetDirectory(runDir.c_str())));
890 std::cout <<
"ERROR, could not cast to directory" << std::endl;
893 hc.addDirectory(
dir, runDirFwd,
files[
i]);
896 std::cout <<
"Accumulated a total of " << hc.size() <<
" histograms." << std::endl;
898 std::cout <<
"Start writing output ..." << std::endl;
902 std::cout <<
"CPU time for histogram merging: (regular histograms)" << std::endl;
906 auto newlbmap = hc.getFileLBMapAndClear();
909 for (
auto& [lbname, newfileset] : newlbmap) {
910 auto& fileset = lbmap[lbname];
911 fileset.merge(newfileset);
914 if (!lbmap.empty() && fillLBDirs) {
915 std::cout <<
"Start merging lb_nnn and lowStat_LB directories (" << lbmap.size() <<
" in total)" << std::endl;
918 hclb.addDirExclusion(m_mergeMatchDirRE);
919 hclb.addHistExclusion(m_mergeMatchHistoRE);
923 std::vector<std::pair<std::string, std::vector<std::string>>> lbToFiles;
924 for (
const auto& [
lb,fileSet] : lbmap) {
925 if (fileSet.size() > 0)
926 lbToFiles.emplace_back(
lb,std::vector<std::string>(fileSet.begin(),fileSet.end()));
930 std::sort(lbToFiles.begin(), lbToFiles.end(),
931 [](
const decltype(lbToFiles)::
value_type&
a,
const decltype(lbToFiles)::
value_type&
b) {
return a.second[0] <
b.second[0]; });
934 std::unique_ptr<TFile> in;
936 std::cout <<
"Merging/copying directory " <<
dir <<
" from " <<
filenames.size() <<
" input file(s) (" << ++
counter <<
"/" << lbToFiles.size() <<
")"
939 if (!in || strcmp(in->GetName(),
fName.c_str()) != 0) {
940 in.reset(TFile::Open(
fName.c_str()));
943 s_dbg(
DEBUG,
"Input file " +
fName +
" already open");
947 std::cout <<
"ERROR, could not open input file " <<
fName << std::endl;
950 TDirectory* tDir = (
dynamic_cast<TDirectory*
>(in->Get(
dir.c_str())));
952 std::cout <<
"ERROR, failed to get directory " <<
dir <<
" from file " <<
fName << std::endl;
954 hclb.addDirectory(tDir,
dir);
959 std::cout <<
"CPU time for histogram merging: (lumiblock-histograms)" << std::endl;
970 typedef std::vector<std::string> FileList_t;
972 const unsigned int nFilesAtOnce = 50;
975 bool success = setListFromFile(
allFiles, listFileName);
977 std::cout <<
"ERROR Failed ot read list of input files" << std::endl;
983 if (
allFiles.size() <= nFilesAtOnce) {
987 FileList_t procFiles, tmpIntermediateFiles;
989 FileList_t::const_iterator filesEnd =
allFiles.end();
990 FileList_t::const_iterator fi =
allFiles.begin();
993 std::string tmpInputFile(
"");
994 std::string tmpOutputFile(
"");
997 while (fi != filesEnd) {
999 procFiles.push_back(*fi);
1002 if (
counter % nFilesAtOnce == 0 || fi == filesEnd) {
1003 std::ostringstream nameStream;
1004 nameStream <<
"tmp_merge_" <<
counter <<
".root";
1005 tmpOutputFile = nameStream.str();
1006 tmpIntermediateFiles.push_back(tmpOutputFile);
1016 for (
const auto& tmpFile : tmpIntermediateFiles) {
1022 void MonitoringFile::printStatistics() {
1024 std::cerr <<
"MonitoringFile::printStatistics(): "
1025 <<
"No input file is open\n";
1031 getAllDirs(indirmap,
m_file,
"");
1033 DirMap_t::const_iterator idirend = indirmap.end();
1034 for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1035 std::string idirName = idir->first;
1037 GatherStatistics stat_shift(idirName);
1038 GatherStatistics stat_all(idirName);
1040 loopOnHistogramsInMetadata(stat_shift, idir->second);
1041 loopOnHistograms(stat_all, idir->second);
1043 std::cout.setf(std::ios_base::left, std::ios_base::adjustfield);
1044 std::cout.width(80);
1045 std::cout << idirName <<
" ";
1047 std::cout.setf(std::ios_base::right, std::ios_base::adjustfield);
1048 std::cout <<
" shift: ";
1050 std::cout << stat_shift.m_nHist1D <<
" ";
1052 std::cout << stat_shift.m_nHist1DBins <<
" ";
1054 std::cout << stat_shift.m_nHist2D <<
" ";
1056 std::cout << stat_shift.m_nHist2DBins <<
" ";
1058 std::cout << stat_shift.m_nGraph <<
" ";
1060 std::cout << stat_shift.m_nGraphPoints <<
" ";
1062 std::cout <<
" all: ";
1063 std::cout << stat_all.m_nHist1D <<
" ";
1065 std::cout << stat_all.m_nHist1DBins <<
" ";
1067 std::cout << stat_all.m_nHist2D <<
" ";
1069 std::cout << stat_all.m_nHist2DBins <<
" ";
1071 std::cout << stat_all.m_nGraph <<
" ";
1073 std::cout << stat_all.m_nGraphPoints <<
"\n";
1079 bool MonitoringFile::copyHistograms(
const std::string&
outFileName,
const std::string&
dirName) {
1085 std::cerr <<
"MonitoringFile::copyHistograms(): "
1086 <<
"No input file is open\n";
1091 DirMap_t reducedmap;
1097 std::cerr <<
"MonitoringFile::copyHistograms(): "
1098 <<
"Directory \'" <<
dirName <<
"\' not found in input file\n";
1102 TDirectory* fromDir =
dynamic_cast<TDirectory*
>(dkey->ReadObj());
1104 DirMap_t::value_type dirmapVal(
dirName, fromDir);
1105 indirmap.insert(dirmapVal);
1107 std::cout <<
"Building list of all TDirectories in file...\n" <<
std::flush;
1108 getAllDirs(indirmap,
m_file,
"");
1111 DirMap_t::const_iterator idirend = indirmap.end();
1112 for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1114 std::string idirName = idir->first;
1115 std::cout <<
"Checking " << idirName <<
"\n" <<
std::flush;
1123 if (!dirHasHistogramsInMetadata(idir->second)) {
1127 reducedmap.insert(*idir);
1132 std::cerr <<
"MonitoringFile::copyHistograms(): "
1133 <<
"Output file not opened\n";
1137 idirend = reducedmap.end();
1138 for (DirMap_t::const_iterator idir = reducedmap.begin(); idir != idirend; ++idir) {
1140 std::string idirName = idir->first;
1141 std::cout <<
"Processing " << idirName <<
"\n" <<
std::flush;
1143 TDirectory* toDir = createDir(outdirmap,
outfile.get(),
"", idirName);
1145 std::cerr <<
"MonitoringFile::copyHistograms(): "
1146 <<
"Directory \'" << idirName <<
"\' not created in output file\n";
1150 CopyHistogram copyFcn(toDir, idirName);
1152 loopOnHistogramsInMetadata(copyFcn, idir->second);
1161 const std::string& hcfg_lowStat,
const std::string& hcfg_medStat) {
1164 std::cout <<
"\nUsing han configurations:\n"
1165 <<
" entire run: " <<
hcfg <<
"\n"
1166 <<
" low stat interval: " << hcfg_lowStat <<
"\n"
1167 <<
" medium stat interval: " << hcfg_medStat <<
"\n\n"
1172 std::cerr <<
"MonitoringFile::getHanResults(): "
1173 <<
"Cannot open input file \"" <<
input <<
"\"\n";
1177 std::vector<std::string> run_dirs;
1178 std::vector<std::string> lowStat_dirs;
1179 std::vector<std::string> medStat_dirs;
1181 TIter next_run(
infile->GetListOfKeys());
1183 while ((key_run =
dynamic_cast<TKey*
>(next_run())) != 0) {
1184 TObject* obj_run = key_run->ReadObj();
1185 TDirectory* tdir_run =
dynamic_cast<TDirectory*
>(obj_run);
1186 if (tdir_run != 0) {
1187 std::string tdir_run_name(tdir_run->GetName());
1188 if (tdir_run_name.find(
"run") != std::string::npos) {
1189 run_dirs.push_back(tdir_run_name);
1190 TIter next_minutes(tdir_run->GetListOfKeys());
1191 TKey* key_minutes(0);
1192 while ((key_minutes =
dynamic_cast<TKey*
>(next_minutes())) != 0) {
1193 TObject* obj_minutes = key_minutes->ReadObj();
1194 TDirectory* tdir_minutes =
dynamic_cast<TDirectory*
>(obj_minutes);
1195 if (tdir_minutes != 0) {
1196 std::string tdir_minutes_name(tdir_minutes->GetName());
1197 if (tdir_minutes_name.find(
"lowStat") != std::string::npos) {
1198 lowStat_dirs.push_back(tdir_run_name +
'/' + tdir_minutes_name);
1199 }
else if (tdir_minutes_name.find(
"medStat") != std::string::npos) {
1200 medStat_dirs.push_back(tdir_run_name +
'/' + tdir_minutes_name);
1215 std::vector<std::string>::const_iterator dirs_end;
1216 std::vector<std::string>::const_iterator
dir;
1218 dirs_end = run_dirs.end();
1219 for (
dir = run_dirs.begin();
dir != dirs_end; ++
dir) {
1220 const std::string& tdir_run_name = *
dir;
1221 std::string han_output_run =
hanResultsDir +
'/' + tdir_run_name +
"_han.root";
1222 std::cout <<
"Calling han( " <<
hcfg <<
", " <<
input <<
", " << tdir_run_name <<
", " << han_output_run <<
" ):\n" <<
std::flush;
1223 han.Analyze(
hcfg,
input, han_output_run, tdir_run_name);
1225 fileList += han_output_run +
" " + tdir_run_name +
"\n";
1228 dirs_end = lowStat_dirs.end();
1229 for (
dir = lowStat_dirs.begin();
dir != dirs_end; ++
dir) {
1230 const std::string& tdir_minutes_path = *
dir;
1232 std::string tdir_minutes_underscore = tdir_minutes_path;
1233 std::string::size_type tdir_minutes_i = tdir_minutes_underscore.find(
'/');
1234 tdir_minutes_underscore.replace(tdir_minutes_i, 1,
"_");
1236 std::string han_output_lowStat =
hanResultsDir +
'/' + tdir_minutes_underscore +
"_han.root";
1237 std::cout <<
"Running han, writing to " << han_output_lowStat <<
":\n" <<
std::flush;
1238 han.Analyze(hcfg_lowStat,
input, han_output_lowStat, tdir_minutes_path);
1240 std::string subdirname(tdir_minutes_path, tdir_minutes_i + 1, std::string::npos);
1241 std::string
dirname(tdir_minutes_path, 0, tdir_minutes_i);
1242 fileList += han_output_lowStat +
" " + subdirname +
" " +
dirname +
" " + subdirname +
"\n";
1245 dirs_end = medStat_dirs.end();
1246 for (
dir = medStat_dirs.begin();
dir != dirs_end; ++
dir) {
1247 const std::string& tdir_minutes_path = *
dir;
1249 std::string tdir_minutes_underscore = tdir_minutes_path;
1250 std::string::size_type tdir_minutes_i = tdir_minutes_underscore.find(
'/');
1251 tdir_minutes_underscore.replace(tdir_minutes_i, 1,
"_");
1253 std::string han_output_medStat =
hanResultsDir +
'/' + tdir_minutes_underscore +
"_han.root";
1254 std::cout <<
"Running han, writing to " << han_output_medStat <<
":\n" <<
std::flush;
1255 han.Analyze(hcfg_medStat,
input, han_output_medStat, tdir_minutes_path);
1257 std::string subdirname(tdir_minutes_path, tdir_minutes_i + 1, std::string::npos);
1258 std::string
dirname(tdir_minutes_path, 0, tdir_minutes_i);
1259 fileList += han_output_medStat +
" " + subdirname +
" " +
dirname +
" " + subdirname +
"\n";
1265 void MonitoringFile::printHanConfig() {
1267 std::cerr <<
"MonitoringFile::printHanConfig(): "
1268 <<
"No input file is open\n";
1274 getAllDirs(indirmap,
m_file,
"");
1276 std::string
indent, indent_p, indent_c;
1277 std::string idirName_p;
1278 DirMap_t::const_iterator idirend = indirmap.end();
1279 for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1280 std::string idirName = idir->first;
1281 std::string::size_type shortNameIndex = idirName.rfind(
'/');
1282 std::string
shortName = idirName.substr(shortNameIndex + 1, std::string::npos);
1284 std::string::size_type fsIndex = idirName.find(
'/');
1285 std::string shortPath;
1286 if (fsIndex != shortNameIndex)
1287 shortPath = idirName.substr(fsIndex + 1, shortNameIndex);
1289 shortPath = idirName.substr(fsIndex + 1, std::string::npos);
1291 std::cout << idirName <<
"\n";
1292 std::cout << shortPath <<
", " <<
shortName <<
"\n";
1330 std::string MonitoringFile::getIndentation(
const std::string&
pathName,
const std::string& leadingSpace) {
1331 std::string space = leadingSpace;
1332 std::string::size_type
i =
pathName.find_first_of(
'/');
1333 if (
i != std::string::npos) {
1334 std::string subPath(
pathName,
i + 1, std::string::npos);
1336 return getIndentation(subPath, space);
1341 std::string MonitoringFile::FindCommon(
const std::string&
name1,
const std::string& name2)
const {
1359 MonitoringFile::CopyHistogram::CopyHistogram(TDirectory *
target,
const std::string&
dirName) : m_target(
target), m_dirName(
dirName), m_metadata(0) {
1360 m_metadata =
new TTree(
"metadata",
"Monitoring Metadata");
1361 m_metadata->SetDirectory(0);
1362 m_metadata->Branch(
"Name", (
void*)
nullptr,
"Name/C");
1363 m_metadata->Branch(
"Interval", (
void*)
nullptr,
"Interval/C");
1364 m_metadata->Branch(
"TriggerChain", (
void*)
nullptr,
"TriggerChain/C");
1365 m_metadata->Branch(
"MergeMethod", (
void*)
nullptr,
"MergeMethod/C");
1368 MonitoringFile::CopyHistogram::~CopyHistogram() {
1370 m_metadata->SetDirectory(m_target);
1371 m_metadata->Write();
1377 hist->SetDirectory(m_target);
1396 void MonitoringFile::CopyHistogram::fillMD(
const MetaData& md) {
1397 std::string
name(md.name);
1399 std::string
chain(md.chain);
1400 std::string
merge(md.merge);
1401 m_metadata->SetBranchAddress(
"Name",
name.data());
1402 m_metadata->SetBranchAddress(
"Interval",
interval.data());
1403 m_metadata->SetBranchAddress(
"TriggerChain",
chain.data());
1404 m_metadata->SetBranchAddress(
"MergeMethod",
merge.data());
1408 bool MonitoringFile::CopyHistogram::executeMD(TH1 *
hist,
const MetaData& md) {
1410 hist->SetDirectory(m_target);
1418 bool MonitoringFile::CopyHistogram::executeMD(TGraph * graph,
const MetaData& md) {
1427 bool MonitoringFile::CopyHistogram::executeMD(TEfficiency *
eff,
const MetaData& md) {
1434 MonitoringFile::GatherStatistics::GatherStatistics(
const std::string&
dirName)
1435 : m_dirName(
dirName), m_nHist1D(0), m_nHist1DBins(0), m_nGraph(0), m_nGraphPoints(0), m_nHist2D(0), m_nHist2DBins(0) {}
1438 TH2* hist2d =
dynamic_cast<TH2*
>(
hist);
1441 m_nHist2DBins += (hist2d->GetNbinsX() * hist2d->GetNbinsY());
1445 m_nHist1DBins +=
hist->GetNbinsX();
1451 m_nGraphPoints += graph->GetMaxSize();
1458 TH1* h_total =
eff->GetCopyPassedHisto();
1459 TH2* h_total2D =
dynamic_cast<TH2*
>(h_total);
1461 if (h_total2D != 0) {
1462 m_nEfficiencyBins += (h_total2D->GetNbinsX() * h_total2D->GetNbinsY());
1465 m_nEfficiencyBins += h_total->GetNbinsX();
1470 MonitoringFile::GatherNames::GatherNames() {}
1473 m_names.push_back(std::string(
hist->GetName()));
1478 m_names.push_back(std::string(graph->GetName()));
1483 m_names.push_back(std::string(
eff->GetName()));
1487 void MonitoringFile::clearData() {
1492 m_fileCompressionLevel = 1;
1496 bool MonitoringFile::dirHasHistogramsInMetadata(TDirectory *
dir) {
1499 TKey* mdKey =
dir->FindKey(
"metadata");
1504 TTree* md =
dynamic_cast<TTree*
>(mdKey->ReadObj());
1516 std::cerr <<
"Exception: \"" <<
e.what() <<
"\" in directory \"" <<
dir->GetName() <<
"\"\n" <<
std::flush;
1527 void MonitoringFile::loopOnHistograms(HistogramOperation & fcn, TDirectory *
dir) {
1528 TIter
next(
dir->GetListOfKeys());
1530 while ((
key =
dynamic_cast<TKey*
>(
next())) != 0) {
1531 TObject*
obj =
key->ReadObj();
1535 if ((
h =
dynamic_cast<TH1*
>(
obj))) {
1537 }
else if ((
g =
dynamic_cast<TGraph*
>(
obj))) {
1539 }
else if ((
e =
dynamic_cast<TEfficiency*
>(
obj))) {
1546 bool MonitoringFile::loopOnHistogramsInMetadata(HistogramOperation & fcn, TDirectory *
dir) {
1548 TKey* mdKey =
dir->FindKey(
"metadata");
1553 TTree* md =
dynamic_cast<TTree*
>(mdKey->ReadObj());
1561 TTreeReaderArray<char> i_name(
reader,
"Name");
1562 TTreeReaderArray<char> i_interval(
reader,
"Interval");
1563 TTreeReaderArray<char> i_chain(
reader,
"TriggerChain");
1564 TTreeReaderArray<char> i_merge(
reader,
"MergeMethod");
1567 const std::string nameStr(
static_cast<char*
>(i_name.GetAddress()));
1569 i_key =
dir->FindKey(
static_cast<char*
>(i_name.GetAddress()));
1571 std::cerr <<
"MonitoringFile::loopOnHistogramsInMetadata(): "
1572 <<
"No \'" << nameStr <<
"\' object found\n";
1575 MetaData md(nameStr,
static_cast<char*
>(i_interval.GetAddress()),
static_cast<char*
>(i_chain.GetAddress()),
static_cast<char*
>(i_merge.GetAddress()));
1576 TObject*
obj = i_key->ReadObj();
1577 TH1*
h =
dynamic_cast<TH1*
>(
obj);
1579 fcn.executeMD(
h, md);
1581 TGraph*
g =
dynamic_cast<TGraph*
>(
obj);
1583 fcn.executeMD(
g, md);
1594 bool MonitoringFile::setListFromFile(std::vector<std::string> &
filelist,
const std::string& listFileName) {
1595 using namespace std;
1599 ifstream listfile(listFileName.c_str());
1601 cerr <<
"MonitoringFile::setListFromFile(): "
1602 <<
"cannot read from file: " << listFileName <<
"\n";
1609 while (getline(listfile,
line)) {
1610 istringstream linestream(
line);
1611 while (linestream.get(
c)) {
1618 linestream.putback(
c);
1621 cerr <<
"MonitoringFile::setListFromFile(): "
1622 <<
"badly formatted line: " <<
line <<
"\n";
1634 int MonitoringFile::mergeLBintervals(
const std::string& inFilename) {
1636 std::cout <<
"Running mergeLBintervals on " << inFilename << std::endl;
1638 std::unique_ptr<TFile>
f(TFile::Open(inFilename.c_str(),
"UPDATE"));
1640 std::cout <<
"ERROR, could not open file " << inFilename <<
" for update" << std::endl;
1643 std::string runDirName;
1644 const std::regex runDirPattern(
"run_[0-9]*");
1645 TIter
next(
f->GetListOfKeys());
1647 while ((
key = (TKey*)
next())) {
1648 const char*
name =
key->GetName();
1649 if (std::regex_match(
name, runDirPattern)) {
1650 if (runDirName.size() > 0) {
1651 std::cout <<
"ERROR More than one run_XXX directory found! Ignoring " <<
name << std::endl;
1658 TDirectory* runDir =
f->GetDirectory(runDirName.c_str());
1659 const auto mapping = buildLBToIntervalMap(runDir);
1661 if (s_dbg.getLvl() ==
VERBOSE) {
1662 std::cout <<
"LB directory mapping:" << std::endl;
1663 for (
const auto& i1 :
mapping) {
1664 std::cout << i1.first;
1665 for (
const auto& i2 : i1.second) {
1666 std::cout <<
"\t" << i2 << std::endl;
1671 for (
const auto& [outDir, inDIrs] :
mapping) {
1672 int stat=mergeLB_processLBinterval(
f.get(), inDIrs, outDir);
1677 f.reset(TFile::Open(inFilename.c_str(),
"UPDATE"));
1678 runDir =
f->GetDirectory(runDirName.c_str());
1680 std::cout <<
"merging lowStat_LB dirs into run-dir" << std::endl;
1681 std::vector<std::string> lowStatDirs;
1682 for (TObject* oKey : *runDir->GetListOfKeys()) {
1683 TKey*
key =
static_cast<TKey*
>(oKey);
1684 const std::string
name =
key->GetName();
1685 const std::string classname =
key->GetClassName();
1686 if (classname.starts_with(
"TDirectory") and
name.starts_with(
"lowStat_LB")) {
1687 lowStatDirs.push_back(runDirName +
"/" +
name);
1688 s_dbg(
VERBOSE,
"Found input: " + runDirName +
"/" +
name);
1692 int stat=mergeLB_processLBinterval(
f.get(), lowStatDirs, runDirName);
1698 std::map<std::string, std::vector<std::string>> MonitoringFile::buildLBToIntervalMap(TDirectory * runDir) {
1700 std::map<std::string, std::vector<std::string>>
ranges;
1703 const std::string runDirName = runDir->GetName();
1704 for (TObject* oKey : *runDir->GetListOfKeys()) {
1705 TKey*
key =
static_cast<TKey*
>(oKey);
1706 const std::string
name =
key->GetName();
1707 const std::string classname =
key->GetClassName();
1708 if (!classname.starts_with(
"TDirectory"))
1710 if (
name.starts_with(
"lb_")) {
1714 }
catch (std::invalid_argument&
e) {
1715 std::cout <<
"ERROR, unexpected directory name " <<
name <<
". Can't parse lb number" << std::endl;
1716 std::cout <<
e.what() << std::endl;
1722 ranges[lbString].push_back(runDirName +
"/" +
name);
1728 int MonitoringFile::mergeLB_processLBinterval(TFile *
file,
const std::vector<std::string>& inputDirNames,
const std::string& outputDirName) {
1730 TDirectory* outDir =
file->GetDirectory(outputDirName.c_str());
1732 outDir =
file->mkdir(outputDirName.c_str());
1735 std::cout <<
"ERROR, can't obtain nor create directory " << outputDirName <<
" in file " <<
file->GetName() << std::endl;
1740 hc.addDirExclusion(m_mergeMatchDirRE);
1741 hc.addHistExclusion(m_mergeMatchHistoRE);
1743 for (
const std::string& inDirName : inputDirNames) {
1744 TDirectory* inDir =
file->GetDirectory(inDirName.c_str());
1745 hc.addDirectory(inDir, outputDirName);
1747 if (hc.size() == 0) {
1748 std::cout <<
"mergeLB_processLBinterval: No new objects found for " << outputDirName << std::endl;
1755 bool MonitoringFile::CheckHistogram(TFile *
f,
const char* HistoName) {
1756 std::unique_ptr<TObject>
obj(
f->Get(HistoName));
1764 int MonitoringFile::getDebugLevel() {
1765 return s_dbg.getLvl();
1767 void MonitoringFile::setDebugLevel(
int level) {
1775 void MonitoringFile::setCheckEquality(
bool value) {dqutils::s_checkEquality=
value;}
1776 std::atomic<int> MonitoringFile::m_fileCompressionLevel = 1;
1777 bool MonitoringFile::m_doTiming =
false;
1778 std::unordered_map<std::string, std::clock_t> MonitoringFile::m_cpuPerHistogram;
1780 std::string MonitoringFile::getPath(TDirectory *
dir) {
1782 std::string
path =
dir->GetPath();
1783 if (
path.find(
':') != std::string::npos)