50 if (
level <= m_currLvl)
51 std::cout <<
msg << std::endl;
60 static dbgPrint s_dbg;
61 static bool s_checkEquality=
false;
63 std::optional<std::regex> checkRegEx(
const std::string&
re) {
70 std::string
test(
"Test String");
71 std::regex_match(
test, reNew);
73 std::cout <<
"ERROR: Invalid RegEx string \"" <<
re <<
"\"." << std::endl;
74 std::cout <<
"See http://www.boost.org/doc/libs/1_42_0/libs/regex/doc/html/boost_regex/syntax.html for allowed regular expression syntax" << std::endl;
85 typedef std::map<std::string, std::vector<std::string>> fileLBMap_t;
86 explicit histCollection(TFile*
out,
bool skipExisting =
false) : m_out{
out}, m_skipExisting(skipExisting) {};
96 void addDirExclusion(
const std::optional<std::regex>& dirEx);
97 void addHistExclusion(
const std::optional<std::regex>& histEx);
99 unsigned size()
const;
102 fileLBMap_t getFileLBMapAndClear() {
104 return std::move(m_fileLBMap);
112 histPerDir_t(
const std::string& nameIn, std::unique_ptr<TObject>&& objIn, TTree* md);
117 std::unique_ptr<TObject>
obj;
118 std::array<std::string, 3>
metadata{
"unset",
"",
"<default>"};
119 std::clock_t cpuSum = 0;
124 bool fillMD(TTree* mdTree);
128 std::unordered_map<std::string, histPerDir_t>
histos;
129 void writeMD(TDirectory* outDir)
const;
135 std::unordered_map<std::string, histDir_t>
m_data;
136 std::optional<std::regex> m_dirExclusion;
137 std::optional<std::regex> m_histExclusion;
138 fileLBMap_t m_fileLBMap;
146 void histCollection::printTiming() {
147 std::vector<std::pair<std::string, clock_t>> cpuPerHistVec;
153 auto ordering = [](std::pair<std::string, clock_t>
a, std::pair<std::string, clock_t>
b) {
return a.second <
b.second; };
155 for (
const auto& [
name,
time] : cpuPerHistVec) {
156 const double tSec =
double(
time) / CLOCKS_PER_SEC;
157 std::cout <<
std::format(
"{:<30} : {:10.3f}",
name, tSec) << std::endl;
162 bool histCollection::histPerDir_t::fillMD(TTree * md) {
164 TTreeReaderArray<char> i_name(
reader,
"Name");
165 TTreeReaderArray<char> i_interval(
reader,
"Interval");
166 TTreeReaderArray<char> i_chain(
reader,
"TriggerChain");
167 TTreeReaderArray<char> i_merge(
reader,
"MergeMethod");
171 const std::string nameStr(
static_cast<char*
>(i_name.GetAddress()));
172 if (
name == nameStr) {
173 metadata = {
static_cast<char*
>(i_interval.GetAddress()),
static_cast<char*
>(i_chain.GetAddress()),
static_cast<char*
>(i_merge.GetAddress())};
183 const std::clock_t cpuStart = std::clock();
184 this->mergeMethod(
obj.get(),
other);
185 cpuSum += std::clock() - cpuStart;
190 void histCollection::histDir_t::writeMD(TDirectory *
out)
const {
193 std::map<std::string, std::array<std::string, 3>> metadatamap;
194 std::unique_ptr<TTree> oldMD((TTree*)
out->Get(
"metadata"));
196 TTreeReader
reader(oldMD.get());
197 TTreeReaderArray<char> i_name(
reader,
"Name");
198 TTreeReaderArray<char> i_interval(
reader,
"Interval");
199 TTreeReaderArray<char> i_chain(
reader,
"TriggerChain");
200 TTreeReaderArray<char> i_merge(
reader,
"MergeMethod");
203 const std::string
name(
static_cast<char*
>(i_name.GetAddress()));
204 metadatamap[
name] = {
static_cast<char*
>(i_interval.GetAddress()),
static_cast<char*
>(i_chain.GetAddress()),
static_cast<char*
>(i_merge.GetAddress())};
209 if (
h.metadata[0]!=
"unset")
210 metadatamap[
key] =
h.metadata;
213 if (metadatamap.empty())
return;
216 std::unique_ptr<TTree> mdTree = std::make_unique<TTree>(
"metadata",
"Monitoring Metadata");
217 mdTree->SetDirectory(
out);
219 mdTree->Branch(
"Name", (
void*)
nullptr,
"Name/C");
220 mdTree->Branch(
"Interval",
interval.data(),
"Interval/C");
221 mdTree->Branch(
"TriggerChain",
chain.data(),
"TriggerChain/C");
222 mdTree->Branch(
"MergeMethod",
merge.data(),
"MergeMethod/C");
224 mdTree->SetBranchAddress(
"Name",
histname);
226 for (
auto& [
key,
h] : metadatamap) {
233 mdTree->Write(0, TObject::kOverwrite);
236 histCollection::~histCollection() {}
238 void histCollection::addDirExclusion(
const std::optional<std::regex>&
dir) {
239 m_dirExclusion =
dir;
243 void histCollection::addHistExclusion(
const std::optional<std::regex>&
dir) {
244 m_histExclusion =
dir;
251 s +=
it.second.histos.size();
258 const histDir_t& hd =
it.second;
259 std::cout <<
"Dir: " <<
it.first <<
" has " << hd.histos.size() <<
" histos" << std::endl;
260 for (
const auto&
it1 : hd.histos)
261 std::cout <<
"\t" <<
it1.second.name << std::endl;
266 template <
class HIST>
268 static_cast<HIST*
>(
a)->Add(
static_cast<const HIST*
>(
b));
273 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
274 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
276 std::cout <<
"ERROR in weightedAverageTH1: Object not of type TH1" << std::endl;
278 if (b1->GetEntries()==0)
return;
285 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
286 const TH1* b1 = (
dynamic_cast<const TH1*
>(
b));
288 std::cout <<
"ERROR in weightedEff: Object not of type TH1" << std::endl;
290 if (b1->GetEntries()==0)
return;
297 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
298 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
300 std::cout <<
"ERROR in mergeRMS: Object not of type TH1" << std::endl;
302 if (b1->GetEntries()==0)
return;
309 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
310 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
312 std::cout <<
"ERROR in RMSpercentDeviation: Object not of type TH1" << std::endl;
314 if (b1->GetEntries()==0)
return;
321 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
322 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
324 std::cout <<
"ERROR in getBinEffPerCent: Object not of type TH1" << std::endl;
326 if (b1->GetEntries()==0)
return;
333 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
334 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
336 std::cout <<
"ERROR in lowerLB: Object not of type TH1" << std::endl;
342 template <
class HIST>
344 if (!s_checkEquality)
346 HIST* a1 = (
dynamic_cast<HIST*
>(
a));
347 const HIST* b1 =
dynamic_cast<const HIST*
>(
b);
349 std::cout <<
"ERROR in identical: Object not of correct type" << std::endl;
356 void merge_rebinned(TObject *
a,
const TObject*
b) {
357 TH1* a1 = (
dynamic_cast<TH1*
>(
a));
358 const TH1* b1 =
dynamic_cast<const TH1*
>(
b);
360 std::cout <<
"ERROR, in merge_rebinned: Object not of type TH1";
363 TH1* b2 =
const_cast<TH1*
>(b1);
368 void merge_eventSample(TObject *
a,
const TObject*
b) {
369 TH2* a1 = (
dynamic_cast<TH2*
>(
a));
370 const TH2* b1 =
dynamic_cast<const TH2*
>(
b);
372 std::cout <<
"ERROR in merge_eventSample: Object not of type TH2" << std::endl;
378 void merge_TEfficency(TObject *
a,
const TObject*
b) {
379 TEfficiency* a1 =
dynamic_cast<TEfficiency*
>(
a);
380 const TEfficiency* b1 =
dynamic_cast<const TEfficiency*
>(
b);
381 TEfficiency* b2 =
const_cast<TEfficiency*
>(b1);
383 std::cout <<
"ERROR in merge_TEfficiency: Object not of type TEfficiency" << std::endl;
392 void merge_TTree(TObject *
a,
const TObject*
b) {
393 TTree* a1 =
dynamic_cast<TTree*
>(
a);
394 const TTree* b1 =
dynamic_cast<const TTree*
>(
b);
396 std::cout <<
"ERROR in merge_TTree: Object not of type TTree" << std::endl;
399 TTree* b2 =
const_cast<TTree*
>(b1);
407 :
name(nameIn),
obj(std::move(objIn)), mergeMethod(nullptr) {
410 std::cout <<
"ERROR while adding " << nameIn <<
": Histogram pointer is NULL" << std::endl;
418 s_dbg(
VERBOSE,
"No matadata found for " +
name +
", use defaults");
420 const std::string& howToMerge =
metadata[2];
421 s_dbg(
VERBOSE,
"Name: " +
name +
" mergeMethod=" + howToMerge);
423 TH1* th1 =
dynamic_cast<TH1*
>(
obj.get());
424 TH2* th2 =
dynamic_cast<TH2*
>(
obj.get());
425 TEfficiency* teff =
dynamic_cast<TEfficiency*
>(
obj.get());
427 th1->SetDirectory(
nullptr);
428 if (howToMerge ==
"<default>") {
430 mergeMethod = &defaultMerge<TH2>;
432 mergeMethod = &defaultMerge<TH1>;
435 else if (howToMerge ==
"weightedAverage" || howToMerge==
"weightedAverage2D")
436 mergeMethod = &weightedAverage;
437 else if (howToMerge ==
"weightedEff")
438 mergeMethod = &weightedEff;
439 else if (howToMerge ==
"mergeRMS")
440 mergeMethod = &mergeRMS;
441 else if (howToMerge ==
"RMSpercentDeviation")
442 mergeMethod = &RMSpercentDeviation;
443 else if (howToMerge ==
"perBinEffPerCent")
444 mergeMethod = &perBinEffPerCent;
445 else if (howToMerge ==
"lowerLB")
447 else if (howToMerge ==
"identical")
449 mergeMethod = &identical<TH2>;
451 mergeMethod = &identical<TH1>;
453 else if ((howToMerge ==
"mergeRebinned") || (howToMerge ==
"merge"))
454 mergeMethod = &merge_rebinned;
456 std::cout <<
"ERROR: Unknown merging method (" << howToMerge <<
") for object of type TH1 named " << nameIn << std::endl;
461 teff->SetDirectory(
nullptr);
462 if (howToMerge ==
"<default>")
463 mergeMethod = &merge_TEfficency;
465 std::cout <<
"ERROR: Unknown merging method (" << howToMerge <<
") for object of type TEfficiency named " << nameIn << std::endl;
467 else if (
nullptr !=
dynamic_cast<TTree*
>(
obj.get())) {
468 mergeMethod = &merge_TTree;
470 std::cout <<
"ERROR Object " <<
name <<
" has unkown type" << std::endl;
478 if (m_dirExclusion && !std::regex_search(
dirName, *m_dirExclusion)) {
483 for (TObject* oKey : *
dir->GetListOfKeys()) {
484 TKey*
key =
static_cast<TKey*
>(oKey);
485 const std::string
name =
key->GetName();
486 const std::string classname =
key->GetClassName();
487 if ((classname ==
"TTree") && (
name ==
"metadata")) {
491 s_dbg(
VERBOSE,
"Found name " +
name +
", classname=" + classname);
496 if (classname.starts_with(
"TH") || classname.starts_with(
"TProfile") || classname.starts_with(
"TEfficiency") || classname ==
"TTree") {
497 if (m_histExclusion && !std::regex_search(
name, *m_histExclusion)) {
498 s_dbg(
DEBUG,
"Histogram with name " +
name +
" is excluded");
503 if (m_skipExisting) {
505 std::unique_ptr<TObject> existingObj(m_out->Get(
newName.c_str()));
510 std::unique_ptr<TTree> md;
511 if (itDir ==
m_data.end()) {
518 auto itH = itDir->second.histos.find(
name);
519 if (itH == itDir->second.histos.end()) {
523 md.reset((TTree*)
dir->Get(
"metadata"));
526 std::unique_ptr<TObject>
obj{
key->ReadObj()};
527 TTree* treeObj =
dynamic_cast<TTree*
>(
obj.get());
529 TDirectory* outDir = m_out->GetDirectory(
dirName.c_str());
531 outDir = m_out->mkdir(
dirName.c_str());
535 TTree* cloneTree = treeObj->CloneTree();
537 obj.reset(cloneTree);
541 itH = itDir->second.histos.emplace(
name, std::move(
histo)).first;
545 std::unique_ptr<TObject>
other(
key->ReadObj());
547 std::cout <<
"ERROR, got NULL key";
549 itH->second.merge(
other.get());
553 }
else if (classname.starts_with(
"TDirectory")) {
554 std::unique_ptr<TObject>
obj(
key->ReadObj());
555 TDirectory*
subdir =
dynamic_cast<TDirectory*
>(
obj.get());
560 if (!
name.starts_with(
"lb_") && !
name.starts_with(
"lowStat_LB")) {
568 std::cout <<
"Ignored objects '" <<
name <<
"' of type " << classname << std::endl;
576 unsigned nIgnored = 0;
580 TDirectory* histDir = m_out->GetDirectory(
fulldir.c_str());
581 if (histDir ==
nullptr) {
582 histDir = m_out->mkdir(
fulldir.c_str());
583 if (histDir ==
nullptr) {
584 std::cout <<
"ERROR, failed to create directory " <<
fulldir << std::endl;
587 s_dbg(
VERBOSE,
"Created directory " +
fulldir +
" in file " + m_out->GetName());
597 std::cout <<
"NOT writing " <<
name <<
". Invalid." << std::endl;
601 it.second.writeMD(histDir);
603 std::cout <<
"Wrote " <<
nWritten <<
" histograms to " << nDirs <<
" directories in output file " << m_out->GetName() << std::endl;
605 std::cout <<
" Omitting " << nIgnored <<
" histograms." << std::endl;
612 MonitoringFile::OutputMetadata::OutputMetadata(TTree *
metadata) : m_metadata(
metadata) {
613 makeBranch(
"Name",
"Name/C");
614 makeBranch(
"Interval",
"Interval/C");
615 makeBranch(
"TriggerChain",
"TriggerChain/C");
616 makeBranch(
"MergeMethod",
"MergeMethod/C");
619 void MonitoringFile::OutputMetadata::makeBranch(
const char* branchName,
const char* branchstr) {
620 if (!m_metadata->GetBranch(branchName)) {
621 m_metadata->Branch(branchName, (
void*)
nullptr, branchstr);
626 const std::string& theMerge) {
627 std::string
name = theName;
629 std::string
chain = theChain;
630 std::string
merge = theMerge;
631 m_metadata->SetBranchAddress(
"Name",
name.data());
632 m_metadata->SetBranchAddress(
"Interval",
interval.data());
633 m_metadata->SetBranchAddress(
"TriggerChain",
chain.data());
634 m_metadata->SetBranchAddress(
"MergeMethod",
merge.data());
638 MonitoringFile::MonitoringFile() :
m_file(0) {
639 m_fileCompressionLevel = 1;
641 MonitoringFile::clearData();
644 bool MonitoringFile::setFile(
const std::string&
fileName) {
652 MonitoringFile::MonitoringFile(
const std::string&
fileName) :
m_file(0) {
653 m_fileCompressionLevel = 1;
655 MonitoringFile::clearData();
659 MonitoringFile::~MonitoringFile() {
665 bool MonitoringFile::setHistogramRegEx(
const std::string&
re) {
666 m_mergeMatchHistoRE = checkRegEx(
re);
667 return m_mergeMatchHistoRE.has_value();
670 bool MonitoringFile::setDirectoryRegEx(
const std::string&
re) {
671 m_mergeMatchDirRE = checkRegEx(
re);
672 return m_mergeMatchDirRE.has_value();
675 void MonitoringFile::getAllDirs(DirMap_t & dirmap, TDirectory *
dir,
const std::string&
dirName) {
681 dirmap.insert(dirmapVal);
684 TIter
next(
dir->GetListOfKeys());
686 while ((
key =
dynamic_cast<TKey*
>(
next())) != 0) {
688 TObject*
obj =
key->ReadObj();
689 TDirectory*
subdir =
dynamic_cast<TDirectory*
>(
obj);
691 std::string subdirName(
subdir->GetName());
692 std::string
fName(
"");
705 TDirectory* MonitoringFile::createDir(DirMap_t & dirmap, TDirectory *
dir,
const std::string&
parent,
const std::string&
path) {
710 DirMap_t::const_iterator diter;
711 std::string::size_type
i =
path.find_first_of(
'/');
712 std::string
fName(
"");
718 if (
i != std::string::npos) {
719 std::string dName(
path, 0,
i);
720 std::string pName(
path,
i + 1, std::string::npos);
723 diter = dirmap.find(
fName);
724 if (diter != dirmap.end()) {
729 dirmap.insert(dirmapVal);
739 diter = dirmap.find(
fName);
740 if (diter != dirmap.end()) {
741 return diter->second;
746 dirmap.insert(dirmapVal);
756 std::string::size_type
i =
path.find_first_of(
'/');
757 if (
i != std::string::npos) {
758 std::string dName(
path, 0,
i);
759 std::string pName(
path,
i + 1, std::string::npos);
761 key =
dir->FindKey(dName.c_str());
763 TDirectory* subDir =
dynamic_cast<TDirectory*
>(
key->ReadObj());
773 return dir->FindKey(
path.c_str());
776 void MonitoringFile::fillMetaDataMap(std::map<std::string, dqutils::MonitoringFile::MetaData> & mdMap, TDirectory *
dir) {
779 TTree* md =
dynamic_cast<TTree*
>(
dir->Get(
"metadata"));
784 TTreeReaderArray<char> i_name(
reader,
"Name");
785 TTreeReaderArray<char> i_interval(
reader,
"Interval");
786 TTreeReaderArray<char> i_chain(
reader,
"TriggerChain");
787 TTreeReaderArray<char> i_merge(
reader,
"MergeMethod");
790 const std::string nameStr(
static_cast<char*
>(i_name.GetAddress()));
791 if (mdMap.find(nameStr) == mdMap.end()) {
792 MetaData md(nameStr,
static_cast<char*
>(i_interval.GetAddress()),
static_cast<char*
>(i_chain.GetAddress()),
static_cast<char*
>(i_merge.GetAddress()));
793 std::map<std::string, MetaData>::value_type mdVal(nameStr, md);
802 std::cout <<
"Writing file: " <<
outFileName << std::endl;
803 std::cout <<
"Start merging [" <<
files.size() <<
"] histogram files" << std::endl;
805 TH1::AddDirectory(
false);
806 if (m_mergeMatchDirRE.has_value() || m_mergeMatchHistoRE.has_value()) {
807 std::cout <<
" ========== Using regular expressions for selective merging ========== " << std::endl;
810 std::cout <<
"CPU time measurement activated " << std::endl;
819 std::cout <<
"Got exactly one input file. Will copy input -> output" << std::endl;
820 if (m_mergeMatchDirRE.has_value() || m_mergeMatchHistoRE.has_value()) {
821 std::cout <<
"regular expressions for selective merging will have no effect!" << std::endl;
826 std::filesystem::copy_file(
inPath,
outPath, std::filesystem::copy_options::overwrite_existing);
832 std::cout <<
" ERROR, cound not open output file " <<
outFileName << std::endl;
835 std::cout <<
"Opened/created output file " <<
outFileName << std::endl;
838 auto TFiledel = [](TFile*
f) {
845 hc.addDirExclusion(m_mergeMatchDirRE);
846 hc.addHistExclusion(m_mergeMatchHistoRE);
849 std::unique_ptr<TFile,decltype(TFiledel)> in1(TFile::Open(
files[0].c_str()));
851 std::cout <<
"ERROR, could not open input file " <<
files[0] << std::endl;
854 std::cout <<
"Working on file 1/" <<
nFiles <<
": " <<
files[0] << std::endl;
855 std::string runDir, runDirFwd;
857 TIter
next(in1->GetListOfKeys());
859 while ((
key = (TKey*)
next())) {
860 const char*
name =
key->GetName();
861 if (std::regex_match(
name, runDirPattern)) {
862 if (runDir.size() > 0) {
863 std::cout <<
"ERROR More than one run_XXX directory found! Ignoring " <<
name << std::endl;
868 if (runDir.empty()) {
869 std::cout <<
"No run-directory found, start with '/'" << std::endl;
874 std::cout <<
"Found run directory " << runDir << std::endl;
879 TDirectory*
dir(
dynamic_cast<TDirectory*
>(in1->GetDirectory(runDir.c_str())));
881 std::cout <<
"ERROR, can't access directory " << runDir;
885 hc.addDirectory(
dir, runDirFwd,
files[0]);
890 for (
size_t i = 1;
i <
files.size(); ++
i) {
891 std::cout <<
"Working on file " << 1+
i <<
"/" <<
nFiles <<
": " <<
files[
i] << std::endl;
892 std::unique_ptr<TFile,decltype(TFiledel)> in(TFile::Open(
files[
i].c_str()));
894 std::cout <<
"ERROR, could not open input file " <<
files[
i] << std::endl;
897 TDirectory*
dir(
dynamic_cast<TDirectory*
>(in->GetDirectory(runDir.c_str())));
899 std::cout <<
"ERROR, could not cast to directory" << std::endl;
902 hc.addDirectory(
dir, runDirFwd,
files[
i]);
905 std::cout <<
"Accumulated a total of " << hc.size() <<
" histograms." << std::endl;
907 std::cout <<
"Start writing output ..." << std::endl;
911 std::cout <<
"CPU time for histogram merging: (regular histograms)" << std::endl;
914 const auto lbmap = hc.getFileLBMapAndClear();
915 if (!lbmap.empty()) {
916 std::cout <<
"Start merging lb_nnn and lowStat_LB directories (" << lbmap.size() <<
" in total)" << std::endl;
918 hclb.addDirExclusion(m_mergeMatchDirRE);
919 hclb.addHistExclusion(m_mergeMatchHistoRE);
923 std::vector<std::pair<std::string, std::vector<std::string>>> lbToFiles;
924 for (
const auto&
val : lbmap) {
925 if (
val.second.size() > 0)
926 lbToFiles.emplace_back(
val);
930 std::sort(lbToFiles.begin(), lbToFiles.end(),
931 [](
const decltype(lbToFiles)::
value_type&
a,
const decltype(lbToFiles)::
value_type&
b) {
return a.second[0] <
b.second[0]; });
934 std::unique_ptr<TFile, decltype(TFiledel)> in;
936 std::cout <<
"Merging/copying directory " <<
dir <<
" from " <<
filenames.size() <<
"input file(s) (" << ++
counter <<
"/" << lbToFiles.size() <<
")"
939 if (!in || strcmp(in->GetName(),
fName.c_str()) != 0) {
940 in.reset(TFile::Open(
fName.c_str()));
943 s_dbg(
DEBUG,
"Input file " +
fName +
" already open");
947 std::cout <<
"ERROR, could not open input file " <<
fName << std::endl;
950 TDirectory* tDir = (
dynamic_cast<TDirectory*
>(in->Get(
dir.c_str())));
952 std::cout <<
"ERROR, failed to get directory " <<
dir <<
" from file " <<
fName << std::endl;
954 hclb.addDirectory(tDir,
dir);
959 std::cout <<
"CPU time for histogram merging: (lumiblock-histograms)" << std::endl;
971 typedef std::vector<std::string> FileList_t;
973 const unsigned int nFilesAtOnce = 50;
976 bool success = setListFromFile(
allFiles, listFileName);
978 std::cout <<
"ERROR Failed ot read list of input files" << std::endl;
982 if (
allFiles.size() <= nFilesAtOnce) {
986 FileList_t procFiles, tmpIntermediateFiles;
988 FileList_t::const_iterator filesEnd =
allFiles.end();
989 FileList_t::const_iterator fi =
allFiles.begin();
992 std::string tmpInputFile(
"");
993 std::string tmpOutputFile(
"");
996 while (fi != filesEnd) {
998 procFiles.push_back(*fi);
1001 if (
counter % nFilesAtOnce == 0 || fi == filesEnd) {
1002 std::ostringstream nameStream;
1003 nameStream <<
"tmp_merge_" <<
counter <<
".root";
1004 tmpOutputFile = nameStream.str();
1005 tmpIntermediateFiles.push_back(tmpOutputFile);
1015 for (
const auto& tmpFile : tmpIntermediateFiles) {
1021 void MonitoringFile::printStatistics() {
1023 std::cerr <<
"MonitoringFile::printStatistics(): "
1024 <<
"No input file is open\n";
1030 getAllDirs(indirmap,
m_file,
"");
1032 DirMap_t::const_iterator idirend = indirmap.end();
1033 for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1034 std::string idirName = idir->first;
1036 GatherStatistics stat_shift(idirName);
1037 GatherStatistics stat_all(idirName);
1039 loopOnHistogramsInMetadata(stat_shift, idir->second);
1040 loopOnHistograms(stat_all, idir->second);
1042 std::cout.setf(std::ios_base::left, std::ios_base::adjustfield);
1043 std::cout.width(80);
1044 std::cout << idirName <<
" ";
1046 std::cout.setf(std::ios_base::right, std::ios_base::adjustfield);
1047 std::cout <<
" shift: ";
1049 std::cout << stat_shift.m_nHist1D <<
" ";
1051 std::cout << stat_shift.m_nHist1DBins <<
" ";
1053 std::cout << stat_shift.m_nHist2D <<
" ";
1055 std::cout << stat_shift.m_nHist2DBins <<
" ";
1057 std::cout << stat_shift.m_nGraph <<
" ";
1059 std::cout << stat_shift.m_nGraphPoints <<
" ";
1061 std::cout <<
" all: ";
1062 std::cout << stat_all.m_nHist1D <<
" ";
1064 std::cout << stat_all.m_nHist1DBins <<
" ";
1066 std::cout << stat_all.m_nHist2D <<
" ";
1068 std::cout << stat_all.m_nHist2DBins <<
" ";
1070 std::cout << stat_all.m_nGraph <<
" ";
1072 std::cout << stat_all.m_nGraphPoints <<
"\n";
1078 bool MonitoringFile::copyHistograms(
const std::string&
outFileName,
const std::string&
dirName) {
1084 std::cerr <<
"MonitoringFile::copyHistograms(): "
1085 <<
"No input file is open\n";
1090 DirMap_t reducedmap;
1096 std::cerr <<
"MonitoringFile::copyHistograms(): "
1097 <<
"Directory \'" <<
dirName <<
"\' not found in input file\n";
1101 TDirectory* fromDir =
dynamic_cast<TDirectory*
>(dkey->ReadObj());
1103 DirMap_t::value_type dirmapVal(
dirName, fromDir);
1104 indirmap.insert(dirmapVal);
1106 std::cout <<
"Building list of all TDirectories in file...\n" <<
std::flush;
1107 getAllDirs(indirmap,
m_file,
"");
1110 DirMap_t::const_iterator idirend = indirmap.end();
1111 for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1113 std::string idirName = idir->first;
1114 std::cout <<
"Checking " << idirName <<
"\n" <<
std::flush;
1122 if (!dirHasHistogramsInMetadata(idir->second)) {
1126 reducedmap.insert(*idir);
1131 std::cerr <<
"MonitoringFile::copyHistograms(): "
1132 <<
"Output file not opened\n";
1136 idirend = reducedmap.end();
1137 for (DirMap_t::const_iterator idir = reducedmap.begin(); idir != idirend; ++idir) {
1139 std::string idirName = idir->first;
1140 std::cout <<
"Processing " << idirName <<
"\n" <<
std::flush;
1142 TDirectory* toDir = createDir(outdirmap,
outfile.get(),
"", idirName);
1144 std::cerr <<
"MonitoringFile::copyHistograms(): "
1145 <<
"Directory \'" << idirName <<
"\' not created in output file\n";
1149 CopyHistogram copyFcn(toDir, idirName);
1151 loopOnHistogramsInMetadata(copyFcn, idir->second);
1162 const std::string& hcfg_lowStat,
const std::string& hcfg_medStat) {
1165 std::cout <<
"\nUsing han configurations:\n"
1166 <<
" entire run: " <<
hcfg <<
"\n"
1167 <<
" low stat interval: " << hcfg_lowStat <<
"\n"
1168 <<
" medium stat interval: " << hcfg_medStat <<
"\n\n"
1173 std::cerr <<
"MonitoringFile::getHanResults(): "
1174 <<
"Cannot open input file \"" <<
input <<
"\"\n";
1178 std::vector<std::string> run_dirs;
1179 std::vector<std::string> lowStat_dirs;
1180 std::vector<std::string> medStat_dirs;
1182 TIter next_run(
infile->GetListOfKeys());
1184 while ((key_run =
dynamic_cast<TKey*
>(next_run())) != 0) {
1185 TObject* obj_run = key_run->ReadObj();
1186 TDirectory* tdir_run =
dynamic_cast<TDirectory*
>(obj_run);
1187 if (tdir_run != 0) {
1188 std::string tdir_run_name(tdir_run->GetName());
1189 if (tdir_run_name.find(
"run") != std::string::npos) {
1190 run_dirs.push_back(tdir_run_name);
1191 TIter next_minutes(tdir_run->GetListOfKeys());
1192 TKey* key_minutes(0);
1193 while ((key_minutes =
dynamic_cast<TKey*
>(next_minutes())) != 0) {
1194 TObject* obj_minutes = key_minutes->ReadObj();
1195 TDirectory* tdir_minutes =
dynamic_cast<TDirectory*
>(obj_minutes);
1196 if (tdir_minutes != 0) {
1197 std::string tdir_minutes_name(tdir_minutes->GetName());
1198 if (tdir_minutes_name.find(
"lowStat") != std::string::npos) {
1199 lowStat_dirs.push_back(tdir_run_name +
'/' + tdir_minutes_name);
1200 }
else if (tdir_minutes_name.find(
"medStat") != std::string::npos) {
1201 medStat_dirs.push_back(tdir_run_name +
'/' + tdir_minutes_name);
1216 std::vector<std::string>::const_iterator dirs_end;
1217 std::vector<std::string>::const_iterator
dir;
1219 dirs_end = run_dirs.end();
1220 for (
dir = run_dirs.begin();
dir != dirs_end; ++
dir) {
1221 const std::string& tdir_run_name = *
dir;
1222 std::string han_output_run =
hanResultsDir +
'/' + tdir_run_name +
"_han.root";
1223 std::cout <<
"Calling han( " <<
hcfg <<
", " <<
input <<
", " << tdir_run_name <<
", " << han_output_run <<
" ):\n" <<
std::flush;
1224 han.Analyze(
hcfg,
input, han_output_run, tdir_run_name);
1226 fileList += han_output_run +
" " + tdir_run_name +
"\n";
1229 dirs_end = lowStat_dirs.end();
1230 for (
dir = lowStat_dirs.begin();
dir != dirs_end; ++
dir) {
1231 const std::string& tdir_minutes_path = *
dir;
1233 std::string tdir_minutes_underscore = tdir_minutes_path;
1234 std::string::size_type tdir_minutes_i = tdir_minutes_underscore.find(
'/');
1235 tdir_minutes_underscore.replace(tdir_minutes_i, 1,
"_");
1237 std::string han_output_lowStat =
hanResultsDir +
'/' + tdir_minutes_underscore +
"_han.root";
1238 std::cout <<
"Running han, writing to " << han_output_lowStat <<
":\n" <<
std::flush;
1239 han.Analyze(hcfg_lowStat,
input, han_output_lowStat, tdir_minutes_path);
1241 std::string subdirname(tdir_minutes_path, tdir_minutes_i + 1, std::string::npos);
1242 std::string
dirname(tdir_minutes_path, 0, tdir_minutes_i);
1243 fileList += han_output_lowStat +
" " + subdirname +
" " +
dirname +
" " + subdirname +
"\n";
1246 dirs_end = medStat_dirs.end();
1247 for (
dir = medStat_dirs.begin();
dir != dirs_end; ++
dir) {
1248 const std::string& tdir_minutes_path = *
dir;
1250 std::string tdir_minutes_underscore = tdir_minutes_path;
1251 std::string::size_type tdir_minutes_i = tdir_minutes_underscore.find(
'/');
1252 tdir_minutes_underscore.replace(tdir_minutes_i, 1,
"_");
1254 std::string han_output_medStat =
hanResultsDir +
'/' + tdir_minutes_underscore +
"_han.root";
1255 std::cout <<
"Running han, writing to " << han_output_medStat <<
":\n" <<
std::flush;
1256 han.Analyze(hcfg_medStat,
input, han_output_medStat, tdir_minutes_path);
1258 std::string subdirname(tdir_minutes_path, tdir_minutes_i + 1, std::string::npos);
1259 std::string
dirname(tdir_minutes_path, 0, tdir_minutes_i);
1260 fileList += han_output_medStat +
" " + subdirname +
" " +
dirname +
" " + subdirname +
"\n";
1266 void MonitoringFile::printHanConfig() {
1268 std::cerr <<
"MonitoringFile::printHanConfig(): "
1269 <<
"No input file is open\n";
1275 getAllDirs(indirmap,
m_file,
"");
1277 std::string
indent, indent_p, indent_c;
1278 std::string idirName_p;
1279 DirMap_t::const_iterator idirend = indirmap.end();
1280 for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1281 std::string idirName = idir->first;
1282 std::string::size_type shortNameIndex = idirName.rfind(
'/');
1283 std::string
shortName = idirName.substr(shortNameIndex + 1, std::string::npos);
1285 std::string::size_type fsIndex = idirName.find(
'/');
1286 std::string shortPath;
1287 if (fsIndex != shortNameIndex)
1288 shortPath = idirName.substr(fsIndex + 1, shortNameIndex);
1290 shortPath = idirName.substr(fsIndex + 1, std::string::npos);
1292 std::cout << idirName <<
"\n";
1293 std::cout << shortPath <<
", " <<
shortName <<
"\n";
1331 std::string MonitoringFile::getIndentation(
const std::string&
pathName,
const std::string& leadingSpace) {
1332 std::string space = leadingSpace;
1333 std::string::size_type
i =
pathName.find_first_of(
'/');
1334 if (
i != std::string::npos) {
1335 std::string subPath(
pathName,
i + 1, std::string::npos);
1337 return getIndentation(subPath, space);
1342 std::string MonitoringFile::FindCommon(
const std::string&
name1,
const std::string& name2)
const {
1360 MonitoringFile::CopyHistogram::CopyHistogram(TDirectory *
target,
const std::string&
dirName) : m_target(
target), m_dirName(
dirName), m_metadata(0) {
1361 m_metadata =
new TTree(
"metadata",
"Monitoring Metadata");
1362 m_metadata->SetDirectory(0);
1363 m_metadata->Branch(
"Name", (
void*)
nullptr,
"Name/C");
1364 m_metadata->Branch(
"Interval", (
void*)
nullptr,
"Interval/C");
1365 m_metadata->Branch(
"TriggerChain", (
void*)
nullptr,
"TriggerChain/C");
1366 m_metadata->Branch(
"MergeMethod", (
void*)
nullptr,
"MergeMethod/C");
1369 MonitoringFile::CopyHistogram::~CopyHistogram() {
1371 m_metadata->SetDirectory(m_target);
1372 m_metadata->Write();
1378 hist->SetDirectory(m_target);
1397 void MonitoringFile::CopyHistogram::fillMD(
const MetaData& md) {
1398 std::string
name(md.name);
1400 std::string
chain(md.chain);
1401 std::string
merge(md.merge);
1402 m_metadata->SetBranchAddress(
"Name",
name.data());
1403 m_metadata->SetBranchAddress(
"Interval",
interval.data());
1404 m_metadata->SetBranchAddress(
"TriggerChain",
chain.data());
1405 m_metadata->SetBranchAddress(
"MergeMethod",
merge.data());
1409 bool MonitoringFile::CopyHistogram::executeMD(TH1 *
hist,
const MetaData& md) {
1411 hist->SetDirectory(m_target);
1419 bool MonitoringFile::CopyHistogram::executeMD(TGraph * graph,
const MetaData& md) {
1428 bool MonitoringFile::CopyHistogram::executeMD(TEfficiency *
eff,
const MetaData& md) {
1435 MonitoringFile::GatherStatistics::GatherStatistics(
const std::string&
dirName)
1436 : m_dirName(
dirName), m_nHist1D(0), m_nHist1DBins(0), m_nGraph(0), m_nGraphPoints(0), m_nHist2D(0), m_nHist2DBins(0) {}
1439 TH2* hist2d =
dynamic_cast<TH2*
>(
hist);
1442 m_nHist2DBins += (hist2d->GetNbinsX() * hist2d->GetNbinsY());
1446 m_nHist1DBins +=
hist->GetNbinsX();
1452 m_nGraphPoints += graph->GetMaxSize();
1459 TH1* h_total =
eff->GetCopyPassedHisto();
1460 TH2* h_total2D =
dynamic_cast<TH2*
>(h_total);
1462 if (h_total2D != 0) {
1463 m_nEfficiencyBins += (h_total2D->GetNbinsX() * h_total2D->GetNbinsY());
1466 m_nEfficiencyBins += h_total->GetNbinsX();
1471 MonitoringFile::GatherNames::GatherNames() {}
1474 m_names.push_back(std::string(
hist->GetName()));
1479 m_names.push_back(std::string(graph->GetName()));
1484 m_names.push_back(std::string(
eff->GetName()));
1488 void MonitoringFile::clearData() {
1493 m_fileCompressionLevel = 1;
1497 bool MonitoringFile::dirHasHistogramsInMetadata(TDirectory *
dir) {
1500 TKey* mdKey =
dir->FindKey(
"metadata");
1505 TTree* md =
dynamic_cast<TTree*
>(mdKey->ReadObj());
1517 std::cerr <<
"Exception: \"" <<
e.what() <<
"\" in directory \"" <<
dir->GetName() <<
"\"\n" <<
std::flush;
1528 void MonitoringFile::loopOnHistograms(HistogramOperation & fcn, TDirectory *
dir) {
1529 TIter
next(
dir->GetListOfKeys());
1531 while ((
key =
dynamic_cast<TKey*
>(
next())) != 0) {
1532 TObject*
obj =
key->ReadObj();
1536 if ((
h =
dynamic_cast<TH1*
>(
obj))) {
1538 }
else if ((
g =
dynamic_cast<TGraph*
>(
obj))) {
1540 }
else if ((
e =
dynamic_cast<TEfficiency*
>(
obj))) {
1547 bool MonitoringFile::loopOnHistogramsInMetadata(HistogramOperation & fcn, TDirectory *
dir) {
1549 TKey* mdKey =
dir->FindKey(
"metadata");
1554 TTree* md =
dynamic_cast<TTree*
>(mdKey->ReadObj());
1562 TTreeReaderArray<char> i_name(
reader,
"Name");
1563 TTreeReaderArray<char> i_interval(
reader,
"Interval");
1564 TTreeReaderArray<char> i_chain(
reader,
"TriggerChain");
1565 TTreeReaderArray<char> i_merge(
reader,
"MergeMethod");
1568 const std::string nameStr(
static_cast<char*
>(i_name.GetAddress()));
1570 i_key =
dir->FindKey(
static_cast<char*
>(i_name.GetAddress()));
1572 std::cerr <<
"MonitoringFile::loopOnHistogramsInMetadata(): "
1573 <<
"No \'" << nameStr <<
"\' object found\n";
1576 MetaData md(nameStr,
static_cast<char*
>(i_interval.GetAddress()),
static_cast<char*
>(i_chain.GetAddress()),
static_cast<char*
>(i_merge.GetAddress()));
1577 TObject*
obj = i_key->ReadObj();
1578 TH1*
h =
dynamic_cast<TH1*
>(
obj);
1580 fcn.executeMD(
h, md);
1582 TGraph*
g =
dynamic_cast<TGraph*
>(
obj);
1584 fcn.executeMD(
g, md);
1595 bool MonitoringFile::setListFromFile(std::vector<std::string> &
filelist,
const std::string& listFileName) {
1596 using namespace std;
1600 ifstream listfile(listFileName.c_str());
1602 cerr <<
"MonitoringFile::setListFromFile(): "
1603 <<
"cannot read from file: " << listFileName <<
"\n";
1610 while (getline(listfile,
line)) {
1611 istringstream linestream(
line);
1612 while (linestream.get(
c)) {
1619 linestream.putback(
c);
1622 cerr <<
"MonitoringFile::setListFromFile(): "
1623 <<
"badly formatted line: " <<
line <<
"\n";
1635 int MonitoringFile::mergeLBintervals(
const std::string& inFilename) {
1637 std::cout <<
"Running mergeLBintervals on " << inFilename << std::endl;
1639 std::unique_ptr<TFile>
f(TFile::Open(inFilename.c_str(),
"UPDATE"));
1641 std::cout <<
"ERROR, could not open file " << inFilename <<
" for update" << std::endl;
1644 std::string runDirName;
1645 const std::regex runDirPattern(
"run_[0-9]*");
1646 TIter
next(
f->GetListOfKeys());
1648 while ((
key = (TKey*)
next())) {
1649 const char*
name =
key->GetName();
1650 if (std::regex_match(
name, runDirPattern)) {
1651 if (runDirName.size() > 0) {
1652 std::cout <<
"ERROR More than one run_XXX directory found! Ignoring " <<
name << std::endl;
1659 TDirectory* runDir =
f->GetDirectory(runDirName.c_str());
1660 const auto mapping = buildLBToIntervalMap(runDir);
1662 if (s_dbg.getLvl() ==
VERBOSE) {
1663 std::cout <<
"LB directory mapping:" << std::endl;
1664 for (
const auto& i1 :
mapping) {
1665 std::cout << i1.first;
1666 for (
const auto& i2 : i1.second) {
1667 std::cout <<
"\t" << i2 << std::endl;
1672 for (
const auto& [outDir, inDIrs] :
mapping) {
1673 int stat=mergeLB_processLBinterval(
f.get(), inDIrs, outDir);
1678 f.reset(TFile::Open(inFilename.c_str(),
"UPDATE"));
1679 runDir =
f->GetDirectory(runDirName.c_str());
1681 std::cout <<
"merging lowStat_LB dirs into run-dir" << std::endl;
1682 std::vector<std::string> lowStatDirs;
1683 for (TObject* oKey : *runDir->GetListOfKeys()) {
1684 TKey*
key =
static_cast<TKey*
>(oKey);
1685 const std::string
name =
key->GetName();
1686 const std::string classname =
key->GetClassName();
1687 if (classname.starts_with(
"TDirectory") and
name.starts_with(
"lowStat_LB")) {
1688 lowStatDirs.push_back(runDirName +
"/" +
name);
1689 s_dbg(
VERBOSE,
"Found input: " + runDirName +
"/" +
name);
1693 int stat=mergeLB_processLBinterval(
f.get(), lowStatDirs, runDirName);
1699 std::map<std::string, std::vector<std::string>> MonitoringFile::buildLBToIntervalMap(TDirectory * runDir) {
1701 std::map<std::string, std::vector<std::string>>
ranges;
1704 const std::string runDirName = runDir->GetName();
1705 for (TObject* oKey : *runDir->GetListOfKeys()) {
1706 TKey*
key =
static_cast<TKey*
>(oKey);
1707 const std::string
name =
key->GetName();
1708 const std::string classname =
key->GetClassName();
1709 if (!classname.starts_with(
"TDirectory"))
1711 if (
name.starts_with(
"lb_")) {
1715 }
catch (std::invalid_argument&
e) {
1716 std::cout <<
"ERROR, unexpected directory name " <<
name <<
". Can't parse lb number" << std::endl;
1717 std::cout <<
e.what() << std::endl;
1723 ranges[lbString].push_back(runDirName +
"/" +
name);
1729 int MonitoringFile::mergeLB_processLBinterval(TFile *
file,
const std::vector<std::string>& inputDirNames,
const std::string& outputDirName) {
1731 TDirectory* outDir =
file->GetDirectory(outputDirName.c_str());
1733 outDir =
file->mkdir(outputDirName.c_str());
1736 std::cout <<
"ERROR, can't obtain nor create directory " << outputDirName <<
" in file " <<
file->GetName() << std::endl;
1741 hc.addDirExclusion(m_mergeMatchDirRE);
1742 hc.addHistExclusion(m_mergeMatchHistoRE);
1744 for (
const std::string& inDirName : inputDirNames) {
1745 TDirectory* inDir =
file->GetDirectory(inDirName.c_str());
1746 hc.addDirectory(inDir, outputDirName);
1748 if (hc.size() == 0) {
1749 std::cout <<
"mergeLB_processLBinterval: No new objects found for " << outputDirName << std::endl;
1756 bool MonitoringFile::CheckHistogram(TFile *
f,
const char* HistoName) {
1757 std::unique_ptr<TObject>
obj(
f->Get(HistoName));
1765 int MonitoringFile::getDebugLevel() {
1766 return s_dbg.getLvl();
1768 void MonitoringFile::setDebugLevel(
int level) {
1776 void MonitoringFile::setCheckEquality(
bool value) {dqutils::s_checkEquality=
value;}
1777 std::atomic<int> MonitoringFile::m_fileCompressionLevel = 1;
1778 bool MonitoringFile::m_doTiming =
false;
1779 std::unordered_map<std::string, std::clock_t> MonitoringFile::m_cpuPerHistogram;
1781 std::string MonitoringFile::getPath(TDirectory *
dir) {
1783 std::string
path =
dir->GetPath();
1784 if (
path.find(
':') != std::string::npos)