ATLAS Offline Software
MonitoringFile.cxx
Go to the documentation of this file.
1 /*
2  Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
3 */
4 
6 
7 #include <TDirectory.h>
8 #include <TEfficiency.h>
9 #include <TFile.h>
10 #include <TGraph.h>
11 #include <TH1.h>
12 #include <TH2.h>
13 #include <TIterator.h>
14 #include <TKey.h>
15 #include <TObject.h>
16 #include <TROOT.h>
17 #include <TSystem.h>
18 #include <TTree.h>
19 
20 #include <cmath>
21 #include <cstdio>
22 #include <ctime>
23 #include <exception>
24 #include <filesystem>
25 #include <format>
26 #include <fstream>
27 #include <iostream>
28 #include <map>
29 #include <memory>
30 #include <set>
31 #include <sstream>
32 #include <vector>
33 
36 #include "TTreeReader.h"
37 #include "TTreeReaderArray.h"
38 
40 
42 
43 namespace dqutils {
44 
45  class dbgPrint {
46 
47  public:
48  dbgPrint(const debugLevel_t setLvl = none) : m_currLvl(setLvl) {};
49  inline void operator()(const debugLevel_t level, const std::string& msg) const {
50  if (level <= m_currLvl)
51  std::cout << msg << std::endl;
52  }
53  void setLvl(const debugLevel_t lvl) { m_currLvl = lvl; }
54  debugLevel_t getLvl() const { return m_currLvl; }
55 
56  private:
57  debugLevel_t m_currLvl;
58  };
59 
60  static dbgPrint s_dbg;
61  static bool s_checkEquality=false;
62 
63  std::optional<std::regex> checkRegEx(const std::string& re) {
64  if (re.empty())
65  return std::nullopt;
66 
67  std::regex reNew(re);
68  try {
69  // this should fail if there are any problems with re!
70  std::string test("Test String");
71  std::regex_match(test, reNew);
72  } catch (std::exception& e) {
73  std::cout << "ERROR: Invalid RegEx string \"" << re << "\"." << std::endl;
74  std::cout << "See http://www.boost.org/doc/libs/1_42_0/libs/regex/doc/html/boost_regex/syntax.html for allowed regular expression syntax" << std::endl;
75  return std::nullopt;
76  }
77  return reNew;
78  }
79 
80  // Internally used data-structures:
81 
82  class histCollection {
83  public:
84  explicit histCollection(TFile* out, bool skipExisting = false) : m_out{out}, m_skipExisting(skipExisting) {};
85  histCollection() = delete;
86 
87  ~histCollection();
88 
89  void addDirectory(TDirectory* dir, const std::string& dirName, const std::string& filename = "");
90  size_t size() { return m_data.size(); };
91  void print();
92  void write(); // Destructive method, will delete internal data after writing
93 
94  void addDirExclusion(const std::optional<std::regex>& dirEx);
95  void addHistExclusion(const std::optional<std::regex>& histEx);
96 
97  unsigned size() const;
98  void clear();
99 
100  fileLBMap_t getFileLBMapAndClear() {
101  m_data.clear();
102  return std::move(m_fileLBMap);
103  }
104 
105  void printTiming();
106 
107  private:
108  class histPerDir_t {
109  public:
110  histPerDir_t(const std::string& nameIn, std::unique_ptr<TObject>&& objIn, TTree* md);
111  histPerDir_t(dqutils::histCollection::histPerDir_t&& other)
112  : name(std::move(other.name)), obj(std::move(other.obj)), metadata(std::move(other.metadata)), mergeMethod(other.mergeMethod) {}
113 
114  std::string name;
115  std::unique_ptr<TObject> obj;
116  std::array<std::string, 3> metadata{"unset","","<default>"};
117  std::clock_t cpuSum = 0;
118  void (*mergeMethod)(TObject* a, const TObject* b) = nullptr;
119  void merge(TObject* other);
120 
121  private:
122  bool fillMD(TTree* mdTree);
123  };
124 
125  struct histDir_t {
126  std::unordered_map<std::string, histPerDir_t> histos;
127  void writeMD(TDirectory* outDir) const;
128  };
129 
130  private:
131  TFile* m_out;
132  bool m_skipExisting;
133  std::unordered_map<std::string, histDir_t> m_data;
134  std::optional<std::regex> m_dirExclusion;
135  std::optional<std::regex> m_histExclusion;
136  fileLBMap_t m_fileLBMap;
137  };
138 
139  void histCollection::clear() {
140  m_data.clear();
141  m_fileLBMap.clear();
142  }
143 
144  void histCollection::printTiming() {
145  std::vector<std::pair<std::string, clock_t>> cpuPerHistVec;
146  for (auto& [dirname, histDir] : m_data) {
147  for (auto& [histname, histo] : histDir.histos) {
148  cpuPerHistVec.emplace_back(dirname + "/" + histname, histo.cpuSum);
149  }
150  }
151  auto ordering = [](std::pair<std::string, clock_t> a, std::pair<std::string, clock_t> b) { return a.second < b.second; };
152  std::sort(cpuPerHistVec.begin(), cpuPerHistVec.end(), ordering);
153  for (const auto& [name, time] : cpuPerHistVec) {
154  const double tSec = double(time) / CLOCKS_PER_SEC;
155  std::cout << std::format("{:<30} : {:10.3f}", name, tSec) << std::endl;
156  }
157  return;
158  }
159 
160  bool histCollection::histPerDir_t::fillMD(TTree * md) {
161  TTreeReader reader(md);
162  TTreeReaderArray<char> i_name(reader, "Name");
163  TTreeReaderArray<char> i_interval(reader, "Interval");
164  TTreeReaderArray<char> i_chain(reader, "TriggerChain");
165  TTreeReaderArray<char> i_merge(reader, "MergeMethod");
166 
167  bool found = false;
168  while (reader.Next()) {
169  const std::string nameStr(static_cast<char*>(i_name.GetAddress()));
170  if (name == nameStr) {
171  metadata = {static_cast<char*>(i_interval.GetAddress()), static_cast<char*>(i_chain.GetAddress()), static_cast<char*>(i_merge.GetAddress())};
172  found = true;
173  break;
174  }
175  }
176  return found;
177  }
178 
180  if (obj && other) {
181  const std::clock_t cpuStart = std::clock();
182  this->mergeMethod(obj.get(), other);
183  cpuSum += std::clock() - cpuStart;
184  }
185  return;
186  }
187 
188  void histCollection::histDir_t::writeMD(TDirectory * out) const {
189 
190  // Check if there is already a metadata-tree. Merge content if necessary
191  std::map<std::string, std::array<std::string, 3>> metadatamap;
192  std::unique_ptr<TTree> oldMD((TTree*)out->Get("metadata"));
193  if (oldMD) {
194  TTreeReader reader(oldMD.get());
195  TTreeReaderArray<char> i_name(reader, "Name");
196  TTreeReaderArray<char> i_interval(reader, "Interval");
197  TTreeReaderArray<char> i_chain(reader, "TriggerChain");
198  TTreeReaderArray<char> i_merge(reader, "MergeMethod");
199 
200  while (reader.Next()) {
201  const std::string name(static_cast<char*>(i_name.GetAddress()));
202  metadatamap[name] = {static_cast<char*>(i_interval.GetAddress()), static_cast<char*>(i_chain.GetAddress()), static_cast<char*>(i_merge.GetAddress())};
203  }
204  }
205 
206  for (const auto& [key, h] : histos) {
207  if (h.metadata[0]!="unset") //Ignore dummy-metadata (eg HLTMon use-case)
208  metadatamap[key] = h.metadata;
209  }
210 
211  if (metadatamap.empty()) return; //Do not write empty metadata tree
212  std::string interval, chain, merge;
213  char histname[1024]; // FIXME, no idea why this works only in this old-fashioned way
214  std::unique_ptr<TTree> mdTree = std::make_unique<TTree>("metadata", "Monitoring Metadata");
215  mdTree->SetDirectory(out);
216 
217  mdTree->Branch("Name", (void*)nullptr, "Name/C");
218  mdTree->Branch("Interval", interval.data(), "Interval/C");
219  mdTree->Branch("TriggerChain", chain.data(), "TriggerChain/C");
220  mdTree->Branch("MergeMethod", merge.data(), "MergeMethod/C");
221 
222  mdTree->SetBranchAddress("Name", histname);
223 
224  for (auto& [key, h] : metadatamap) {
225  strncpy(histname, key.c_str(), 1023);
226  interval = h[0];
227  chain = h[1];
228  merge = h[2];
229  mdTree->Fill();
230  }
231  mdTree->Write(0, TObject::kOverwrite);
232  }
233 
234  histCollection::~histCollection() {}
235 
236  void histCollection::addDirExclusion(const std::optional<std::regex>& dir) {
237  m_dirExclusion = dir;
238  return;
239  }
240 
241  void histCollection::addHistExclusion(const std::optional<std::regex>& dir) {
242  m_histExclusion = dir;
243  return;
244  }
245 
246  unsigned histCollection::size() const {
247  unsigned s = 0;
248  for (const auto& it : m_data) {
249  s += it.second.histos.size();
250  }
251  return s;
252  }
253 
254  void histCollection::print() {
255  for (const auto& it : m_data) {
256  const histDir_t& hd = it.second;
257  std::cout << "Dir: " << it.first << " has " << hd.histos.size() << " histos" << std::endl;
258  for (const auto& it1 : hd.histos)
259  std::cout << "\t" << it1.second.name << std::endl;
260  }
261  return;
262  }
263 
264  template <class HIST>
265  void defaultMerge(TObject * a, const TObject* b) {
266  static_cast<HIST*>(a)->Add(static_cast<const HIST*>(b));
267  return;
268  }
269 
270  void weightedAverage ATLAS_NOT_THREAD_SAFE(TObject * a, const TObject* b) {
271  TH1* a1 = (dynamic_cast<TH1*>(a));
272  const TH1* b1 = dynamic_cast<const TH1*>(b);
273  if (!b1 || !a1)
274  std::cout << "ERROR in weightedAverageTH1: Object not of type TH1" << std::endl;
275  else {
276  if (b1->GetEntries()==0) return;
278  }
279  return;
280  }
281 
282  void weightedEff ATLAS_NOT_THREAD_SAFE(TObject * a, const TObject* b) {
283  TH1* a1 = (dynamic_cast<TH1*>(a));
284  const TH1* b1 = (dynamic_cast<const TH1*>(b));
285  if (!b1 || !a1)
286  std::cout << "ERROR in weightedEff: Object not of type TH1" << std::endl;
287  else {
288  if (b1->GetEntries()==0) return;
290  }
291  return;
292  }
293 
294  void mergeRMS ATLAS_NOT_THREAD_SAFE(TObject * a, const TObject* b) {
295  TH1* a1 = (dynamic_cast<TH1*>(a));
296  const TH1* b1 = dynamic_cast<const TH1*>(b);
297  if (!b1 || !a1)
298  std::cout << "ERROR in mergeRMS: Object not of type TH1" << std::endl;
299  else {
300  if (b1->GetEntries()==0) return;
302  }
303  return;
304  }
305 
306  void RMSpercentDeviation ATLAS_NOT_THREAD_SAFE(TObject * a, const TObject* b) {
307  TH1* a1 = (dynamic_cast<TH1*>(a));
308  const TH1* b1 = dynamic_cast<const TH1*>(b);
309  if (!b1 || !a1)
310  std::cout << "ERROR in RMSpercentDeviation: Object not of type TH1" << std::endl;
311  else {
312  if (b1->GetEntries()==0) return;
314  }
315  return;
316  }
317 
318  void perBinEffPerCent ATLAS_NOT_THREAD_SAFE(TObject * a, const TObject* b) {
319  TH1* a1 = (dynamic_cast<TH1*>(a));
320  const TH1* b1 = dynamic_cast<const TH1*>(b);
321  if (!b1 || !a1)
322  std::cout << "ERROR in getBinEffPerCent: Object not of type TH1" << std::endl;
323  else {
324  if (b1->GetEntries()==0) return;
326  }
327  return;
328  }
329 
330  void lowerLB ATLAS_NOT_THREAD_SAFE(TObject * a, const TObject* b) {
331  TH1* a1 = (dynamic_cast<TH1*>(a));
332  const TH1* b1 = dynamic_cast<const TH1*>(b);
333  if (!b1 || !a1)
334  std::cout << "ERROR in lowerLB: Object not of type TH1" << std::endl;
335  else
337  return;
338  }
339 
340  template <class HIST>
341  void identical(TObject * a, const TObject* b) {
342  if (!s_checkEquality)
343  return; //quasi null-operation
344  HIST* a1 = (dynamic_cast<HIST*>(a));
345  const HIST* b1 = dynamic_cast<const HIST*>(b);
346  if (!b1 || !a1){
347  std::cout << "ERROR in identical: Object not of correct type" << std::endl;
348  return;
349  }
351  return;
352  }
353 
354  void merge_rebinned(TObject * a, const TObject* b) {
355  TH1* a1 = (dynamic_cast<TH1*>(a));
356  const TH1* b1 = dynamic_cast<const TH1*>(b);
357  if (!a1 || !b1) {
358  std::cout << "ERROR, in merge_rebinned: Object not of type TH1";
359  return;
360  }
361  TH1* b2 = const_cast<TH1*>(b1);
363  return;
364  }
365 
366  void merge_eventSample(TObject * a, const TObject* b) {
367  TH2* a1 = (dynamic_cast<TH2*>(a));
368  const TH2* b1 = dynamic_cast<const TH2*>(b);
369  if (!a1 || !b1) {
370  std::cout << "ERROR in merge_eventSample: Object not of type TH2" << std::endl;
371  return;
372  }
374  }
375 
376  void merge_TEfficency(TObject * a, const TObject* b) {
377  TEfficiency* a1 = dynamic_cast<TEfficiency*>(a);
378  const TEfficiency* b1 = dynamic_cast<const TEfficiency*>(b);
379  TEfficiency* b2 = const_cast<TEfficiency*>(b1);
380  if (!a1 || !b1) {
381  std::cout << "ERROR in merge_TEfficiency: Object not of type TEfficiency" << std::endl;
382  return;
383  }
384  TList listE;
385  listE.Add(b2);
386  a1->Merge(&listE);
387  listE.Clear();
388  }
389 
390  void merge_TTree(TObject * a, const TObject* b) {
391  TTree* a1 = dynamic_cast<TTree*>(a);
392  const TTree* b1 = dynamic_cast<const TTree*>(b);
393  if (!a1 || !b1) {
394  std::cout << "ERROR in merge_TTree: Object not of type TTree" << std::endl;
395  return;
396  }
397  TTree* b2 = const_cast<TTree*>(b1);
398  TList listT;
399  listT.Add(b2);
400  a1->Merge(&listT);
401  listT.Clear();
402  }
403 
404  histCollection::histPerDir_t::histPerDir_t(const std::string& nameIn, std::unique_ptr<TObject>&& objIn, TTree* mdTree)
405  : name(nameIn), obj(std::move(objIn)), mergeMethod(nullptr) {
406  // Some sanity checks:
407  if (!obj) {
408  std::cout << "ERROR while adding " << nameIn << ": Histogram pointer is NULL" << std::endl;
409  return;
410  }
411 
412  if (mdTree) {
413  fillMD(mdTree);
414  }
415  else {
416  s_dbg(VERBOSE,"No matadata found for " + name +", use defaults");
417  }
418  const std::string& howToMerge = metadata[2];
419  s_dbg(VERBOSE, "Name: " + name + " mergeMethod=" + howToMerge);
420 
421  TH1* th1 = dynamic_cast<TH1*>(obj.get());
422  TH2* th2 = dynamic_cast<TH2*>(obj.get());
423  TEfficiency* teff = dynamic_cast<TEfficiency*>(obj.get());
424  if (th1) {
425  th1->SetDirectory(nullptr); // Get ownership of this hist
426  if (howToMerge == "<default>") {
427  if (th2) {
428  mergeMethod = &defaultMerge<TH2>;
429  } else { //TH1 case
430  mergeMethod = &defaultMerge<TH1>;
431  }
432  }
433  else if (howToMerge == "weightedAverage" || howToMerge=="weightedAverage2D")
434  mergeMethod = &weightedAverage;
435  else if (howToMerge == "weightedEff")
436  mergeMethod = &weightedEff;
437  else if (howToMerge == "mergeRMS")
438  mergeMethod = &mergeRMS;
439  else if (howToMerge == "RMSpercentDeviation")
440  mergeMethod = &RMSpercentDeviation;
441  else if (howToMerge == "perBinEffPerCent")
442  mergeMethod = &perBinEffPerCent;
443  else if (howToMerge == "lowerLB")
444  mergeMethod = &lowerLB;
445  else if (howToMerge == "identical")
446  if (th2) {
447  mergeMethod = &identical<TH2>;
448  } else { //TH1 case
449  mergeMethod = &identical<TH1>;
450  }
451  else if ((howToMerge == "mergeRebinned") || (howToMerge == "merge"))
452  mergeMethod = &merge_rebinned;
453  else {
454  std::cout << "ERROR: Unknown merging method (" << howToMerge << ") for object of type TH1 named " << nameIn << std::endl;
455  obj.reset(nullptr);
456  }
457  } // end if TH1
458  else if (teff) {
459  teff->SetDirectory(nullptr);
460  if (howToMerge == "<default>")
461  mergeMethod = &merge_TEfficency;
462  else
463  std::cout << "ERROR: Unknown merging method (" << howToMerge << ") for object of type TEfficiency named " << nameIn << std::endl;
464  } // end if TEfficiency
465  else if (nullptr != dynamic_cast<TTree*>(obj.get())) {
466  mergeMethod = &merge_TTree;
467  } else {
468  std::cout << "ERROR Object " << name << " has unkown type" << std::endl;
469  obj.reset(nullptr);
470  }
471  }
472 
473  void histCollection::addDirectory(TDirectory * dir, const std::string& dirName, const std::string& filename) {
474 
475  s_dbg(VERBOSE, "Working on directory " + dirName);
476  if (m_dirExclusion && !std::regex_search(dirName, *m_dirExclusion)) {
477  s_dbg(DEBUG, "Path " + dirName + " is excluded");
478  return;
479  }
480 
481  for (TObject* oKey : *dir->GetListOfKeys()) {
482  TKey* key = static_cast<TKey*>(oKey);
483  const std::string name = key->GetName();
484  const std::string classname = key->GetClassName();
485  if ((classname == "TTree") && (name == "metadata")) {
486  continue;
487  }
488 
489  s_dbg(VERBOSE, "Found name " + name + ", classname=" + classname);
490 
491  const std::string newName = dirName.empty() ? name : dirName + "/" + name;
492  auto itDir = m_data.find(dirName);
493 
494  if (classname.starts_with("TH") || classname.starts_with("TProfile") || classname.starts_with("TEfficiency") || classname == "TTree") {
495  if (m_histExclusion && !std::regex_search(name, *m_histExclusion)) {
496  s_dbg(DEBUG, "Histogram with name " + name + " is excluded");
497  continue;
498  }
499 
500  // arrive here if we have at least one histogram in this directory
501  if (m_skipExisting) {
502  // Check if this object exists already in the output-file
503  std::unique_ptr<TObject> existingObj(m_out->Get(newName.c_str()));
504  if (existingObj)
505  continue;
506  }
507 
508  std::unique_ptr<TTree> md;
509  if (itDir == m_data.end()) {
510  // Have not seen this dirName yet
511  itDir = m_data.emplace(dirName, histDir_t()).first;
512  s_dbg(VERBOSE, "Registering new directory " + dirName);
513  }
514 
515  // Check if we already have this histogram in the list
516  auto itH = itDir->second.histos.find(name);
517  if (itH == itDir->second.histos.end()) {
518  // New histogram (or Tree):
519  if (!md) {
520  // Metadata tree not yet read in this directory
521  md.reset((TTree*)dir->Get("metadata"));
522  }
523 
524  std::unique_ptr<TObject> obj{key->ReadObj()};
525  TTree* treeObj = dynamic_cast<TTree*>(obj.get());
526  if (treeObj) {
527  TDirectory* outDir = m_out->GetDirectory(dirName.c_str());
528  if (!outDir)
529  outDir = m_out->mkdir(dirName.c_str());
530  // TTree need special treatment ...
531  TDirectory* currentDir = gDirectory;
532  outDir->cd();
533  TTree* cloneTree = treeObj->CloneTree();
534  // this disconnects parent tree
535  obj.reset(cloneTree);
536  currentDir->cd();
537  }
538  histPerDir_t histo(name, std::move(obj), md.get());
539  itH = itDir->second.histos.emplace(name, std::move(histo)).first; //Take owernship of object here!
540  s_dbg(VERBOSE, "Cloning histogram " + name + " in dir " + dirName);
541  } else {
542  // Histogram already known .. merge it
543  std::unique_ptr<TObject> other(key->ReadObj());
544  if (!other) {
545  std::cout << "ERROR, got NULL key";
546  } else {
547  itH->second.merge(other.get()); // Release object in this case
548  s_dbg(VERBOSE, "Merging histogram " + name + " in dir " + dirName);
549  }
550  }
551  } else if (classname.starts_with("TDirectory")) {
552  std::unique_ptr<TObject> obj(key->ReadObj());
553  TDirectory* subdir = dynamic_cast<TDirectory*>(obj.get());
554  if (subdir) {
555  if (filename.empty()) {
556  this->addDirectory(subdir, newName, filename);
557  } else {
558  if (!name.starts_with("lb_") && !name.starts_with("lowStat_LB")) {
559  this->addDirectory(subdir, newName, filename);
560  } else {
561  m_fileLBMap[newName].insert(filename);
562  }
563  }
564  }
565  } else {
566  std::cout << "Ignored objects '" << name << "' of type " << classname << std::endl;
567  }
568  }
569  return;
570  }
571 
572  void histCollection::write() {
573  unsigned nWritten = 0;
574  unsigned nIgnored = 0;
575  unsigned nDirs = 0;
576  for (auto& it : m_data) {
577  const std::string fulldir = it.first;
578  TDirectory* histDir = m_out->GetDirectory(fulldir.c_str());
579  if (histDir == nullptr) { // Create the directory if it doesn't exist yet
580  histDir = m_out->mkdir(fulldir.c_str());
581  if (histDir == nullptr) {
582  std::cout << "ERROR, failed to create directory " << fulldir << std::endl;
583  break;
584  } else {
585  s_dbg(VERBOSE, "Created directory " + fulldir + " in file " + m_out->GetName());
586  }
587  }
588  m_out->cd(fulldir.c_str());
589  ++nDirs;
590  for (auto& [name, histo] : it.second.histos) {
591  if (histo.obj) {
592  histo.obj->Write();
593  ++nWritten;
594  } else {
595  std::cout << "NOT writing " << name << ". Invalid." << std::endl;
596  ++nIgnored;
597  }
598  } // End loop over histograms in one directory
599  it.second.writeMD(histDir);
600  } // End loop over directories;
601  std::cout << "Wrote " << nWritten << " histograms to " << nDirs << " directories in output file " << m_out->GetName() << std::endl;
602  if (nIgnored)
603  std::cout << " Omitting " << nIgnored << " histograms." << std::endl;
604  }
605 
606  // *********************************************************************
607  // Public Methods
608  // *********************************************************************
609 
611  makeBranch("Name", "Name/C");
612  makeBranch("Interval", "Interval/C");
613  makeBranch("TriggerChain", "TriggerChain/C");
614  makeBranch("MergeMethod", "MergeMethod/C");
615  }
616 
617  void MonitoringFile::OutputMetadata::makeBranch(const char* branchName, const char* branchstr) {
618  if (!m_metadata->GetBranch(branchName)) {
619  m_metadata->Branch(branchName, (void*)nullptr, branchstr);
620  }
621  }
622 
623  void MonitoringFile::OutputMetadata::fill(const std::string& theName, const std::string& theInterval, const std::string& theChain,
624  const std::string& theMerge) {
625  std::string name = theName;
626  std::string interval = theInterval;
627  std::string chain = theChain;
628  std::string merge = theMerge;
629  m_metadata->SetBranchAddress("Name", name.data());
630  m_metadata->SetBranchAddress("Interval", interval.data());
631  m_metadata->SetBranchAddress("TriggerChain", chain.data());
632  m_metadata->SetBranchAddress("MergeMethod", merge.data());
633  m_metadata->Fill();
634  }
635 
638  m_doTiming = false;
640  }
641 
642  bool MonitoringFile::setFile(const std::string& fileName) {
643  clearData();
644  m_file = TFile::Open(fileName.c_str());
645  if (m_file != 0)
646  return true;
647  return false;
648  }
649 
650  MonitoringFile::MonitoringFile(const std::string& fileName) : m_file(0) {
652  m_doTiming = false;
655  }
656 
659 
660  delete m_file;
661  }
662 
663  bool MonitoringFile::setHistogramRegEx(const std::string& re) {
664  m_mergeMatchHistoRE = checkRegEx(re);
665  return m_mergeMatchHistoRE.has_value();
666  }
667 
668  bool MonitoringFile::setDirectoryRegEx(const std::string& re) {
669  m_mergeMatchDirRE = checkRegEx(re);
670  return m_mergeMatchDirRE.has_value();
671  }
672 
673  void MonitoringFile::getAllDirs(DirMap_t & dirmap, TDirectory * dir, const std::string& dirName) {
674  if (dir == 0)
675  return;
676 
677  if (dirName != "") {
678  DirMap_t::value_type dirmapVal(dirName, dir);
679  dirmap.insert(dirmapVal);
680  }
681 
682  TIter next(dir->GetListOfKeys());
683  TKey* key;
684  while ((key = dynamic_cast<TKey*>(next())) != 0) {
685  // don't delete TDirectories
686  TObject* obj = key->ReadObj();
687  TDirectory* subdir = dynamic_cast<TDirectory*>(obj);
688  if (subdir != 0) {
689  std::string subdirName(subdir->GetName());
690  std::string fName("");
691  if (dirName != "") {
692  fName += dirName;
693  fName += '/';
694  }
695  fName += subdirName;
696  getAllDirs(dirmap, subdir, fName);
697  } else {
698  delete obj;
699  }
700  }
701  }
702 
703  TDirectory* MonitoringFile::createDir(DirMap_t & dirmap, TDirectory * dir, const std::string& parent, const std::string& path) {
704  if (dir == 0)
705  return 0;
706 
707  TDirectory* subdir(0);
708  DirMap_t::const_iterator diter;
709  std::string::size_type i = path.find_first_of('/');
710  std::string fName("");
711  if (parent != "") {
712  fName += parent;
713  fName += '/';
714  }
715 
716  if (i != std::string::npos) {
717  std::string dName(path, 0, i);
718  std::string pName(path, i + 1, std::string::npos);
719  fName += dName;
720  if (dName != "") {
721  diter = dirmap.find(fName);
722  if (diter != dirmap.end()) {
723  subdir = diter->second;
724  } else {
725  subdir = dir->mkdir(dName.c_str());
726  DirMap_t::value_type dirmapVal(fName, subdir);
727  dirmap.insert(dirmapVal);
728  }
729  } else {
730  subdir = dir;
731  }
732  return createDir(dirmap, subdir, fName, pName);
733  }
734 
735  fName += path;
736 
737  diter = dirmap.find(fName);
738  if (diter != dirmap.end()) {
739  return diter->second;
740  }
741 
742  subdir = dir->mkdir(path.c_str());
743  DirMap_t::value_type dirmapVal(fName, subdir);
744  dirmap.insert(dirmapVal);
745  return subdir;
746  }
747 
748  TKey* MonitoringFile::getObjKey(TDirectory * dir, const std::string& path) {
749  if (dir == 0)
750  return 0;
751 
752  TKey* key(0);
753 
754  std::string::size_type i = path.find_first_of('/');
755  if (i != std::string::npos) {
756  std::string dName(path, 0, i);
757  std::string pName(path, i + 1, std::string::npos);
758  if (dName != "") {
759  key = dir->FindKey(dName.c_str());
760  if (key != 0) {
761  TDirectory* subDir = dynamic_cast<TDirectory*>(key->ReadObj());
762  if (subDir) {
763  return getObjKey(subDir, pName);
764  } // else fall through
765  }
766  return 0;
767  }
768  return getObjKey(dir, pName);
769  }
770 
771  return dir->FindKey(path.c_str());
772  }
773 
774  void MonitoringFile::fillMetaDataMap(std::map<std::string, dqutils::MonitoringFile::MetaData> & mdMap, TDirectory * dir) {
775  if (dir == 0)
776  return;
777  TTree* md = dynamic_cast<TTree*>(dir->Get("metadata"));
778  if (md == 0)
779  return;
780 
781  TTreeReader reader(md);
782  TTreeReaderArray<char> i_name(reader, "Name");
783  TTreeReaderArray<char> i_interval(reader, "Interval");
784  TTreeReaderArray<char> i_chain(reader, "TriggerChain");
785  TTreeReaderArray<char> i_merge(reader, "MergeMethod");
786 
787  while (reader.Next()) {
788  const std::string nameStr(static_cast<char*>(i_name.GetAddress()));
789  if (mdMap.find(nameStr) == mdMap.end()) {
790  MetaData md(nameStr, static_cast<char*>(i_interval.GetAddress()), static_cast<char*>(i_chain.GetAddress()), static_cast<char*>(i_merge.GetAddress()));
791  std::map<std::string, MetaData>::value_type mdVal(nameStr, md);
792  mdMap.insert(mdVal);
793  }
794  }
795 
796  delete md;
797  }
798 
799  int MonitoringFile::mergeFiles(const std::string & outFileName, const std::vector<std::string>& files, fileLBMap_t& lbmap, bool fillLBDirs) {
800  std::cout << "Writing file: " << outFileName << std::endl;
801  std::cout << "Start merging [" << files.size() << "] histogram files" << std::endl;
803  TH1::AddDirectory(false);
804  if (m_mergeMatchDirRE.has_value() || m_mergeMatchHistoRE.has_value()) {
805  std::cout << " ========== Using regular expressions for selective merging ========== " << std::endl;
806  }
807  if (m_doTiming) {
808  std::cout << "CPU time measurement activated " << std::endl;
809  }
810 
811  const size_t nFiles = files.size();
812 
813  if (nFiles < 1)
814  return -1;
815 
816  if (nFiles == 1) {
817  std::cout << "Got exactly one input file. Will copy input -> output" << std::endl;
818  if (m_mergeMatchDirRE.has_value() || m_mergeMatchHistoRE.has_value()) {
819  std::cout << "regular expressions for selective merging will have no effect!" << std::endl;
820  }
821 
824  std::filesystem::copy_file(inPath, outPath, std::filesystem::copy_options::overwrite_existing);
825  return 0;
826  }
827 
828  std::unique_ptr<TFile> outfile(TFile::Open(outFileName.c_str(), "RECREATE", outFileName.c_str(), m_fileCompressionLevel));
829  if (outfile.get() == 0) {
830  std::cout << " ERROR, cound not open output file " << outFileName << std::endl;
831  return -1;
832  }
833  std::cout << "Opened/created output file " << outFileName << std::endl;
834 
835  histCollection hc(outfile.get());
836  hc.addDirExclusion(m_mergeMatchDirRE);
837  hc.addHistExclusion(m_mergeMatchHistoRE);
838 
839  // Open first input file, mostly to get the run-directory
840  std::unique_ptr<TFile> in1(TFile::Open(files[0].c_str()));
841  if (!in1) {
842  std::cout << "ERROR, could not open input file " << files[0] << std::endl;
843  return -1;
844  }
845  std::cout << "Working on file 1/" << nFiles << ": " << files[0] << std::endl;
846  std::string runDir, runDirFwd;
847  const std::regex runDirPattern("run_[0-9]*");
848  TIter next(in1->GetListOfKeys());
849  TKey* key;
850  while ((key = (TKey*)next())) {
851  const char* name = key->GetName();
852  if (std::regex_match(name, runDirPattern)) {
853  if (runDir.size() > 0) {
854  std::cout << "ERROR More than one run_XXX directory found! Ignoring " << name << std::endl;
855  } else
856  runDir = name;
857  }
858  }
859  if (runDir.empty()) {
860  std::cout << "No run-directory found, start with '/'" << std::endl;
861  runDir="/";
862  runDirFwd="";
863  }
864  else {
865  std::cout << "Found run directory " << runDir << std::endl;
866  runDirFwd=runDir;
867  }
868 
869 
870  TDirectory* dir(dynamic_cast<TDirectory*>(in1->GetDirectory(runDir.c_str())));
871  if (!dir) {
872  std::cout << "ERROR, can't access directory " << runDir;
873  return -1;
874  }
875 
876  hc.addDirectory(dir, runDirFwd, files[0]);
877 
878  // Close first input file
879  in1.reset(nullptr);
880 
881  for (size_t i = 1; i < files.size(); ++i) {
882  std::cout << "Working on file " << 1+i << "/" << nFiles << ": " << files[i] << std::endl;
883  std::unique_ptr<TFile> in(TFile::Open(files[i].c_str()));
884  if (!in) {
885  std::cout << "ERROR, could not open input file " << files[i] << std::endl;
886  return -1;
887  }
888  TDirectory* dir(dynamic_cast<TDirectory*>(in->GetDirectory(runDir.c_str())));
889  if (not dir){
890  std::cout << "ERROR, could not cast to directory" << std::endl;
891  return -1;
892  }
893  hc.addDirectory(dir, runDirFwd, files[i]);
894  }
895 
896  std::cout << "Accumulated a total of " << hc.size() << " histograms." << std::endl;
897 
898  std::cout << "Start writing output ..." << std::endl;
899  hc.write();
900 
901  if (m_doTiming) {
902  std::cout << "CPU time for histogram merging: (regular histograms)" << std::endl;
903  hc.printTiming();
904  }
905 
906  auto newlbmap = hc.getFileLBMapAndClear();
907 
908  // Update file-to-lbdir map pass as argument with the new map from these files
909  for (auto& [lbname, newfileset] : newlbmap) {
910  auto& fileset = lbmap[lbname];
911  fileset.merge(newfileset);
912  }
913 
914  if (!lbmap.empty() && fillLBDirs) {
915  std::cout << "Start merging lb_nnn and lowStat_LB directories (" << lbmap.size() << " in total)" << std::endl;
916 
917  histCollection hclb(outfile.get());
918  hclb.addDirExclusion(m_mergeMatchDirRE);
919  hclb.addHistExclusion(m_mergeMatchHistoRE);
920 
921  // Sort lb/file list by file-name to avoid re-oping the same files:
922  // Copy map to vector<pair> ...
923  std::vector<std::pair<std::string, std::vector<std::string>>> lbToFiles;
924  for (const auto& [lb,fileSet] : lbmap) {
925  if (fileSet.size() > 0)
926  lbToFiles.emplace_back(lb,std::vector<std::string>(fileSet.begin(),fileSet.end()));
927  }
928 
929  //..and sort the vector
930  std::sort(lbToFiles.begin(), lbToFiles.end(),
931  [](const decltype(lbToFiles)::value_type& a, const decltype(lbToFiles)::value_type& b) { return a.second[0] < b.second[0]; });
932 
933  size_t counter = 0;
934  std::unique_ptr<TFile> in;
935  for (const auto& [dir, filenames] : lbToFiles) {
936  std::cout << "Merging/copying directory " << dir << " from " << filenames.size() << " input file(s) (" << ++counter << "/" << lbToFiles.size() << ")"
937  << std::endl;
938  for (const std::string& fName : filenames) {
939  if (!in || strcmp(in->GetName(), fName.c_str()) != 0) {
940  in.reset(TFile::Open(fName.c_str()));
941  s_dbg(DEBUG, "Opening input file " + fName);
942  } else {
943  s_dbg(DEBUG, "Input file " + fName + " already open");
944  }
945 
946  if (!in) {
947  std::cout << "ERROR, could not open input file " << fName << std::endl;
948  return -1;
949  }
950  TDirectory* tDir = (dynamic_cast<TDirectory*>(in->Get(dir.c_str())));
951  if (!tDir) {
952  std::cout << "ERROR, failed to get directory " << dir << " from file " << fName << std::endl;
953  } else {
954  hclb.addDirectory(tDir, dir);
955  }
956  } // end loop over filenames
957  hclb.write();
958  if (m_doTiming) {
959  std::cout << "CPU time for histogram merging: (lumiblock-histograms)" << std::endl;
960  hclb.printTiming();
961  }
962  hclb.clear();
963  } // end loop over lbmap
964  in.reset(nullptr);
965  }
966  return 0;
967  }
968 
969  int MonitoringFile::mergeFiles(const std::string& outFileName, const std::string& listFileName) {
970  typedef std::vector<std::string> FileList_t;
971 
972  const unsigned int nFilesAtOnce = 50;
973 
974  FileList_t allFiles;
975  bool success = setListFromFile(allFiles, listFileName);
976  if (!success) {
977  std::cout << "ERROR Failed ot read list of input files" << std::endl;
978  return -1;
979  }
980 
981 
982  fileLBMap_t fileLBMap;
983  if (allFiles.size() <= nFilesAtOnce) {
984  return mergeFiles(outFileName, allFiles, fileLBMap, true);
985  }
986 
987  FileList_t procFiles, tmpIntermediateFiles;
988 
989  FileList_t::const_iterator filesEnd = allFiles.end();
990  FileList_t::const_iterator fi = allFiles.begin();
991 
992  unsigned int counter = 0;
993  std::string tmpInputFile("");
994  std::string tmpOutputFile("");
995 
996  // new logic: merge intermediately, then merge intermediate files
997  while (fi != filesEnd) {
998 
999  procFiles.push_back(*fi);
1000  ++counter;
1001  ++fi;
1002  if (counter % nFilesAtOnce == 0 || fi == filesEnd) {
1003  std::ostringstream nameStream;
1004  nameStream << "tmp_merge_" << counter << ".root";
1005  tmpOutputFile = nameStream.str();
1006  tmpIntermediateFiles.push_back(tmpOutputFile);
1007  int stat=mergeFiles(tmpOutputFile, procFiles, fileLBMap,false);
1008  if (stat) return stat;
1009  procFiles.clear();
1010  }
1011  }
1012 
1013  int stat=mergeFiles(outFileName, tmpIntermediateFiles,fileLBMap,true);
1014  if (stat) return stat;
1015 
1016  for (const auto& tmpFile : tmpIntermediateFiles) {
1017  std::remove(tmpFile.c_str());
1018  }
1019  return 0;
1020  }
1021 
1023  if (m_file == 0) {
1024  std::cerr << "MonitoringFile::printStatistics(): "
1025  << "No input file is open\n";
1026  return;
1027  }
1028 
1029  DirMap_t indirmap;
1030 
1031  getAllDirs(indirmap, m_file, "");
1032 
1033  DirMap_t::const_iterator idirend = indirmap.end();
1034  for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1035  std::string idirName = idir->first;
1036 
1037  GatherStatistics stat_shift(idirName);
1038  GatherStatistics stat_all(idirName);
1039 
1040  loopOnHistogramsInMetadata(stat_shift, idir->second);
1041  loopOnHistograms(stat_all, idir->second);
1042 
1043  std::cout.setf(std::ios_base::left, std::ios_base::adjustfield);
1044  std::cout.width(80);
1045  std::cout << idirName << " ";
1046 
1047  std::cout.setf(std::ios_base::right, std::ios_base::adjustfield);
1048  std::cout << " shift: ";
1049  std::cout.width(3);
1050  std::cout << stat_shift.m_nHist1D << " ";
1051  std::cout.width(5);
1052  std::cout << stat_shift.m_nHist1DBins << " ";
1053  std::cout.width(3);
1054  std::cout << stat_shift.m_nHist2D << " ";
1055  std::cout.width(7);
1056  std::cout << stat_shift.m_nHist2DBins << " ";
1057  std::cout.width(3);
1058  std::cout << stat_shift.m_nGraph << " ";
1059  std::cout.width(5);
1060  std::cout << stat_shift.m_nGraphPoints << " ";
1061 
1062  std::cout << " all: ";
1063  std::cout << stat_all.m_nHist1D << " ";
1064  std::cout.width(5);
1065  std::cout << stat_all.m_nHist1DBins << " ";
1066  std::cout.width(3);
1067  std::cout << stat_all.m_nHist2D << " ";
1068  std::cout.width(7);
1069  std::cout << stat_all.m_nHist2DBins << " ";
1070  std::cout.width(3);
1071  std::cout << stat_all.m_nGraph << " ";
1072  std::cout.width(5);
1073  std::cout << stat_all.m_nGraphPoints << "\n";
1074 
1075  std::cout << std::flush;
1076  }
1077  }
1078 
1079  bool MonitoringFile::copyHistograms(const std::string& outFileName, const std::string& dirName) {
1081  // bool useRecursiveDelete = gROOT->MustClean();
1082  // gROOT->SetMustClean(false);
1083 
1084  if (m_file == 0) {
1085  std::cerr << "MonitoringFile::copyHistograms(): "
1086  << "No input file is open\n";
1087  return false;
1088  }
1089 
1090  DirMap_t indirmap;
1091  DirMap_t reducedmap;
1092  DirMap_t outdirmap;
1093 
1094  if (dirName != "all") {
1095  TKey* dkey = getObjKey(m_file, dirName);
1096  if (dkey == 0) {
1097  std::cerr << "MonitoringFile::copyHistograms(): "
1098  << "Directory \'" << dirName << "\' not found in input file\n";
1099  return false;
1100  }
1101 
1102  TDirectory* fromDir = dynamic_cast<TDirectory*>(dkey->ReadObj());
1103 
1104  DirMap_t::value_type dirmapVal(dirName, fromDir);
1105  indirmap.insert(dirmapVal);
1106  } else {
1107  std::cout << "Building list of all TDirectories in file...\n" << std::flush;
1108  getAllDirs(indirmap, m_file, "");
1109  }
1110 
1111  DirMap_t::const_iterator idirend = indirmap.end();
1112  for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1113 
1114  std::string idirName = idir->first;
1115  std::cout << "Checking " << idirName << "\n" << std::flush;
1116  // std::string::size_type j = idirName.find( "L1Calo/1_PPr_EmFADCTiming" );
1117  // if( j != std::string::npos ) {
1118  // std::cerr << "Skipping directory \"" << idirName << "\"\n";
1119  // std::cerr << std::flush;
1120  // continue;
1121  // }
1122 
1123  if (!dirHasHistogramsInMetadata(idir->second)) {
1124  continue;
1125  }
1126 
1127  reducedmap.insert(*idir);
1128  }
1129 
1130  std::unique_ptr<TFile> outfile(TFile::Open(outFileName.c_str(), "RECREATE", outFileName.c_str(), m_fileCompressionLevel));
1131  if (outfile.get() == 0) {
1132  std::cerr << "MonitoringFile::copyHistograms(): "
1133  << "Output file not opened\n";
1134  return false;
1135  }
1136 
1137  idirend = reducedmap.end();
1138  for (DirMap_t::const_iterator idir = reducedmap.begin(); idir != idirend; ++idir) {
1139 
1140  std::string idirName = idir->first;
1141  std::cout << "Processing " << idirName << "\n" << std::flush;
1142 
1143  TDirectory* toDir = createDir(outdirmap, outfile.get(), "", idirName);
1144  if (toDir == 0) {
1145  std::cerr << "MonitoringFile::copyHistograms(): "
1146  << "Directory \'" << idirName << "\' not created in output file\n";
1147  return false;
1148  }
1149 
1150  CopyHistogram copyFcn(toDir, idirName);
1151 
1152  loopOnHistogramsInMetadata(copyFcn, idir->second);
1153  }
1154 
1155  outfile->Write();
1156  // gROOT->SetMustClean(useRecursiveDelete);
1157  return true;
1158  }
1159 
1160  std::string MonitoringFile::getHanResults(const std::string& hanResultsDir, const std::string& input, const std::string& hcfg,
1161  const std::string& hcfg_lowStat, const std::string& hcfg_medStat) {
1162  // DisableMustClean disabled;
1163 
1164  std::cout << "\nUsing han configurations:\n"
1165  << " entire run: " << hcfg << "\n"
1166  << " low stat interval: " << hcfg_lowStat << "\n"
1167  << " medium stat interval: " << hcfg_medStat << "\n\n"
1168  << std::flush;
1169 
1170  TFile* infile = TFile::Open(input.c_str());
1171  if (infile == 0) {
1172  std::cerr << "MonitoringFile::getHanResults(): "
1173  << "Cannot open input file \"" << input << "\"\n";
1174  return "";
1175  }
1176 
1177  std::vector<std::string> run_dirs;
1178  std::vector<std::string> lowStat_dirs;
1179  std::vector<std::string> medStat_dirs;
1180 
1181  TIter next_run(infile->GetListOfKeys());
1182  TKey* key_run(0);
1183  while ((key_run = dynamic_cast<TKey*>(next_run())) != 0) {
1184  TObject* obj_run = key_run->ReadObj();
1185  TDirectory* tdir_run = dynamic_cast<TDirectory*>(obj_run);
1186  if (tdir_run != 0) {
1187  std::string tdir_run_name(tdir_run->GetName());
1188  if (tdir_run_name.find("run") != std::string::npos) {
1189  run_dirs.push_back(tdir_run_name);
1190  TIter next_minutes(tdir_run->GetListOfKeys());
1191  TKey* key_minutes(0);
1192  while ((key_minutes = dynamic_cast<TKey*>(next_minutes())) != 0) {
1193  TObject* obj_minutes = key_minutes->ReadObj();
1194  TDirectory* tdir_minutes = dynamic_cast<TDirectory*>(obj_minutes);
1195  if (tdir_minutes != 0) {
1196  std::string tdir_minutes_name(tdir_minutes->GetName());
1197  if (tdir_minutes_name.find("lowStat") != std::string::npos) {
1198  lowStat_dirs.push_back(tdir_run_name + '/' + tdir_minutes_name);
1199  } else if (tdir_minutes_name.find("medStat") != std::string::npos) {
1200  medStat_dirs.push_back(tdir_run_name + '/' + tdir_minutes_name);
1201  }
1202  }
1203  delete obj_minutes;
1204  }
1205  }
1206  }
1207  delete obj_run;
1208  }
1209 
1210  delete infile;
1211 
1212  dqi::HanApp han;
1213 
1214  std::string fileList = " ";
1215  std::vector<std::string>::const_iterator dirs_end;
1216  std::vector<std::string>::const_iterator dir;
1217 
1218  dirs_end = run_dirs.end();
1219  for (dir = run_dirs.begin(); dir != dirs_end; ++dir) {
1220  const std::string& tdir_run_name = *dir;
1221  std::string han_output_run = hanResultsDir + '/' + tdir_run_name + "_han.root";
1222  std::cout << "Calling han( " << hcfg << ", " << input << ", " << tdir_run_name << ", " << han_output_run << " ):\n" << std::flush;
1223  han.Analyze(hcfg, input, han_output_run, tdir_run_name);
1224  std::cout << "\n";
1225  fileList += han_output_run + " " + tdir_run_name + "\n";
1226  }
1227 
1228  dirs_end = lowStat_dirs.end();
1229  for (dir = lowStat_dirs.begin(); dir != dirs_end; ++dir) {
1230  const std::string& tdir_minutes_path = *dir;
1231 
1232  std::string tdir_minutes_underscore = tdir_minutes_path;
1233  std::string::size_type tdir_minutes_i = tdir_minutes_underscore.find('/');
1234  tdir_minutes_underscore.replace(tdir_minutes_i, 1, "_");
1235 
1236  std::string han_output_lowStat = hanResultsDir + '/' + tdir_minutes_underscore + "_han.root";
1237  std::cout << "Running han, writing to " << han_output_lowStat << ":\n" << std::flush;
1238  han.Analyze(hcfg_lowStat, input, han_output_lowStat, tdir_minutes_path);
1239  std::cout << "\n";
1240  std::string subdirname(tdir_minutes_path, tdir_minutes_i + 1, std::string::npos);
1241  std::string dirname(tdir_minutes_path, 0, tdir_minutes_i);
1242  fileList += han_output_lowStat + " " + subdirname + " " + dirname + " " + subdirname + "\n";
1243  }
1244 
1245  dirs_end = medStat_dirs.end();
1246  for (dir = medStat_dirs.begin(); dir != dirs_end; ++dir) {
1247  const std::string& tdir_minutes_path = *dir;
1248 
1249  std::string tdir_minutes_underscore = tdir_minutes_path;
1250  std::string::size_type tdir_minutes_i = tdir_minutes_underscore.find('/');
1251  tdir_minutes_underscore.replace(tdir_minutes_i, 1, "_");
1252 
1253  std::string han_output_medStat = hanResultsDir + '/' + tdir_minutes_underscore + "_han.root";
1254  std::cout << "Running han, writing to " << han_output_medStat << ":\n" << std::flush;
1255  han.Analyze(hcfg_medStat, input, han_output_medStat, tdir_minutes_path);
1256  std::cout << "\n";
1257  std::string subdirname(tdir_minutes_path, tdir_minutes_i + 1, std::string::npos);
1258  std::string dirname(tdir_minutes_path, 0, tdir_minutes_i);
1259  fileList += han_output_medStat + " " + subdirname + " " + dirname + " " + subdirname + "\n";
1260  }
1261 
1262  return fileList;
1263  }
1264 
1266  if (m_file == 0) {
1267  std::cerr << "MonitoringFile::printHanConfig(): "
1268  << "No input file is open\n";
1269  return;
1270  }
1271 
1272  DirMap_t indirmap;
1273 
1274  getAllDirs(indirmap, m_file, "");
1275 
1276  std::string indent, indent_p, indent_c;
1277  std::string idirName_p;
1278  DirMap_t::const_iterator idirend = indirmap.end();
1279  for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1280  std::string idirName = idir->first;
1281  std::string::size_type shortNameIndex = idirName.rfind('/');
1282  std::string shortName = idirName.substr(shortNameIndex + 1, std::string::npos);
1283 
1284  std::string::size_type fsIndex = idirName.find('/');
1285  std::string shortPath;
1286  if (fsIndex != shortNameIndex)
1287  shortPath = idirName.substr(fsIndex + 1, shortNameIndex);
1288  else
1289  shortPath = idirName.substr(fsIndex + 1, std::string::npos);
1290 
1291  std::cout << idirName << "\n";
1292  std::cout << shortPath << ", " << shortName << "\n";
1293  /*
1294  indent = getIndentation(idirName,"");
1295  if(int(indent.size())==in_p){
1296  std::cout << indent << "} \n";
1297  std::cout << indent << "dir " << shortName << " { \n";
1298  std::cout << indent << " output " << idirName << "\n";
1299  std::cout << indent << " hist all_in_dir { \n " << indent << " } \n";
1300  }
1301  else if (int(indent.size()) > in_p){
1302  std::cout << indent << "dir " << shortName << " { \n";
1303  std::cout << indent << " output " << idirName << "\n";
1304  std::cout << indent << " hist all_in_dir { \n " << indent << " } \n";
1305  }
1306  else{
1307  //find common part + number of common '/'
1308  std::string common = FindCommon(idirName,idirName_p);
1309  indent_c = getIndentation(common,"");
1310  int counter = (indent_p.size() - indent_c.size())/2;
1311  for (int i = counter; i>0; i--){
1312  std::string temp = indent_c;
1313  for (int j = 0; j< i; j++){
1314  temp+=" ";
1315  }
1316  std::cout << temp << "} \n" ;
1317  }
1318  std::cout << indent << "} \n";
1319  std::cout << indent << "dir " << shortName << " { \n";
1320  std::cout << indent << " output " << idirName << "\n";
1321  std::cout << indent << " hist all_in_dir { \n " << indent << " } \n";
1322  }
1323  indent_p = indent;
1324  in_p = indent_p.size();
1325  idirName_p = idirName;
1326  */
1327  }
1328  }
1329 
1330  std::string MonitoringFile::getIndentation(const std::string& pathName, const std::string& leadingSpace) {
1331  std::string space = leadingSpace;
1332  std::string::size_type i = pathName.find_first_of('/');
1333  if (i != std::string::npos) {
1334  std::string subPath(pathName, i + 1, std::string::npos);
1335  space += " ";
1336  return getIndentation(subPath, space);
1337  }
1338  return space;
1339  }
1340 
1341  std::string MonitoringFile::FindCommon(const std::string& name1, const std::string& name2) const {
1342  int length = (name1.size() < name2.size()) ? name1.size() : name2.size();
1343  bool found = true;
1344  int count = 0;
1345  while (found == true && count < length) {
1346  if (name1[count] == name2[count]) {
1347  count++;
1348  } else {
1349  found = false;
1350  }
1351  }
1352  return (name1.substr(0, count));
1353  }
1354 
1355  // *********************************************************************
1356  // Protected Methods
1357  // *********************************************************************
1358 
1359  MonitoringFile::CopyHistogram::CopyHistogram(TDirectory * target, const std::string& dirName) : m_target(target), m_dirName(dirName), m_metadata(0) {
1360  m_metadata = new TTree("metadata", "Monitoring Metadata");
1361  m_metadata->SetDirectory(0);
1362  m_metadata->Branch("Name", (void*)nullptr, "Name/C");
1363  m_metadata->Branch("Interval", (void*)nullptr, "Interval/C");
1364  m_metadata->Branch("TriggerChain", (void*)nullptr, "TriggerChain/C");
1365  m_metadata->Branch("MergeMethod", (void*)nullptr, "MergeMethod/C");
1366  }
1367 
1369  m_target->cd();
1370  m_metadata->SetDirectory(m_target);
1371  m_metadata->Write();
1372  delete m_metadata;
1373  }
1374 
1376  m_target->cd();
1377  hist->SetDirectory(m_target);
1378  hist->Write();
1379 
1380  return true;
1381  }
1382 
1383  bool MonitoringFile::CopyHistogram::execute(TGraph * graph) {
1384  m_target->cd();
1385  graph->Write();
1386 
1387  return true;
1388  }
1389 
1390  bool MonitoringFile::CopyHistogram::execute(TEfficiency * eff) {
1391  m_target->cd();
1392  eff->Write();
1393  return true;
1394  }
1395 
1396  void MonitoringFile::CopyHistogram::fillMD(const MetaData& md) {
1397  std::string name(md.name);
1398  std::string interval(md.interval);
1399  std::string chain(md.chain);
1400  std::string merge(md.merge);
1401  m_metadata->SetBranchAddress("Name", name.data());
1402  m_metadata->SetBranchAddress("Interval", interval.data());
1403  m_metadata->SetBranchAddress("TriggerChain", chain.data());
1404  m_metadata->SetBranchAddress("MergeMethod", merge.data());
1405  m_metadata->Fill();
1406  }
1407 
1408  bool MonitoringFile::CopyHistogram::executeMD(TH1 * hist, const MetaData& md) {
1409  m_target->cd();
1410  hist->SetDirectory(m_target);
1411  hist->Write();
1412 
1413  fillMD(md);
1414 
1415  return true;
1416  }
1417 
1418  bool MonitoringFile::CopyHistogram::executeMD(TGraph * graph, const MetaData& md) {
1419  m_target->cd();
1420  graph->Write();
1421 
1422  fillMD(md);
1423 
1424  return true;
1425  }
1426 
1427  bool MonitoringFile::CopyHistogram::executeMD(TEfficiency * eff, const MetaData& md) {
1428  m_target->cd();
1429  eff->Write();
1430  fillMD(md);
1431  return true;
1432  }
1433 
1435  : m_dirName(dirName), m_nHist1D(0), m_nHist1DBins(0), m_nGraph(0), m_nGraphPoints(0), m_nHist2D(0), m_nHist2DBins(0) {}
1436 
1438  TH2* hist2d = dynamic_cast<TH2*>(hist);
1439  if (hist2d != 0) {
1440  ++m_nHist2D;
1441  m_nHist2DBins += (hist2d->GetNbinsX() * hist2d->GetNbinsY());
1442  return true;
1443  }
1444  ++m_nHist1D;
1445  m_nHist1DBins += hist->GetNbinsX();
1446  return true;
1447  }
1448 
1449  bool MonitoringFile::GatherStatistics::execute(TGraph * graph) {
1450  ++m_nGraph;
1451  m_nGraphPoints += graph->GetMaxSize();
1452  return true;
1453  }
1454 
1455  bool MonitoringFile::GatherStatistics::execute(TEfficiency * eff) {
1456  ++m_nEfficiency;
1457 
1458  TH1* h_total = eff->GetCopyPassedHisto();
1459  TH2* h_total2D = dynamic_cast<TH2*>(h_total);
1460 
1461  if (h_total2D != 0) {
1462  m_nEfficiencyBins += (h_total2D->GetNbinsX() * h_total2D->GetNbinsY());
1463  return true;
1464  } else {
1465  m_nEfficiencyBins += h_total->GetNbinsX();
1466  return true;
1467  }
1468  }
1469 
1471 
1473  m_names.push_back(std::string(hist->GetName()));
1474  return true;
1475  }
1476 
1477  bool MonitoringFile::GatherNames::execute(TGraph * graph) {
1478  m_names.push_back(std::string(graph->GetName()));
1479  return true;
1480  }
1481 
1482  bool MonitoringFile::GatherNames::execute(TEfficiency * eff) {
1483  m_names.push_back(std::string(eff->GetName()));
1484  return true;
1485  }
1486 
1487  void MonitoringFile::clearData() {
1489 
1490  delete m_file;
1491  m_file = 0;
1493  m_doTiming = false;
1494  }
1495 
1496  bool MonitoringFile::dirHasHistogramsInMetadata(TDirectory * dir) {
1497  dir->cd();
1498 
1499  TKey* mdKey = dir->FindKey("metadata");
1500  if (mdKey == 0) {
1501  return false;
1502  }
1503 
1504  TTree* md = dynamic_cast<TTree*>(mdKey->ReadObj());
1505  if (md == 0) {
1506  return false;
1507  }
1508 
1509  int counter = 0;
1510  int nEntries = int(md->GetEntries());
1511 
1512  while (counter < nEntries) {
1513  try {
1514  md->GetEntry(counter);
1515  } catch (const std::exception& e) {
1516  std::cerr << "Exception: \"" << e.what() << "\" in directory \"" << dir->GetName() << "\"\n" << std::flush;
1517  return false;
1518  }
1519 
1520  return true;
1521  ++counter;
1522  }
1523 
1524  return false;
1525  }
1526 
1527  void MonitoringFile::loopOnHistograms(HistogramOperation & fcn, TDirectory * dir) {
1528  TIter next(dir->GetListOfKeys());
1529  TKey* key;
1530  while ((key = dynamic_cast<TKey*>(next())) != 0) {
1531  TObject* obj = key->ReadObj();
1532  TH1* h(0);
1533  TGraph* g(0);
1534  TEfficiency* e(0);
1535  if ((h = dynamic_cast<TH1*>(obj))) {
1536  fcn.execute(h);
1537  } else if ((g = dynamic_cast<TGraph*>(obj))) {
1538  fcn.execute(g);
1539  } else if ((e = dynamic_cast<TEfficiency*>(obj))) {
1540  fcn.execute(e);
1541  }
1542  delete obj;
1543  }
1544  }
1545 
1546  bool MonitoringFile::loopOnHistogramsInMetadata(HistogramOperation & fcn, TDirectory * dir) {
1547  dir->cd();
1548  TKey* mdKey = dir->FindKey("metadata");
1549  if (mdKey == 0) {
1550  return false;
1551  }
1552 
1553  TTree* md = dynamic_cast<TTree*>(mdKey->ReadObj());
1554  if (md == 0) {
1555  return false;
1556  }
1557 
1558  TKey* i_key;
1559 
1560  TTreeReader reader(md);
1561  TTreeReaderArray<char> i_name(reader, "Name");
1562  TTreeReaderArray<char> i_interval(reader, "Interval");
1563  TTreeReaderArray<char> i_chain(reader, "TriggerChain");
1564  TTreeReaderArray<char> i_merge(reader, "MergeMethod");
1565 
1566  while (reader.Next()) {
1567  const std::string nameStr(static_cast<char*>(i_name.GetAddress()));
1568  dir->cd();
1569  i_key = dir->FindKey(static_cast<char*>(i_name.GetAddress()));
1570  if (i_key == 0) {
1571  std::cerr << "MonitoringFile::loopOnHistogramsInMetadata(): "
1572  << "No \'" << nameStr << "\' object found\n";
1573  return false;
1574  }
1575  MetaData md(nameStr, static_cast<char*>(i_interval.GetAddress()), static_cast<char*>(i_chain.GetAddress()), static_cast<char*>(i_merge.GetAddress()));
1576  TObject* obj = i_key->ReadObj();
1577  TH1* h = dynamic_cast<TH1*>(obj);
1578  if (h != 0) {
1579  fcn.executeMD(h, md);
1580  } else {
1581  TGraph* g = dynamic_cast<TGraph*>(obj);
1582  if (g != 0) {
1583  fcn.executeMD(g, md);
1584  }
1585  }
1586  delete obj;
1587  }
1588 
1589  delete md;
1590 
1591  return true;
1592  }
1593 
1594  bool MonitoringFile::setListFromFile(std::vector<std::string> & filelist, const std::string& listFileName) {
1595  using namespace std;
1596 
1597  filelist.clear();
1598 
1599  ifstream listfile(listFileName.c_str());
1600  if (!listfile) {
1601  cerr << "MonitoringFile::setListFromFile(): "
1602  << "cannot read from file: " << listFileName << "\n";
1603  return false;
1604  }
1605 
1606  string line;
1607  char c;
1608  string filename;
1609  while (getline(listfile, line)) {
1610  istringstream linestream(line);
1611  while (linestream.get(c)) {
1612  if (!isspace(c)) {
1613  // ignore comments
1614  if (c == '#') {
1615  break;
1616  }
1617 
1618  linestream.putback(c);
1619  linestream >> filename;
1620  if (!linestream) {
1621  cerr << "MonitoringFile::setListFromFile(): "
1622  << "badly formatted line: " << line << "\n";
1623  break;
1624  }
1625 
1626  filelist.push_back(filename);
1627  }
1628  }
1629  }
1630 
1631  return true;
1632  }
1633 
1634  int MonitoringFile::mergeLBintervals(const std::string& inFilename) {
1635 
1636  std::cout << "Running mergeLBintervals on " << inFilename << std::endl;
1637 
1638  std::unique_ptr<TFile> f(TFile::Open(inFilename.c_str(), "UPDATE"));
1639  if (!f) {
1640  std::cout << "ERROR, could not open file " << inFilename << " for update" << std::endl;
1641  return -1;
1642  }
1643  std::string runDirName;
1644  const std::regex runDirPattern("run_[0-9]*");
1645  TIter next(f->GetListOfKeys());
1646  TKey* key;
1647  while ((key = (TKey*)next())) {
1648  const char* name = key->GetName();
1649  if (std::regex_match(name, runDirPattern)) {
1650  if (runDirName.size() > 0) {
1651  std::cout << "ERROR More than one run_XXX directory found! Ignoring " << name << std::endl;
1652  } else
1653  runDirName = name;
1654  }
1655  break;
1656  }
1657 
1658  TDirectory* runDir = f->GetDirectory(runDirName.c_str());
1659  const auto mapping = buildLBToIntervalMap(runDir);
1660 
1661  if (s_dbg.getLvl() == VERBOSE) {
1662  std::cout << "LB directory mapping:" << std::endl;
1663  for (const auto& i1 : mapping) {
1664  std::cout << i1.first;
1665  for (const auto& i2 : i1.second) {
1666  std::cout << "\t" << i2 << std::endl;
1667  }
1668  }
1669  }
1670 
1671  for (const auto& [outDir, inDIrs] : mapping) {
1672  int stat=mergeLB_processLBinterval(f.get(), inDIrs, outDir);
1673  if (stat) return stat;
1674  }
1675 
1676  f->Close();
1677  f.reset(TFile::Open(inFilename.c_str(), "UPDATE"));
1678  runDir = f->GetDirectory(runDirName.c_str());
1679 
1680  std::cout << "merging lowStat_LB dirs into run-dir" << std::endl;
1681  std::vector<std::string> lowStatDirs;
1682  for (TObject* oKey : *runDir->GetListOfKeys()) {
1683  TKey* key = static_cast<TKey*>(oKey);
1684  const std::string name = key->GetName();
1685  const std::string classname = key->GetClassName();
1686  if (classname.starts_with("TDirectory") and name.starts_with("lowStat_LB")) {
1687  lowStatDirs.push_back(runDirName + "/" + name);
1688  s_dbg(VERBOSE, "Found input: " + runDirName + "/" + name);
1689  }
1690  }
1691 
1692  int stat=mergeLB_processLBinterval(f.get(), lowStatDirs, runDirName);
1693 
1694  f->Close();
1695  return stat;
1696  }
1697 
1698  std::map<std::string, std::vector<std::string>> MonitoringFile::buildLBToIntervalMap(TDirectory * runDir) {
1699 
1700  std::map<std::string, std::vector<std::string>> ranges;
1701 
1702  // No recusion here, everything we care out is run_NNNNN/lb_nnnn (and run_NNNNN/lowStat_nn-mm)
1703  const std::string runDirName = runDir->GetName();
1704  for (TObject* oKey : *runDir->GetListOfKeys()) {
1705  TKey* key = static_cast<TKey*>(oKey);
1706  const std::string name = key->GetName();
1707  const std::string classname = key->GetClassName();
1708  if (!classname.starts_with("TDirectory"))
1709  continue;
1710  if (name.starts_with("lb_")) {
1711  unsigned lumiBlock = 0;
1712  try {
1713  lumiBlock = std::stol(name.substr(3));
1714  } catch (std::invalid_argument& e) {
1715  std::cout << "ERROR, unexpected directory name " << name << ". Can't parse lb number" << std::endl;
1716  std::cout << e.what() << std::endl;
1717  continue;
1718  }
1719  // Copied from Control/AthenaMonitoringKernel/src/HistogramFiller/OfflineHistogramProvider.h
1720  const unsigned lbBase = lumiBlock - (((int64_t)lumiBlock - 1) % 20);
1721  const std::string lbString = runDirName + "/lowStat_LB" + std::to_string(lbBase) + "-" + std::to_string(lbBase + 19);
1722  ranges[lbString].push_back(runDirName + "/" + name);
1723  } // end if lb_NNNN dir
1724  } // end loop over directories under run_NNNNN
1725  return ranges;
1726  }
1727 
1728  int MonitoringFile::mergeLB_processLBinterval(TFile * file, const std::vector<std::string>& inputDirNames, const std::string& outputDirName) {
1729 
1730  TDirectory* outDir = file->GetDirectory(outputDirName.c_str());
1731  if (!outDir) {
1732  outDir = file->mkdir(outputDirName.c_str());
1733  }
1734  if (!outDir) {
1735  std::cout << "ERROR, can't obtain nor create directory " << outputDirName << " in file " << file->GetName() << std::endl;
1736  return -1;
1737  }
1738 
1739  histCollection hc(file, true);
1740  hc.addDirExclusion(m_mergeMatchDirRE);
1741  hc.addHistExclusion(m_mergeMatchHistoRE);
1742 
1743  for (const std::string& inDirName : inputDirNames) {
1744  TDirectory* inDir = file->GetDirectory(inDirName.c_str());
1745  hc.addDirectory(inDir, outputDirName);
1746  }
1747  if (hc.size() == 0) {
1748  std::cout << "mergeLB_processLBinterval: No new objects found for " << outputDirName << std::endl;
1749  } else {
1750  hc.write();
1751  }
1752  return 0;
1753  }
1754 
1755  bool MonitoringFile::CheckHistogram(TFile * f, const char* HistoName) {
1756  std::unique_ptr<TObject> obj(f->Get(HistoName));
1757  if (!obj.get()) {
1758  // std::cerr<<"No such histogram \""<< HistoName << "\"\n";
1759  return false;
1760  } else
1761  return true;
1762  }
1763 
1765  return s_dbg.getLvl();
1766  }
1768  s_dbg.setLvl((debugLevel_t)(level));
1769  }
1770  void MonitoringFile::doTiming() {
1771  m_doTiming = true;
1772  }
1773 
1774 
1775  void MonitoringFile::setCheckEquality(bool value) {dqutils::s_checkEquality=value;}
1776  std::atomic<int> MonitoringFile::m_fileCompressionLevel = 1;
1777  bool MonitoringFile::m_doTiming = false;
1778  std::unordered_map<std::string, std::clock_t> MonitoringFile::m_cpuPerHistogram;
1779 
1780  std::string MonitoringFile::getPath(TDirectory * dir) {
1781 
1782  std::string path = dir->GetPath();
1783  if (path.find(':') != std::string::npos)
1784  path = path.substr(path.rfind(':') + 1);
1785 
1786  return path;
1787  }
1788 
1789 } // namespace dqutils
dqutils::MonitoringFile::mergeLBintervals
int mergeLBintervals(const std::string &inFilename)
dqutils::MonitoringFile::getAllDirs
static void getAllDirs(DirMap_t &dirmap, TDirectory *dir, const std::string &dirName)
dqutils::debugLevel_t
debugLevel_t
Definition: MonitoringFile.h:56
histCollection
Definition: LArQuickHistMerge.cxx:50
dqutils::ATLAS_NOT_THREAD_SAFE
void getImageBuffer ATLAS_NOT_THREAD_SAFE(TImage **img, TCanvas *myC, char **x, int *y)
Definition: HanOutputFile.cxx:1130
AllowedVariables::e
e
Definition: AsgElectronSelectorTool.cxx:37
trigbs_pickEvents.ranges
ranges
Definition: trigbs_pickEvents.py:60
HanApp.h
fitman.nWritten
int nWritten
Definition: fitman.py:420
dqutils::MonitoringFile::copyHistograms
virtual bool copyHistograms(const std::string &outFileName, const std::string &dirName="all")
Copy the indicated set of histograms to an output file.
ReadCellNoiseFromCool.name1
name1
Definition: ReadCellNoiseFromCool.py:233
athena.path
path
python interpreter configuration --------------------------------------—
Definition: athena.py:128
runLayerRecalibration.chain
chain
Definition: runLayerRecalibration.py:175
dqutils::MonitoringFile::createDir
static TDirectory * createDir(DirMap_t &dirmap, TDirectory *dir, const std::string &parent, const std::string &path)
dqutils::MonitoringFile
Definition: MonitoringFile.h:59
run.infile
string infile
Definition: run.py:13
dqutils::MonitoringFile::setDebugLevel
static void setDebugLevel(int level)
vtune_athena.format
format
Definition: vtune_athena.py:14
FullCPAlgorithmsTest_eljob.flush
flush
Definition: FullCPAlgorithmsTest_eljob.py:194
dqutils::MonitoringFile::GatherNames::GatherNames
GatherNames()
dqutils::MonitoringFile::CheckHistogram
static bool CheckHistogram(TFile *f, const char *HistoName)
rootconvert.fName
string fName
Definition: rootconvert.py:5
dqutils::MonitoringFile::printHanConfig
virtual void printHanConfig()
dqutils::MonitoringFile::CopyHistogram::execute
virtual bool execute(TH1 *hist)
histCollection::print
void print()
Definition: LArQuickHistMerge.cxx:112
m_data
std::vector< T > m_data
Definition: TrackTruthMatchingBaseAlg.cxx:660
lumiFormat.inPath
string inPath
Definition: lumiFormat.py:68
plotmaker.hist
hist
Definition: plotmaker.py:148
dqutils::DEBUG
@ DEBUG
Definition: MonitoringFile.h:56
dqutils::MonitoringFile::mergeLB_processLBinterval
int mergeLB_processLBinterval(TFile *file, const std::vector< std::string > &inputDirNames, const std::string &outputDirName)
histCollection::write
void write(TFile *out)
Definition: LArQuickHistMerge.cxx:376
skel.it
it
Definition: skel.GENtoEVGEN.py:407
dqutils::MonitoringFile::merge_lowerLB
static void merge_lowerLB(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:703
dirname
std::string dirname(std::string name)
Definition: utils.cxx:200
dqutils::MonitoringFile::CopyHistogram::executeMD
virtual bool executeMD(TH1 *hist, const MetaData &md)
python.AthDsoLogger.out
out
Definition: AthDsoLogger.py:70
dqutils::MonitoringFile::CopyHistogram::CopyHistogram
CopyHistogram(TDirectory *target, const std::string &dirName)
dqutils::MonitoringFile::merge_Rebinned
static void merge_Rebinned(TH1 &a, TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:501
dqutils::MonitoringFile::getDebugLevel
static int getDebugLevel()
CscCalibQuery.fileList
fileList
Definition: CscCalibQuery.py:329
dqutils::MonitoringFile::clearData
virtual void clearData()
athena.value
value
Definition: athena.py:124
dqutils::none
@ none
Definition: MonitoringFile.h:56
TrigInDetValidation_Base.test
test
Definition: TrigInDetValidation_Base.py:142
dqutils::MonitoringFile::DirMap_t
std::map< std::string, TDirectory * > DirMap_t
Definition: MonitoringFile.h:95
PrintTrkAnaSummary.dirName
dirName
Definition: PrintTrkAnaSummary.py:275
BchCleanup.identical
identical
Definition: BchCleanup.py:304
python.base_data.hcfg
hcfg
Definition: base_data.py:21
dqutils::MonitoringFile::getObjKey
static TKey * getObjKey(TDirectory *dir, const std::string &path)
defaultMerge
void defaultMerge(TObject *a, const TObject *b)
Definition: LArQuickHistMerge.cxx:134
histCollection::size
size_t size()
Definition: LArQuickHistMerge.cxx:56
dqi::DisableMustClean
Definition: HanUtils.h:30
dqutils::MonitoringFile::CopyHistogram::fillMD
void fillMD(const MetaData &md)
find_tgc_unfilled_channelids.mapping
mapping
Definition: find_tgc_unfilled_channelids.py:17
dq_defect_bulk_create_defects.line
line
Definition: dq_defect_bulk_create_defects.py:27
XMLtoHeader.count
count
Definition: XMLtoHeader.py:84
dqutils::MonitoringFile::MonitoringFile
MonitoringFile()
python.iconfTool.models.loaders.level
level
Definition: loaders.py:20
dqutils::MonitoringFile::getHanResults
static std::string getHanResults(const std::string &hanResultsDir, const std::string &input, const std::string &hcfg, const std::string &hcfg_min10, const std::string &hcfg_min30)
dqutils::MonitoringFile::~MonitoringFile
virtual ~MonitoringFile()
PrepareReferenceFile.regex
regex
Definition: PrepareReferenceFile.py:43
python.checkMetadata.metadata
metadata
Definition: checkMetadata.py:175
PixelModuleFeMask_create_db.remove
string remove
Definition: PixelModuleFeMask_create_db.py:83
python.setupRTTAlg.size
int size
Definition: setupRTTAlg.py:39
dqutils::MonitoringFile::fillMetaDataMap
static void fillMetaDataMap(std::map< std::string, dqutils::MonitoringFile::MetaData > &mdMap, TDirectory *dir)
dqutils::MonitoringFile::loopOnHistograms
static void loopOnHistograms(HistogramOperation &fcn, TDirectory *dir)
dqutils::MonitoringFile::m_cpuPerHistogram
static std::unordered_map< std::string, std::clock_t > m_cpuPerHistogram
Definition: MonitoringFile.h:442
dqutils::MonitoringFile::loopOnHistogramsInMetadata
static bool loopOnHistogramsInMetadata(HistogramOperation &fcn, TDirectory *dir)
covarianceTool.pathName
pathName
Definition: covarianceTool.py:704
PixelAthClusterMonAlgCfg.histname
histname
Definition: PixelAthClusterMonAlgCfg.py:106
dqutils::MonitoringFile::OutputMetadata::makeBranch
void makeBranch(const char *branchName, const char *branchstr)
dqutils::MonitoringFile::CopyHistogram::~CopyHistogram
virtual ~CopyHistogram()
geometry_dat_to_json.indent
indent
Definition: geometry_dat_to_json.py:37
python.BunchSpacingUtils.lb
lb
Definition: BunchSpacingUtils.py:86
dqutils::MonitoringFile::m_mergeMatchDirRE
std::optional< std::regex > m_mergeMatchDirRE
Definition: MonitoringFile.h:439
fillPileUpNoiseLumi.next
next
Definition: fillPileUpNoiseLumi.py:52
ParseInputs.gDirectory
gDirectory
Definition: Final2012/ParseInputs.py:133
dqutils::MonitoringFile::OutputMetadata::fill
virtual void fill(const std::string &name, const std::string &interval, const std::string &chain, const std::string &merge)
lumiFormat.i
int i
Definition: lumiFormat.py:85
dqutils::MonitoringFile::m_file
TFile * m_file
Definition: MonitoringFile.h:420
python.CaloCondTools.g
g
Definition: CaloCondTools.py:15
HanUtils.h
extractSporadic.h
list h
Definition: extractSporadic.py:96
dqutils::MonitoringFile::setHistogramRegEx
bool setHistogramRegEx(const std::string &re)
getLatestRuns.interval
interval
Definition: getLatestRuns.py:24
python.ByteStreamConfig.write
def write
Definition: Event/ByteStreamCnvSvc/python/ByteStreamConfig.py:248
generateReferenceFile.files
files
Definition: generateReferenceFile.py:12
calibdata.exception
exception
Definition: calibdata.py:495
file
TFile * file
Definition: tile_monitor.h:29
test_pyathena.parent
parent
Definition: test_pyathena.py:15
dqutils::MonitoringFile::getPath
static std::string getPath(TDirectory *dir)
hist_file_dump.f
f
Definition: hist_file_dump.py:140
SCT_Monitoring::disabled
@ disabled
Definition: SCT_MonitoringNumbers.h:60
hotSpotInTAG.lowerLB
int lowerLB
Definition: hotSpotInTAG.py:174
index.currentDir
currentDir
Definition: index.py:37
xAOD::double
double
Definition: CompositeParticle_v1.cxx:159
dqutils
Definition: CoolMdt.h:76
skel.allFiles
list allFiles
Definition: skel.GENtoEVGEN.py:767
beamspotman.stat
stat
Definition: beamspotman.py:262
dqutils::MonitoringFile::mergeFiles
int mergeFiles(const std::string &outFileName, const std::vector< std::string > &files, fileLBMap_t &lbMap, bool fillLBDirs=true)
beamspotman.dir
string dir
Definition: beamspotman.py:619
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:26
dqutils::MonitoringFile::doTiming
void doTiming()
dqutils::MonitoringFile::merge_RMS
static void merge_RMS(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:586
dqutils::MonitoringFile::merge_weightedAverage
static void merge_weightedAverage(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:303
dqutils::MonitoringFile::GatherStatistics::GatherStatistics
GatherStatistics(const std::string &dirName)
han
Definition: han.py:1
name
std::string name
Definition: Control/AthContainers/Root/debug.cxx:240
ActsTrk::to_string
std::string to_string(const DetectorType &type)
Definition: GeometryDefs.h:34
plotBeamSpotMon.b
b
Definition: plotBeamSpotMon.py:76
dqutils::MonitoringFile::setCheckEquality
static void setCheckEquality(bool value)
MakeNewFileFromOldAndSubstitution.newName
dictionary newName
Definition: ICHEP2016/MakeNewFileFromOldAndSubstitution.py:95
dumpNswErrorDb.outFileName
string outFileName
Definition: dumpNswErrorDb.py:131
checkTriggerxAOD.found
found
Definition: checkTriggerxAOD.py:328
dqi::HanApp
Definition: HanApp.h:21
dqutils::MonitoringFile::dirHasHistogramsInMetadata
static bool dirHasHistogramsInMetadata(TDirectory *dir)
dqutils::MonitoringFile::GatherNames::execute
virtual bool execute(TH1 *hist)
grepfile.filenames
list filenames
Definition: grepfile.py:34
dqutils::MonitoringFile::OutputMetadata::OutputMetadata
OutputMetadata(TTree *metadata)
lumiFormat.outPath
string outPath
Definition: lumiFormat.py:64
dqutils::MonitoringFile::setDirectoryRegEx
bool setDirectoryRegEx(const std::string &re)
MonitoringFile.h
dqutils::MonitoringFile::m_doTiming
static bool m_doTiming
Definition: MonitoringFile.h:441
a
TList * a
Definition: liststreamerinfos.cxx:10
InDetDD::other
@ other
Definition: InDetDD_Defs.h:16
VKalVrtAthena::varHolder_detail::clear
void clear(T &var)
Definition: NtupleVars.h:48
h
python.CaloAddPedShiftConfig.int
int
Definition: CaloAddPedShiftConfig.py:45
CaloSwCorrections.time
def time(flags, cells_name, *args, **kw)
Definition: CaloSwCorrections.py:242
dqutils::MonitoringFile::m_fileCompressionLevel
static std::atomic< int > m_fileCompressionLevel
Definition: MonitoringFile.h:440
copySelective.target
string target
Definition: copySelective.py:36
dqutils::MonitoringFile::merge_identical
static void merge_identical(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:752
python.envutil.filelist
filelist
print ("Checking files %s..." % fullfile)
Definition: envutil.py:133
re
const boost::regex re(r_e)
ATLAS_NO_CHECK_FILE_THREAD_SAFETY
ATLAS_NO_CHECK_FILE_THREAD_SAFETY
Definition: MonitoringFile.cxx:39
python.utility.LHE.merge
def merge(input_file_pattern, output_file)
Merge many input LHE files into a single output file.
Definition: LHE.py:29
dqutils::MonitoringFile::OutputMetadata::m_metadata
TTree * m_metadata
Definition: MonitoringFile.h:83
CaloCellTimeCorrFiller.filename
filename
Definition: CaloCellTimeCorrFiller.py:23
dqutils::VERBOSE
@ VERBOSE
Definition: MonitoringFile.h:56
python.base_data.hanResultsDir
hanResultsDir
Definition: base_data.py:31
dqutils::MonitoringFile::FindCommon
virtual std::string FindCommon(const std::string &name1, const std::string &name2) const
python.SystemOfUnits.s
float s
Definition: SystemOfUnits.py:147
dqutils::MonitoringFile::GatherStatistics::execute
virtual bool execute(TH1 *hist)
dqutils::fileLBMap_t
std::map< std::string, std::set< std::string > > fileLBMap_t
Definition: MonitoringFile.h:53
ClassImp
ClassImp(dqutils::MonitoringFile) namespace dqutils
Definition: MonitoringFile.cxx:41
dqutils::MonitoringFile::getIndentation
static std::string getIndentation(const std::string &pathName, const std::string &leadingSpace="")
DiTauMassTools::MMCFitMethod::shortName
const std::string shortName[MAX]
Definition: PhysicsAnalysis/TauID/DiTauMassTools/DiTauMassTools/HelperFunctions.h:48
dqt_zlumi_alleff_HIST.eff
int eff
Definition: dqt_zlumi_alleff_HIST.py:113
checkCorrelInHIST.histos
dictionary histos
Definition: checkCorrelInHIST.py:413
dqutils::MonitoringFile::printStatistics
virtual void printStatistics()
dqutils::MonitoringFile::merge_perBinEffPerCent
static void merge_perBinEffPerCent(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:135
jobOptions.fileName
fileName
Definition: jobOptions.SuperChic_ALP2.py:39
python.envutil.fulldir
fulldir
Definition: envutil.py:153
collisions.reader
reader
read the goodrunslist xml file(s)
Definition: collisions.py:22
python.LumiCalcRecover.subdir
subdir
Definition: LumiCalcRecover.py:25
test_pyathena.counter
counter
Definition: test_pyathena.py:15
xAOD::lumiBlock
setTeId lumiBlock
Definition: L2StandAloneMuon_v1.cxx:328
plotBeamSpotCompare.histo
histo
Definition: plotBeamSpotCompare.py:414
columnar::operator()
decltype(auto) operator()(ObjectId< CI, CM > id) const noexcept
Definition: ColumnAccessor.h:173
dqBeamSpot.nEntries
int nEntries
Definition: dqBeamSpot.py:72
python.PyAthena.obj
obj
Definition: PyAthena.py:132
dqutils::MonitoringFile::merge_weightedEff
static void merge_weightedEff(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:421
value_type
Definition: EDM_MasterSearch.h:11
PrepareReferenceFile.outfile
outfile
Definition: PrepareReferenceFile.py:42
python.compressB64.c
def c
Definition: compressB64.py:93
length
double length(const pvec &v)
Definition: FPGATrackSimLLPDoubletHoughTransformTool.cxx:26
python.AutoConfigFlags.msg
msg
Definition: AutoConfigFlags.py:7
hist_file_dump.ordering
ordering
Definition: hist_file_dump.py:85
dqutils::MonitoringFile::setFile
virtual bool setFile(const std::string &fileName)
Clears all previous data and opens the file with the given name for analysis, returning a boolean ind...
dqutils::MonitoringFile::merge_RMSpercentDeviation
static void merge_RMSpercentDeviation(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:643
histCollection::addDirectory
void addDirectory(TDirectory *dir, const std::string &dirName)
Definition: LArQuickHistMerge.cxx:296
histCollection::histPerDir_t::histPerDir_t
histPerDir_t(const std::string &nameIn, TObject *objIn, TTree *md, bool dbg=false)
Definition: LArQuickHistMerge.cxx:226
merge
Definition: merge.py:1
dqutils::MonitoringFile::setListFromFile
static bool setListFromFile(std::vector< std::string > &filelist, const std::string &listFileName)
dqutils::MonitoringFile::merge_eventSample
static void merge_eventSample(TH2 &a, const TH2 &b)
Definition: MonitoringFile_MergeAlgs.cxx:550
dqutils::MonitoringFile::buildLBToIntervalMap
std::map< std::string, std::vector< std::string > > buildLBToIntervalMap(TDirectory *runDir)
CscCalibQuery.nFiles
int nFiles
Definition: CscCalibQuery.py:331
mapkey::key
key
Definition: TElectronEfficiencyCorrectionTool.cxx:37
dqutils::MonitoringFile::m_mergeMatchHistoRE
std::optional< std::regex > m_mergeMatchHistoRE
Definition: MonitoringFile.h:438