Loading [MathJax]/extensions/tex2jax.js
ATLAS Offline Software
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
MonitoringFile.cxx
Go to the documentation of this file.
1 /*
2  Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
3 */
4 
6 
7 #include <TDirectory.h>
8 #include <TEfficiency.h>
9 #include <TFile.h>
10 #include <TGraph.h>
11 #include <TH1.h>
12 #include <TH2.h>
13 #include <TIterator.h>
14 #include <TKey.h>
15 #include <TObject.h>
16 #include <TROOT.h>
17 #include <TSystem.h>
18 #include <TTree.h>
19 
20 #include <cmath>
21 #include <cstdio>
22 #include <ctime>
23 #include <exception>
24 #include <filesystem>
25 #include <format>
26 #include <fstream>
27 #include <iostream>
28 #include <map>
29 #include <memory>
30 #include <set>
31 #include <sstream>
32 #include <vector>
33 
36 #include "TTreeReader.h"
37 #include "TTreeReaderArray.h"
38 
40 
42 
43 namespace dqutils {
44 
45  class dbgPrint {
46 
47  public:
48  dbgPrint(const debugLevel_t setLvl = none) : m_currLvl(setLvl) {};
49  inline void operator()(const debugLevel_t level, const std::string& msg) const {
50  if (level <= m_currLvl)
51  std::cout << msg << std::endl;
52  }
53  void setLvl(const debugLevel_t lvl) { m_currLvl = lvl; }
54  debugLevel_t getLvl() const { return m_currLvl; }
55 
56  private:
57  debugLevel_t m_currLvl;
58  };
59 
60  static dbgPrint s_dbg;
61  static bool s_checkEquality=false;
62 
63  std::optional<std::regex> checkRegEx(const std::string& re) {
64  if (re.empty())
65  return std::nullopt;
66 
67  std::regex reNew(re);
68  try {
69  // this should fail if there are any problems with re!
70  std::string test("Test String");
71  std::regex_match(test, reNew);
72  } catch (std::exception& e) {
73  std::cout << "ERROR: Invalid RegEx string \"" << re << "\"." << std::endl;
74  std::cout << "See http://www.boost.org/doc/libs/1_42_0/libs/regex/doc/html/boost_regex/syntax.html for allowed regular expression syntax" << std::endl;
75  return std::nullopt;
76  }
77  return reNew;
78  }
79 
80  // Internally used data-structures:
81 
82  class histCollection {
83 
84  public:
85  typedef std::map<std::string, std::vector<std::string>> fileLBMap_t;
86  explicit histCollection(TFile* out, bool skipExisting = false) : m_out{out}, m_skipExisting(skipExisting) {};
87  histCollection() = delete;
88 
89  ~histCollection();
90 
91  void addDirectory(TDirectory* dir, const std::string& dirName, const std::string& filename = "");
92  size_t size() { return m_data.size(); };
93  void print();
94  void write(); // Destructive method, will delete internal data after writing
95 
96  void addDirExclusion(const std::optional<std::regex>& dirEx);
97  void addHistExclusion(const std::optional<std::regex>& histEx);
98 
99  unsigned size() const;
100  void clear();
101 
102  fileLBMap_t getFileLBMapAndClear() {
103  m_data.clear();
104  return std::move(m_fileLBMap);
105  }
106 
107  void printTiming();
108 
109  private:
110  class histPerDir_t {
111  public:
112  histPerDir_t(const std::string& nameIn, std::unique_ptr<TObject>&& objIn, TTree* md);
113  histPerDir_t(dqutils::histCollection::histPerDir_t&& other)
114  : name(std::move(other.name)), obj(std::move(other.obj)), metadata(std::move(other.metadata)), mergeMethod(other.mergeMethod) {}
115 
116  std::string name;
117  std::unique_ptr<TObject> obj;
118  std::array<std::string, 3> metadata{"unset","","<default>"};
119  std::clock_t cpuSum = 0;
120  void (*mergeMethod)(TObject* a, const TObject* b) = nullptr;
121  void merge(TObject* other);
122 
123  private:
124  bool fillMD(TTree* mdTree);
125  };
126 
127  struct histDir_t {
128  std::unordered_map<std::string, histPerDir_t> histos;
129  void writeMD(TDirectory* outDir) const;
130  };
131 
132  private:
133  TFile* m_out;
134  bool m_skipExisting;
135  std::unordered_map<std::string, histDir_t> m_data;
136  std::optional<std::regex> m_dirExclusion;
137  std::optional<std::regex> m_histExclusion;
138  fileLBMap_t m_fileLBMap;
139  };
140 
141  void histCollection::clear() {
142  m_data.clear();
143  m_fileLBMap.clear();
144  }
145 
146  void histCollection::printTiming() {
147  std::vector<std::pair<std::string, clock_t>> cpuPerHistVec;
148  for (auto& [dirname, histDir] : m_data) {
149  for (auto& [histname, histo] : histDir.histos) {
150  cpuPerHistVec.emplace_back(dirname + "/" + histname, histo.cpuSum);
151  }
152  }
153  auto ordering = [](std::pair<std::string, clock_t> a, std::pair<std::string, clock_t> b) { return a.second < b.second; };
154  std::sort(cpuPerHistVec.begin(), cpuPerHistVec.end(), ordering);
155  for (const auto& [name, time] : cpuPerHistVec) {
156  const double tSec = double(time) / CLOCKS_PER_SEC;
157  std::cout << std::format("{:<30} : {:10.3f}", name, tSec) << std::endl;
158  }
159  return;
160  }
161 
162  bool histCollection::histPerDir_t::fillMD(TTree * md) {
163  TTreeReader reader(md);
164  TTreeReaderArray<char> i_name(reader, "Name");
165  TTreeReaderArray<char> i_interval(reader, "Interval");
166  TTreeReaderArray<char> i_chain(reader, "TriggerChain");
167  TTreeReaderArray<char> i_merge(reader, "MergeMethod");
168 
169  bool found = false;
170  while (reader.Next()) {
171  const std::string nameStr(static_cast<char*>(i_name.GetAddress()));
172  if (name == nameStr) {
173  metadata = {static_cast<char*>(i_interval.GetAddress()), static_cast<char*>(i_chain.GetAddress()), static_cast<char*>(i_merge.GetAddress())};
174  found = true;
175  break;
176  }
177  }
178  return found;
179  }
180 
182  if (obj && other) {
183  const std::clock_t cpuStart = std::clock();
184  this->mergeMethod(obj.get(), other);
185  cpuSum += std::clock() - cpuStart;
186  }
187  return;
188  }
189 
190  void histCollection::histDir_t::writeMD(TDirectory * out) const {
191 
192  // Check if there is already a metadata-tree. Merge content if necessary
193  std::map<std::string, std::array<std::string, 3>> metadatamap;
194  std::unique_ptr<TTree> oldMD((TTree*)out->Get("metadata"));
195  if (oldMD) {
196  TTreeReader reader(oldMD.get());
197  TTreeReaderArray<char> i_name(reader, "Name");
198  TTreeReaderArray<char> i_interval(reader, "Interval");
199  TTreeReaderArray<char> i_chain(reader, "TriggerChain");
200  TTreeReaderArray<char> i_merge(reader, "MergeMethod");
201 
202  while (reader.Next()) {
203  const std::string name(static_cast<char*>(i_name.GetAddress()));
204  metadatamap[name] = {static_cast<char*>(i_interval.GetAddress()), static_cast<char*>(i_chain.GetAddress()), static_cast<char*>(i_merge.GetAddress())};
205  }
206  }
207 
208  for (const auto& [key, h] : histos) {
209  if (h.metadata[0]!="unset") //Ignore dummy-metadata (eg HLTMon use-case)
210  metadatamap[key] = h.metadata;
211  }
212 
213  if (metadatamap.empty()) return; //Do not write empty metadata tree
214  std::string interval, chain, merge;
215  char histname[1024]; // FIXME, no idea why this works only in this old-fashioned way
216  std::unique_ptr<TTree> mdTree = std::make_unique<TTree>("metadata", "Monitoring Metadata");
217  mdTree->SetDirectory(out);
218 
219  mdTree->Branch("Name", (void*)nullptr, "Name/C");
220  mdTree->Branch("Interval", interval.data(), "Interval/C");
221  mdTree->Branch("TriggerChain", chain.data(), "TriggerChain/C");
222  mdTree->Branch("MergeMethod", merge.data(), "MergeMethod/C");
223 
224  mdTree->SetBranchAddress("Name", histname);
225 
226  for (auto& [key, h] : metadatamap) {
227  strncpy(histname, key.c_str(), 1023);
228  interval = h[0];
229  chain = h[1];
230  merge = h[2];
231  mdTree->Fill();
232  }
233  mdTree->Write(0, TObject::kOverwrite);
234  }
235 
236  histCollection::~histCollection() {}
237 
238  void histCollection::addDirExclusion(const std::optional<std::regex>& dir) {
239  m_dirExclusion = dir;
240  return;
241  }
242 
243  void histCollection::addHistExclusion(const std::optional<std::regex>& dir) {
244  m_histExclusion = dir;
245  return;
246  }
247 
248  unsigned histCollection::size() const {
249  unsigned s = 0;
250  for (const auto& it : m_data) {
251  s += it.second.histos.size();
252  }
253  return s;
254  }
255 
256  void histCollection::print() {
257  for (const auto& it : m_data) {
258  const histDir_t& hd = it.second;
259  std::cout << "Dir: " << it.first << " has " << hd.histos.size() << " histos" << std::endl;
260  for (const auto& it1 : hd.histos)
261  std::cout << "\t" << it1.second.name << std::endl;
262  }
263  return;
264  }
265 
266  template <class HIST>
267  void defaultMerge(TObject * a, const TObject* b) {
268  ((HIST*)a)->Add((HIST*)b);
269  return;
270  }
271 
272  void weightedAverage ATLAS_NOT_THREAD_SAFE(TObject * a, const TObject* b) {
273  TH1* a1 = (dynamic_cast<TH1*>(a));
274  const TH1* b1 = dynamic_cast<const TH1*>(b);
275  if (!b1 || !a1)
276  std::cout << "ERROR in weightedAverageTH1: Object not of type TH1" << std::endl;
277  else {
278  if (b1->GetEntries()==0) return;
280  }
281  return;
282  }
283 
284  void weightedEff ATLAS_NOT_THREAD_SAFE(TObject * a, const TObject* b) {
285  TH1* a1 = (dynamic_cast<TH1*>(a));
286  const TH1* b1 = (dynamic_cast<const TH1*>(b));
287  if (!b1 || !a1)
288  std::cout << "ERROR in weightedEff: Object not of type TH1" << std::endl;
289  else {
290  if (b1->GetEntries()==0) return;
292  }
293  return;
294  }
295 
296  void mergeRMS ATLAS_NOT_THREAD_SAFE(TObject * a, const TObject* b) {
297  TH1* a1 = (dynamic_cast<TH1*>(a));
298  const TH1* b1 = dynamic_cast<const TH1*>(b);
299  if (!b1 || !a1)
300  std::cout << "ERROR in mergeRMS: Object not of type TH1" << std::endl;
301  else {
302  if (b1->GetEntries()==0) return;
304  }
305  return;
306  }
307 
308  void RMSpercentDeviation ATLAS_NOT_THREAD_SAFE(TObject * a, const TObject* b) {
309  TH1* a1 = (dynamic_cast<TH1*>(a));
310  const TH1* b1 = dynamic_cast<const TH1*>(b);
311  if (!b1 || !a1)
312  std::cout << "ERROR in RMSpercentDeviation: Object not of type TH1" << std::endl;
313  else {
314  if (b1->GetEntries()==0) return;
316  }
317  return;
318  }
319 
320  void perBinEffPerCent ATLAS_NOT_THREAD_SAFE(TObject * a, const TObject* b) {
321  TH1* a1 = (dynamic_cast<TH1*>(a));
322  const TH1* b1 = dynamic_cast<const TH1*>(b);
323  if (!b1 || !a1)
324  std::cout << "ERROR in getBinEffPerCent: Object not of type TH1" << std::endl;
325  else {
326  if (b1->GetEntries()==0) return;
328  }
329  return;
330  }
331 
332  void lowerLB ATLAS_NOT_THREAD_SAFE(TObject * a, const TObject* b) {
333  TH1* a1 = (dynamic_cast<TH1*>(a));
334  const TH1* b1 = dynamic_cast<const TH1*>(b);
335  if (!b1 || !a1)
336  std::cout << "ERROR in lowerLB: Object not of type TH1" << std::endl;
337  else
339  return;
340  }
341 
342  template <class HIST>
343  void identical(TObject * a, const TObject* b) {
344  if (!s_checkEquality)
345  return; //quasi null-operation
346  HIST* a1 = (dynamic_cast<HIST*>(a));
347  const HIST* b1 = dynamic_cast<const HIST*>(b);
348  if (!b1 || !a1){
349  std::cout << "ERROR in identical: Object not of correct type" << std::endl;
350  return;
351  }
353  return;
354  }
355 
356  void merge_rebinned(TObject * a, const TObject* b) {
357  TH1* a1 = (dynamic_cast<TH1*>(a));
358  const TH1* b1 = dynamic_cast<const TH1*>(b);
359  if (!a1 || !b1) {
360  std::cout << "ERROR, in merge_rebinned: Object not of type TH1";
361  return;
362  }
363  TH1* b2 = const_cast<TH1*>(b1);
365  return;
366  }
367 
368  void merge_eventSample(TObject * a, const TObject* b) {
369  TH2* a1 = (dynamic_cast<TH2*>(a));
370  const TH2* b1 = dynamic_cast<const TH2*>(b);
371  if (!a1 || !b1) {
372  std::cout << "ERROR in merge_eventSample: Object not of type TH2" << std::endl;
373  return;
374  }
376  }
377 
378  void merge_TEfficency(TObject * a, const TObject* b) {
379  TEfficiency* a1 = dynamic_cast<TEfficiency*>(a);
380  const TEfficiency* b1 = dynamic_cast<const TEfficiency*>(b);
381  TEfficiency* b2 = const_cast<TEfficiency*>(b1);
382  if (!a1 || !b1) {
383  std::cout << "ERROR in merge_TEfficiency: Object not of type TEfficiency" << std::endl;
384  return;
385  }
386  TList listE;
387  listE.Add(b2);
388  a1->Merge(&listE);
389  listE.Clear();
390  }
391 
392  void merge_TTree(TObject * a, const TObject* b) {
393  TTree* a1 = dynamic_cast<TTree*>(a);
394  const TTree* b1 = dynamic_cast<const TTree*>(b);
395  if (!a1 || !b1) {
396  std::cout << "ERROR in merge_TTree: Object not of type TTree" << std::endl;
397  return;
398  }
399  TTree* b2 = const_cast<TTree*>(b1);
400  TList listT;
401  listT.Add(b2);
402  a1->Merge(&listT);
403  listT.Clear();
404  }
405 
406  histCollection::histPerDir_t::histPerDir_t(const std::string& nameIn, std::unique_ptr<TObject>&& objIn, TTree* mdTree)
407  : name(nameIn), obj(std::move(objIn)), mergeMethod(nullptr) {
408  // Some sanity checks:
409  if (!obj) {
410  std::cout << "ERROR while adding " << nameIn << ": Histogram pointer is NULL" << std::endl;
411  return;
412  }
413 
414  if (mdTree) {
415  fillMD(mdTree);
416  }
417  else {
418  s_dbg(VERBOSE,"No matadata found for " + name +", use defaults");
419  }
420  const std::string& howToMerge = metadata[2];
421  s_dbg(VERBOSE, "Name: " + name + " mergeMethod=" + howToMerge);
422 
423  TH1* th1 = dynamic_cast<TH1*>(obj.get());
424  TH2* th2 = dynamic_cast<TH2*>(obj.get());
425  TEfficiency* teff = dynamic_cast<TEfficiency*>(obj.get());
426  if (th1) {
427  th1->SetDirectory(nullptr); // Get ownership of this hist
428  if (howToMerge == "<default>") {
429  if (th2) {
430  mergeMethod = &defaultMerge<TH2>;
431  } else { //TH1 case
432  mergeMethod = &defaultMerge<TH1>;
433  }
434  }
435  else if (howToMerge == "weightedAverage" || howToMerge=="weightedAverage2D")
436  mergeMethod = &weightedAverage;
437  else if (howToMerge == "weightedEff")
438  mergeMethod = &weightedEff;
439  else if (howToMerge == "mergeRMS")
440  mergeMethod = &mergeRMS;
441  else if (howToMerge == "RMSpercentDeviation")
442  mergeMethod = &RMSpercentDeviation;
443  else if (howToMerge == "perBinEffPerCent")
444  mergeMethod = &perBinEffPerCent;
445  else if (howToMerge == "lowerLB")
446  mergeMethod = &lowerLB;
447  else if (howToMerge == "identical")
448  if (th2) {
449  mergeMethod = &identical<TH2>;
450  } else { //TH1 case
451  mergeMethod = &identical<TH1>;
452  }
453  else if ((howToMerge == "mergeRebinned") || (howToMerge == "merge"))
454  mergeMethod = &merge_rebinned;
455  else {
456  std::cout << "ERROR: Unknown merging method (" << howToMerge << ") for object of type TH1 named " << nameIn << std::endl;
457  obj.reset(nullptr);
458  }
459  } // end if TH1
460  else if (teff) {
461  teff->SetDirectory(nullptr);
462  if (howToMerge == "<default>")
463  mergeMethod = &merge_TEfficency;
464  else
465  std::cout << "ERROR: Unknown merging method (" << howToMerge << ") for object of type TEfficiency named " << nameIn << std::endl;
466  } // end if TEfficiency
467  else if (nullptr != dynamic_cast<TTree*>(obj.get())) {
468  mergeMethod = &merge_TTree;
469  } else {
470  std::cout << "ERROR Object " << name << " has unkown type" << std::endl;
471  obj.reset(nullptr);
472  }
473  }
474 
475  void histCollection::addDirectory(TDirectory * dir, const std::string& dirName, const std::string& filename) {
476 
477  s_dbg(VERBOSE, "Working on directory " + dirName);
478  if (m_dirExclusion && !std::regex_search(dirName, *m_dirExclusion)) {
479  s_dbg(DEBUG, "Path " + dirName + " is excluded");
480  return;
481  }
482 
483  for (TObject* oKey : *dir->GetListOfKeys()) {
484  TKey* key = static_cast<TKey*>(oKey);
485  const std::string name = key->GetName();
486  const std::string classname = key->GetClassName();
487  if ((classname == "TTree") && (name == "metadata")) {
488  continue;
489  }
490 
491  s_dbg(VERBOSE, "Found name " + name + ", classname=" + classname);
492 
493  const std::string newName = dirName.empty() ? name : dirName + "/" + name;
494  auto itDir = m_data.find(dirName);
495 
496  if (classname.starts_with("TH") || classname.starts_with("TProfile") || classname.starts_with("TEfficiency") || classname == "TTree") {
497  if (m_histExclusion && !std::regex_search(name, *m_histExclusion)) {
498  s_dbg(DEBUG, "Histogram with name " + name + " is excluded");
499  continue;
500  }
501 
502  // arrive here if we have at least one histogram in this directory
503  if (m_skipExisting) {
504  // Check if this object exists already in the output-file
505  std::unique_ptr<TObject> existingObj(m_out->Get(newName.c_str()));
506  if (existingObj)
507  continue;
508  }
509 
510  std::unique_ptr<TTree> md;
511  if (itDir == m_data.end()) {
512  // Have not seen this dirName yet
513  itDir = m_data.emplace(dirName, histDir_t()).first;
514  s_dbg(VERBOSE, "Registering new directory " + dirName);
515  }
516 
517  // Check if we already have this histogram in the list
518  auto itH = itDir->second.histos.find(name);
519  if (itH == itDir->second.histos.end()) {
520  // New histogram (or Tree):
521  if (!md) {
522  // Metadata tree not yet read in this directory
523  md.reset((TTree*)dir->Get("metadata"));
524  }
525 
526  std::unique_ptr<TObject> obj{key->ReadObj()};
527  TTree* treeObj = dynamic_cast<TTree*>(obj.get());
528  if (treeObj) {
529  TDirectory* outDir = m_out->GetDirectory(dirName.c_str());
530  if (!outDir)
531  outDir = m_out->mkdir(dirName.c_str());
532  // TTree need special treatment ...
533  TDirectory* currentDir = gDirectory;
534  outDir->cd();
535  TTree* cloneTree = treeObj->CloneTree();
536  // this disconnects parent tree
537  obj.reset(cloneTree);
538  currentDir->cd();
539  }
540  histPerDir_t histo(name, std::move(obj), md.get());
541  itH = itDir->second.histos.emplace(name, std::move(histo)).first; //Take owernship of object here!
542  s_dbg(VERBOSE, "Cloning histogram " + name + " in dir " + dirName);
543  } else {
544  // Histogram already known .. merge it
545  std::unique_ptr<TObject> other(key->ReadObj());
546  if (!other) {
547  std::cout << "ERROR, got NULL key";
548  } else {
549  itH->second.merge(other.get()); // Release object in this case
550  s_dbg(VERBOSE, "Merging histogram " + name + " in dir " + dirName);
551  }
552  }
553  } else if (classname.starts_with("TDirectory")) {
554  std::unique_ptr<TObject> obj(key->ReadObj());
555  TDirectory* subdir = dynamic_cast<TDirectory*>(obj.get());
556  if (subdir) {
557  if (filename.empty()) {
558  this->addDirectory(subdir, newName, filename);
559  } else {
560  if (!name.starts_with("lb_") && !name.starts_with("lowStat_LB")) {
561  this->addDirectory(subdir, newName, filename);
562  } else {
563  m_fileLBMap[newName].push_back(filename);
564  }
565  }
566  }
567  } else {
568  std::cout << "Ignored objects '" << name << "' of type " << classname << std::endl;
569  }
570  }
571  return;
572  }
573 
574  void histCollection::write() {
575  unsigned nWritten = 0;
576  unsigned nIgnored = 0;
577  unsigned nDirs = 0;
578  for (auto& it : m_data) {
579  const std::string fulldir = it.first;
580  TDirectory* histDir = m_out->GetDirectory(fulldir.c_str());
581  if (histDir == nullptr) { // Create the directory if it doesn't exist yet
582  histDir = m_out->mkdir(fulldir.c_str());
583  if (histDir == nullptr) {
584  std::cout << "ERROR, failed to create directory " << fulldir << std::endl;
585  break;
586  } else {
587  s_dbg(VERBOSE, "Created directory " + fulldir + " in file " + m_out->GetName());
588  }
589  }
590  m_out->cd(fulldir.c_str());
591  ++nDirs;
592  for (auto& [name, histo] : it.second.histos) {
593  if (histo.obj) {
594  histo.obj->Write();
595  ++nWritten;
596  } else {
597  std::cout << "NOT writing " << name << ". Invalid." << std::endl;
598  ++nIgnored;
599  }
600  } // End loop over histograms in one directory
601  it.second.writeMD(histDir);
602  } // End loop over directories;
603  std::cout << "Wrote " << nWritten << " histograms to " << nDirs << " directories in output file " << m_out->GetName() << std::endl;
604  if (nIgnored)
605  std::cout << " Omitting " << nIgnored << " histograms." << std::endl;
606  }
607 
608  // *********************************************************************
609  // Public Methods
610  // *********************************************************************
611 
613  makeBranch("Name", "Name/C");
614  makeBranch("Interval", "Interval/C");
615  makeBranch("TriggerChain", "TriggerChain/C");
616  makeBranch("MergeMethod", "MergeMethod/C");
617  }
618 
619  void MonitoringFile::OutputMetadata::makeBranch(const char* branchName, const char* branchstr) {
620  if (!m_metadata->GetBranch(branchName)) {
621  m_metadata->Branch(branchName, (void*)nullptr, branchstr);
622  }
623  }
624 
625  void MonitoringFile::OutputMetadata::fill(const std::string& theName, const std::string& theInterval, const std::string& theChain,
626  const std::string& theMerge) {
627  std::string name = theName;
628  std::string interval = theInterval;
629  std::string chain = theChain;
630  std::string merge = theMerge;
631  m_metadata->SetBranchAddress("Name", name.data());
632  m_metadata->SetBranchAddress("Interval", interval.data());
633  m_metadata->SetBranchAddress("TriggerChain", chain.data());
634  m_metadata->SetBranchAddress("MergeMethod", merge.data());
635  m_metadata->Fill();
636  }
637 
640  m_doTiming = false;
642  }
643 
644  bool MonitoringFile::setFile(const std::string& fileName) {
645  clearData();
646  m_file = TFile::Open(fileName.c_str());
647  if (m_file != 0)
648  return true;
649  return false;
650  }
651 
652  MonitoringFile::MonitoringFile(const std::string& fileName) : m_file(0) {
654  m_doTiming = false;
657  }
658 
661 
662  delete m_file;
663  }
664 
665  bool MonitoringFile::setHistogramRegEx(const std::string& re) {
666  m_mergeMatchHistoRE = checkRegEx(re);
667  return m_mergeMatchHistoRE.has_value();
668  }
669 
670  bool MonitoringFile::setDirectoryRegEx(const std::string& re) {
671  m_mergeMatchDirRE = checkRegEx(re);
672  return m_mergeMatchDirRE.has_value();
673  }
674 
675  void MonitoringFile::getAllDirs(DirMap_t & dirmap, TDirectory * dir, const std::string& dirName) {
676  if (dir == 0)
677  return;
678 
679  if (dirName != "") {
680  DirMap_t::value_type dirmapVal(dirName, dir);
681  dirmap.insert(dirmapVal);
682  }
683 
684  TIter next(dir->GetListOfKeys());
685  TKey* key;
686  while ((key = dynamic_cast<TKey*>(next())) != 0) {
687  // don't delete TDirectories
688  TObject* obj = key->ReadObj();
689  TDirectory* subdir = dynamic_cast<TDirectory*>(obj);
690  if (subdir != 0) {
691  std::string subdirName(subdir->GetName());
692  std::string fName("");
693  if (dirName != "") {
694  fName += dirName;
695  fName += '/';
696  }
697  fName += subdirName;
698  getAllDirs(dirmap, subdir, fName);
699  } else {
700  delete obj;
701  }
702  }
703  }
704 
705  TDirectory* MonitoringFile::createDir(DirMap_t & dirmap, TDirectory * dir, const std::string& parent, const std::string& path) {
706  if (dir == 0)
707  return 0;
708 
709  TDirectory* subdir(0);
710  DirMap_t::const_iterator diter;
711  std::string::size_type i = path.find_first_of('/');
712  std::string fName("");
713  if (parent != "") {
714  fName += parent;
715  fName += '/';
716  }
717 
718  if (i != std::string::npos) {
719  std::string dName(path, 0, i);
720  std::string pName(path, i + 1, std::string::npos);
721  fName += dName;
722  if (dName != "") {
723  diter = dirmap.find(fName);
724  if (diter != dirmap.end()) {
725  subdir = diter->second;
726  } else {
727  subdir = dir->mkdir(dName.c_str());
728  DirMap_t::value_type dirmapVal(fName, subdir);
729  dirmap.insert(dirmapVal);
730  }
731  } else {
732  subdir = dir;
733  }
734  return createDir(dirmap, subdir, fName, pName);
735  }
736 
737  fName += path;
738 
739  diter = dirmap.find(fName);
740  if (diter != dirmap.end()) {
741  return diter->second;
742  }
743 
744  subdir = dir->mkdir(path.c_str());
745  DirMap_t::value_type dirmapVal(fName, subdir);
746  dirmap.insert(dirmapVal);
747  return subdir;
748  }
749 
750  TKey* MonitoringFile::getObjKey(TDirectory * dir, const std::string& path) {
751  if (dir == 0)
752  return 0;
753 
754  TKey* key(0);
755 
756  std::string::size_type i = path.find_first_of('/');
757  if (i != std::string::npos) {
758  std::string dName(path, 0, i);
759  std::string pName(path, i + 1, std::string::npos);
760  if (dName != "") {
761  key = dir->FindKey(dName.c_str());
762  if (key != 0) {
763  TDirectory* subDir = dynamic_cast<TDirectory*>(key->ReadObj());
764  if (subDir) {
765  return getObjKey(subDir, pName);
766  } // else fall through
767  }
768  return 0;
769  }
770  return getObjKey(dir, pName);
771  }
772 
773  return dir->FindKey(path.c_str());
774  }
775 
776  void MonitoringFile::fillMetaDataMap(std::map<std::string, dqutils::MonitoringFile::MetaData> & mdMap, TDirectory * dir) {
777  if (dir == 0)
778  return;
779  TTree* md = dynamic_cast<TTree*>(dir->Get("metadata"));
780  if (md == 0)
781  return;
782 
783  TTreeReader reader(md);
784  TTreeReaderArray<char> i_name(reader, "Name");
785  TTreeReaderArray<char> i_interval(reader, "Interval");
786  TTreeReaderArray<char> i_chain(reader, "TriggerChain");
787  TTreeReaderArray<char> i_merge(reader, "MergeMethod");
788 
789  while (reader.Next()) {
790  const std::string nameStr(static_cast<char*>(i_name.GetAddress()));
791  if (mdMap.find(nameStr) == mdMap.end()) {
792  MetaData md(nameStr, static_cast<char*>(i_interval.GetAddress()), static_cast<char*>(i_chain.GetAddress()), static_cast<char*>(i_merge.GetAddress()));
793  std::map<std::string, MetaData>::value_type mdVal(nameStr, md);
794  mdMap.insert(mdVal);
795  }
796  }
797 
798  delete md;
799  }
800 
801  int MonitoringFile::mergeFiles(const std::string& outFileName, const std::vector<std::string>& files) {
802  std::cout << "Writing file: " << outFileName << std::endl;
803  std::cout << "Start merging [" << files.size() << "] histogram files" << std::endl;
805  TH1::AddDirectory(false);
806  if (m_mergeMatchDirRE.has_value() || m_mergeMatchHistoRE.has_value()) {
807  std::cout << " ========== Using regular expressions for selective merging ========== " << std::endl;
808  }
809  if (m_doTiming) {
810  std::cout << "CPU time measurement activated " << std::endl;
811  }
812 
813  const size_t nFiles = files.size();
814 
815  if (nFiles < 1)
816  return -1;
817 
818  if (nFiles == 1) {
819  std::cout << "Got exactly one input file. Will copy input -> output" << std::endl;
820  if (m_mergeMatchDirRE.has_value() || m_mergeMatchHistoRE.has_value()) {
821  std::cout << "regular expressions for selective merging will have no effect!" << std::endl;
822  }
823 
826  std::filesystem::copy_file(inPath, outPath, std::filesystem::copy_options::overwrite_existing);
827  return 0;
828  }
829 
830  std::unique_ptr<TFile> outfile(TFile::Open(outFileName.c_str(), "RECREATE", outFileName.c_str(), m_fileCompressionLevel));
831  if (outfile.get() == 0) {
832  std::cout << " ERROR, cound not open output file " << outFileName << std::endl;
833  return -1;
834  }
835  std::cout << "Opened/created output file " << outFileName << std::endl;
836 
837  histCollection hc(outfile.get());
838  hc.addDirExclusion(m_mergeMatchDirRE);
839  hc.addHistExclusion(m_mergeMatchHistoRE);
840 
841  // Open first input file, mostly to get the run-directory
842  std::unique_ptr<TFile> in1(TFile::Open(files[0].c_str()));
843  if (!in1) {
844  std::cout << "ERROR, could not open input file " << files[0] << std::endl;
845  return -1;
846  }
847  std::cout << "Working on file 1/" << nFiles << ": " << files[0] << std::endl;
848  std::string runDir, runDirFwd;
849  const std::regex runDirPattern("run_[0-9]*");
850  TIter next(in1->GetListOfKeys());
851  TKey* key;
852  while ((key = (TKey*)next())) {
853  const char* name = key->GetName();
854  if (std::regex_match(name, runDirPattern)) {
855  if (runDir.size() > 0) {
856  std::cout << "ERROR More than one run_XXX directory found! Ignoring " << name << std::endl;
857  } else
858  runDir = name;
859  }
860  }
861  if (runDir.empty()) {
862  std::cout << "No run-directory found, start with '/'" << std::endl;
863  runDir="/";
864  runDirFwd="";
865  }
866  else {
867  std::cout << "Found run directory " << runDir << std::endl;
868  runDirFwd=runDir;
869  }
870 
871 
872  TDirectory* dir(dynamic_cast<TDirectory*>(in1->GetDirectory(runDir.c_str())));
873  if (!dir) {
874  std::cout << "ERROR, can't access directory " << runDir;
875  return -1;
876  }
877 
878  hc.addDirectory(dir, runDirFwd, files[0]);
879 
880  // Close first input file
881  in1->Delete("");
882  in1->Close();
883  in1.reset(nullptr);
884 
885  for (size_t i = 1; i < files.size(); ++i) {
886  std::cout << "Working on file " << 1+i << "/" << nFiles << ": " << files[i] << std::endl;
887  std::unique_ptr<TFile> in(TFile::Open(files[i].c_str()));
888  if (!in) {
889  std::cout << "ERROR, could not open input file " << files[i] << std::endl;
890  return -1;
891  }
892  TDirectory* dir(dynamic_cast<TDirectory*>(in->GetDirectory(runDir.c_str())));
893  if (not dir){
894  std::cout << "ERROR, could not cast to directory" << std::endl;
895  return -1;
896  }
897  hc.addDirectory(dir, runDirFwd, files[i]);
898  in->Delete("");
899  in->Close();
900  }
901 
902  std::cout << "Accumulated a total of " << hc.size() << " histograms." << std::endl;
903 
904  std::cout << "Start writing output ..." << std::endl;
905  hc.write();
906 
907  if (m_doTiming) {
908  std::cout << "CPU time for histogram merging: (regular histograms)" << std::endl;
909  hc.printTiming();
910  }
911  const auto lbmap = hc.getFileLBMapAndClear();
912  if (!lbmap.empty()) {
913  std::cout << "Start merging lb_nnn and lowStat_LB directories (" << lbmap.size() << " in total)" << std::endl;
914  histCollection hclb(outfile.get());
915  hc.addDirExclusion(m_mergeMatchDirRE);
916  hc.addHistExclusion(m_mergeMatchHistoRE);
917 
918  for (const auto& [dir, filenames] : lbmap) {
919  std::cout << "Merging/copying directory " << dir << std::endl;
920  for (const std::string& fName : filenames) {
921  std::unique_ptr<TFile> in(TFile::Open(fName.c_str()));
922  if (!in) {
923  std::cout << "ERROR, could not open input file " << fName << std::endl;
924  return -1;
925  }
926  TDirectory* tDir = (dynamic_cast<TDirectory*>(in->Get(dir.c_str())));
927  if (!tDir) {
928  std::cout << "ERROR, failed to get directory " << dir << " from file " << fName << std::endl;
929  } else {
930  hclb.addDirectory(tDir, dir);
931  }
932  in->Delete("");
933  in->Close();
934  }
935  hclb.write();
936  if (m_doTiming) {
937  std::cout << "CPU time for histogram merging: (lumiblock-histograms)" << std::endl;
938  hclb.printTiming();
939  }
940  hclb.clear();
941  }
942  }
943  outfile->Close();
944  return 0;
945  }
946 
947  int MonitoringFile::mergeFiles(const std::string& outFileName, const std::string& listFileName) {
948  typedef std::vector<std::string> FileList_t;
949 
950  const unsigned int nFilesAtOnce = 50;
951 
952  FileList_t allFiles;
953  bool success = setListFromFile(allFiles, listFileName);
954  if (!success) {
955  std::cout << "ERROR Failed ot read list of input files" << std::endl;
956  return -1;
957  }
958 
959  if (allFiles.size() <= nFilesAtOnce) {
961  }
962 
963  FileList_t procFiles, tmpIntermediateFiles;
964 
965  FileList_t::const_iterator filesEnd = allFiles.end();
966  FileList_t::const_iterator fi = allFiles.begin();
967 
968  unsigned int counter = 0;
969  std::string tmpInputFile("");
970  std::string tmpOutputFile("");
971 
972  // new logic: merge intermediately, then merge intermediate files
973  while (fi != filesEnd) {
974 
975  procFiles.push_back(*fi);
976  ++counter;
977  ++fi;
978  if (counter % nFilesAtOnce == 0 || fi == filesEnd) {
979  std::ostringstream nameStream;
980  nameStream << "tmp_merge_" << counter << ".root";
981  tmpOutputFile = nameStream.str();
982  tmpIntermediateFiles.push_back(tmpOutputFile);
983  int stat=mergeFiles(tmpOutputFile, procFiles);
984  if (stat) return stat;
985  procFiles.clear();
986  }
987  }
988 
989  int stat=mergeFiles(outFileName, tmpIntermediateFiles);
990  if (stat) return stat;
991 
992  for (const auto& tmpFile : tmpIntermediateFiles) {
993  std::remove(tmpFile.c_str());
994  }
995  return 0;
996  }
997 
999  if (m_file == 0) {
1000  std::cerr << "MonitoringFile::printStatistics(): "
1001  << "No input file is open\n";
1002  return;
1003  }
1004 
1005  DirMap_t indirmap;
1006 
1007  getAllDirs(indirmap, m_file, "");
1008 
1009  DirMap_t::const_iterator idirend = indirmap.end();
1010  for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1011  std::string idirName = idir->first;
1012 
1013  GatherStatistics stat_shift(idirName);
1014  GatherStatistics stat_all(idirName);
1015 
1016  loopOnHistogramsInMetadata(stat_shift, idir->second);
1017  loopOnHistograms(stat_all, idir->second);
1018 
1019  std::cout.setf(std::ios_base::left, std::ios_base::adjustfield);
1020  std::cout.width(80);
1021  std::cout << idirName << " ";
1022 
1023  std::cout.setf(std::ios_base::right, std::ios_base::adjustfield);
1024  std::cout << " shift: ";
1025  std::cout.width(3);
1026  std::cout << stat_shift.m_nHist1D << " ";
1027  std::cout.width(5);
1028  std::cout << stat_shift.m_nHist1DBins << " ";
1029  std::cout.width(3);
1030  std::cout << stat_shift.m_nHist2D << " ";
1031  std::cout.width(7);
1032  std::cout << stat_shift.m_nHist2DBins << " ";
1033  std::cout.width(3);
1034  std::cout << stat_shift.m_nGraph << " ";
1035  std::cout.width(5);
1036  std::cout << stat_shift.m_nGraphPoints << " ";
1037 
1038  std::cout << " all: ";
1039  std::cout << stat_all.m_nHist1D << " ";
1040  std::cout.width(5);
1041  std::cout << stat_all.m_nHist1DBins << " ";
1042  std::cout.width(3);
1043  std::cout << stat_all.m_nHist2D << " ";
1044  std::cout.width(7);
1045  std::cout << stat_all.m_nHist2DBins << " ";
1046  std::cout.width(3);
1047  std::cout << stat_all.m_nGraph << " ";
1048  std::cout.width(5);
1049  std::cout << stat_all.m_nGraphPoints << "\n";
1050 
1051  std::cout << std::flush;
1052  }
1053  }
1054 
1055  bool MonitoringFile::copyHistograms(const std::string& outFileName, const std::string& dirName) {
1057  // bool useRecursiveDelete = gROOT->MustClean();
1058  // gROOT->SetMustClean(false);
1059 
1060  if (m_file == 0) {
1061  std::cerr << "MonitoringFile::copyHistograms(): "
1062  << "No input file is open\n";
1063  return false;
1064  }
1065 
1066  DirMap_t indirmap;
1067  DirMap_t reducedmap;
1068  DirMap_t outdirmap;
1069 
1070  if (dirName != "all") {
1071  TKey* dkey = getObjKey(m_file, dirName);
1072  if (dkey == 0) {
1073  std::cerr << "MonitoringFile::copyHistograms(): "
1074  << "Directory \'" << dirName << "\' not found in input file\n";
1075  return false;
1076  }
1077 
1078  TDirectory* fromDir = dynamic_cast<TDirectory*>(dkey->ReadObj());
1079 
1080  DirMap_t::value_type dirmapVal(dirName, fromDir);
1081  indirmap.insert(dirmapVal);
1082  } else {
1083  std::cout << "Building list of all TDirectories in file...\n" << std::flush;
1084  getAllDirs(indirmap, m_file, "");
1085  }
1086 
1087  DirMap_t::const_iterator idirend = indirmap.end();
1088  for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1089 
1090  std::string idirName = idir->first;
1091  std::cout << "Checking " << idirName << "\n" << std::flush;
1092  // std::string::size_type j = idirName.find( "L1Calo/1_PPr_EmFADCTiming" );
1093  // if( j != std::string::npos ) {
1094  // std::cerr << "Skipping directory \"" << idirName << "\"\n";
1095  // std::cerr << std::flush;
1096  // continue;
1097  // }
1098 
1099  if (!dirHasHistogramsInMetadata(idir->second)) {
1100  continue;
1101  }
1102 
1103  reducedmap.insert(*idir);
1104  }
1105 
1106  std::unique_ptr<TFile> outfile(TFile::Open(outFileName.c_str(), "RECREATE", outFileName.c_str(), m_fileCompressionLevel));
1107  if (outfile.get() == 0) {
1108  std::cerr << "MonitoringFile::copyHistograms(): "
1109  << "Output file not opened\n";
1110  return false;
1111  }
1112 
1113  idirend = reducedmap.end();
1114  for (DirMap_t::const_iterator idir = reducedmap.begin(); idir != idirend; ++idir) {
1115 
1116  std::string idirName = idir->first;
1117  std::cout << "Processing " << idirName << "\n" << std::flush;
1118 
1119  TDirectory* toDir = createDir(outdirmap, outfile.get(), "", idirName);
1120  if (toDir == 0) {
1121  std::cerr << "MonitoringFile::copyHistograms(): "
1122  << "Directory \'" << idirName << "\' not created in output file\n";
1123  return false;
1124  }
1125 
1126  CopyHistogram copyFcn(toDir, idirName);
1127 
1128  loopOnHistogramsInMetadata(copyFcn, idir->second);
1129  }
1130 
1131  outfile->Write();
1132  outfile->Close();
1133 
1134  // gROOT->SetMustClean(useRecursiveDelete);
1135  return true;
1136  }
1137 
1138  std::string MonitoringFile::getHanResults(const std::string& hanResultsDir, const std::string& input, const std::string& hcfg,
1139  const std::string& hcfg_lowStat, const std::string& hcfg_medStat) {
1140  // DisableMustClean disabled;
1141 
1142  std::cout << "\nUsing han configurations:\n"
1143  << " entire run: " << hcfg << "\n"
1144  << " low stat interval: " << hcfg_lowStat << "\n"
1145  << " medium stat interval: " << hcfg_medStat << "\n\n"
1146  << std::flush;
1147 
1148  TFile* infile = TFile::Open(input.c_str());
1149  if (infile == 0) {
1150  std::cerr << "MonitoringFile::getHanResults(): "
1151  << "Cannot open input file \"" << input << "\"\n";
1152  return "";
1153  }
1154 
1155  std::vector<std::string> run_dirs;
1156  std::vector<std::string> lowStat_dirs;
1157  std::vector<std::string> medStat_dirs;
1158 
1159  TIter next_run(infile->GetListOfKeys());
1160  TKey* key_run(0);
1161  while ((key_run = dynamic_cast<TKey*>(next_run())) != 0) {
1162  TObject* obj_run = key_run->ReadObj();
1163  TDirectory* tdir_run = dynamic_cast<TDirectory*>(obj_run);
1164  if (tdir_run != 0) {
1165  std::string tdir_run_name(tdir_run->GetName());
1166  if (tdir_run_name.find("run") != std::string::npos) {
1167  run_dirs.push_back(tdir_run_name);
1168  TIter next_minutes(tdir_run->GetListOfKeys());
1169  TKey* key_minutes(0);
1170  while ((key_minutes = dynamic_cast<TKey*>(next_minutes())) != 0) {
1171  TObject* obj_minutes = key_minutes->ReadObj();
1172  TDirectory* tdir_minutes = dynamic_cast<TDirectory*>(obj_minutes);
1173  if (tdir_minutes != 0) {
1174  std::string tdir_minutes_name(tdir_minutes->GetName());
1175  if (tdir_minutes_name.find("lowStat") != std::string::npos) {
1176  lowStat_dirs.push_back(tdir_run_name + '/' + tdir_minutes_name);
1177  } else if (tdir_minutes_name.find("medStat") != std::string::npos) {
1178  medStat_dirs.push_back(tdir_run_name + '/' + tdir_minutes_name);
1179  }
1180  }
1181  delete obj_minutes;
1182  }
1183  }
1184  }
1185  delete obj_run;
1186  }
1187 
1188  delete infile;
1189 
1190  dqi::HanApp han;
1191 
1192  std::string fileList = " ";
1193  std::vector<std::string>::const_iterator dirs_end;
1194  std::vector<std::string>::const_iterator dir;
1195 
1196  dirs_end = run_dirs.end();
1197  for (dir = run_dirs.begin(); dir != dirs_end; ++dir) {
1198  const std::string& tdir_run_name = *dir;
1199  std::string han_output_run = hanResultsDir + '/' + tdir_run_name + "_han.root";
1200  std::cout << "Calling han( " << hcfg << ", " << input << ", " << tdir_run_name << ", " << han_output_run << " ):\n" << std::flush;
1201  han.Analyze(hcfg, input, han_output_run, tdir_run_name);
1202  std::cout << "\n";
1203  fileList += han_output_run + " " + tdir_run_name + "\n";
1204  }
1205 
1206  dirs_end = lowStat_dirs.end();
1207  for (dir = lowStat_dirs.begin(); dir != dirs_end; ++dir) {
1208  const std::string& tdir_minutes_path = *dir;
1209 
1210  std::string tdir_minutes_underscore = tdir_minutes_path;
1211  std::string::size_type tdir_minutes_i = tdir_minutes_underscore.find('/');
1212  tdir_minutes_underscore.replace(tdir_minutes_i, 1, "_");
1213 
1214  std::string han_output_lowStat = hanResultsDir + '/' + tdir_minutes_underscore + "_han.root";
1215  std::cout << "Running han, writing to " << han_output_lowStat << ":\n" << std::flush;
1216  han.Analyze(hcfg_lowStat, input, han_output_lowStat, tdir_minutes_path);
1217  std::cout << "\n";
1218  std::string subdirname(tdir_minutes_path, tdir_minutes_i + 1, std::string::npos);
1219  std::string dirname(tdir_minutes_path, 0, tdir_minutes_i);
1220  fileList += han_output_lowStat + " " + subdirname + " " + dirname + " " + subdirname + "\n";
1221  }
1222 
1223  dirs_end = medStat_dirs.end();
1224  for (dir = medStat_dirs.begin(); dir != dirs_end; ++dir) {
1225  const std::string& tdir_minutes_path = *dir;
1226 
1227  std::string tdir_minutes_underscore = tdir_minutes_path;
1228  std::string::size_type tdir_minutes_i = tdir_minutes_underscore.find('/');
1229  tdir_minutes_underscore.replace(tdir_minutes_i, 1, "_");
1230 
1231  std::string han_output_medStat = hanResultsDir + '/' + tdir_minutes_underscore + "_han.root";
1232  std::cout << "Running han, writing to " << han_output_medStat << ":\n" << std::flush;
1233  han.Analyze(hcfg_medStat, input, han_output_medStat, tdir_minutes_path);
1234  std::cout << "\n";
1235  std::string subdirname(tdir_minutes_path, tdir_minutes_i + 1, std::string::npos);
1236  std::string dirname(tdir_minutes_path, 0, tdir_minutes_i);
1237  fileList += han_output_medStat + " " + subdirname + " " + dirname + " " + subdirname + "\n";
1238  }
1239 
1240  return fileList;
1241  }
1242 
1244  if (m_file == 0) {
1245  std::cerr << "MonitoringFile::printHanConfig(): "
1246  << "No input file is open\n";
1247  return;
1248  }
1249 
1250  DirMap_t indirmap;
1251 
1252  getAllDirs(indirmap, m_file, "");
1253 
1254  std::string indent, indent_p, indent_c;
1255  std::string idirName_p;
1256  DirMap_t::const_iterator idirend = indirmap.end();
1257  for (DirMap_t::const_iterator idir = indirmap.begin(); idir != idirend; ++idir) {
1258  std::string idirName = idir->first;
1259  std::string::size_type shortNameIndex = idirName.rfind('/');
1260  std::string shortName = idirName.substr(shortNameIndex + 1, std::string::npos);
1261 
1262  std::string::size_type fsIndex = idirName.find('/');
1263  std::string shortPath;
1264  if (fsIndex != shortNameIndex)
1265  shortPath = idirName.substr(fsIndex + 1, shortNameIndex);
1266  else
1267  shortPath = idirName.substr(fsIndex + 1, std::string::npos);
1268 
1269  std::cout << idirName << "\n";
1270  std::cout << shortPath << ", " << shortName << "\n";
1271  /*
1272  indent = getIndentation(idirName,"");
1273  if(int(indent.size())==in_p){
1274  std::cout << indent << "} \n";
1275  std::cout << indent << "dir " << shortName << " { \n";
1276  std::cout << indent << " output " << idirName << "\n";
1277  std::cout << indent << " hist all_in_dir { \n " << indent << " } \n";
1278  }
1279  else if (int(indent.size()) > in_p){
1280  std::cout << indent << "dir " << shortName << " { \n";
1281  std::cout << indent << " output " << idirName << "\n";
1282  std::cout << indent << " hist all_in_dir { \n " << indent << " } \n";
1283  }
1284  else{
1285  //find common part + number of common '/'
1286  std::string common = FindCommon(idirName,idirName_p);
1287  indent_c = getIndentation(common,"");
1288  int counter = (indent_p.size() - indent_c.size())/2;
1289  for (int i = counter; i>0; i--){
1290  std::string temp = indent_c;
1291  for (int j = 0; j< i; j++){
1292  temp+=" ";
1293  }
1294  std::cout << temp << "} \n" ;
1295  }
1296  std::cout << indent << "} \n";
1297  std::cout << indent << "dir " << shortName << " { \n";
1298  std::cout << indent << " output " << idirName << "\n";
1299  std::cout << indent << " hist all_in_dir { \n " << indent << " } \n";
1300  }
1301  indent_p = indent;
1302  in_p = indent_p.size();
1303  idirName_p = idirName;
1304  */
1305  }
1306  }
1307 
1308  std::string MonitoringFile::getIndentation(const std::string& pathName, const std::string& leadingSpace) {
1309  std::string space = leadingSpace;
1310  std::string::size_type i = pathName.find_first_of('/');
1311  if (i != std::string::npos) {
1312  std::string subPath(pathName, i + 1, std::string::npos);
1313  space += " ";
1314  return getIndentation(subPath, space);
1315  }
1316  return space;
1317  }
1318 
1319  std::string MonitoringFile::FindCommon(const std::string& name1, const std::string& name2) const {
1320  int length = (name1.size() < name2.size()) ? name1.size() : name2.size();
1321  bool found = true;
1322  int count = 0;
1323  while (found == true && count < length) {
1324  if (name1[count] == name2[count]) {
1325  count++;
1326  } else {
1327  found = false;
1328  }
1329  }
1330  return (name1.substr(0, count));
1331  }
1332 
1333  // *********************************************************************
1334  // Protected Methods
1335  // *********************************************************************
1336 
1337  MonitoringFile::CopyHistogram::CopyHistogram(TDirectory * target, const std::string& dirName) : m_target(target), m_dirName(dirName), m_metadata(0) {
1338  m_metadata = new TTree("metadata", "Monitoring Metadata");
1339  m_metadata->SetDirectory(0);
1340  m_metadata->Branch("Name", (void*)nullptr, "Name/C");
1341  m_metadata->Branch("Interval", (void*)nullptr, "Interval/C");
1342  m_metadata->Branch("TriggerChain", (void*)nullptr, "TriggerChain/C");
1343  m_metadata->Branch("MergeMethod", (void*)nullptr, "MergeMethod/C");
1344  }
1345 
1347  m_target->cd();
1348  m_metadata->SetDirectory(m_target);
1349  m_metadata->Write();
1350  delete m_metadata;
1351  }
1352 
1354  m_target->cd();
1355  hist->SetDirectory(m_target);
1356  hist->Write();
1357 
1358  return true;
1359  }
1360 
1361  bool MonitoringFile::CopyHistogram::execute(TGraph * graph) {
1362  m_target->cd();
1363  graph->Write();
1364 
1365  return true;
1366  }
1367 
1368  bool MonitoringFile::CopyHistogram::execute(TEfficiency * eff) {
1369  m_target->cd();
1370  eff->Write();
1371  return true;
1372  }
1373 
1374  void MonitoringFile::CopyHistogram::fillMD(const MetaData& md) {
1375  std::string name(md.name);
1376  std::string interval(md.interval);
1377  std::string chain(md.chain);
1378  std::string merge(md.merge);
1379  m_metadata->SetBranchAddress("Name", name.data());
1380  m_metadata->SetBranchAddress("Interval", interval.data());
1381  m_metadata->SetBranchAddress("TriggerChain", chain.data());
1382  m_metadata->SetBranchAddress("MergeMethod", merge.data());
1383  m_metadata->Fill();
1384  }
1385 
1386  bool MonitoringFile::CopyHistogram::executeMD(TH1 * hist, const MetaData& md) {
1387  m_target->cd();
1388  hist->SetDirectory(m_target);
1389  hist->Write();
1390 
1391  fillMD(md);
1392 
1393  return true;
1394  }
1395 
1396  bool MonitoringFile::CopyHistogram::executeMD(TGraph * graph, const MetaData& md) {
1397  m_target->cd();
1398  graph->Write();
1399 
1400  fillMD(md);
1401 
1402  return true;
1403  }
1404 
1405  bool MonitoringFile::CopyHistogram::executeMD(TEfficiency * eff, const MetaData& md) {
1406  m_target->cd();
1407  eff->Write();
1408  fillMD(md);
1409  return true;
1410  }
1411 
1413  : m_dirName(dirName), m_nHist1D(0), m_nHist1DBins(0), m_nGraph(0), m_nGraphPoints(0), m_nHist2D(0), m_nHist2DBins(0) {}
1414 
1416  TH2* hist2d = dynamic_cast<TH2*>(hist);
1417  if (hist2d != 0) {
1418  ++m_nHist2D;
1419  m_nHist2DBins += (hist2d->GetNbinsX() * hist2d->GetNbinsY());
1420  return true;
1421  }
1422  ++m_nHist1D;
1423  m_nHist1DBins += hist->GetNbinsX();
1424  return true;
1425  }
1426 
1427  bool MonitoringFile::GatherStatistics::execute(TGraph * graph) {
1428  ++m_nGraph;
1429  m_nGraphPoints += graph->GetMaxSize();
1430  return true;
1431  }
1432 
1433  bool MonitoringFile::GatherStatistics::execute(TEfficiency * eff) {
1434  ++m_nEfficiency;
1435 
1436  TH1* h_total = eff->GetCopyPassedHisto();
1437  TH2* h_total2D = dynamic_cast<TH2*>(h_total);
1438 
1439  if (h_total2D != 0) {
1440  m_nEfficiencyBins += (h_total2D->GetNbinsX() * h_total2D->GetNbinsY());
1441  return true;
1442  } else {
1443  m_nEfficiencyBins += h_total->GetNbinsX();
1444  return true;
1445  }
1446  }
1447 
1449 
1451  m_names.push_back(std::string(hist->GetName()));
1452  return true;
1453  }
1454 
1455  bool MonitoringFile::GatherNames::execute(TGraph * graph) {
1456  m_names.push_back(std::string(graph->GetName()));
1457  return true;
1458  }
1459 
1460  bool MonitoringFile::GatherNames::execute(TEfficiency * eff) {
1461  m_names.push_back(std::string(eff->GetName()));
1462  return true;
1463  }
1464 
1465  void MonitoringFile::clearData() {
1467 
1468  delete m_file;
1469  m_file = 0;
1471  m_doTiming = false;
1472  }
1473 
1474  bool MonitoringFile::dirHasHistogramsInMetadata(TDirectory * dir) {
1475  dir->cd();
1476 
1477  TKey* mdKey = dir->FindKey("metadata");
1478  if (mdKey == 0) {
1479  return false;
1480  }
1481 
1482  TTree* md = dynamic_cast<TTree*>(mdKey->ReadObj());
1483  if (md == 0) {
1484  return false;
1485  }
1486 
1487  int counter = 0;
1488  int nEntries = int(md->GetEntries());
1489 
1490  while (counter < nEntries) {
1491  try {
1492  md->GetEntry(counter);
1493  } catch (const std::exception& e) {
1494  std::cerr << "Exception: \"" << e.what() << "\" in directory \"" << dir->GetName() << "\"\n" << std::flush;
1495  return false;
1496  }
1497 
1498  return true;
1499  ++counter;
1500  }
1501 
1502  return false;
1503  }
1504 
1505  void MonitoringFile::loopOnHistograms(HistogramOperation & fcn, TDirectory * dir) {
1506  TIter next(dir->GetListOfKeys());
1507  TKey* key;
1508  while ((key = dynamic_cast<TKey*>(next())) != 0) {
1509  TObject* obj = key->ReadObj();
1510  TH1* h(0);
1511  TGraph* g(0);
1512  TEfficiency* e(0);
1513  if ((h = dynamic_cast<TH1*>(obj))) {
1514  fcn.execute(h);
1515  } else if ((g = dynamic_cast<TGraph*>(obj))) {
1516  fcn.execute(g);
1517  } else if ((e = dynamic_cast<TEfficiency*>(obj))) {
1518  fcn.execute(e);
1519  }
1520  delete obj;
1521  }
1522  }
1523 
1524  bool MonitoringFile::loopOnHistogramsInMetadata(HistogramOperation & fcn, TDirectory * dir) {
1525  dir->cd();
1526  TKey* mdKey = dir->FindKey("metadata");
1527  if (mdKey == 0) {
1528  return false;
1529  }
1530 
1531  TTree* md = dynamic_cast<TTree*>(mdKey->ReadObj());
1532  if (md == 0) {
1533  return false;
1534  }
1535 
1536  TKey* i_key;
1537 
1538  TTreeReader reader(md);
1539  TTreeReaderArray<char> i_name(reader, "Name");
1540  TTreeReaderArray<char> i_interval(reader, "Interval");
1541  TTreeReaderArray<char> i_chain(reader, "TriggerChain");
1542  TTreeReaderArray<char> i_merge(reader, "MergeMethod");
1543 
1544  while (reader.Next()) {
1545  const std::string nameStr(static_cast<char*>(i_name.GetAddress()));
1546  dir->cd();
1547  i_key = dir->FindKey(static_cast<char*>(i_name.GetAddress()));
1548  if (i_key == 0) {
1549  std::cerr << "MonitoringFile::loopOnHistogramsInMetadata(): "
1550  << "No \'" << nameStr << "\' object found\n";
1551  return false;
1552  }
1553  MetaData md(nameStr, static_cast<char*>(i_interval.GetAddress()), static_cast<char*>(i_chain.GetAddress()), static_cast<char*>(i_merge.GetAddress()));
1554  TObject* obj = i_key->ReadObj();
1555  TH1* h = dynamic_cast<TH1*>(obj);
1556  if (h != 0) {
1557  fcn.executeMD(h, md);
1558  } else {
1559  TGraph* g = dynamic_cast<TGraph*>(obj);
1560  if (g != 0) {
1561  fcn.executeMD(g, md);
1562  }
1563  }
1564  delete obj;
1565  }
1566 
1567  delete md;
1568 
1569  return true;
1570  }
1571 
1572  bool MonitoringFile::setListFromFile(std::vector<std::string> & filelist, const std::string& listFileName) {
1573  using namespace std;
1574 
1575  filelist.clear();
1576 
1577  ifstream listfile(listFileName.c_str());
1578  if (!listfile) {
1579  cerr << "MonitoringFile::setListFromFile(): "
1580  << "cannot read from file: " << listFileName << "\n";
1581  return false;
1582  }
1583 
1584  string line;
1585  char c;
1586  string filename;
1587  while (getline(listfile, line)) {
1588  istringstream linestream(line);
1589  while (linestream.get(c)) {
1590  if (!isspace(c)) {
1591  // ignore comments
1592  if (c == '#') {
1593  break;
1594  }
1595 
1596  linestream.putback(c);
1597  linestream >> filename;
1598  if (!linestream) {
1599  cerr << "MonitoringFile::setListFromFile(): "
1600  << "badly formatted line: " << line << "\n";
1601  break;
1602  }
1603 
1604  filelist.push_back(filename);
1605  }
1606  }
1607  }
1608 
1609  return true;
1610  }
1611 
1612  int MonitoringFile::mergeLBintervals(const std::string& inFilename) {
1613 
1614  std::cout << "Running mergeLBintervals on " << inFilename << std::endl;
1615 
1616  std::unique_ptr<TFile> f(TFile::Open(inFilename.c_str(), "UPDATE"));
1617  if (!f) {
1618  std::cout << "ERROR, could not open file " << inFilename << " for update" << std::endl;
1619  return -1;
1620  }
1621  std::string runDirName;
1622  const std::regex runDirPattern("run_[0-9]*");
1623  TIter next(f->GetListOfKeys());
1624  TKey* key;
1625  while ((key = (TKey*)next())) {
1626  const char* name = key->GetName();
1627  if (std::regex_match(name, runDirPattern)) {
1628  if (runDirName.size() > 0) {
1629  std::cout << "ERROR More than one run_XXX directory found! Ignoring " << name << std::endl;
1630  } else
1631  runDirName = name;
1632  }
1633  break;
1634  }
1635 
1636  TDirectory* runDir = f->GetDirectory(runDirName.c_str());
1637  const auto mapping = buildLBToIntervalMap(runDir);
1638 
1639  if (s_dbg.getLvl() == VERBOSE) {
1640  std::cout << "LB directory mapping:" << std::endl;
1641  for (const auto& i1 : mapping) {
1642  std::cout << i1.first;
1643  for (const auto& i2 : i1.second) {
1644  std::cout << "\t" << i2 << std::endl;
1645  }
1646  }
1647  }
1648 
1649  for (const auto& [outDir, inDIrs] : mapping) {
1650  int stat=mergeLB_processLBinterval(f.get(), inDIrs, outDir);
1651  if (stat) return stat;
1652  }
1653 
1654  f->Close();
1655  f.reset(TFile::Open(inFilename.c_str(), "UPDATE"));
1656  runDir = f->GetDirectory(runDirName.c_str());
1657 
1658  std::cout << "merging lowStat_LB dirs into run-dir" << std::endl;
1659  std::vector<std::string> lowStatDirs;
1660  for (TObject* oKey : *runDir->GetListOfKeys()) {
1661  TKey* key = static_cast<TKey*>(oKey);
1662  const std::string name = key->GetName();
1663  const std::string classname = key->GetClassName();
1664  if (classname.starts_with("TDirectory") and name.starts_with("lowStat_LB")) {
1665  lowStatDirs.push_back(runDirName + "/" + name);
1666  s_dbg(VERBOSE, "Found input: " + runDirName + "/" + name);
1667  }
1668  }
1669 
1670  int stat=mergeLB_processLBinterval(f.get(), lowStatDirs, runDirName);
1671 
1672  f->Close();
1673  return stat;
1674  }
1675 
1676  std::map<std::string, std::vector<std::string>> MonitoringFile::buildLBToIntervalMap(TDirectory * runDir) {
1677 
1678  std::map<std::string, std::vector<std::string>> ranges;
1679 
1680  // No recusion here, everything we care out is run_NNNNN/lb_nnnn (and run_NNNNN/lowStat_nn-mm)
1681  const std::string runDirName = runDir->GetName();
1682  for (TObject* oKey : *runDir->GetListOfKeys()) {
1683  TKey* key = static_cast<TKey*>(oKey);
1684  const std::string name = key->GetName();
1685  const std::string classname = key->GetClassName();
1686  if (!classname.starts_with("TDirectory"))
1687  continue;
1688  if (name.starts_with("lb_")) {
1689  unsigned lumiBlock = 0;
1690  try {
1691  lumiBlock = std::stol(name.substr(3));
1692  } catch (std::invalid_argument& e) {
1693  std::cout << "ERROR, unexpected directory name " << name << ". Can't parse lb number" << std::endl;
1694  std::cout << e.what() << std::endl;
1695  continue;
1696  }
1697  // Copied from Control/AthenaMonitoringKernel/src/HistogramFiller/OfflineHistogramProvider.h
1698  const unsigned lbBase = lumiBlock - (((int64_t)lumiBlock - 1) % 20);
1699  const std::string lbString = runDirName + "/lowStat_LB" + std::to_string(lbBase) + "-" + std::to_string(lbBase + 19);
1700  ranges[lbString].push_back(runDirName + "/" + name);
1701  } // end if lb_NNNN dir
1702  } // end loop over directories under run_NNNNN
1703  return ranges;
1704  }
1705 
1706  int MonitoringFile::mergeLB_processLBinterval(TFile * file, const std::vector<std::string>& inputDirNames, const std::string& outputDirName) {
1707 
1708  TDirectory* outDir = file->GetDirectory(outputDirName.c_str());
1709  if (!outDir) {
1710  outDir = file->mkdir(outputDirName.c_str());
1711  }
1712  if (!outDir) {
1713  std::cout << "ERROR, can't obtain nor create directory " << outputDirName << " in file " << file->GetName() << std::endl;
1714  return -1;
1715  }
1716 
1717  histCollection hc(file, true);
1718  hc.addDirExclusion(m_mergeMatchDirRE);
1719  hc.addHistExclusion(m_mergeMatchHistoRE);
1720 
1721  for (const std::string& inDirName : inputDirNames) {
1722  TDirectory* inDir = file->GetDirectory(inDirName.c_str());
1723  hc.addDirectory(inDir, outputDirName);
1724  }
1725  if (hc.size() == 0) {
1726  std::cout << "mergeLB_processLBinterval: No new objects found for " << outputDirName << std::endl;
1727  } else {
1728  hc.write();
1729  }
1730  return 0;
1731  }
1732 
1733  bool MonitoringFile::CheckHistogram(TFile * f, const char* HistoName) {
1734  std::unique_ptr<TObject> obj(f->Get(HistoName));
1735  if (!obj.get()) {
1736  // std::cerr<<"No such histogram \""<< HistoName << "\"\n";
1737  return false;
1738  } else
1739  return true;
1740  }
1741 
1743  return s_dbg.getLvl();
1744  }
1746  s_dbg.setLvl((debugLevel_t)(level));
1747  }
1748  void MonitoringFile::doTiming() {
1749  m_doTiming = true;
1750  }
1751 
1752 
1753  void MonitoringFile::setCheckEquality(bool value) {dqutils::s_checkEquality=value;}
1754  std::atomic<int> MonitoringFile::m_fileCompressionLevel = 1;
1755  bool MonitoringFile::m_doTiming = false;
1756  std::unordered_map<std::string, std::clock_t> MonitoringFile::m_cpuPerHistogram;
1757 
1758  std::string MonitoringFile::getPath(TDirectory * dir) {
1759 
1760  std::string path = dir->GetPath();
1761  if (path.find(':') != std::string::npos)
1762  path = path.substr(path.rfind(':') + 1);
1763 
1764  return path;
1765  }
1766 
1767 } // namespace dqutils
dqutils::MonitoringFile::mergeLBintervals
int mergeLBintervals(const std::string &inFilename)
dqutils::MonitoringFile::getAllDirs
static void getAllDirs(DirMap_t &dirmap, TDirectory *dir, const std::string &dirName)
dqutils::debugLevel_t
debugLevel_t
Definition: MonitoringFile.h:55
histCollection
Definition: LArQuickHistMerge.cxx:50
dqutils::ATLAS_NOT_THREAD_SAFE
void getImageBuffer ATLAS_NOT_THREAD_SAFE(TImage **img, TCanvas *myC, char **x, int *y)
Definition: HanOutputFile.cxx:1130
AllowedVariables::e
e
Definition: AsgElectronSelectorTool.cxx:37
trigbs_pickEvents.ranges
ranges
Definition: trigbs_pickEvents.py:60
HanApp.h
fitman.nWritten
int nWritten
Definition: fitman.py:420
dqutils::MonitoringFile::copyHistograms
virtual bool copyHistograms(const std::string &outFileName, const std::string &dirName="all")
Copy the indicated set of histograms to an output file.
ReadCellNoiseFromCool.name1
name1
Definition: ReadCellNoiseFromCool.py:233
athena.path
path
python interpreter configuration --------------------------------------—
Definition: athena.py:128
runLayerRecalibration.chain
chain
Definition: runLayerRecalibration.py:175
dqutils::MonitoringFile::createDir
static TDirectory * createDir(DirMap_t &dirmap, TDirectory *dir, const std::string &parent, const std::string &path)
dqutils::MonitoringFile
Definition: MonitoringFile.h:58
run.infile
string infile
Definition: run.py:13
dqutils::MonitoringFile::setDebugLevel
static void setDebugLevel(int level)
vtune_athena.format
format
Definition: vtune_athena.py:14
FullCPAlgorithmsTest_eljob.flush
flush
Definition: FullCPAlgorithmsTest_eljob.py:194
dqutils::MonitoringFile::GatherNames::GatherNames
GatherNames()
collListGuids.line
string line
Definition: collListGuids.py:77
dqutils::MonitoringFile::CheckHistogram
static bool CheckHistogram(TFile *f, const char *HistoName)
rootconvert.fName
string fName
Definition: rootconvert.py:5
dqutils::MonitoringFile::printHanConfig
virtual void printHanConfig()
dqutils::MonitoringFile::CopyHistogram::execute
virtual bool execute(TH1 *hist)
histCollection::print
void print()
Definition: LArQuickHistMerge.cxx:112
m_data
std::vector< T > m_data
Definition: TrackTruthMatchingBaseAlg.cxx:660
lumiFormat.inPath
string inPath
Definition: lumiFormat.py:68
plotmaker.hist
hist
Definition: plotmaker.py:148
dqutils::DEBUG
@ DEBUG
Definition: MonitoringFile.h:55
dqutils::MonitoringFile::mergeLB_processLBinterval
int mergeLB_processLBinterval(TFile *file, const std::vector< std::string > &inputDirNames, const std::string &outputDirName)
histCollection::write
void write(TFile *out)
Definition: LArQuickHistMerge.cxx:376
skel.it
it
Definition: skel.GENtoEVGEN.py:407
dqutils::MonitoringFile::merge_lowerLB
static void merge_lowerLB(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:702
dirname
std::string dirname(std::string name)
Definition: utils.cxx:200
dqutils::MonitoringFile::CopyHistogram::executeMD
virtual bool executeMD(TH1 *hist, const MetaData &md)
python.AthDsoLogger.out
out
Definition: AthDsoLogger.py:71
dqutils::MonitoringFile::CopyHistogram::CopyHistogram
CopyHistogram(TDirectory *target, const std::string &dirName)
dqutils::MonitoringFile::merge_Rebinned
static void merge_Rebinned(TH1 &a, TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:501
dqutils::MonitoringFile::getDebugLevel
static int getDebugLevel()
CscCalibQuery.fileList
fileList
Definition: CscCalibQuery.py:330
dqutils::MonitoringFile::clearData
virtual void clearData()
athena.value
value
Definition: athena.py:124
dqutils::none
@ none
Definition: MonitoringFile.h:55
TrigInDetValidation_Base.test
test
Definition: TrigInDetValidation_Base.py:147
dqutils::MonitoringFile::DirMap_t
std::map< std::string, TDirectory * > DirMap_t
Definition: MonitoringFile.h:94
PrintTrkAnaSummary.dirName
dirName
Definition: PrintTrkAnaSummary.py:137
BchCleanup.identical
identical
Definition: BchCleanup.py:304
python.base_data.hcfg
hcfg
Definition: base_data.py:22
dqutils::MonitoringFile::getObjKey
static TKey * getObjKey(TDirectory *dir, const std::string &path)
defaultMerge
void defaultMerge(TObject *a, const TObject *b)
Definition: LArQuickHistMerge.cxx:134
histCollection::size
size_t size()
Definition: LArQuickHistMerge.cxx:56
dqi::DisableMustClean
Definition: HanUtils.h:30
dqutils::MonitoringFile::CopyHistogram::fillMD
void fillMD(const MetaData &md)
find_tgc_unfilled_channelids.mapping
mapping
Definition: find_tgc_unfilled_channelids.py:17
XMLtoHeader.count
count
Definition: XMLtoHeader.py:85
dqutils::MonitoringFile::MonitoringFile
MonitoringFile()
python.iconfTool.models.loaders.level
level
Definition: loaders.py:20
dqutils::MonitoringFile::getHanResults
static std::string getHanResults(const std::string &hanResultsDir, const std::string &input, const std::string &hcfg, const std::string &hcfg_min10, const std::string &hcfg_min30)
dqutils::MonitoringFile::~MonitoringFile
virtual ~MonitoringFile()
PrepareReferenceFile.regex
regex
Definition: PrepareReferenceFile.py:43
python.checkMetadata.metadata
metadata
Definition: checkMetadata.py:175
PixelModuleFeMask_create_db.remove
string remove
Definition: PixelModuleFeMask_create_db.py:83
python.setupRTTAlg.size
int size
Definition: setupRTTAlg.py:39
dqutils::MonitoringFile::fillMetaDataMap
static void fillMetaDataMap(std::map< std::string, dqutils::MonitoringFile::MetaData > &mdMap, TDirectory *dir)
dqutils::MonitoringFile::loopOnHistograms
static void loopOnHistograms(HistogramOperation &fcn, TDirectory *dir)
dqutils::MonitoringFile::m_cpuPerHistogram
static std::unordered_map< std::string, std::clock_t > m_cpuPerHistogram
Definition: MonitoringFile.h:441
dqutils::MonitoringFile::loopOnHistogramsInMetadata
static bool loopOnHistogramsInMetadata(HistogramOperation &fcn, TDirectory *dir)
covarianceTool.pathName
pathName
Definition: covarianceTool.py:704
PixelAthClusterMonAlgCfg.histname
histname
Definition: PixelAthClusterMonAlgCfg.py:106
dqutils::MonitoringFile::OutputMetadata::makeBranch
void makeBranch(const char *branchName, const char *branchstr)
dqutils::MonitoringFile::CopyHistogram::~CopyHistogram
virtual ~CopyHistogram()
geometry_dat_to_json.indent
indent
Definition: geometry_dat_to_json.py:37
dqutils::MonitoringFile::m_mergeMatchDirRE
std::optional< std::regex > m_mergeMatchDirRE
Definition: MonitoringFile.h:438
fillPileUpNoiseLumi.next
next
Definition: fillPileUpNoiseLumi.py:52
ParseInputs.gDirectory
gDirectory
Definition: Final2012/ParseInputs.py:133
dqutils::MonitoringFile::OutputMetadata::fill
virtual void fill(const std::string &name, const std::string &interval, const std::string &chain, const std::string &merge)
lumiFormat.i
int i
Definition: lumiFormat.py:85
dqutils::MonitoringFile::m_file
TFile * m_file
Definition: MonitoringFile.h:419
python.CaloCondTools.g
g
Definition: CaloCondTools.py:15
DetDescrDictionaryDict::it1
std::vector< HWIdentifier >::iterator it1
Definition: DetDescrDictionaryDict.h:17
HanUtils.h
extractSporadic.h
list h
Definition: extractSporadic.py:97
dqutils::MonitoringFile::setHistogramRegEx
bool setHistogramRegEx(const std::string &re)
getLatestRuns.interval
interval
Definition: getLatestRuns.py:24
python.ByteStreamConfig.write
def write
Definition: Event/ByteStreamCnvSvc/python/ByteStreamConfig.py:248
generateReferenceFile.files
files
Definition: generateReferenceFile.py:12
calibdata.exception
exception
Definition: calibdata.py:496
file
TFile * file
Definition: tile_monitor.h:29
test_pyathena.parent
parent
Definition: test_pyathena.py:15
dqutils::MonitoringFile::getPath
static std::string getPath(TDirectory *dir)
hist_file_dump.f
f
Definition: hist_file_dump.py:141
SCT_Monitoring::disabled
@ disabled
Definition: SCT_MonitoringNumbers.h:60
hotSpotInTAG.lowerLB
int lowerLB
Definition: hotSpotInTAG.py:175
index.currentDir
currentDir
Definition: index.py:37
dqutils::MonitoringFile::mergeFiles
int mergeFiles(const std::string &outFileName, const std::vector< std::string > &files)
xAOD::double
double
Definition: CompositeParticle_v1.cxx:159
dqutils
Definition: CoolMdt.h:76
skel.allFiles
list allFiles
Definition: skel.GENtoEVGEN.py:763
beamspotman.stat
stat
Definition: beamspotman.py:266
beamspotman.dir
string dir
Definition: beamspotman.py:623
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:25
dqutils::MonitoringFile::doTiming
void doTiming()
dqutils::MonitoringFile::merge_RMS
static void merge_RMS(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:585
dqutils::MonitoringFile::merge_weightedAverage
static void merge_weightedAverage(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:303
dqutils::MonitoringFile::GatherStatistics::GatherStatistics
GatherStatistics(const std::string &dirName)
han
Definition: han.py:1
name
std::string name
Definition: Control/AthContainers/Root/debug.cxx:240
ActsTrk::to_string
std::string to_string(const DetectorType &type)
Definition: GeometryDefs.h:34
plotBeamSpotMon.b
b
Definition: plotBeamSpotMon.py:77
dqutils::MonitoringFile::setCheckEquality
static void setCheckEquality(bool value)
MakeNewFileFromOldAndSubstitution.newName
dictionary newName
Definition: ICHEP2016/MakeNewFileFromOldAndSubstitution.py:95
dumpNswErrorDb.outFileName
string outFileName
Definition: dumpNswErrorDb.py:131
checkTriggerxAOD.found
found
Definition: checkTriggerxAOD.py:328
dqi::HanApp
Definition: HanApp.h:21
columnar::operator()
decltype(auto) operator()(ObjectId< OT, CM > id) const noexcept
Definition: ColumnAccessor.h:175
dqutils::MonitoringFile::dirHasHistogramsInMetadata
static bool dirHasHistogramsInMetadata(TDirectory *dir)
dqutils::MonitoringFile::GatherNames::execute
virtual bool execute(TH1 *hist)
grepfile.filenames
list filenames
Definition: grepfile.py:34
dqutils::MonitoringFile::OutputMetadata::OutputMetadata
OutputMetadata(TTree *metadata)
lumiFormat.outPath
string outPath
Definition: lumiFormat.py:64
dqutils::MonitoringFile::setDirectoryRegEx
bool setDirectoryRegEx(const std::string &re)
MonitoringFile.h
dqutils::MonitoringFile::m_doTiming
static bool m_doTiming
Definition: MonitoringFile.h:440
a
TList * a
Definition: liststreamerinfos.cxx:10
InDetDD::other
@ other
Definition: InDetDD_Defs.h:16
VKalVrtAthena::varHolder_detail::clear
void clear(T &var)
Definition: NtupleVars.h:48
h
python.CaloAddPedShiftConfig.int
int
Definition: CaloAddPedShiftConfig.py:45
dqutils::MonitoringFile::m_fileCompressionLevel
static std::atomic< int > m_fileCompressionLevel
Definition: MonitoringFile.h:439
copySelective.target
string target
Definition: copySelective.py:37
dqutils::MonitoringFile::merge_identical
static void merge_identical(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:751
python.envutil.filelist
filelist
print ("Checking files %s..." % fullfile)
Definition: envutil.py:133
re
const boost::regex re(r_e)
ATLAS_NO_CHECK_FILE_THREAD_SAFETY
ATLAS_NO_CHECK_FILE_THREAD_SAFETY
Definition: MonitoringFile.cxx:39
python.utility.LHE.merge
def merge(input_file_pattern, output_file)
Merge many input LHE files into a single output file.
Definition: LHE.py:29
dqutils::MonitoringFile::OutputMetadata::m_metadata
TTree * m_metadata
Definition: MonitoringFile.h:82
CaloCellTimeCorrFiller.filename
filename
Definition: CaloCellTimeCorrFiller.py:24
dqutils::VERBOSE
@ VERBOSE
Definition: MonitoringFile.h:55
python.base_data.hanResultsDir
hanResultsDir
Definition: base_data.py:32
dqutils::MonitoringFile::FindCommon
virtual std::string FindCommon(const std::string &name1, const std::string &name2) const
python.SystemOfUnits.s
float s
Definition: SystemOfUnits.py:146
dqutils::MonitoringFile::GatherStatistics::execute
virtual bool execute(TH1 *hist)
ClassImp
ClassImp(dqutils::MonitoringFile) namespace dqutils
Definition: MonitoringFile.cxx:41
dqutils::MonitoringFile::getIndentation
static std::string getIndentation(const std::string &pathName, const std::string &leadingSpace="")
DiTauMassTools::MMCFitMethod::shortName
const std::string shortName[MAX]
Definition: PhysicsAnalysis/TauID/DiTauMassTools/DiTauMassTools/HelperFunctions.h:48
dqt_zlumi_alleff_HIST.eff
int eff
Definition: dqt_zlumi_alleff_HIST.py:113
checkCorrelInHIST.histos
dictionary histos
Definition: checkCorrelInHIST.py:413
dqutils::MonitoringFile::printStatistics
virtual void printStatistics()
dqutils::MonitoringFile::merge_perBinEffPerCent
static void merge_perBinEffPerCent(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:135
jobOptions.fileName
fileName
Definition: jobOptions.SuperChic_ALP2.py:39
python.envutil.fulldir
fulldir
Definition: envutil.py:153
collisions.reader
reader
read the goodrunslist xml file(s)
Definition: collisions.py:22
python.LumiCalcRecover.subdir
subdir
Definition: LumiCalcRecover.py:25
test_pyathena.counter
counter
Definition: test_pyathena.py:15
xAOD::lumiBlock
setTeId lumiBlock
Definition: L2StandAloneMuon_v1.cxx:327
plotBeamSpotCompare.histo
histo
Definition: plotBeamSpotCompare.py:415
dqBeamSpot.nEntries
int nEntries
Definition: dqBeamSpot.py:73
python.PyAthena.obj
obj
Definition: PyAthena.py:132
dqutils::MonitoringFile::merge_weightedEff
static void merge_weightedEff(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:421
PrepareReferenceFile.outfile
outfile
Definition: PrepareReferenceFile.py:42
python.compressB64.c
def c
Definition: compressB64.py:93
length
double length(const pvec &v)
Definition: FPGATrackSimLLPDoubletHoughTransformTool.cxx:26
python.AutoConfigFlags.msg
msg
Definition: AutoConfigFlags.py:7
hist_file_dump.ordering
ordering
Definition: hist_file_dump.py:86
dqutils::MonitoringFile::setFile
virtual bool setFile(const std::string &fileName)
Clears all previous data and opens the file with the given name for analysis, returning a boolean ind...
dqutils::MonitoringFile::merge_RMSpercentDeviation
static void merge_RMSpercentDeviation(TH1 &a, const TH1 &b)
Definition: MonitoringFile_MergeAlgs.cxx:642
histCollection::addDirectory
void addDirectory(TDirectory *dir, const std::string &dirName)
Definition: LArQuickHistMerge.cxx:296
histCollection::histPerDir_t::histPerDir_t
histPerDir_t(const std::string &nameIn, TObject *objIn, TTree *md, bool dbg=false)
Definition: LArQuickHistMerge.cxx:226
merge
Definition: merge.py:1
dqutils::MonitoringFile::setListFromFile
static bool setListFromFile(std::vector< std::string > &filelist, const std::string &listFileName)
dqutils::MonitoringFile::merge_eventSample
static void merge_eventSample(TH2 &a, const TH2 &b)
Definition: MonitoringFile_MergeAlgs.cxx:549
dqutils::MonitoringFile::buildLBToIntervalMap
std::map< std::string, std::vector< std::string > > buildLBToIntervalMap(TDirectory *runDir)
CscCalibQuery.nFiles
int nFiles
Definition: CscCalibQuery.py:332
mapkey::key
key
Definition: TElectronEfficiencyCorrectionTool.cxx:37
dqutils::MonitoringFile::m_mergeMatchHistoRE
std::optional< std::regex > m_mergeMatchHistoRE
Definition: MonitoringFile.h:437