ATLAS Offline Software
hist_file_dump.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 # Copyright (C) 2002-2020 CERN for the benefit of the ATLAS collaboration
3 from __future__ import print_function
4 
5 import ROOT
6 import sys, os
7 import argparse
8 import zlib
9 import json
10 from PyUtils.fprint import _formatFloat
11 from typing import List, Tuple, Callable
12 
13 def fixprecision(x, precision=15):
14  import math
15  if not isinstance(x, float):
16  return x
17  else:
18  mantissa, exponent = math.frexp(x)
19  sm = '%.12g' % mantissa
20  return _formatFloat (float(sm[:precision]) * 2**exponent)
21 
22 def jsonfixup(instr, fuzzyarray=False):
23  if not isinstance(instr, str):
24  instr = instr.Data()
25  j=json.loads(instr)
26  # the following are very subject to floating point numeric effects
27  # are doubles, keep 15 decimal digits of precision
28  for badkey in ('fTsumw', 'fTsumwx', 'fTsumw2', 'fTsumwx2', 'fTsumwy', 'fTsumwy2', 'fTsumwxy',
29  'fTsumwz', 'fTsumwz2', 'fTsumwxz', 'fTsumwyz' ):
30  if badkey in j:
31  j[badkey] = fixprecision(float(j[badkey]))
32  #print(type(j["fTsumwx"]))
33  # member, digits of precision
34  arrkeys: List[Tuple[str,int,Callable]] = [('fSumw2', 15, float)]
35  if fuzzyarray:
36  arrkeys += [('fBinEntries', 15, float), ('fBinSumw2', 15, float)]
37  # apply different precision for fArray depending on object type
38  if '_typename' in j:
39  if j['_typename'] in ('TH1F', 'TH2F', 'TH3F'):
40  arrkeys.append(('fArray', 6, float))
41  elif j['_typename'] in ('TH1C', 'TH1I', 'TH1S', 'TH2C', 'TH2I', 'TH2S',
42  'TH3C', 'TH3I', 'TH3S'):
43  # precision here isn't relevant, *should* be an integer
44  arrkeys.append(('fArray', 15, lambda x: x))
45  else:
46  arrkeys.append(('fArray', 15, float))
47  else:
48  arrkeys.append(('fArray', 15, float))
49  for badkey, precision, func in arrkeys:
50  if badkey in j:
51  j[badkey] = [fixprecision(func(_), precision) for _ in j[badkey]]
52  # the following ignores small layout fluctuations in TTrees
53  if 'fBranches' in j:
54  for branch in j['fBranches']['arr']:
55  branch['fBasketSeek'] = []
56  return json.dumps(j, sort_keys=True)
57 
58 parser=argparse.ArgumentParser()
59 parser.add_argument('filename',
60  help='Input HIST file name')
61 parser.add_argument('-r', '--rankorder', default='onfile',
62  choices=['onfile', 'uncompressed', 'name'],
63  help='rankorder is "onfile" (default), "uncompressed" or "name"')
64 parser.add_argument('-p', '--path',
65  help='Only look under this directory')
66 parser.add_argument('--hash', action='store_true',
67  help='Print hashes of objects')
68 parser.add_argument('--metadata', action='store_true',
69  help='Include metadata trees')
70 parser.add_argument('--no_onfile', action='store_true',
71  help="Don't show on file size")
72 parser.add_argument('--no_inmem', action='store_true',
73  help="Don't show in memory size")
74 parser.add_argument('--tree_entries', action='store_true',
75  help="Use more robust hash of TTree branches + entries")
76 parser.add_argument('--fuzzy_histbins', action='store_true',
77  help="Allow small variations in histogram bin content")
78 args=parser.parse_args()
79 
80 ordering = args.rankorder
81 
82 accounting = {}; hashes = {}; types = {}
83 
84 ROOT.gInterpreter.LoadText("UInt_t bufferhash(TKey* key) { key->SetBuffer(); key->ReadFile(); UInt_t rv = TString::Hash(key->GetBuffer()+key->GetKeylen(), key->GetNbytes()-key->GetKeylen()); key->DeleteBuffer(); return rv; }")
85 ROOT.gInterpreter.LoadText("void* getbuffer(TKey* key) { key->SetBuffer(); key->ReadFile(); return (void*) (key->GetBuffer()+key->GetKeylen()); }")
86 ROOT.gInterpreter.LoadText("UInt_t bufferhash2(TKey* key) { TObject* obj = key->ReadObj(); TMessage msg(kMESS_OBJECT); msg.WriteObject(obj); UInt_t rv = TString::Hash(msg.Buffer(), msg.Length()); delete obj; return rv; }")
87 ROOT.gInterpreter.LoadText("UInt_t bufferhash3(TKey* key) { TObject* obj = key->ReadObj(); UInt_t rv = obj->Hash(); delete obj; return rv; }")
88 ROOT.gInterpreter.LoadText("TString getjson(TKey* key) { TObject* obj = key->ReadObj(); auto rv = TBufferJSON::ConvertToJSON(obj); delete obj; return rv; }")
89 
90 ROOT.gSystem.Load('libDataQualityUtils')
91 
92 def fuzzytreehash(tkey):
93  t = tkey.ReadObj()
94  rv = zlib.adler32(((' '.join(_.GetName() for _ in t.GetListOfBranches()))
95  + (' '.join(_.GetName() + _.GetTypeName() for _ in t.GetListOfLeaves()))
96  + ' ' + str(t.GetEntries())).encode()
97  )
98  del t
99  return rv
100 
101 def dumpdir(d):
102  thispath = d.GetPath()
103  if ':' in thispath:
104  thispath = thispath.split(':', 1)[1]
105  #print thispath
106  subdirs = []
107  for k in d.GetListOfKeys():
108  if not args.metadata and k.GetName() == 'metadata' and k.GetClassName() == 'TTree':
109  continue
110  if k.GetClassName().startswith('TDirectory'):
111  subdirs.append(k)
112  else:
113  if args.tree_entries and k.GetClassName() == 'TTree':
114  lhash = fuzzytreehash(k)
115  elif args.hash:
116  if k.GetClassName() != 'TEfficiency':
117  fixedjson = jsonfixup(ROOT.getjson(k), args.fuzzy_histbins)
118  else:
119  j0 = json.loads(ROOT.getjson(k).Data())
120  for subh in ('fPassedHistogram', 'fTotalHistogram'):
121  j0[subh] = json.loads(jsonfixup(json.dumps(j0[subh])))
122  fixedjson = json.dumps(j0, sort_keys=True)
123  lhash = zlib.adler32(fixedjson.encode())
124  if lhash < 0:
125  lhash += 2**32
126  else:
127  lhash = 0
128  idxname = os.path.join(thispath, k.GetName())
129  accounting[idxname] = (k.GetObjlen(), k.GetNbytes()-k.GetKeylen())
130  hashes[idxname] = lhash
131  types[idxname] = k.GetClassName()
132  for k in subdirs:
133  dumpdir(k.ReadObj())
134 
135 f = ROOT.TFile.Open(args.filename)
136 if args.path:
137  d = f.Get(args.path.rstrip('/'))
138  if not d:
139  print("Can't access path", args.path, "- exiting")
140  sys.exit(1)
141 else:
142  d = f
143 dumpdir(d)
144 
145 if ordering == 'onfile':
146  def key(x): return (x[1][1], x[1][0], x[0])
147 elif ordering == 'uncompressed':
148  def key(x): return (x[1][0], x[1][1], x[0])
149 else:
150  def key(x): return (x[0], x[1][1], x[1][0])
151 sortedl = sorted(accounting.items(), key=key, reverse=True)
152 if args.hash:
153  print('\n'.join(('%s %s: '
154  + ('%d uncompressed' % b if not args.no_inmem else '')
155  + (', %d on file ' % c if not args.no_onfile else ' ')
156  + '(hash %s)')
157  % (types[a], a, hashes[a]) for a, (b, c) in sortedl)
158  )
159 else:
160  print('\n'.join(('%s %s: '
161  + ('%d uncompressed' % b if not args.no_inmem else '')
162  + (', %d on file' % c if not args.no_onfile else ' '))
163  % (types[a], a) for a, (b, c) in sortedl)
164  )
165 
hist_file_dump.jsonfixup
def jsonfixup(instr, fuzzyarray=False)
Definition: hist_file_dump.py:22
hist_file_dump.key
def key(x)
Definition: hist_file_dump.py:146
hist_file_dump.dumpdir
def dumpdir(d)
Definition: hist_file_dump.py:101
Data
@ Data
Definition: BaseObject.h:11
AtlasMcWeight::encode
number_type encode(double weight)
Definition: AtlasMcWeight.cxx:65
DerivationFramework::TriggerMatchingUtils::sorted
std::vector< typename T::value_type > sorted(T begin, T end)
Helper function to create a sorted vector from an unsorted one.
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:25
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
hist_file_dump.fixprecision
def fixprecision(x, precision=15)
Definition: hist_file_dump.py:13
hist_file_dump.fuzzytreehash
def fuzzytreehash(tkey)
Definition: hist_file_dump.py:92
str
Definition: BTagTrackIpAccessor.cxx:11
readCCLHist.float
float
Definition: readCCLHist.py:83