6 Transate arbitrary root file into a han config file
7 @author: ponyisi@utexas.edu
9 Adapted for physics validation 14 May 2014
12 from __future__
import print_function
14 from DQConfMakerBase.DQElements
import DQRegion, DQReference, DQAlgorithm, DQAlgorithmParameter
15 from DQConfMakerBase.Helpers
import make_thresholds
16 from DataQualityUtils.hanwriter
import writeHanConfiguration
19 repeatalgorithm = DQAlgorithm(id=
'RepeatAlgorithm',
20 libname=
'libdqm_algorithms.so')
21 worst = DQAlgorithm(id=
'WorstCaseSummary',libname=
'libdqm_summaries.so')
27 algorithmparameters = [DQAlgorithmParameter(
'AuxAlgName--Chi2Test_Chi2_per_NDF', 1),
28 DQAlgorithmParameter(
'RepeatAlgorithm--ResultsNEntries', 1)]
31 norefalgorithm = DQAlgorithm(id=
'GatherData',
32 libname=
'libdqm_algorithms.so')
35 thresh = make_thresholds(
'Chi2_per_NDF', 1.0, 1.50,
'Chi2Thresholds')
38 def recurse(rdir, dqregion, ignorepath, modelrefs=[], displaystring='Draw=PE', displaystring2D='Draw=COLZ', regex=None, startpath=None, hists=None, manglefunc=None):
39 if manglefunc
is None:
40 manglefunc =
lambda a, b: a
41 for key
in rdir.GetListOfKeys():
42 cl = key.GetClassName(); rcl = ROOT.TClass.GetClass(cl)
43 if ' ' in key.GetName():
44 print(
'WARNING: cannot have spaces in histogram names for han config; not including %s %s' % (cl, key.GetName()))
46 if rcl.InheritsFrom(
'TH1')
or rcl.InheritsFrom(
'TGraph')
or rcl.InheritsFrom(
'TEfficiency'):
47 if '/' in key.GetName():
48 print(
'WARNING: cannot have slashes in histogram names, encountered in directory %s, histogram %s' % (rdir.GetPath(), key.GetName()))
50 if key.GetName() ==
'summary':
51 print(
'WARNING: cannot have histogram named summary, encountered in %s' % rdir.GetPath())
53 fpath = rdir.GetPath().
replace(ignorepath,
'')
54 name = (fpath +
'/' + key.GetName()).lstrip(
'/')
61 if not match:
continue
63 if not regex.match(name):
continue
64 dqpargs = {
'id' : (
'' if fpath
else 'top_level/') + name,
65 'inputdatasource': (startpath +
'/' if startpath
else '') + name,
69 for mref
in modelrefs:
70 newref = DQReference(manglefunc(mref.getReference().
replace(
'same_name', (startpath +
'/' if startpath
else '') + name), mref.id))
71 newref.addAnnotation(
'info', mref.id)
72 lnewrefs.append(newref)
73 dqpargs.update({
'algorithm': repeatalgorithm,
74 'algorithmparameters': algorithmparameters,
76 'references': lnewrefs
79 dqpargs[
'algorithm'] = norefalgorithm
80 dqpar = dqregion.newDQParameter( **dqpargs)
82 if not options.normalize: drawstrs.append(
'NoNorm')
83 if options.logy
and (cl.startswith(
'TH1')
or cl==
'TProfile'): drawstrs.append(
'LogY')
84 if options.logy
and (cl.startswith(
'TH2')
or cl==
'TProfile2D'): drawstrs.append(
'LogZ')
85 if cl.startswith(
'TH1'): drawstrs.append(displaystring)
86 if cl ==
'TProfile': drawstrs.append(displaystring)
87 if cl.startswith(
'TH2')
or cl==
'TProfile2D': drawstrs.append(displaystring2D)
88 if options.scaleref != 1: drawstrs.append(
'ScaleRef=%f' % options.scaleref)
89 if options.ratio: drawstrs.append(
'RatioPad')
91 if options.ratio2D: drawstrs.append(
'Ref2DRatio')
92 if options.ratiorange
is not None:
93 drawstrs.append(
'delta(%f)' % options.ratiorange)
95 drawstrs.append(
'DataName=%s' % options.title)
96 dqpar.addAnnotation(
'display',
','.
join(drawstrs))
98 elif rcl.InheritsFrom(
'TDirectory'):
99 newregion = dqregion.newDQRegion( key.GetName(), algorithm=worst )
100 recurse(key.ReadObj(), newregion, ignorepath, modelrefs, displaystring, displaystring2D, regex, startpath, hists, manglefunc)
104 returns True if we should kill this node
105 False if we should not
107 params = dqregion.getDQParameters()
110 subregions = dqregion.getSubRegions()
111 if subregions
is None:
114 subregions = subregions[:]
116 for sr
in subregions:
120 dqregion.delRelation(
'DQRegions', sr)
121 subregions = dqregion.getSubRegions()
122 if subregions
is None:
124 if len(subregions) + len(params) == 0:
130 params = dqregion.getDQParameters()
133 subregions = dqregion.getSubRegions()
134 if subregions
is None:
137 return len(params) +
sum([
paramcount(region)
for region
in subregions])
139 def process(infname, confname, options, refs=None):
141 f = ROOT.TFile.Open(infname,
'READ')
143 print(
'ERROR: cannot open %s' % infname)
146 top_level = DQRegion(id=
'topRegion',algorithm=worst)
147 print(
'Building tree...')
148 refpairs = refs.split(
',')
if refs
else []
150 refdict = dict(_.split(
':', 1)
for _
in refpairs)
151 except Exception
as e:
154 dqrs = [DQReference(reference=
'%s:same_name' % v, id=k)
155 for k, v
in list(refdict.items())]
156 displaystring = options.drawopt
157 if options.refdrawopt:
158 displaystring +=
',' + (
','.
join(
'DrawRef=%s' % _
for _
in options.refdrawopt.split(
',')))
159 displaystring2D = options.drawopt2D
160 if options.drawrefopt2D:
161 displaystring2D +=
',' + (
','.
join(
'DrawRef2D=%s' % _
for _
in options.drawrefopt2D.split(
',')))
163 if options.startpath:
164 topindir = f.Get(options.startpath)
166 raise ValueError(
"Path %s doesn't exist in input file" % options.startpath)
167 topindirname = f.GetPath() + options.startpath.strip(
'/')
168 startpath = options.startpath.strip(
'/')
171 topindirname = f.GetPath()
175 refstartpaths = options.refstartpath.split(
',')
if options.refstartpath
else []
177 refstartpathdict = dict(_.split(
':')
for _
in refstartpaths)
178 for k, v
in refstartpathdict.items():
179 refstartpathdict[k] = v.strip(
'/')
180 except Exception
as e:
182 def refpath_manglefunc(path, id):
184 pfx = refstartpathdict[id]
186 return path.replace(
':' + (startpath +
'/' if startpath
else ''),
':' + (pfx +
'/' if pfx
else ''), 1)
191 if options.histlistfile:
192 hists = [re.compile(line.rstrip(
'\n'))
for line
in open(options.histlistfile)]
193 if options.pathregex:
print(
"histlistfile given, pathregex is ignored")
194 if options.refmangle:
196 sys.path.append(os.getcwd())
198 manglefunc = importlib.import_module(options.refmangle).mangle
200 manglefunc = refpath_manglefunc
201 recurse(topindir, top_level, topindirname, dqrs, displaystring, displaystring2D,
202 re.compile(options.pathregex), startpath, hists, manglefunc=manglefunc)
203 print(
'Pruning dead branches...')
207 sublevel = top_level.getSubRegions()[:]
209 top_level.delRelation(
'DQRegions', x)
211 print(
'Writing output')
216 import shutil, os, sys, contextlib
218 han_is_found = (ROOT.gSystem.Load(
'libDataQualityInterfaces') != 1)
220 print(
'ERROR: unable to load offline DQMF; unable to proceed')
222 bname = os.path.basename(fname)
229 prebuilt_hcfg =
False
231 @contextlib.contextmanager
234 td = tempfile.mkdtemp()
238 with tmpdir()
as hantmpdir:
240 print(
'====> Processing file %s' % (fname))
241 print(
'====> Generating han configuration file')
242 hantmpinput = os.path.join(hantmpdir, bname)
243 shutil.copyfile(fname, hantmpinput)
244 haninput = hantmpinput
245 hanconfig = os.path.join(hantmpdir,
'han.config')
246 rv =
process(hantmpinput, hanconfig, options, options.reffile)
251 print(
'No histograms to display; exiting with code 0')
254 print(
'====> Compiling han configuration')
255 hanhcfg = os.path.join(hantmpdir,
'han.hcfg')
256 ROOT.dqi.HanConfig().AssembleAndSave( hanconfig, hanhcfg )
257 print(
'====> Executing han')
259 memlimit = resource.getrlimit(resource.RLIMIT_AS)
260 resource.setrlimit(resource.RLIMIT_AS, (memlimit[1], memlimit[1]))
261 hanoutput = haninput.rpartition(
'.')[0] +
'_han.root'
263 rv = ROOT.dqi.HanApp().Analyze( hanhcfg, haninput, hanoutput )
265 raise Exception(
'failure in han')
266 print(
'====> Dumping web display output')
267 from DataQualityUtils
import handimod
268 handimod.handiWithComparisons( options.title,
272 'https://atlasdqm.web.cern.ch/atlasdqm/js/',
273 3
if options.jsRoot
else 1)
285 except Exception
as e:
288 traceback.print_exc()
289 if 'canonical format' not in str(e):
293 if not prebuilt_hcfg:
294 os.unlink(hantmpinput)
304 if __name__==
"__main__":
305 import sys, optparse, os
306 os.environ[
'TDAQ_ERS_NO_SIGNAL_HANDLERS']=
'1'
307 parser = optparse.OptionParser(usage=
'usage: %prog [options] inputfile')
308 parser.add_option(
'--reffile', default=
None,
309 help=
'Reference files to use. Must have same structure as inputfile. Format: tag1:reffile1.root,tag2:reffile2.root,...')
310 parser.add_option(
'--outdir', default=
'./handi',
311 help=
'Directory for web ouptut')
312 parser.add_option(
'--normalize', default=
False, action=
'store_true',
313 help=
'Normalize reference histograms for display')
314 parser.add_option(
'--title', default=
'Summary',
315 help=
'Title for histograms being tested')
316 parser.add_option(
'--drawopt', default=
'Draw=PE',
317 help=
'Draw options for tested histograms (only use if you know what you are doing)')
318 parser.add_option(
'--refdrawopt',
319 help=
'ROOT Draw option for reference histograms (e.g. HIST)')
320 parser.add_option(
'--drawopt2D', default=
'Draw=COLZ',
321 help=
'Draw options for tested TH2 histograms (only use if you know what you are doing)')
322 parser.add_option(
'--drawrefopt2D', default=
None,
323 help=
'Draw options for reference TH2 histograms. If nothing is specified, no 2D reference histograms are drawn. If you want to draw both test and reference histo, recommended settings are --drawopt2D="Draw=BOX" --drawrefopt2D="COLZ"')
324 parser.add_option(
'--logy', action=
'store_true',
325 help=
'Display on log Y scale')
326 parser.add_option(
'--pathregex', default=
'.*',
327 help=
'Specify regex to match histograms, e.g. "(Btag|Jets)"')
328 parser.add_option(
'--startpath', default=
None,
329 help=
'Start from this subdirectory of the file')
330 parser.add_option(
'--refstartpath', default=
None,
331 help=
'Start from this subdirectory of reference files. By default is the same as startpath. Format: tag1:dir1,tag2:dir2,...')
332 parser.add_option(
'--histlistfile',
333 help=
'text file with a list of regexes/histogram names')
334 parser.add_option(
'--scaleref', type=
"float", default=1,
335 help=
'Scale references by this value')
336 parser.add_option(
'--Kolmogorov', default=
False, action=
'store_true',
337 help=
'Run Kolmogorov test instead of Chi2 test')
338 parser.add_option(
'--ratio', default=
False, action=
'store_true',
339 help=
'Draw histograms with ratio plots')
340 parser.add_option(
'--ratio2D', default=
False, action=
'store_true',
341 help=
'Draw 2D histograms with ratio plots')
342 parser.add_option(
'--jsRoot',action=
'store_true', default=
False,
343 help=
"make interactive jsRoot displays")
344 parser.add_option(
'--ratiorange', default=
None, type=
"float",
345 help=
'set range for ratio plots (as delta to 1.0)')
346 parser.add_option(
'--refmangle', default=
None, type=
"string",
347 help=
'provide a Python module to translate histogram names between test and reference files. Module should provide\na function mangle(testhistoname, reflabel)')
349 options, args = parser.parse_args()
351 if not 1 == len(args):
355 if options.Kolmogorov:
356 algorithmparameters = [DQAlgorithmParameter(
'AuxAlgName--KolmogorovTest_Prob', 1),
357 DQAlgorithmParameter(
'RepeatAlgorithm--ResultsNEntries', 1)]
358 thresh = make_thresholds(
'P', 0.05, 0.01,
'pThresholds')