ATLAS Offline Software
difftuple.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
2 
3 #
4 # File: D3PDMakerTest/python/difftuple.py
5 # Author: snyder@bnl.gov
6 # Date: Feb, 2010
7 # Purpose: Diff two root tuple files.
8 #
9 
10 from __future__ import print_function
11 
12 # Always run in batch mode.
13 import os
14 if 'DISPLAY' in os.environ:
15  del os.environ['DISPLAY']
16 
17 import ROOT
18 import types
19 import os
20 from fnmatch import fnmatch
21 from PyUtils.Helpers import ROOT6Setup
22 ROOT6Setup()
23 
24 # new : old
25 renames = {
26 # 'jet_antikt4h1topo_LArQuality' : 'jet_antikt4h1topo_quality',
27 # 'jet_antikt4h1tower_LArQuality' : 'jet_antikt4h1tower_quality',
28 # 'jet_antikt4truth_LArQuality' : 'jet_antikt4truth_quality',
29 # 'jet_antikt6h1tower_LArQuality' : 'jet_antikt6h1tower_quality',
30 
31 # 'METJetsInfo_JetPtWeightedEventEMFraction' : 'MET_JetsInfo_JetPtWeightedEventEMFraction',
32 # 'METJetsInfo_JetPtWeightedNumAssociatedTracks' : 'MET_JetsInfo_JetPtWeightedNumAssociatedTracks',
33 # 'METJetsInfo_JetPtWeightedSize' : 'MET_JetsInfo_JetPtWeightedSize',
34 # 'METJetsInfo_LeadingJetEt' : 'MET_JetsInfo_LeadingJetEt',
35 # 'METJetsInfo_LeadingJetEta' : 'MET_JetsInfo_LeadingJetEta',
36 
37 # 'MET_DM_All_et' : 'MET_DMAll_et',
38 # 'MET_DM_All_etx' : 'MET_DMAll_etx',
39 # 'MET_DM_All_ety' : 'MET_DMAll_ety',
40 # 'MET_DM_All_phi' : 'MET_DMAll_phi',
41 # 'MET_DM_All_sumet' : 'MET_DMAll_sumet',
42 
43 # 'MET_DM_Crack1_et' : 'MET_DMCrack1_et',
44 # 'MET_DM_Crack1_etx' : 'MET_DMCrack1_etx',
45 # 'MET_DM_Crack1_ety' : 'MET_DMCrack1_ety',
46 # 'MET_DM_Crack1_phi' : 'MET_DMCrack1_phi',
47 # 'MET_DM_Crack1_sumet' : 'MET_DMCrack1_sumet',
48 
49 # 'MET_DM_Crack2_et' : 'MET_DMCrack2_et',
50 # 'MET_DM_Crack2_etx' : 'MET_DMCrack2_etx',
51 # 'MET_DM_Crack2_ety' : 'MET_DMCrack2_ety',
52 # 'MET_DM_Crack2_phi' : 'MET_DMCrack2_phi',
53 # 'MET_DM_Crack2_sumet' : 'MET_DMCrack2_sumet',
54 
55 # 'MET_DM_Cryo_et' : 'MET_DMCryo_et',
56 # 'MET_DM_Cryo_etx' : 'MET_DMCryo_etx',
57 # 'MET_DM_Cryo_ety' : 'MET_DMCryo_ety',
58 # 'MET_DM_Cryo_phi' : 'MET_DMCryo_phi',
59 # 'MET_DM_Cryo_sumet' : 'MET_DMCryo_sumet',
60 
61 # 'MET_Muid_Spectro_et' : 'MET_MuidSpectro_et',
62 # 'MET_Muid_Spectro_etx' : 'MET_MuidSpectro_etx',
63 # 'MET_Muid_Spectro_ety' : 'MET_MuidSpectro_ety',
64 # 'MET_Muid_Spectro_phi' : 'MET_MuidSpectro_phi',
65 # 'MET_Muid_Spectro_sumet' : 'MET_MuidSpectro_sumet',
66 
67 # 'MET_Muid_Track_et' : 'MET_MuidTrack_et',
68 # 'MET_Muid_Track_etx' : 'MET_MuidTrack_etx',
69 # 'MET_Muid_Track_ety' : 'MET_MuidTrack_ety',
70 # 'MET_Muid_Track_phi' : 'MET_MuidTrack_phi',
71 # 'MET_Muid_Track_sumet' : 'MET_MuidTrack_sumet',
72 
73 # 'MET_MuonBoy_Spectro_et' : 'MET_MuonBoySpectro_et',
74 # 'MET_MuonBoy_Spectro_etx' : 'MET_MuonBoySpectro_etx',
75 # 'MET_MuonBoy_Spectro_ety' : 'MET_MuonBoySpectro_ety',
76 # 'MET_MuonBoy_Spectro_phi' : 'MET_MuonBoySpectro_phi',
77 # 'MET_MuonBoy_Spectro_sumet' : 'MET_MuonBoySpectro_sumet',
78 
79 # 'MET_MuonBoy_Track_et' : 'MET_MuonBoyTrack_et',
80 # 'MET_MuonBoy_Track_etx' : 'MET_MuonBoyTrack_etx',
81 # 'MET_MuonBoy_Track_ety' : 'MET_MuonBoyTrack_ety',
82 # 'MET_MuonBoy_Track_phi' : 'MET_MuonBoyTrack_phi',
83 # 'MET_MuonBoy_Track_sumet' : 'MET_MuonBoyTrack_sumet',
84 
85 # 'MET_RefMuon_Track_et' : 'MET_RefMuonTrack_et',
86 # 'MET_RefMuon_Track_etx' : 'MET_RefMuonTrack_etx',
87 # 'MET_RefMuon_Track_ety' : 'MET_RefMuonTrack_ety',
88 # 'MET_RefMuon_Track_phi' : 'MET_RefMuonTrack_phi',
89 # 'MET_RefMuon_Track_sumet' : 'MET_RefMuonTrack_sumet',
90 
91  #'vxp_trk_phi' : 'vxp_trk_phi0',
92  #'ef_el_trackcov_d0' : 'ef_el_trackcovd0',
93  #'ef_el_trackcov_d0_phi' : 'ef_el_trackcovd0phi',
94  #'ef_el_trackcov_d0_qoverp' : 'ef_el_trackcovd0qoverp',
95  #'ef_el_trackcov_d0_theta' : 'ef_el_trackcovd0theta',
96  #'ef_el_trackcov_d0_z0' : 'ef_el_trackcovd0z0',
97  #'ef_el_trackcov_phi' : 'ef_el_trackcovphi',
98  #'ef_el_trackcov_phi_qoverp' : 'ef_el_trackcovphiqoverp',
99  #'ef_el_trackcov_phi_theta' : 'ef_el_trackcovphitheta',
100  #'ef_el_trackcov_theta' : 'ef_el_trackcovtheta',
101  #'ef_el_trackcov_theta_qoverp' : 'ef_el_trackcovthetaqoverp',
102  #'ef_el_trackcov_qoverp' : 'ef_el_trackcovqoverp',
103  #'ef_el_trackcov_z0' : 'ef_el_trackcovz0',
104  #'ef_el_trackcov_z0_phi' : 'ef_el_trackcovz0phi',
105  #'ef_el_trackcov_z0_qoverp' : 'ef_el_trackcovz0qoverp',
106  #'ef_el_trackcov_z0_theta' : 'ef_el_trackcovz0theta',#
107 
108  #'el_trackcov_d0' : 'el_trackcovd0',
109  #'el_trackcov_d0_phi' : 'el_trackcovd0phi',
110  #'el_trackcov_d0_qoverp' : 'el_trackcovd0qoverp',
111  #'el_trackcov_d0_theta' : 'el_trackcovd0theta',
112  #'el_trackcov_d0_z0' : 'el_trackcovd0z0',
113  #'el_trackcov_phi' : 'el_trackcovphi',
114  #'el_trackcov_phi_qoverp' : 'el_trackcovphiqoverp',
115  #'el_trackcov_phi_theta' : 'el_trackcovphitheta',
116  #'el_trackcov_theta' : 'el_trackcovtheta',
117  #'el_trackcov_theta_qoverp' : 'el_trackcovthetaqoverp',
118  #'el_trackcov_qoverp' : 'el_trackcovqoverp',
119  #'el_trackcov_z0' : 'el_trackcovz0',
120  #'el_trackcov_z0_phi' : 'el_trackcovz0phi',
121  #'el_trackcov_z0_qoverp' : 'el_trackcovz0qoverp',
122  #'el_trackcov_z0_theta' : 'el_trackcovz0theta',
123 
124  #'elfilt_trackcov_d0' : 'elfilt_trackcovd0',
125  #'elfilt_trackcov_d0_phi' : 'elfilt_trackcovd0phi',
126  #'elfilt_trackcov_d0_qoverp' : 'elfilt_trackcovd0qoverp',
127  #'elfilt_trackcov_d0_theta' : 'elfilt_trackcovd0theta',
128  #'elfilt_trackcov_d0_z0' : 'elfilt_trackcovd0z0',
129  #'elfilt_trackcov_phi' : 'elfilt_trackcovphi',
130  #'elfilt_trackcov_phi_qoverp' : 'elfilt_trackcovphiqoverp',
131  #'elfilt_trackcov_phi_theta' : 'elfilt_trackcovphitheta',
132  #'elfilt_trackcov_theta' : 'elfilt_trackcovtheta',
133  #'elfilt_trackcov_theta_qoverp' : 'elfilt_trackcovthetaqoverp',
134  #'elfilt_trackcov_qoverp' : 'elfilt_trackcovqoverp',
135  #'elfilt_trackcov_z0' : 'elfilt_trackcovz0',
136  #'elfilt_trackcov_z0_phi' : 'elfilt_trackcovz0phi',
137  #'elfilt_trackcov_z0_qoverp' : 'elfilt_trackcovz0qoverp',
138  #'elfilt_trackcov_z0_theta' : 'elfilt_trackcovz0theta',
139 
140  #'mu_trackcov_d0' : 'mu_trackcovd0',
141  #'mu_trackcov_d0_phi' : 'mu_trackcovd0phi',
142  #'mu_trackcov_d0_qoverp' : 'mu_trackcovd0qoverp',
143  #'mu_trackcov_d0_theta' : 'mu_trackcovd0theta',
144  #'mu_trackcov_d0_z0' : 'mu_trackcovd0z0',
145  #'mu_trackcov_phi' : 'mu_trackcovphi',
146  #'mu_trackcov_phi_qoverp' : 'mu_trackcovphiqoverp',
147  #'mu_trackcov_phi_theta' : 'mu_trackcovphitheta',
148  #'mu_trackcov_theta' : 'mu_trackcovtheta',
149  #'mu_trackcov_theta_qoverp' : 'mu_trackcovthetaqoverp',
150  #'mu_trackcov_qoverp' : 'mu_trackcovqoverp',
151  #'mu_trackcov_z0' : 'mu_trackcovz0',
152  #'mu_trackcov_z0_phi' : 'mu_trackcovz0phi',
153  #'mu_trackcov_z0_qoverp' : 'mu_trackcovz0qoverp',
154  #'mu_trackcov_z0_theta' : 'mu_trackcovz0theta',
155 
156  #'trk_cov_d0' : 'trk_covd0',
157  #'trk_cov_phi' : 'trk_covphi',
158  #'trk_cov_qoverp' : 'trk_covqoverp',
159  #'trk_cov_theta' : 'trk_covtheta',
160  #'trk_cov_z0' : 'trk_covz0',
161  #'trk_cov_d0_phi' : 'trk_covd0phi',
162  #'trk_cov_d0_qoverp' : 'trk_covd0qoverp',
163  #'trk_cov_d0_theta' : 'trk_covd0theta',
164  #'trk_cov_d0_z0' : 'trk_covd0z0',
165  #'trk_cov_z0_phi' : 'trk_covz0phi',
166  #'trk_cov_z0_qoverp' : 'trk_covz0qoverp',
167  #'trk_cov_z0_theta' : 'trk_covz0theta',
168  #'trk_cov_phi_qoverp' : 'trk_covphiqoverp',
169  #'trk_cov_phi_theta' : 'trk_covphitheta',
170  #'trk_cov_theta_qoverp' : 'trk_covthetaqoverp',
171 
172  #'vxp_x' : 'vxp_vertx',
173  #'vxp_y' : 'vxp_verty',
174  #'vxp_z' : 'vxp_vertz',
175  #'vxp_err_x' : 'vxp_errx',
176  #'vxp_err_y' : 'vxp_erry',
177  #'vxp_err_z' : 'vxp_errz',
178  #'vxp_E' : 'vxp_E',
179  #'vxp_m' : 'vxp_vertM',
180  #'vxp_px' : 'vxp_vertPx',
181  #'vxp_py' : 'vxp_vertPy',
182  #'vxp_pz' : 'vxp_vertPz',
183  #'vxp_sumPt' : 'vxp_vertSumPt',
184  #'vxp_nTracks' : 'vxp_vertNtrk',
185  }
186 
187 # List of branches to ignore for various versions.
188 # Format is a list of tuples: (V, V, V, [BRANCHES...])
189 # If any of the strings in V appear in CMTPATH, then
190 # we do not ignore the branches. Otherwise, we ignore
191 # branches matching any of the glob paths in BRANCHES.
192 _ignore_branches = [
193  ('15.6',
194  ['mb_eta',
195  'mb_phi',
196  'tau_calcVars_*',
197  'el_rawcl_*',
198  'ph_rawcl_*',
199  'elfilt_rawcl_*',
200  'ph_mvaptcone*',
201  'MET_Goodness_N_Jets',
202  'MET_Goodness_DeltaTrackTopo',
203  'MET_Goodness_isBad_Jet',
204  'MET_Goodness_isGood_Jet',
205  'MET_Goodness_isUgly_Jet',
206  'MET_Goodness_tileGap3F_Jet',
207  'MET_Goodness_MET_CorrTopo_etx',
208  'MET_Goodness_MET_CorrTopo_ety',
209  'MET_Goodness_MET_CorrTopo_sumet',
210  'MET_Goodness_MET_Topo_etx',
211  'MET_Goodness_MET_Topo_ety',
212  'MET_Goodness_MET_Topo_sumet',
213  'jet_*',
214  '*_OriginIndex',
215  '*_Timing',
216  '*_timing',
217  '*ntrk',
218  '*sumPtTrk',
219  'el_medium',
220  'elfilt_medium',
221  'trk_blayerPrediction_locX',
222  ]
223  ),
224  ('15.6', '16.0',
225  [
226  ]
227  )
228  ]
229 
230 # Should a branch be ignored?
231 def ignore_p (br):
232  ver = os.environ.get ('CMTPATH')
233 
234  for vlist in _ignore_branches:
235  for v in vlist[:-1]:
236  found = False
237  if ver and ver.find (v) >= 0:
238  found = True
239  break
240  if found: continue
241  for b in vlist[-1]:
242  if fnmatch (br, b):
243  return True
244  return False
245 
246 
247 
248 def topy (o):
249  if type(o).__name__.startswith ('vector<'):
250  ll = list(o)
251  if ll and type(ll[0]).__name__.startswith ('vector<'):
252  ll = [list(l) for l in ll]
253  return ll
254  return o
255 
256 
257 inttypes = [types.IntType, types.LongType]
258 def compare (o1, o2, thresh = 1e-6, ithresh = None):
259  # Allow comparing int/long int.
260  if type(o1) in inttypes and type(o2) in inttypes:
261  if o1 < 0: o1 = o1 + (1<<32)
262  if o2 < 0: o2 = o2 + (1<<32)
263  return o1 == o2
264  if type(o1) is not type(o2):
265  return False
266  if isinstance(o1,list):
267  if len(o1) != len(o2):
268  return False
269  for i in range(len(o1)):
270  if not compare (o1[i], o2[i],
271  thresh=thresh, ithresh=ithresh): return False
272  return True
273  if type(o1).__name__ in ['map<string,int>',
274  'map<string,float>',
275  'map<string,string>']:
276  return ROOT.D3PDTest.MapDumper.equal (o1, o2)
277  if isinstance(o1, float):
278  if ithresh and abs(o1) < ithresh and abs(o2) < ithresh:
279  return True
280  num = o1-o2
281  den = abs(o1)+abs(o2)
282  if den == 0: return True
283  x = abs(num / den)
284  if callable(thresh): thresh = thresh(den)
285  if x > thresh:
286  print ('fmismatch', o1, o2, x)
287  return False
288  return True
289  return o1 == o2
290 
291 
292 def mc_eta_thresh (x):
293  if x > 36: return 1e-2
294  if x > 34: return 2e-3
295  if x > 32: return 2e-4
296  if x > 30: return 1e-4
297  if x > 28: return 1e-5
298  return 1e-6
299 
300 
301 def diff_trees (t1, t2):
302  n1 = t1.GetEntries()
303  n2 = t2.GetEntries()
304  if n1 != n2:
305  print ('Different nentries for tree', t1.GetName(), ': ', n1, n2)
306  n1 = min(n1, n2)
307  b1 = [b.GetName() for b in t1.GetListOfBranches()]
308  b2 = [b.GetName() for b in t2.GetListOfBranches()]
309  b1.sort()
310  b2.sort()
311  branchmap = renames.copy()
312  for b in b1:
313  if b not in b2:
314  bb = branchmap.get (b)
315  if not bb or bb not in b2:
316  if not ignore_p(b):
317  print ('Branch', b, 'in first tree but not in second.')
318  if bb: del branchmap[b]
319  else:
320  b2.remove (bb)
321  else:
322  b2.remove (b)
323  branchmap[b] = b
324  for b in b2:
325  if not ignore_p(b):
326  print ('Branch', b, 'in second tree but not in first.')
327 
328  for i in range (n1):
329  t1.GetEntry(i)
330  t2.GetEntry(i)
331  for b in b1:
332  if ignore_p(b): continue
333  bb = branchmap.get(b)
334  if not bb: continue
335  o1 = topy (getattr(t1, b))
336  o2 = topy (getattr(t2, bb))
337 
338  ithresh = None
339  thresh = 1e-6
340  if b.find('jet_')>=0 and b.endswith ('_m'): ithresh = 0.1
341  if b == 'mc_m': ithresh = 0.1
342  if b.find ('_rawcl_etas') >= 0: thresh = 2e-4
343  if b.endswith ('_convIP'): thresh = 3e-5
344  if b.endswith ('_emscale_E'): thresh = 9e-5
345  if b.endswith ('_emscale_eta'): thresh = 9e-5
346  if b.endswith ('_emscale_m'): ithresh = 0.1
347  if b.endswith ('_constscale_E'): thresh = 9e-5
348  if b.endswith ('_constscale_eta'): thresh = 9e-5
349  if b == 'mc_eta': thresh = mc_eta_thresh
350  if b.endswith ('_seg_locX'): ithresh = 2e-12
351  if b.endswith ('_seg_locY'): ithresh = 2e-12
352  if b == 'MET_Goodness_DeltaEt_JetAlgs_Jet': ithresh = 2e-11
353  if b == 'MET_Goodness_EEM_Jet': thresh = 2e-5
354  if b == 'MET_Goodness_HECf_Jet': thresh = 3e-6
355  if b.find ('_blayerPrediction') >= 0: thresh = 4e-4
356  if not compare (o1, o2, thresh = thresh, ithresh = ithresh):
357  print ('Branch mismatch', b, 'entry', i, ':', ithresh)
358  print (o1)
359  print (o2)
360 
361  return
362 
363 
364 def diff_objstrings (k, s1, s2):
365  # nb. != not working correctly for TObjString in 5.26.00c_python2.6
366  if not (s1 == s2):
367  print ('Objstring', k, 'mismatch:')
368  print (repr(s1))
369  print (repr(s2))
370  return
371 
372 
373 def diff_dirs (f1, f2):
374  k1 = [k.GetName() for k in f1.GetListOfKeys()]
375  k2 = [k.GetName() for k in f2.GetListOfKeys()]
376  k1.sort()
377  k2.sort()
378  if k1 != k2:
379  print ("Key list mismatch for", f1.GetName(), f2.GetName(), ":")
380  print (k1)
381  print (k2)
382  for k in k1:
383  if k not in k2: continue
384  o1 = f1.Get(k)
385  o2 = f2.Get(k)
386  if type(o1) is not type(o2):
387  print ('Type mismatch for ', k)
388  print (o1, o2)
389  if k == 'Schema':
390  pass
391  elif isinstance (o1, ROOT.TTree):
392  diff_trees (o1, o2)
393  elif isinstance (o1, ROOT.TDirectory):
394  diff_dirs (o1, o2)
395  elif isinstance (o1, ROOT.TObjString):
396  diff_objstrings (k, o1, o2)
397  else:
398  print (k, type(o1))
399  return
400 
401 
402 if __name__ == '__main__':
403  import sys
404  f1 = ROOT.TFile (sys.argv[1])
405  f2 = ROOT.TFile (sys.argv[2])
406  diff_dirs (f1, f2)
python.difftuple.ignore_p
def ignore_p(br)
Definition: difftuple.py:231
physval_make_web_display.thresh
thresh
Definition: physval_make_web_display.py:35
python.difftuple.mc_eta_thresh
def mc_eta_thresh(x)
Definition: difftuple.py:292
python.difftuple.diff_dirs
def diff_dirs(f1, f2)
Definition: difftuple.py:373
python.difftuple.diff_trees
def diff_trees(t1, t2)
Definition: difftuple.py:301
plotBeamSpotVxVal.range
range
Definition: plotBeamSpotVxVal.py:195
python.Helpers.ROOT6Setup
def ROOT6Setup(batch=False)
Definition: Tools/PyUtils/python/Helpers.py:19
PyAthena::repr
std::string repr(PyObject *o)
returns the string representation of a python object equivalent of calling repr(o) in python
Definition: PyAthenaUtils.cxx:106
python.difftuple.compare
def compare(o1, o2, thresh=1e-6, ithresh=None)
Definition: difftuple.py:258
histSizes.list
def list(name, path='/')
Definition: histSizes.py:38
min
#define min(a, b)
Definition: cfImp.cxx:40
python.difftuple.diff_objstrings
def diff_objstrings(k, s1, s2)
Definition: difftuple.py:364
python.CaloScaleNoiseConfig.type
type
Definition: CaloScaleNoiseConfig.py:78
python.difftuple.topy
def topy(o)
Definition: difftuple.py:248
Generated on Thu Oct 3 2024 21:13:45 for ATLAS Offline Software by   doxygen 1.8.18