ATLAS Offline Software
Loading...
Searching...
No Matches
python.difftuple_text Namespace Reference

Classes

class  Reader
class  Tree
class  String
class  Hist
class  Dumpreader

Functions

 atlasProduction (ver)
 compile_list (ignore_list)
 get_ver ()
 ignore_p (match, ignore_list)
 ignore_br_p (br)
 ignore_key_p (key)
 read_val (val)
 dictkey_diff (d1, d2, msg, filter=None)
 apply_renames (d1, d2, renames)
 diff_string (k, s1, s2)
 compare (o1, o2, thresh=1e-6, ithresh=None, eltcmp=None)
 mc_eta_thresh (x)
 eg_pid_compare (o1, o2)
 compare_detElementId (o1, o2)
 compare_pixid (o1, o2)
 compare_abs (o1, o2)
 diff_branch (k, kb, v1, v2)
 diff_tree (k, t1, t2)
 diff_hist (k, h1, h2)
 diff_files (d1, d2)

Variables

bool ignore_event_differences = False
dict renames
dict rename_keys
list _ignore_branches
list _ignore_keys
list inttypes = [types.IntType, types.LongType]
list branch_thresh
 p
 d1 = Dumpreader (p.stdout)
 d2 = Dumpreader (open (sys.argv[2]))

Function Documentation

◆ apply_renames()

apply_renames ( d1,
d2,
renames )

Definition at line 1285 of file difftuple_text.py.

1285def apply_renames (d1, d2, renames):
1286 for (knew, kold) in renames.items():
1287 if (d2.has_key (kold) and not d2.has_key (knew) and
1288 not d1.has_key (kold) and d1.has_key (knew)):
1289 d2[knew] = d2[kold]
1290 del d2[kold]
1291 return
1292
1293

◆ atlasProduction()

atlasProduction ( ver)

Definition at line 34 of file difftuple_text.py.

34def atlasProduction(ver):
35 return (ver and
36 (ver.find('17.2')>=0 or
37 ver.find('17.3')>=0) and
38 #ver.find('AtlasProduction')>=0 and
39 ver.find('AtlasPhysics')==-1 and
40 ver.find('usatlas+u/snyder')==-1)
41
42
43# List of branches to ignore for various versions.
44# Format is a list of tuples: (V, V, V, [BRANCHES...])
45# We consider the strings V in turn and see if they're contained
46# in CMTPATH. If the first V is `-', then we ignore the branches
47# if there are any matches. Otherwise, we ignore the branches
48# unless there are any matches.
49# If V is callable, then instead of looking for at match in CMTPATH,
50# we call it with CMTPATH as an argument.
51# The list of branches can include glob patterns.

◆ compare()

compare ( o1,
o2,
thresh = 1e-6,
ithresh = None,
eltcmp = None )

Definition at line 1302 of file difftuple_text.py.

1302def compare (o1, o2, thresh = 1e-6, ithresh = None, eltcmp = None):
1303 if eltcmp and not isinstance(o1, list):
1304 return eltcmp (o1, o2)
1305 # Allow comparing int/long int.
1306 if type(o1) in inttypes and type(o2) in inttypes:
1307 if o1 < 0: o1 = o1 + (1<<32)
1308 if o2 < 0: o2 = o2 + (1<<32)
1309 return o1 == o2
1310 # Promote int to float if needed.
1311 if type(o1) in inttypes and isinstance(o2, float):
1312 o1 = float(o1)
1313 elif type(o2) in inttypes and isinstance(o1, float):
1314 o2 = float(o2)
1315
1316 if type(o1) is not type(o2):
1317 return False
1318 if isinstance(o1, list):
1319 if len(o1) != len(o2):
1320 return False
1321
1322 # Inline this to speed up this common case.
1323 if len(o1) > 0 and isinstance(o1[0], float) and not ithresh and not callable(thresh) and not eltcmp:
1324 xabs = abs
1325 for i in range(len(o1)):
1326 x1 = o1[i]
1327 x2 = o2[i]
1328 ax1 = xabs(x1)
1329 ax2 = xabs(x2)
1330 num = x1-x2
1331 den = ax1 + ax2
1332 if den == 0: continue
1333 x = xabs(num / den)
1334 if x > thresh:
1335 print ('fnmismatch', x1, x2, x, thresh)
1336 return False
1337 return True
1338
1339 for i in range(len(o1)):
1340 if not compare (o1[i], o2[i],
1341 thresh=thresh,
1342 ithresh=ithresh,
1343 eltcmp=eltcmp):
1344 return False
1345 return True
1346 if type(o1).__name__ in ['map<string,int>',
1347 'map<string,float>',
1348 'map<string,string>']:
1349 return ROOT.D3PDTest.MapDumper.equal (o1, o2)
1350 if isinstance(o1, float):
1351 if callable(ithresh):
1352 ret = ithresh(o1, o2)
1353 if ret is True or ret is False: return ret
1354 elif ithresh and abs(o1) < ithresh and abs(o2) < ithresh:
1355 return True
1356 num = o1-o2
1357 den = abs(o1)+abs(o2)
1358 if den == 0: return True
1359 x = abs(num / den)
1360 if callable(thresh): thresh = thresh(den)
1361 if x > thresh:
1362 print ('fmismatch', o1, o2, x, thresh)
1363 return False
1364 return True
1365 return o1 == o2
1366
1367

◆ compare_abs()

compare_abs ( o1,
o2 )

Definition at line 1395 of file difftuple_text.py.

1395def compare_abs (o1, o2):
1396 return compare (abs(o1), abs(o2))
1397
1398

◆ compare_detElementId()

compare_detElementId ( o1,
o2 )

Definition at line 1382 of file difftuple_text.py.

1382def compare_detElementId (o1, o2):
1383 if o1 == o2: return True
1384 if o1 == (o2 << 32): return True
1385 return False
1386
1387

◆ compare_pixid()

compare_pixid ( o1,
o2 )

Definition at line 1388 of file difftuple_text.py.

1388def compare_pixid (o1, o2):
1389 if o1 == o2: return True
1390 o2a = ((o2&0x7fffffff)-1)<<28
1391 if o1 == o2a : return True
1392 return False
1393
1394

◆ compile_list()

compile_list ( ignore_list)

Definition at line 1014 of file difftuple_text.py.

1014def compile_list (ignore_list):
1015 for vlist in ignore_list:
1016 l = vlist[-1]
1017 for i in range(len(l)):
1018 l[i] = re.compile (fnmatch.translate (l[i]))
1019 return
1020
1021

◆ dictkey_diff()

dictkey_diff ( d1,
d2,
msg,
filter = None )

Definition at line 1274 of file difftuple_text.py.

1274def dictkey_diff (d1, d2, msg, filter = None):
1275 keys = [k for k in d1.keys() if k not in d2]
1276 if filter: keys = [k for k in keys if not filter(k)]
1277 if keys:
1278 keys.sort()
1279 print (msg)
1280 for k in keys:
1281 print (' ', k)
1282 return
1283
1284

◆ diff_branch()

diff_branch ( k,
kb,
v1,
v2 )

Definition at line 1531 of file difftuple_text.py.

1531def diff_branch (k, kb, v1, v2):
1532 head = '%s branch %s:' % (k, kb)
1533 ev1 = v1.keys()
1534 ev2 = v2.keys()
1535 ev1.sort()
1536 ev2.sort()
1537 if ev1 != ev2:
1538 if ignore_event_differences:
1539 ev1 = list(set(ev1).intersection(set(ev2)))
1540 ev1.sort()
1541 else:
1542 print (head, 'Different set of events; new file: ',
1543 ev1, 'reference file', ev2)
1544 return
1545
1546 ithresh = None
1547 eltcmp = None
1548
1549 thresh = 1e-6
1550 for (pat, th) in branch_thresh:
1551 if pat.match (kb):
1552 if isinstance(th, tuple):
1553 thresh = th[0]
1554 ithresh = th[1]
1555 else:
1556 thresh = th
1557 break
1558
1559 # GSF reco run in d3pd job, so we have dbg/opt diffs.
1560 if kb.startswith ('el_gsf_'): thresh = 1e-4
1561 if kb == 'el_gsf_deltaeta2': thresh = 3e-3
1562 if kb == 'el_gsf_deltaphi2': thresh = 2e-1
1563 if kb == 'el_gsf_deltaphiRescaled': thresh = 2e-2
1564 if kb == 'el_gsf_deltaeta1': thresh = 3e-3
1565 if kb == 'el_gsf_trackz0pv': thresh = 1e-1
1566 if kb == 'el_gsf_covqoverp': thresh = 5e-2
1567 if kb == 'el_gsf_L2_dr': thresh = 3e-2
1568 if kb == 'el_gsf_EF_dr': thresh = 7e-2
1569 if kb == 'el_gsf_covd0theta': thresh = 3e-4
1570 if kb == 'el_gsf_electronweight': thresh = 2e-2
1571 if kb == 'el_gsf_electronbgweight': thresh = 3e-1
1572 if kb == 'el_gsf_adaboost': thresh = 7e-2
1573 if kb == 'el_gsf_neuralnet': thresh = 2e-1
1574 if kb == 'el_gsf_isolationlikelihoodhqelectrons': thresh = 4e-2
1575 if kb == 'el_gsf_jet_dr': thresh = 2e-2
1576 if kb == 'el_gsf_Et': thresh = 3e-3
1577 if kb == 'el_gsf_E': thresh = 4e-3
1578 if kb == 'sc_yCells': thresh = 2e-6
1579 if kb.startswith ('el_gsf_p'): thresh = 3e-3
1580 if kb.startswith ('el_gsf_refittedTrack_cov'): thresh = 3e-1
1581 if kb.startswith ('el_gsf_trackcov'): thresh = 5e-1
1582 if kb.startswith ('el_gsf_tracksig'): thresh = 4e-2
1583 if kb.startswith ('el_gsf_trackd0'): thresh = 6e-2
1584 if kb.startswith ('el_gsf_trackz0'): thresh = 3e-2
1585 if kb.startswith ('el_gsf_trackfitchi2'): thresh = 7e-3
1586 if kb.startswith ('el_gsf_trackqoverp'): thresh = 2e-2
1587 if kb.startswith ('el_gsf_trackpt'): thresh = 2e-2
1588 if kb.startswith ('el_gsf_refittedTrack_d0'): thresh = 3e-3
1589 if kb.startswith ('el_gsf_refittedTrack_LMqoverp'): thresh = 3e-3
1590 if kb.startswith ('el_gsf_refittedTrack_qoverp'): thresh = 2e-2
1591 if kb.startswith ('el_gsf_refittedTrack_covphitheta'): thresh = 5e-1
1592 if kb.startswith ('el_gsf_mvaptcone'): ithresh = 1e-10
1593 if kb == 'el_gsf_m':
1594 thresh = 2e-2
1595 ithresh = 0.01
1596
1597 if kb.find('jet_')>=0 and kb.endswith ('_m'): ithresh = 0.1
1598 if kb.find('jetcone_')>=0 and kb.endswith ('_m'): ithresh = 1e-3
1599 if kb.find('muonTruth_')>=0 and kb.endswith ('_m'): ithresh = 0.5
1600 if kb == 'mc_m':
1601 ithresh = 0.1
1602 thresh = 2e-5
1603 if kb.endswith ('_effTopoInvMass'): thresh = 1e-2
1604 if kb.endswith ('_topoInvMass'): thresh = 1e-2
1605 if (kb.endswith ('_topoMeanDeltaR') or
1606 kb.endswith ('_effTopoMeanDeltaR')):
1607 ithresh = 1e-6
1608 if kb.find ('_rawcl_etas') >= 0: thresh = 2e-4
1609 if kb.endswith ('_convIP'): thresh = 4e-4
1610 if kb.endswith ('_convIPRev'): thresh = 6e-5
1611 if kb.endswith ('_emscale_E'): thresh = 9e-5
1612 if kb.endswith ('_emscale_eta'): thresh = 9e-5
1613 if kb.endswith ('_emscale_m'): ithresh = 0.1
1614 if kb.endswith ('_constscale_E'): thresh = 9e-5
1615 if kb.endswith ('_constscale_eta'): thresh = 9e-5
1616 if kb == 'mc_eta': thresh = mc_eta_thresh
1617 if kb.endswith ('_seg_locX'): ithresh = 2e-12
1618 if kb.endswith ('_seg_locY'): ithresh = 2e-12
1619 if kb == 'MET_Goodness_DeltaEt_JetAlgs_Jet': ithresh = 3e-11
1620 if kb == 'MET_Goodness_EEM_Jet': thresh = 2e-5
1621 if kb == 'MET_Goodness_HECf_Jet': thresh = 3e-6
1622 if kb.find ('_blayerPrediction') >= 0: thresh = 1e-2
1623 if kb.endswith ('_hecf'): thresh = 4e-6
1624 if kb.endswith ('_SpaceTime_t'): thresh = 4e-6
1625 if kb.endswith ('_SpaceTime_weight'): thresh = 2e-5
1626 if kb.endswith ('_SpaceTime_tError'): thresh = 8e-6
1627 if kb.endswith ('_calcVars_ChPiEMEOverCaloEME'): thresh = 1e-5
1628 if kb.endswith ('_vx_m'): ithresh = 1e-2
1629 if kb.endswith ('MET_Goodness_DeltaEt_JetAlgs_Jet'): ithresh = 1e-10
1630
1631 # Diffs btn 32/64-bit
1632 if kb == 'mc_perigee_theta': thresh = 1
1633 if kb == 'mc_perigee_d0': thresh = 1
1634 if kb == 'mc_perigee_phi': thresh = 1 #(ev 2,9)
1635 if kb == 'mc_perigee_z0': thresh = 1
1636 if kb == 'tau_likelihood': thresh = 1e-5
1637 if kb == 'tau_SafeLikelihood': thresh = 1e-5
1638 if kb == 'el_gsf_softeweight': thresh = 1e-1
1639 if kb == 'el_gsf_softebgweight': thresh = 2e-2
1640
1641 if kb.endswith ('_mvaptcone20'): ithresh = 1e-10
1642 if kb.endswith ('_mvaptcone30'): ithresh = 1e-10
1643 if kb.endswith ('_mvaptcone40'): ithresh = 1e-10
1644
1645 # more 32/64 dbg/opt diffs
1646 if kb == 'tau_SafeLikelihood': thresh = 0.4
1647 if kb == 'tau_likelihood': thresh = 0.4
1648 if kb == 'tau_BDTJetScore': thresh = 1.0
1649 if kb == 'tau_calcVars_corrFTrk': thresh = 0.02
1650 if kb == 'tau_calcVars_corrCentFrac': thresh = 0.4
1651 if kb == 'tau_JetBDTSigLoose': thresh = 1.0
1652 if kb == 'tau_JetBDTSigMedium': thresh = 1.0
1653 if kb == 'tau_JetBDTSigTight': thresh = 1.0
1654 if kb == 'tau_tauLlhMedium': thresh = 1.0
1655 if kb == 'tau_pi0_cl1_pt': thresh = 1e-5
1656
1657 if kb.endswith ('_neighbourSig'): thresh = 1e-4
1658 if kb.endswith ('_curvatureSig'): thresh = 1e-4
1659
1660 if kb == 'pixClus_detElementId': eltcmp = compare_detElementId
1661 if kb == 'pixClus_id': eltcmp = compare_pixid
1662 if kb == 'trig_EF_emcl_pt': eltcmp = compare_abs
1663
1664 for e in ev1:
1665 head = '%s branch %s event %d:' % (k, kb, e)
1666 b1 = v1[e]
1667 b2 = v2[e]
1668
1669 # Promote int to float if needed.
1670 if type(b1) in inttypes and isinstance(b2, float):
1671 b1 = float(b1)
1672 elif type(b2) in inttypes and isinstance(b1, float):
1673 b2 = float(b2)
1674
1675 if type(b1) is not type(b2):
1676 print (head, 'Type differs; new file: ', type(b1).__name__,
1677 'reference file:', type(b2).__name__)
1678 elif not compare (b1, b2, thresh = thresh, ithresh = ithresh,
1679 eltcmp = eltcmp):
1680 print (head, 'Branch mismatch')
1681 print (' new file:', b1)
1682 print (' ref file:', b2)
1683
1684 return
1685
STL class.
std::vector< std::string > intersection(std::vector< std::string > &v1, std::vector< std::string > &v2)

◆ diff_files()

diff_files ( d1,
d2 )

Definition at line 1712 of file difftuple_text.py.

1712def diff_files (d1, d2):
1713 apply_renames (d1.keys, d2.keys, rename_keys)
1714 dictkey_diff (d1.keys, d2.keys, 'Keys only in new file:',
1715 ignore_key_p)
1716 dictkey_diff (d2.keys, d1.keys, 'Keys only in reference file:',
1717 ignore_key_p)
1718 for k in d1.keys.keys():
1719 if k in d2.keys and not ignore_key_p (k):
1720 v1 = d1.keys[k]
1721 v2 = d2.keys[k]
1722 if type(v1) is not type(v2):
1723 print ('Key', k, 'has type', type(v1).__name__,
1724 'in new file, but type', type(v2).__name__,
1725 'in reference file')
1726 elif type(v1) is String:
1727 diff_string (k, v1, v2)
1728 elif type(v1) is Tree:
1729 diff_tree (k, v1, v2)
1730 elif type(v1) is Hist:
1731 diff_hist (k, v1, v2)
1732 else:
1733 print ('Unknown type for diff:', type(v1).__name__)
1734 return
1735
1736

◆ diff_hist()

diff_hist ( k,
h1,
h2 )

Definition at line 1700 of file difftuple_text.py.

1700def diff_hist (k, h1, h2):
1701 if not compare (h1.bins, h2.bins):
1702 print ('Hist', k, 'has bins', h1.bins)
1703 print ('in new file but bins', h2.bins)
1704 print ('in reference file')
1705 if not compare (h1.errs, h2.errs):
1706 print ('Hist', k, 'has errs', h1.errs)
1707 print ('in new file but errs', h2.errs)
1708 print ('in reference file')
1709 return
1710
1711

◆ diff_string()

diff_string ( k,
s1,
s2 )

Definition at line 1294 of file difftuple_text.py.

1294def diff_string (k, s1, s2):
1295 if s1.val != s2.val:
1296 print ('String', k, 'has value', s1.val,
1297 'in new file but value', s2.val, 'in reference file')
1298 return
1299
1300

◆ diff_tree()

diff_tree ( k,
t1,
t2 )

Definition at line 1686 of file difftuple_text.py.

1686def diff_tree (k, t1, t2):
1687 if t1.dummy and t2.dummy: return
1688 apply_renames (t1.branches, t2.branches, renames)
1689 dictkey_diff (t1.branches, t2.branches, '%s: Branches only in new file:'%k,
1690 ignore_br_p)
1691 dictkey_diff (t2.branches, t1.branches, '%s: Branches only in reference file:'%k,
1692 ignore_br_p)
1693 for kb in t1.branches:
1694 if t2.branches.has_key (kb):
1695 if not ignore_br_p(kb):
1696 diff_branch (k, kb, t1.branches[kb], t2.branches[kb])
1697 return
1698
1699

◆ eg_pid_compare()

eg_pid_compare ( o1,
o2 )

Definition at line 1377 of file difftuple_text.py.

1377def eg_pid_compare (o1, o2):
1378 if o1 == 0xffffffff and o2 == 0: return True
1379 return None
1380
1381

◆ get_ver()

get_ver ( )

Definition at line 1026 of file difftuple_text.py.

1026def get_ver():
1027 ver = os.environ.get ('CMTPATH')
1028 if ver:
1029 ver = string.split (ver, ':')
1030 ver = [s for s in ver if s.find('AtlasSetup') < 0]
1031 ver.append (os.environ.get ('CMTCONFIG'))
1032 ver = string.join (ver, ':')
1033 else:
1034 ver = 'localbuild'
1035 return ver
1036
1037

◆ ignore_br_p()

ignore_br_p ( br)

Definition at line 1060 of file difftuple_text.py.

1060def ignore_br_p (br):
1061 return ignore_p (br, _ignore_branches)
1062
1063

◆ ignore_key_p()

ignore_key_p ( key)

Definition at line 1064 of file difftuple_text.py.

1064def ignore_key_p (key):
1065 return ignore_p (key, _ignore_keys)
1066
1067

◆ ignore_p()

ignore_p ( match,
ignore_list )

Definition at line 1038 of file difftuple_text.py.

1038def ignore_p (match, ignore_list):
1039 ver = get_ver()
1040
1041 for vlist in ignore_list:
1042 vv = vlist[:-1]
1043 ignore = True
1044 if vv and vv[0] == '-':
1045 ignore = False
1046 vv = vv[1:]
1047 for v in vv:
1048 if (callable(v) and v(ver) or
1049 not callable(v) and ver and ver.find (v) >= 0):
1050 ignore = not ignore
1051 break
1052 if not ignore: continue
1053 for b in vlist[-1]:
1054 if b.match (match):
1055 return True
1056 return False
1057
1058
1059# Should a branch be ignored?

◆ mc_eta_thresh()

mc_eta_thresh ( x)

Definition at line 1368 of file difftuple_text.py.

1368def mc_eta_thresh (x):
1369 if x > 36: return 1e-2
1370 if x > 34: return 2e-3
1371 if x > 32: return 2e-4
1372 if x > 30: return 1e-4
1373 if x > 28: return 1e-5
1374 return 1e-6
1375
1376

◆ read_val()

read_val ( val)

Definition at line 1068 of file difftuple_text.py.

1068def read_val (val):
1069 if not val: return val
1070
1071 if val.find ('+') >= 0:
1072 psplit = split_list1 (val, '+')
1073 if len(psplit) > 1:
1074 #psplit = split_list1 (val, '+')
1075 #if len(psplit) > 1:
1076 good = True
1077 for i in range(len(psplit)):
1078 psplit[i] = read_val (psplit[i])
1079 if not isinstance(psplit[i], list):
1080 good = False
1081 break
1082 if good:
1083 return reduce (operator.__add__, psplit)
1084
1085 if val[-1] == 'L':
1086 try:
1087 return int(val)
1088 except TypeError:
1089 pass
1090
1091 if val[0] in '0123456789-':
1092 if val.find ('.') < 0:
1093 try:
1094 return int(val)
1095 except TypeError:
1096 pass
1097 except ValueError:
1098 pass
1099
1100 try:
1101 return float (val)
1102 except TypeError:
1103 pass
1104 except ValueError:
1105 pass
1106
1107 try:
1108 (count, elt) = val.split ('*', 1)
1109 count = int (count)
1110 if elt.startswith('[') and elt.endswith(']'):
1111 return count * [read_val(elt[1:-1])]
1112 except ValueError:
1113 pass
1114
1115 if val[0] == '[':
1116 return [read_val(s) for s in split_list(val)]
1117 return val
1118
1119

Variable Documentation

◆ _ignore_branches

list python.difftuple_text._ignore_branches
protected

Definition at line 52 of file difftuple_text.py.

◆ _ignore_keys

list python.difftuple_text._ignore_keys
protected

Definition at line 934 of file difftuple_text.py.

◆ branch_thresh

list python.difftuple_text.branch_thresh

Definition at line 1399 of file difftuple_text.py.

◆ d1

python.difftuple_text.d1 = Dumpreader (p.stdout)

Definition at line 1750 of file difftuple_text.py.

◆ d2

python.difftuple_text.d2 = Dumpreader (open (sys.argv[2]))

Definition at line 1752 of file difftuple_text.py.

◆ ignore_event_differences

bool ignore_event_differences = False

Definition at line 20 of file difftuple_text.py.

◆ inttypes

list python.difftuple_text.inttypes = [types.IntType, types.LongType]

Definition at line 1301 of file difftuple_text.py.

◆ p

python.difftuple_text.p
Initial value:
= subprocess.Popen ('python -m D3PDMakerTest.dumptuple_any %s' %
sys.argv[1],
shell = True,
stdout = subprocess.PIPE)

Definition at line 1746 of file difftuple_text.py.

◆ rename_keys

dict python.difftuple_text.rename_keys
Initial value:
= {
# Hack for ordering difference when scell tuple is on.
'caloMeta/EventInfoD3PDObject_000005' : 'caloMeta/EventInfoD3PDObject_000004',
'physicsMeta/EventInfoD3PDObject_000003' : 'physicsMeta/EventInfoD3PDObject_000002',
'physicsTrigDecMeta/EventInfoD3PDObject_000004' : 'physicsTrigDecMeta/EventInfoD3PDObject_000003',
}

Definition at line 27 of file difftuple_text.py.

◆ renames

dict python.difftuple_text.renames
Initial value:
= {
}

Definition at line 24 of file difftuple_text.py.