python.difftuple_text Namespace Reference

Classes
class	Reader
class	Tree
class	String
class	Hist
class	Dumpreader

Functions
	atlasProduction (ver)
	compile_list (ignore_list)
	get_ver ()
	ignore_p (match, ignore_list)
	ignore_br_p (br)
	ignore_key_p (key)
	read_val (val)
	dictkey_diff (d1, d2, msg, filter=None)
	apply_renames (d1, d2, renames)
	diff_string (k, s1, s2)
	compare (o1, o2, thresh=1e-6, ithresh=None, eltcmp=None)
	mc_eta_thresh (x)
	eg_pid_compare (o1, o2)
	compare_detElementId (o1, o2)
	compare_pixid (o1, o2)
	compare_abs (o1, o2)
	diff_branch (k, kb, v1, v2)
	diff_tree (k, t1, t2)
	diff_hist (k, h1, h2)
	diff_files (d1, d2)

Variables
bool	ignore_event_differences = False
dict	renames
dict	rename_keys
list	_ignore_branches
list	_ignore_keys
list	inttypes = [types.IntType, types.LongType]
list	branch_thresh
	p
	d1 = Dumpreader (p.stdout)
	d2 = Dumpreader (open (sys.argv[2]))

Function Documentation

apply_renames()

apply_renames	(		d1,
			d2,
			renames )

Definition at line 1286 of file difftuple_text.py.

def apply_renames (d1, d2, renames):
    for (knew, kold) in renames.items():
        if (d2.has_key (kold) and not d2.has_key (knew) and
            not d1.has_key (kold) and d1.has_key (knew)):
            d2[knew] = d2[kold]
            del d2[kold]
    return
 
 

atlasProduction()

atlasProduction

(

ver

)

Definition at line 34 of file difftuple_text.py.

def atlasProduction(ver):
    return (ver and
            (ver.find('17.2')>=0 or
             ver.find('17.3')>=0) and
            #ver.find('AtlasProduction')>=0 and
            ver.find('AtlasPhysics')==-1 and
            ver.find('usatlas+u/snyder')==-1)
 
 
# List of branches to ignore for various versions.
# Format is a list of tuples: (V, V, V, [BRANCHES...])
# We consider the strings V in turn and see if they're contained
# in CMTPATH.  If the first V is `-', then we ignore the branches
# if there are any matches.  Otherwise, we ignore the branches
# unless there are any matches.
# If V is callable, then instead of looking for at match in CMTPATH,
# we call it with CMTPATH as an argument.
# The list of branches can include glob patterns.

compare()

compare	(		o1,
			o2,
			thresh = 1e-6,
			ithresh = None,
			eltcmp = None )

Definition at line 1303 of file difftuple_text.py.

def compare (o1, o2, thresh = 1e-6, ithresh = None, eltcmp = None):
    if eltcmp and not isinstance(o1, list):
        return eltcmp (o1, o2)
    # Allow comparing int/long int.
    if type(o1) in inttypes and type(o2) in inttypes:
        if o1 < 0: o1 = o1 + (1<<32)
        if o2 < 0: o2 = o2 + (1<<32)
        return o1 == o2
    # Promote int to float if needed.
    if type(o1) in inttypes and isinstance(o2, float):
        o1 = float(o1)
    elif type(o2) in inttypes and isinstance(o1, float):
        o2 = float(o2)
 
    if type(o1) is not type(o2):
        return False
    if isinstance(o1, list):
        if len(o1) != len(o2):
            return False
 
        # Inline this to speed up this common case.
        if len(o1) > 0 and isinstance(o1[0], float) and not ithresh and not callable(thresh) and not eltcmp:
            xabs = abs
            for i in range(len(o1)):
                x1 = o1[i]
                x2 = o2[i]
                ax1 = xabs(x1)
                ax2 = xabs(x2)
                num = x1-x2
                den = ax1 + ax2
                if den == 0: continue
                x = xabs(num / den)
                if x > thresh:
                    print ('fnmismatch', x1, x2, x, thresh)
                    return False
            return True
 
        for i in range(len(o1)):
            if not compare (o1[i], o2[i],
                            thresh=thresh,
                            ithresh=ithresh,
                            eltcmp=eltcmp):
                return False
        return True
    if type(o1).__name__ in ['map<string,int>',
                             'map<string,float>',
                             'map<string,string>']:
        return ROOT.D3PDTest.MapDumper.equal (o1, o2)
    if isinstance(o1, float):
        if callable(ithresh):
            ret = ithresh(o1, o2)
            if ret is True or ret is False: return ret
        elif ithresh and abs(o1) < ithresh and abs(o2) < ithresh:
            return True
        num = o1-o2
        den = abs(o1)+abs(o2)
        if den == 0: return True
        x = abs(num / den)
        if callable(thresh): thresh = thresh(den)
        if x > thresh:
            print ('fmismatch', o1, o2, x, thresh)
            return False
        return True
    return o1 == o2
 
 

compare_abs()

compare_abs	(		o1,
			o2 )

Definition at line 1396 of file difftuple_text.py.

def compare_abs (o1, o2):
    return compare (abs(o1), abs(o2))
 
 

compare_detElementId()

compare_detElementId	(		o1,
			o2 )

Definition at line 1383 of file difftuple_text.py.

def compare_detElementId (o1, o2):
    if o1 == o2: return True
    if o1 == (o2 << 32): return True
    return False
 
 

compare_pixid()

compare_pixid	(		o1,
			o2 )

Definition at line 1389 of file difftuple_text.py.

def compare_pixid (o1, o2):
    if o1 == o2: return True
    o2a = ((o2&0x7fffffff)-1)<<28
    if o1 == o2a : return True
    return False
 
 

compile_list()

compile_list

(

ignore_list

)

Definition at line 1015 of file difftuple_text.py.

def compile_list (ignore_list):
    for vlist in ignore_list:
        l = vlist[-1]
        for i in range(len(l)):
            l[i] = re.compile (fnmatch.translate (l[i]))
    return
 
 

dictkey_diff()

dictkey_diff	(		d1,
			d2,
			msg,
			filter = None )

Definition at line 1275 of file difftuple_text.py.

def dictkey_diff (d1, d2, msg, filter = None):
    keys = [k for k in d1.keys() if k not in d2]
    if filter: keys = [k for k in keys if not filter(k)]
    if keys:
        keys.sort()
        print (msg)
        for k in keys:
            print ('  ', k)
    return
 
 

diff_branch()

diff_branch	(		k,
			kb,
			v1,
			v2 )

Definition at line 1532 of file difftuple_text.py.

def diff_branch (k, kb, v1, v2):
    head = '%s branch %s:' % (k, kb)
    ev1 = v1.keys()
    ev2 = v2.keys()
    ev1.sort()
    ev2.sort()
    if ev1 != ev2:
        if ignore_event_differences:
            ev1 = list(set(ev1).intersection(set(ev2)))
            ev1.sort()
        else:
            print (head, 'Different set of events; new file: ',
                   ev1, 'reference file', ev2)
            return
 
    ithresh = None
    eltcmp = None
 
    thresh = 1e-6
    for (pat, th) in branch_thresh:
        if pat.match (kb):
            if isinstance(th, tuple):
                thresh = th[0]
                ithresh = th[1]
            else:
                thresh = th
            break
 
    # GSF reco run in d3pd job, so we have dbg/opt diffs.
    if kb.startswith ('el_gsf_'): thresh = 1e-4
    if kb == 'el_gsf_deltaeta2': thresh = 3e-3
    if kb == 'el_gsf_deltaphi2': thresh = 2e-1
    if kb == 'el_gsf_deltaphiRescaled': thresh = 2e-2
    if kb == 'el_gsf_deltaeta1': thresh = 3e-3
    if kb == 'el_gsf_trackz0pv': thresh = 1e-1
    if kb == 'el_gsf_covqoverp': thresh = 5e-2
    if kb == 'el_gsf_L2_dr': thresh = 3e-2
    if kb == 'el_gsf_EF_dr': thresh = 7e-2
    if kb == 'el_gsf_covd0theta': thresh = 3e-4
    if kb == 'el_gsf_electronweight': thresh = 2e-2
    if kb == 'el_gsf_electronbgweight': thresh = 3e-1
    if kb == 'el_gsf_adaboost': thresh = 7e-2
    if kb == 'el_gsf_neuralnet': thresh = 2e-1
    if kb == 'el_gsf_isolationlikelihoodhqelectrons': thresh = 4e-2
    if kb == 'el_gsf_jet_dr': thresh = 2e-2
    if kb == 'el_gsf_Et': thresh = 3e-3
    if kb == 'el_gsf_E': thresh = 4e-3
    if kb == 'sc_yCells': thresh = 2e-6
    if kb.startswith ('el_gsf_p'): thresh = 3e-3
    if kb.startswith ('el_gsf_refittedTrack_cov'): thresh = 3e-1
    if kb.startswith ('el_gsf_trackcov'): thresh = 5e-1
    if kb.startswith ('el_gsf_tracksig'): thresh = 4e-2
    if kb.startswith ('el_gsf_trackd0'): thresh = 6e-2
    if kb.startswith ('el_gsf_trackz0'): thresh = 3e-2
    if kb.startswith ('el_gsf_trackfitchi2'): thresh = 7e-3
    if kb.startswith ('el_gsf_trackqoverp'): thresh = 2e-2
    if kb.startswith ('el_gsf_trackpt'): thresh = 2e-2
    if kb.startswith ('el_gsf_refittedTrack_d0'): thresh = 3e-3
    if kb.startswith ('el_gsf_refittedTrack_LMqoverp'): thresh = 3e-3
    if kb.startswith ('el_gsf_refittedTrack_qoverp'): thresh = 2e-2
    if kb.startswith ('el_gsf_refittedTrack_covphitheta'): thresh = 5e-1
    if kb.startswith ('el_gsf_mvaptcone'): ithresh = 1e-10
    if kb == 'el_gsf_m':
        thresh = 2e-2
        ithresh = 0.01
 
    if kb.find('jet_')>=0 and kb.endswith ('_m'): ithresh = 0.1
    if kb.find('jetcone_')>=0 and kb.endswith ('_m'): ithresh = 1e-3
    if kb.find('muonTruth_')>=0 and kb.endswith ('_m'): ithresh = 0.5
    if kb == 'mc_m':
        ithresh = 0.1
        thresh = 2e-5
    if kb.endswith ('_effTopoInvMass'): thresh = 1e-2
    if kb.endswith ('_topoInvMass'): thresh = 1e-2
    if (kb.endswith ('_topoMeanDeltaR') or
        kb.endswith ('_effTopoMeanDeltaR')):
        ithresh = 1e-6
    if kb.find ('_rawcl_etas') >= 0: thresh = 2e-4
    if kb.endswith ('_convIP'): thresh = 4e-4
    if kb.endswith ('_convIPRev'): thresh = 6e-5
    if kb.endswith ('_emscale_E'): thresh = 9e-5
    if kb.endswith ('_emscale_eta'): thresh = 9e-5
    if kb.endswith ('_emscale_m'): ithresh = 0.1
    if kb.endswith ('_constscale_E'): thresh = 9e-5
    if kb.endswith ('_constscale_eta'): thresh = 9e-5
    if kb == 'mc_eta': thresh = mc_eta_thresh
    if kb.endswith ('_seg_locX'): ithresh = 2e-12
    if kb.endswith ('_seg_locY'): ithresh = 2e-12
    if kb == 'MET_Goodness_DeltaEt_JetAlgs_Jet': ithresh = 3e-11
    if kb == 'MET_Goodness_EEM_Jet': thresh = 2e-5
    if kb == 'MET_Goodness_HECf_Jet': thresh = 3e-6
    if kb.find ('_blayerPrediction') >= 0: thresh = 1e-2
    if kb.endswith ('_hecf'): thresh = 4e-6
    if kb.endswith ('_SpaceTime_t'): thresh = 4e-6
    if kb.endswith ('_SpaceTime_weight'): thresh = 2e-5
    if kb.endswith ('_SpaceTime_tError'): thresh = 8e-6
    if kb.endswith ('_calcVars_ChPiEMEOverCaloEME'): thresh = 1e-5
    if kb.endswith ('_vx_m'): ithresh = 1e-2
    if kb.endswith ('MET_Goodness_DeltaEt_JetAlgs_Jet'): ithresh = 1e-10
 
    # Diffs btn 32/64-bit
    if kb == 'mc_perigee_theta': thresh = 1
    if kb == 'mc_perigee_d0': thresh = 1
    if kb == 'mc_perigee_phi': thresh = 1 #(ev 2,9)
    if kb == 'mc_perigee_z0': thresh = 1
    if kb == 'tau_likelihood': thresh = 1e-5
    if kb == 'tau_SafeLikelihood': thresh = 1e-5
    if kb == 'el_gsf_softeweight': thresh = 1e-1
    if kb == 'el_gsf_softebgweight': thresh = 2e-2
 
    if kb.endswith ('_mvaptcone20'): ithresh = 1e-10
    if kb.endswith ('_mvaptcone30'): ithresh = 1e-10
    if kb.endswith ('_mvaptcone40'): ithresh = 1e-10
 
    # more 32/64 dbg/opt diffs
    if kb == 'tau_SafeLikelihood': thresh = 0.4
    if kb == 'tau_likelihood': thresh = 0.4
    if kb == 'tau_BDTJetScore': thresh = 1.0
    if kb == 'tau_calcVars_corrFTrk': thresh = 0.02
    if kb == 'tau_calcVars_corrCentFrac': thresh = 0.4
    if kb == 'tau_JetBDTSigLoose': thresh = 1.0
    if kb == 'tau_JetBDTSigMedium': thresh = 1.0
    if kb == 'tau_JetBDTSigTight': thresh = 1.0
    if kb == 'tau_tauLlhMedium': thresh = 1.0
    if kb == 'tau_pi0_cl1_pt': thresh = 1e-5
 
    if kb.endswith ('_neighbourSig'): thresh = 1e-4
    if kb.endswith ('_curvatureSig'): thresh = 1e-4
 
    if kb == 'pixClus_detElementId': eltcmp = compare_detElementId
    if kb == 'pixClus_id': eltcmp = compare_pixid
    if kb == 'trig_EF_emcl_pt': eltcmp = compare_abs
 
    for e in ev1:
        head = '%s branch %s event %d:' % (k, kb, e)
        b1 = v1[e]
        b2 = v2[e]
 
        # Promote int to float if needed.
        if type(b1) in inttypes and isinstance(b2, float):
            b1 = float(b1)
        elif type(b2) in inttypes and isinstance(b1, float):
            b2 = float(b2)
 
        if type(b1) is not type(b2):
            print (head, 'Type differs; new file: ', type(b1).__name__,
                   'reference file:', type(b2).__name__)
        elif not compare (b1, b2, thresh = thresh, ithresh = ithresh,
                          eltcmp = eltcmp):
            print (head, 'Branch mismatch')
            print ('  new file:', b1)
            print ('  ref file:', b2)
        
    return
 

diff_files()

diff_files	(		d1,
			d2 )

Definition at line 1713 of file difftuple_text.py.

def diff_files (d1, d2):
    apply_renames (d1.keys, d2.keys, rename_keys)
    dictkey_diff (d1.keys, d2.keys, 'Keys only in new file:',
                  ignore_key_p)
    dictkey_diff (d2.keys, d1.keys, 'Keys only in reference file:',
                  ignore_key_p)
    for k in d1.keys.keys():
        if k in d2.keys and not ignore_key_p (k):
            v1 = d1.keys[k]
            v2 = d2.keys[k]
            if type(v1) is not type(v2):
                print ('Key', k, 'has type', type(v1).__name__,
                       'in new file, but type', type(v2).__name__,
                       'in reference file')
            elif type(v1) is String:
                diff_string (k, v1, v2)
            elif type(v1) is Tree:
                diff_tree (k, v1, v2)
            elif type(v1) is Hist:
                diff_hist (k, v1, v2)
            else:
                print ('Unknown type for diff:', type(v1).__name__)
    return
 
 

diff_hist()

diff_hist	(		k,
			h1,
			h2 )

Definition at line 1701 of file difftuple_text.py.

def diff_hist (k, h1, h2):
    if not compare (h1.bins, h2.bins):
        print ('Hist', k, 'has bins', h1.bins)
        print ('in new file but bins', h2.bins)
        print ('in reference file')
    if not compare (h1.errs, h2.errs):
        print ('Hist', k, 'has errs', h1.errs)
        print ('in new file but errs', h2.errs)
        print ('in reference file')
    return
 
 

diff_string()

diff_string	(		k,
			s1,
			s2 )

Definition at line 1295 of file difftuple_text.py.

def diff_string (k, s1, s2):
    if s1.val != s2.val:
        print ('String', k, 'has value', s1.val,
               'in new file but value', s2.val, 'in reference file')
    return
 
 

diff_tree()

diff_tree	(		k,
			t1,
			t2 )

Definition at line 1687 of file difftuple_text.py.

def diff_tree (k, t1, t2):
    if t1.dummy and t2.dummy: return
    apply_renames (t1.branches, t2.branches, renames)
    dictkey_diff (t1.branches, t2.branches, '%s: Branches only in new file:'%k,
                  ignore_br_p)
    dictkey_diff (t2.branches, t1.branches, '%s: Branches only in reference file:'%k,
                  ignore_br_p)
    for kb in t1.branches:
        if t2.branches.has_key (kb):
            if not ignore_br_p(kb):
                diff_branch (k, kb, t1.branches[kb], t2.branches[kb])
    return
 
 

eg_pid_compare()

eg_pid_compare	(		o1,
			o2 )

Definition at line 1378 of file difftuple_text.py.

def eg_pid_compare (o1, o2):
    if o1 == 0xffffffff and o2 == 0: return True
    return None
 
 

get_ver()

get_ver

(

)

Definition at line 1027 of file difftuple_text.py.

def get_ver():
    ver = os.environ.get ('CMTPATH')
    if ver:
        ver = string.split (ver, ':')
        ver = [s for s in ver if s.find('AtlasSetup') < 0]
        ver.append (os.environ.get ('CMTCONFIG'))
        ver = string.join (ver, ':')
    else:
        ver = 'localbuild'
    return ver
    
 

ignore_br_p()

ignore_br_p

(

br

)

Definition at line 1061 of file difftuple_text.py.

def ignore_br_p (br):
    return ignore_p (br, _ignore_branches)
 
 

ignore_key_p()

ignore_key_p

(

key

)

Definition at line 1065 of file difftuple_text.py.

def ignore_key_p (key):
    return ignore_p (key, _ignore_keys)
 
 

ignore_p()

ignore_p	(		match,
			ignore_list )

Definition at line 1039 of file difftuple_text.py.

def ignore_p (match, ignore_list):
    ver = get_ver()
 
    for vlist in ignore_list:
        vv = vlist[:-1]
        ignore = True
        if vv and vv[0] == '-':
            ignore = False
            vv = vv[1:]
        for v in vv:
            if (callable(v) and v(ver) or
                not callable(v) and ver and ver.find (v) >= 0):
                ignore = not ignore
                break
        if not ignore: continue
        for b in vlist[-1]:
            if b.match (match):
                return True
    return False
 
 
# Should a branch be ignored?

mc_eta_thresh()

mc_eta_thresh

(

x

)

Definition at line 1369 of file difftuple_text.py.

def mc_eta_thresh (x):
    if x > 36: return 1e-2
    if x > 34: return 2e-3
    if x > 32: return 2e-4
    if x > 30: return 1e-4
    if x > 28: return 1e-5
    return 1e-6
 
 

read_val()

read_val

(

val

)

Definition at line 1069 of file difftuple_text.py.

def read_val (val):
    if not val: return val
 
    if val.find ('+') >= 0:
      psplit = split_list1 (val, '+')
      if len(psplit) > 1:
      #psplit = split_list1 (val, '+')
      #if len(psplit) > 1:
        good = True
        for i in range(len(psplit)):
            psplit[i] = read_val (psplit[i])
            if not isinstance(psplit[i], list):
                good = False
                break
        if good:
            return reduce (operator.__add__, psplit)
 
    if val[-1] == 'L':
        try:
            return int(val)
        except TypeError:
            pass
 
    if val[0] in '0123456789-':
        if val.find ('.') < 0:
            try:
                return int(val)
            except TypeError:
                pass
            except ValueError:
                pass
 
        try:
            return float (val)
        except TypeError:
            pass
        except ValueError:
            pass
 
    try:
        (count, elt) = val.split ('*', 1)
        count = int (count)
        if elt.startswith('[') and elt.endswith(']'):
            return count * [read_val(elt[1:-1])]
    except ValueError:
        pass
 
    if val[0] == '[':
        return [read_val(s) for s in split_list(val)]
    return val
 
 

Variable Documentation

_ignore_branches

list python.difftuple_text._ignore_branches

protected

Definition at line 52 of file difftuple_text.py.

_ignore_keys

list python.difftuple_text._ignore_keys

protected

Definition at line 934 of file difftuple_text.py.

branch_thresh

list python.difftuple_text.branch_thresh

Definition at line 1400 of file difftuple_text.py.

d1

python.difftuple_text.d1 = Dumpreader (p.stdout)

Definition at line 1751 of file difftuple_text.py.

d2

python.difftuple_text.d2 = Dumpreader (open (sys.argv[2]))

Definition at line 1753 of file difftuple_text.py.

ignore_event_differences

bool ignore_event_differences = False

Definition at line 20 of file difftuple_text.py.

inttypes

list python.difftuple_text.inttypes = [types.IntType, types.LongType]

Definition at line 1302 of file difftuple_text.py.

p

python.difftuple_text.p

Initial value:

=  subprocess.Popen ('python -m D3PDMakerTest.dumptuple_any %s' %
                            sys.argv[1],
                          shell = True,
                          stdout = subprocess.PIPE)

Definition at line 1747 of file difftuple_text.py.

rename_keys

dict python.difftuple_text.rename_keys

Initial value:

=  {
    # Hack for ordering difference when scell tuple is on.
    'caloMeta/EventInfoD3PDObject_000005' : 'caloMeta/EventInfoD3PDObject_000004',
    'physicsMeta/EventInfoD3PDObject_000003' : 'physicsMeta/EventInfoD3PDObject_000002',
    'physicsTrigDecMeta/EventInfoD3PDObject_000004' : 'physicsTrigDecMeta/EventInfoD3PDObject_000003',
    }

Definition at line 27 of file difftuple_text.py.

renames

dict python.difftuple_text.renames

Initial value:

=  {
    }

Definition at line 24 of file difftuple_text.py.