python.trfFileUtils Namespace Reference

Functions
	AthenaLiteFileInfo (filename, filetype, retrieveKeys=athFileInterestingKeys)
	New lightweight interface to getting a single file's metadata.
	HISTEntries (fileName)
	Determines number of events in a HIST file.
	NTUPEntries (fileName, treeNames)
	Determines number of entries in NTUP file with given tree names.
	PRWEntries (fileName, integral=False)
	Determines number of entries in PRW file.
	PHYSVALEntries (fileName, integral=False)
	Determines number of entries in NTUP_PHYSVAL file.
	ROOTGetSize (filename)
	Get the size of a file via ROOT's TFile.
	urlType (filename)
	Return the LAN access type for a file URL.

Variables
	msg = logging.getLogger(__name__)
list	athFileInterestingKeys = ['file_size', 'file_guid', 'file_type', 'nentries']

Function Documentation

AthenaLiteFileInfo()

AthenaLiteFileInfo	(		filename,
			filetype,
			retrieveKeys = athFileInterestingKeys )

New lightweight interface to getting a single file's metadata.

Definition at line 23 of file trfFileUtils.py.

def AthenaLiteFileInfo(filename, filetype, retrieveKeys = athFileInterestingKeys):
    msg.debug('Calling AthenaLiteFileInfo for {0} (type {1})'.format(filename, filetype))
    from PyUtils.MetaReader import read_metadata
 
    metaDict = {}
    try:
        meta = read_metadata(filename,None,'lite')[filename]
        msg.debug('read_metadata came back for {0}'.format(filename))
        metaDict[filename] = {}
        for key in retrieveKeys:
            msg.debug('Looking for key {0}'.format(key))
            try:
                metaval = meta[key]
                metaDict[filename][key] = metaval.lower() if key == 'file_type' else metaval
            except KeyError:
                msg.warning('Missing key in athFile info: {0}'.format(key))
    except (ValueError, AssertionError, ReferenceError):
        msg.error('Problem in getting metadata for {0}'.format(filename))
        return None
    msg.debug('Returning {0}'.format(metaDict))
    return metaDict    
 

HISTEntries()

HISTEntries

(

fileName

)

Determines number of events in a HIST file.

Basically taken from PyJobTransformsCore.trfutil.MonitorHistFile

Parameters

fileName

Path to the HIST file.

Returns

• Number of events.
• None if the determination failed.

Note

Use the PyUtils forking decorator to ensure that ROOT is run completely within a child process and will not 'pollute' the parent python process with unthread-safe bits of code (otherwise strange hangs are observed on subsequent uses of ROOT)

Definition at line 55 of file trfFileUtils.py.

def HISTEntries(fileName):
 
    root = import_root()
 
    fname = root.TFile.Open(fileName, 'READ')
    
    if not (isinstance(fname, root.TFile) and fname.IsOpen()):
        return None
 
    # HIST_HLTMON case
    htrig_path = 'HLTFramework/HltEventLoopMgr/TotalTime'
    htrig = fname.Get(htrig_path)
    if isinstance( htrig, root.TH1 ):
        nev_trig = htrig.GetEntries()
        fname.Close()
        msg.debug( 'Retrieved %s in file %s, found %i entries', htrig_path, fileName,  nev_trig)
        return nev_trig
 
    rundir = None
    keys = fname.GetListOfKeys()
    
    for key in keys:
        
        name=key.GetName()
        
        if name.startswith('run_') and name != 'run_multiple':
            
            if rundir is not None:
                msg.warning('Found two run_ directories in HIST file %s: %s and %s', fileName, rundir, name)
                return None
            else:
                rundir = name
                
        del name
       
    if rundir is None:
        msg.warning( 'Unable to find run directory in HIST file %s', fileName )
        fname.Close()
        return None
    
    msg.info( 'Using run directory %s for event counting of HIST file %s. ', rundir, fileName )
    
    hpath = '%s/GLOBAL/DQTDataFlow/events_lb' % rundir
    possibleLBs = []
    if 'tmp.HIST_' in fileName:
        msg.info( 'Special case for temporary HIST file {0}. '.format( fileName ) )
        h = fname.Get('{0}'.format(rundir))
        for directories in h.GetListOfKeys() :
            if 'lb' in directories.GetName():
                msg.info( 'Using {0} in tmp HIST file {1}. '.format(directories.GetName(),  fileName ) )
                hpath = rundir+'/'+str(directories.GetName())+'/GLOBAL/DQTDataFlow/events_lb' 
                possibleLBs.append(hpath)
    else:
        msg.info( 'Classical case for HIST file {0}. '.format( fileName ) )
        possibleLBs.append(hpath)
    nev = 0
    if len(possibleLBs) == 0:
        msg.warning( 'Unable to find events_lb histogram in HIST file %s', fileName )
        fname.Close()
        return None
    for hpath in possibleLBs:
        h = fname.Get(hpath)
        
        if not isinstance( h, root.TH1 ):
            msg.warning( 'Unable to retrieve %s in HIST file %s.', hpath, fileName )
            fname.Close()
            return None
        
        nBinsX = h.GetNbinsX()
        nevLoc = 0
        
        for i in range(1, nBinsX):
            
            if h[i] < 0:
                msg.warning( 'Negative number of events for step %s in HIST file %s.', h.GetXaxis().GetBinLabel(i), fileName )
                fname.Close()
                return None
            
            elif h[i] == 0:
                continue
            
            if nevLoc == 0:
                nevLoc = h[i]
                
            else:
                if nevLoc != h[i]:
                    msg.warning( 'Mismatch in events per step in HIST file %s; most recent step seen is %s.', fileName, h.GetXaxis().GetBinLabel(i) )
                    fname.Close()
                    return None
        nev += nevLoc        
    fname.Close()
    return nev
 
 
 

NTUPEntries()

NTUPEntries	(		fileName,
			treeNames )

Determines number of entries in NTUP file with given tree names.

Basically taken from PyJobTransformsCore.trfutil.ntup_entries.

Parameters

fileName	Path to the NTUP file.
treeNames	Tree name or list of tree names. In the latter case it is checked if all trees contain the same number of events

Returns

• Number of entries.
• None if the determination failed.

Note

Use the PyUtils forking decorator to ensure that ROOT is run completely within a child process and will not 'pollute' the parent python process with unthread-safe bits of code (otherwise strange hangs are observed on subsequent uses of ROOT)

Definition at line 162 of file trfFileUtils.py.

def NTUPEntries(fileName, treeNames):
    
    if not isinstance( treeNames, list ):
        treeNames=[treeNames]
        
    root = import_root()
    
    fname = root.TFile.Open(fileName, 'READ')
    
    if not (isinstance(fname, root.TFile) and fname.IsOpen()):
        return None
    
    prevNum=None
    prevTree=None
               
    for treeName in treeNames:
            
        tree = fname.Get(treeName)
            
        if not isinstance(tree, root.TTree):
            return None
        
        num = tree.GetEntriesFast()
 
        if not num>=0:
            msg.warning('GetEntriesFast returned non positive value for tree %s in NTUP file %s.', treeName, fileName )
            return None
                
        if prevNum is not None and prevNum != num:
            msg.warning( "Found diffferent number of entries in tree %s and tree %s of file %s.", treeName, prevTree, fileName )
            return None
        
        numberOfEntries=num
        prevTree=treeName
        del num
        del tree
 
    fname.Close()
 
    return numberOfEntries
 
 

PHYSVALEntries()

PHYSVALEntries	(		fileName,
			integral = False )

Determines number of entries in NTUP_PHYSVAL file.

Parameters

fileName	Path to the PHYSVAL file.
integral	Returns sum of weights if true

Returns

• Number of entries.
• Sum of weights if integral is true.
• None if the determination failed.

Note

Use the PyCmt forking decorator to ensure that ROOT is run completely within a child process and will not 'pollute' the parent python process with unthread-safe bits of code (otherwise strange hangs are observed on subsequent uses of ROOT)

Definition at line 258 of file trfFileUtils.py.

def PHYSVALEntries(fileName, integral=False):
 
    root = import_root()
 
    fname = root.TFile.Open(fileName, 'READ')
 
    if not (isinstance(fname, root.TFile) and fname.IsOpen()):
        return None
 
    aipc = fname.Get("/EventInfo/EventInfo_actualInteractionsPerCrossing")
 
    if not aipc:
        # Not PHYSVAL...
        return None
 
    # If we want the weights, give us the weights
    if integral:
        return aipc.Integral()
 
    # Otherwise we just want the entries
    return int(aipc.GetEntries())
 
 

PRWEntries()

PRWEntries	(		fileName,
			integral = False )

Determines number of entries in PRW file.

Parameters

fileName	Path to the PRW file.
integral	Returns sum of weights if true

Returns

• Number of entries.
• Sum of weights if integral is true.
• None if the determination failed.

Note

Use the PyCmt forking decorator to ensure that ROOT is run completely within a child process and will not 'pollute' the parent python process with unthread-safe bits of code (otherwise strange hangs are observed on subsequent uses of ROOT)

Definition at line 215 of file trfFileUtils.py.

def PRWEntries(fileName, integral=False):
 
    root = import_root()
 
    fname = root.TFile.Open(fileName, 'READ')
 
    if not (isinstance(fname, root.TFile) and fname.IsOpen()):
        return None
 
    rundir = None
 
    for key in fname.GetListOfKeys():
        if key.GetName()=='PileupReweighting':
            rundir = fname.Get('PileupReweighting')
            break
        # Not PRW...
 
    if rundir is None: return None
 
    total = 0
    for key in rundir.GetListOfKeys():
        if 'pileup' in key.GetName():
            msg.debug('Working on file '+fileName+' histo '+key.GetName())
            if integral:
                total += rundir.Get(key.GetName()).Integral()
            else:
                total += rundir.Get(key.GetName()).GetEntries()
        # Was not one of our histograms
    # Make sure we return an int for the number of events
    return int(total)
 
 

ROOTGetSize()

ROOTGetSize

(

filename

)

Get the size of a file via ROOT's TFile.

Use TFile.Open to retrieve a ROOT filehandle, which will deal with all non-posix filesystems. Return the GetSize() value. The option filetype=raw is added to ensure this works for non-ROOT files too (e.g. BS)

Note

Use the PyUtils forking decorator to ensure that ROOT is run completely within a child process and will not 'pollute' the parent python process with unthread-safe bits of code (otherwise strange hangs are observed on subsequent uses of ROOT)

Parameters

filename

Filename to get size of

Returns

fileSize or None if there was a problem

Definition at line 291 of file trfFileUtils.py.

def ROOTGetSize(filename):
    root = import_root()
    
    try:
        msg.debug('Calling TFile.Open for {0}'.format(filename))
        extraparam = '?filetype=raw'
        if filename.startswith("https") or filename.startswith("davs"):
            try:
                pos = filename.find("?")
                if pos>=0:
                    extraparam = '&filetype=raw'
                else:
                    extraparam = '?filetype=raw'
            except Exception:
                extraparam = '?filetype=raw'
        fname = root.TFile.Open(filename + extraparam, 'READ')
        fsize = fname.GetSize()
        msg.debug('Got size {0} from TFile.GetSize'.format(fsize))
    except ReferenceError:
        msg.error('Failed to get size of {0}'.format(filename))
        return None
    
    fname.Close()
    del root
    return fsize
    
 

urlType()

urlType

(

filename

)

Return the LAN access type for a file URL.

Parameters

filename

Name of file to examine

Returns

• String with LAN protocol

Definition at line 322 of file trfFileUtils.py.

def urlType(filename):
    if filename.startswith('dcap:'):
        return 'dcap'
    if filename.startswith('root:'):
        return 'root'
    if filename.startswith('rfio:'):
        return 'rfio'
    if filename.startswith('file:'):
        return 'posix'
    if filename.startswith('https:'):
        return 'root'
    if filename.startswith('davs:'):
        return 'root'
    return 'posix'
 

Variable Documentation

athFileInterestingKeys

list python.trfFileUtils.athFileInterestingKeys = ['file_size', 'file_guid', 'file_type', 'nentries']

Definition at line 20 of file trfFileUtils.py.

msg

python.trfFileUtils.msg = logging.getLogger(__name__)

Definition at line 10 of file trfFileUtils.py.