python.PoolFile Namespace Reference

Classes
class	Counter
class	DiffFiles
class	PoolFile
class	PoolFileCatalog
class	PoolOpts
class	PoolRecord
class	Units
	— data ---------------------------------------------------------------— More...

Functions
def	isRNTuple (obj)
def	_get_total_size (branch)
def	file_name (fname)
def	_setup_ssl (root)
def	_root_open (fname)
def	retrieveBranchInfos (branch, poolRecord, ident="")
def	make_pool_record (branch, dirType)
def	extract_items (pool_file, verbose=True, items_type='eventdata')
def	poolRecord (self, name)
def	saveReport (self, fileName)
def	_save_shelve_report (self, fileName)
def	_save_csv_report (self, fileName)
def	__del__ (self)

Variables
	__author__
	__all__
	— data ---------------------------------------------------------------— More...
	stdout
	out.write( "#### Map ####" + os.linesep ) out.flush() self.poolFile.Map() out.write( "#" * 80 + os.linesep ) More...
	poolFile

Function Documentation

__del__()

def python.PoolFile.__del__

(

self

)

Definition at line 972 of file PoolFile.py.

    def __del__(self):
        if self.poolFile and hasattr(self.poolFile, 'Close'):
            try:
                self.poolFile.Close()
                self.poolFile = None
            except Exception as err:
                print("WARNING:",err)
                pass
            

_get_total_size()

def python.PoolFile._get_total_size (   branch )

private

Definition at line 309 of file PoolFile.py.

def _get_total_size (branch):
   if PoolOpts.FAST_MODE:
       return -1.
   if not PoolOpts.SUPER_DETAILED_BRANCH_SZ:
       return branch.GetTotalSize()
   brSize = 0
   branch.LoadBaskets()
   for bnum in range(0, branch.GetWriteBasket()):
       basket = branch.GetBasket(bnum)
       brSize += basket.GetObjlen() - 8
   return brSize
 

_root_open()

def python.PoolFile._root_open (   fname )

private

Definition at line 395 of file PoolFile.py.

def _root_open(fname):
    import PyUtils.RootUtils as ru
    root = ru.import_root()
    import re
 
    with ShutUp(filters=[
        re.compile('TClass::TClass:0: RuntimeWarning: no dictionary for class.*') ]):
        root.gSystem.Load('libRootCollection')
        root_open = root.TFile.Open
 
        # we need to get back the protocol b/c of the special
        # case of secure-http which needs to open TFiles as TWebFiles...
        protocol, _ = file_name(fname)
        if protocol == 'https':
            _setup_ssl(root)
            root_open = root.TWebFile.Open
 
        f = root_open(fname, 'READ')
        if f is None or not f:
            import errno
            raise IOError(errno.ENOENT,
                          'No such file or directory',fname)
        return f
    return
 

_save_csv_report()

def python.PoolFile._save_csv_report (   self,   fileName  )

private

Save all the gathered informations into a CSV file

Definition at line 950 of file PoolFile.py.

    def _save_csv_report(self, fileName):
        """
        Save all the gathered informations into a CSV file
        """
        import csv, os
        if os.path.exists (fileName):
            os.unlink (fileName)
        args = {'newline' : ''}
        f = open (fileName, 'w', **args)
        o = csv.writer (f)
        o.writerow (['file name', self._fileInfos['name']])
        o.writerow (['file size', self._fileInfos['size']])
        o.writerow (['nbr evts',  self.dataHeader.nEntries])
        o.writerow (['mem size', 'disk size', 'mem size nozip', 'items',
                     'container name', 'branch type'])
 
        for d in self.data:
            o.writerow ([d.memSize, d.diskSize, d.memSizeNoZip,
                         d.nEntries, d.name, d.dirType])
        f.close()
        return
 

_save_shelve_report()

def python.PoolFile._save_shelve_report (   self,   fileName  )

private

Save all the gathered informations into a python shelve
Data can then be read like so:
 >>> import shelve
 >>> db = shelve.open( 'myfile.dat', 'r' )
 >>> report = db['report']
 >>> print ('fileSize:',report['fileSize'])
 >>> print ('dataHeader/memSize:',report['dataHeader'].memSize)
 >>> for d in report['data']:
 ...   print ('data:',d.name,d.nEntries,d.memSize)

Definition at line 925 of file PoolFile.py.

    def _save_shelve_report(self, fileName):
        """
        Save all the gathered informations into a python shelve
        Data can then be read like so:
         >>> import shelve
         >>> db = shelve.open( 'myfile.dat', 'r' )
         >>> report = db['report']
         >>> print ('fileSize:',report['fileSize'])
         >>> print ('dataHeader/memSize:',report['dataHeader'].memSize)
         >>> for d in report['data']:
         ...   print ('data:',d.name,d.nEntries,d.memSize)
        """
        import shelve, os
        if os.path.exists (fileName):
            os.unlink (fileName)
        db = shelve.open (fileName)
        db['report'] = {
            'fileInfos'  : self._fileInfos,
            'nbrEvts'    : self.dataHeader.nEntries,
            'dataHeader' : self.dataHeader,
            'data'       : self.data
            }
        db.close()
        return
 

_setup_ssl()

def python.PoolFile._setup_ssl (   root )

private

Definition at line 381 of file PoolFile.py.

def _setup_ssl(root):
    x509_proxy = os.environ.get('X509_USER_PROXY', '')
    if x509_proxy:
        # setup proper credentials
        root.TSSLSocket.SetUpSSL(
            x509_proxy,
            "/etc/grid-security/certificates",
            x509_proxy,
            x509_proxy)
    else:
        print("## warning: protocol https is requested but no X509_USER_PROXY was found! (opening the file might fail.)")
        pass
    return
 

extract_items()

def python.PoolFile.extract_items	(		pool_file,
			verbose = True,
			items_type = 'eventdata'
	)

Helper function to read a POOL file and extract the item-list from the
DataHeader content.
@params
  `pool_file`  the name of the pool file to inspect
  `verbose`    self-explanatory
  `items_type` what kind of items one is interested in
               allowed values: 'eventdata' 'metadata'
Note: this function is actually executed in a forked sub-process
      if `fork` is True

Definition at line 454 of file PoolFile.py.

def extract_items(pool_file, verbose=True, items_type='eventdata'):
    """Helper function to read a POOL file and extract the item-list from the
    DataHeader content.
    @params
      `pool_file`  the name of the pool file to inspect
      `verbose`    self-explanatory
      `items_type` what kind of items one is interested in
                   allowed values: 'eventdata' 'metadata'
    Note: this function is actually executed in a forked sub-process
          if `fork` is True
    """
    _allowed_values = ('eventdata',
                       'metadata',)
    if items_type not in _allowed_values:
        err = "".join([
            "invalid argument for 'items_type'. ",
            "got: [%s] " % items_type,
            "(allowed values: %r)" % _allowed_values
            ])
        raise ValueError(err)
 
    key = '%s_items' % items_type
    f_root = _root_open(pool_file)
    import PyUtils.FilePeekerTool as fpt
    fp = fpt.FilePeekerTool(f_root)
    items = fp.getPeekedData(key)
 
    if items is None:
        items = []
    return items
 

file_name()

def python.PoolFile.file_name

(

fname

)

take a file name, return the pair (protocol, 'real' file name)

Definition at line 321 of file PoolFile.py.

def file_name(fname):
    """take a file name, return the pair (protocol, 'real' file name)
    """
    fname = os.path.expanduser(os.path.expandvars(fname))
 
    def _normalize_uri(uri):
        if uri.startswith('/'):
            return 'file:'+uri
        return uri
 
    from urllib.parse import urlsplit
    url = urlsplit(_normalize_uri(fname))
    protocol = url.scheme
    def _normalize(fname):
        from posixpath import normpath
        fname = normpath(fname)
        if fname.startswith('//'): fname = fname[1:]
        return fname
 
    if protocol in ('', 'file', 'pfn'):
        protocol = ''
        fname = _normalize(url.path)
 
        
        if fname.startswith('/castor/'):
            protocol = 'rfio'
            fname = protocol + ':' + fname
 
    elif protocol in ('rfio', 'castor'):
        protocol = 'rfio'
        fname = _normalize(url.path)
        fname = protocol+':'+fname
 
    elif protocol in ('root','dcap', 'dcache', 'http', 'https', 'dav', 'davs'):
        pass
 
    elif protocol in ('gsidcap',):
        protocol = 'gfal:gsidcap'
        pass
 
    elif protocol in ('lfn','fid',):
        # percolate through the PoolFileCatalog
        from PyUtils.PoolFile import PoolFileCatalog as pfc
        fname = pfc().pfn(protocol+':'+url.path)
        pass
 
    elif protocol in ('ami',):
        # !! keep order of tokens !
        for token in ('ami:', '//', '/'):
            if fname.startswith(token):
                fname = fname[len(token):]
            fname = 'ami://' + fname
            pass
 
    else:
        print(f'## warning: unknown protocol [{protocol}]. we will just return our input')
        pass
 
    return (protocol, fname)
 

isRNTuple()

def python.PoolFile.isRNTuple

(

obj

)

Definition at line 36 of file PoolFile.py.

def isRNTuple(obj):
    # MN: remove the "try" after migration to ROOT 6.34
    try: from ROOT import RNTuple
    except(ImportError): from ROOT.Experimental import RNTuple
    return isinstance( obj, RNTuple )
 
 

make_pool_record()

def python.PoolFile.make_pool_record	(		branch,
			dirType
	)

Definition at line 441 of file PoolFile.py.

def make_pool_record (branch, dirType):
    memSize = _get_total_size (branch) / Units.kb
    zipBytes = branch.GetZipBytes()
    memSizeNoZip = memSize if zipBytes < 0.001 else 0.
    diskSize     = branch.GetZipBytes() / Units.kb
    typeName = branch.GetClassName()
    if not typeName and (leaf := branch.GetListOfLeaves().At(0)):
        typeName = leaf.GetTypeName()
    return PoolRecord(branch.GetName(), memSize, diskSize, memSizeNoZip,
                      branch.GetEntries(),
                      dirType=dirType,
                      typeName=typeName)
 

poolRecord()

def python.PoolFile.poolRecord	(		self,
			name
	)

Return a PoolRecord according to its (branch) name
Raise KeyError if no match is found

Definition at line 905 of file PoolFile.py.

    def poolRecord(self, name):
        """
        Return a PoolRecord according to its (branch) name
        Raise KeyError if no match is found
        """
        for data in self.data:
            if data.name == name:
                return data
        raise KeyError("No PoolRecord with name [%s]" % name)
 

retrieveBranchInfos()

def python.PoolFile.retrieveBranchInfos	(		branch,
			poolRecord,
			ident = ""
	)

Definition at line 420 of file PoolFile.py.

def retrieveBranchInfos( branch, poolRecord, ident = "" ):
    fmt = "%s %3i %8.3f %8.3f %8.3f %s"
    if 0:
        out = fmt % ( ident,
                      branch.GetListOfBranches().GetSize(),
                      _get_total_size (branch), 
                      branch.GetTotBytes(),
                      branch.GetZipBytes(),
                      branch.GetName() )
        print(out)
        
    branches  = branch.GetListOfBranches()
    for b in branches:
        poolRecord.memSize  += _get_total_size (b) / Units.kb
        if (b.GetZipBytes() < 0.001):
            poolRecord.memSizeNoZip  += _get_total_size (b) / Units.kb
        poolRecord.diskSize += b.GetZipBytes() / Units.kb
        poolRecord = retrieveBranchInfos ( b, poolRecord, ident+"  " )
        
    return poolRecord
 

saveReport()

def python.PoolFile.saveReport	(		self,
			fileName
	)

Save all the gathered informations into a python shelve or a CSV file
(depending on the @param `fileName` extension)

Definition at line 915 of file PoolFile.py.

    def saveReport (self, fileName):
        """
        Save all the gathered informations into a python shelve or a CSV file
        (depending on the @param `fileName` extension)
        """
        import os
        if os.path.splitext(fileName)[-1] == '.csv':
            return self._save_csv_report (fileName)
        return self._save_shelve_report (fileName)
    

Variable Documentation

__all__

python.PoolFile.__all__

private

— data ---------------------------------------------------------------—

Definition at line 11 of file PoolFile.py.

__author__

python.PoolFile.__author__

private

Definition at line 8 of file PoolFile.py.

poolFile

python.PoolFile.poolFile

Definition at line 976 of file PoolFile.py.

stdout

python.PoolFile.stdout

out.write( "#### Map ####" + os.linesep ) out.flush() self.poolFile.Map() out.write( "#" * 80 + os.linesep )

Definition at line 902 of file PoolFile.py.