python.PoolFile Namespace Reference

Classes
class	Counter
class	DiffFiles
class	PoolFile
class	PoolFileCatalog
class	PoolOpts
class	PoolRecord
class	Units
	— data ---------------------------------------------------------------— More...

Functions
	isRNTuple (obj)
	_get_total_size (branch)
	file_name (fname)
	_setup_ssl (root)
	_root_open (fname)
	retrieveBranchInfos (branch, poolRecord, ident="")
	make_pool_record (branch, dirType)
	poolRecord (self, name)
	saveReport (self, fileName)
	_save_shelve_report (self, fileName)
	_save_csv_report (self, fileName)
	__del__ (self)

Variables
str	__author__ = "Sebastien Binet <binet@cern.ch>"
list	__all__
	— data ---------------------------------------------------------------—
	stdout
	data
	poolFile

Function Documentation

__del__()

python.PoolFile.__del__

(

self

)

Definition at line 951 of file PoolFile.py.

    def __del__(self):
        if self.poolFile and hasattr(self.poolFile, 'Close'):
            try:
                self.poolFile.Close()
                self.poolFile = None
            except Exception as err:
                print("WARNING:",err)
                pass
            

_get_total_size()

python.PoolFile._get_total_size ( branch )

protected

Definition at line 313 of file PoolFile.py.

def _get_total_size (branch):
   if PoolOpts.FAST_MODE:
       return -1.
   if not PoolOpts.SUPER_DETAILED_BRANCH_SZ:
       return branch.GetTotalSize()
   brSize = 0
   branch.LoadBaskets()
   for bnum in range(0, branch.GetWriteBasket()):
       basket = branch.GetBasket(bnum)
       brSize += basket.GetObjlen() - 8
   return brSize
 

_root_open()

python.PoolFile._root_open ( fname )

protected

Definition at line 399 of file PoolFile.py.

def _root_open(fname):
    import PyUtils.RootUtils as ru
    root = ru.import_root()
    import re
 
    with ShutUp(filters=[
        re.compile('TClass::TClass:0: RuntimeWarning: no dictionary for class.*') ]):
        root.gSystem.Load('libRootCollection')
        root_open = root.TFile.Open
 
        # we need to get back the protocol b/c of the special
        # case of secure-http which needs to open TFiles as TWebFiles...
        protocol, _ = file_name(fname)
        if protocol == 'https':
            _setup_ssl(root)
            root_open = root.TWebFile.Open
 
        f = root_open(fname, 'READ')
        if f is None or not f:
            import errno
            raise IOError(errno.ENOENT,
                          'No such file or directory',fname)
        return f
    return
 

_save_csv_report()

python.PoolFile._save_csv_report ( self, fileName )

protected

Save all the gathered informations into a CSV file

Definition at line 929 of file PoolFile.py.

    def _save_csv_report(self, fileName):
        """
        Save all the gathered informations into a CSV file
        """
        import csv, os
        if os.path.exists (fileName):
            os.unlink (fileName)
        args = {'newline' : ''}
        f = open (fileName, 'w', **args)
        o = csv.writer (f)
        o.writerow (['file name', self._fileInfos['name']])
        o.writerow (['file size', self._fileInfos['size']])
        o.writerow (['nbr evts',  self.dataHeader.nEntries])
        o.writerow (['mem size', 'disk size', 'mem size nozip', 'items',
                     'container name', 'branch type'])
 
        for d in self.data:
            o.writerow ([d.memSize, d.diskSize, d.memSizeNoZip,
                         d.nEntries, d.name, d.dirType])
        f.close()
        return
 

_save_shelve_report()

python.PoolFile._save_shelve_report ( self, fileName )

protected

Save all the gathered informations into a python shelve
Data can then be read like so:
 >>> import shelve
 >>> db = shelve.open( 'myfile.dat', 'r' )
 >>> report = db['report']
 >>> print ('fileSize:',report['fileSize'])
 >>> print ('dataHeader/memSize:',report['dataHeader'].memSize)
 >>> for d in report['data']:
 ...   print ('data:',d.name,d.nEntries,d.memSize)

Definition at line 904 of file PoolFile.py.

    def _save_shelve_report(self, fileName):
        """
        Save all the gathered informations into a python shelve
        Data can then be read like so:
         >>> import shelve
         >>> db = shelve.open( 'myfile.dat', 'r' )
         >>> report = db['report']
         >>> print ('fileSize:',report['fileSize'])
         >>> print ('dataHeader/memSize:',report['dataHeader'].memSize)
         >>> for d in report['data']:
         ...   print ('data:',d.name,d.nEntries,d.memSize)
        """
        import shelve, os
        if os.path.exists (fileName):
            os.unlink (fileName)
        db = shelve.open (fileName)
        db['report'] = {
            'fileInfos'  : self._fileInfos,
            'nbrEvts'    : self.dataHeader.nEntries,
            'dataHeader' : self.dataHeader,
            'data'       : self.data
            }
        db.close()
        return
 

_setup_ssl()

python.PoolFile._setup_ssl ( root )

protected

Definition at line 385 of file PoolFile.py.

def _setup_ssl(root):
    x509_proxy = os.environ.get('X509_USER_PROXY', '')
    if x509_proxy:
        # setup proper credentials
        root.TSSLSocket.SetUpSSL(
            x509_proxy,
            "/etc/grid-security/certificates",
            x509_proxy,
            x509_proxy)
    else:
        print("## warning: protocol https is requested but no X509_USER_PROXY was found! (opening the file might fail.)")
        pass
    return
 

file_name()

python.PoolFile.file_name

(

fname

)

take a file name, return the pair (protocol, 'real' file name)

Definition at line 325 of file PoolFile.py.

def file_name(fname):
    """take a file name, return the pair (protocol, 'real' file name)
    """
    fname = os.path.expanduser(os.path.expandvars(fname))
 
    def _normalize_uri(uri):
        if uri.startswith('/'):
            return 'file:'+uri
        return uri
 
    from urllib.parse import urlsplit
    url = urlsplit(_normalize_uri(fname))
    protocol = url.scheme
    def _normalize(fname):
        from posixpath import normpath
        fname = normpath(fname)
        if fname.startswith('//'): fname = fname[1:]
        return fname
 
    if protocol in ('', 'file', 'pfn'):
        protocol = ''
        fname = _normalize(url.path)
 
        
        if fname.startswith('/castor/'):
            protocol = 'rfio'
            fname = protocol + ':' + fname
 
    elif protocol in ('rfio', 'castor'):
        protocol = 'rfio'
        fname = _normalize(url.path)
        fname = protocol+':'+fname
 
    elif protocol in ('root','dcap', 'dcache', 'http', 'https', 'dav', 'davs'):
        pass
 
    elif protocol in ('gsidcap',):
        protocol = 'gfal:gsidcap'
        pass
 
    elif protocol in ('lfn','fid',):
        # percolate through the PoolFileCatalog
        from PyUtils.PoolFile import PoolFileCatalog as pfc
        fname = pfc().pfn(protocol+':'+url.path)
        pass
 
    elif protocol in ('ami',):
        # !! keep order of tokens !
        for token in ('ami:', '//', '/'):
            if fname.startswith(token):
                fname = fname[len(token):]
            fname = 'ami://' + fname
            pass
 
    else:
        print(f'## warning: unknown protocol [{protocol}]. we will just return our input')
        pass
 
    return (protocol, fname)
 

isRNTuple()

python.PoolFile.isRNTuple

(

obj

)

Definition at line 35 of file PoolFile.py.

def isRNTuple(obj):
    # MN: remove the "try" after migration to ROOT 6.34
    try: from ROOT import RNTuple
    except(ImportError): from ROOT.Experimental import RNTuple
    return isinstance( obj, RNTuple )
 
 

make_pool_record()

python.PoolFile.make_pool_record	(		branch,
			dirType )

Definition at line 445 of file PoolFile.py.

def make_pool_record (branch, dirType):
    memSize = _get_total_size (branch) / Units.kb
    zipBytes = branch.GetZipBytes()
    memSizeNoZip = memSize if zipBytes < 0.001 else 0.
    diskSize     = branch.GetZipBytes() / Units.kb
    typeName = branch.GetClassName()
    if not typeName and (leaf := branch.GetListOfLeaves().At(0)):
        typeName = leaf.GetTypeName()
    return PoolRecord(branch.GetName(), memSize, diskSize, memSizeNoZip,
                      branch.GetEntries(),
                      dirType=dirType,
                      typeName=typeName)
 

poolRecord()

python.PoolFile.poolRecord	(		self,
			name )

Return a PoolRecord according to its (branch) name
Raise KeyError if no match is found

Definition at line 884 of file PoolFile.py.

    def poolRecord(self, name):
        """
        Return a PoolRecord according to its (branch) name
        Raise KeyError if no match is found
        """
        for data in self.data:
            if data.name == name:
                return data
        raise KeyError("No PoolRecord with name [%s]" % name)
 

retrieveBranchInfos()

python.PoolFile.retrieveBranchInfos	(		branch,
			poolRecord,
			ident = "" )

Definition at line 424 of file PoolFile.py.

def retrieveBranchInfos( branch, poolRecord, ident = "" ):
    fmt = "%s %3i %8.3f %8.3f %8.3f %s"
    if 0:
        out = fmt % ( ident,
                      branch.GetListOfBranches().GetSize(),
                      _get_total_size (branch), 
                      branch.GetTotBytes(),
                      branch.GetZipBytes(),
                      branch.GetName() )
        print(out)
        
    branches  = branch.GetListOfBranches()
    for b in branches:
        poolRecord.memSize  += _get_total_size (b) / Units.kb
        if (b.GetZipBytes() < 0.001):
            poolRecord.memSizeNoZip  += _get_total_size (b) / Units.kb
        poolRecord.diskSize += b.GetZipBytes() / Units.kb
        poolRecord = retrieveBranchInfos ( b, poolRecord, ident+"  " )
        
    return poolRecord
 

saveReport()

python.PoolFile.saveReport	(		self,
			fileName )

Save all the gathered informations into a python shelve or a CSV file
(depending on the @param `fileName` extension)

Definition at line 894 of file PoolFile.py.

    def saveReport (self, fileName):
        """
        Save all the gathered informations into a python shelve or a CSV file
        (depending on the @param `fileName` extension)
        """
        import os
        if os.path.splitext(fileName)[-1] == '.csv':
            return self._save_csv_report (fileName)
        return self._save_shelve_report (fileName)
    

Variable Documentation

__all__

list python.PoolFile.__all__

private

Initial value:

=  [
    'PoolFileCatalog',
    'PoolOpts',
    'isRNTuple',
    'PoolRecord',
    'PoolFile',
    'DiffFiles',
    ]

— data ---------------------------------------------------------------—

Definition at line 11 of file PoolFile.py.

__author__

str python.PoolFile.__author__ = "Sebastien Binet <binet@cern.ch>"

private

Definition at line 8 of file PoolFile.py.

data

python.PoolFile.data

Definition at line 889 of file PoolFile.py.

poolFile

python.PoolFile.poolFile

Definition at line 952 of file PoolFile.py.

stdout

python.PoolFile.stdout

Definition at line 881 of file PoolFile.py.