python.PoolFile.PoolFile Class Reference

Inheritance diagram for python.PoolFile.PoolFile:

Collaboration diagram for python.PoolFile.PoolFile:

Public Member Functions
	__init__ (self, fileName, verbose=True)
	fileInfos (self)
	checkFile (self, sorting=PoolRecord.Sorter.DiskSize)
	detailedDump (self, bufferName=None)

Public Attributes
	keys = None
	dataHeader
	augNames = set()
dict	dataHeaderA = {}
list	data = []
	verbose = verbose
	poolFile = None
	ROOT = ROOT

Protected Attributes
dict	_fileInfos = None

Private Member Functions
	__openPoolFile (self, fileName)
	__processFile (self)

Detailed Description

A simple class to retrieve informations about the content of a POOL file.
It should be abstracted from the underlying technology used to create this
POOL file (Db, ROOT,...).
Right now, we are using the easy and loosy solution: going straight to the
ROOT 'API'.

Definition at line 453 of file PoolFile.py.

Constructor & Destructor Documentation

__init__()

python.PoolFile.PoolFile.__init__	(		self,
			fileName,
			verbose = True )

Definition at line 462 of file PoolFile.py.

    def __init__(self, fileName, verbose=True):
        object.__init__(self)
 
        self._fileInfos = None
        self.keys       = None
        self.dataHeader = PoolRecord("DataHeader", 0, 0, 0,
                                     nEntries = 0,
                                     dirType = "T")
        self.augNames   = set()
        self.dataHeaderA = {}
        self.data       = []
        self.verbose    = verbose
 
        # get the "final" file name (handles all kind of protocols)
        try:
            protocol, fileName = file_name(fileName)
        except Exception as err:
            print("## warning: problem opening PoolFileCatalog:\n%s"%err)
            import traceback
            traceback.print_exc(err)
            pass
        
        self.poolFile = None
        dbFileName = whichdb( fileName )
        if dbFileName not in ( None, '' ):
            if self.verbose is True:
                print("## opening file [%s]..." % str(fileName))
            db = shelve.open( fileName, 'r' )
            if self.verbose is True:
                print("## opening file [OK]")
            report = db['report']
            self._fileInfos = report['fileInfos']
            self.dataHeader = report['dataHeader']
            self.data       = report['data']
        else:
            if self.verbose is True:
                print("## opening file [%s]..." % str(fileName))
            self.__openPoolFile( fileName )
            if self.verbose is True:
                print("## opening file [OK]")
            self.__processFile()
            
        return
 

Member Function Documentation

__openPoolFile()

python.PoolFile.PoolFile.__openPoolFile ( self, fileName )

private

Definition at line 506 of file PoolFile.py.

    def __openPoolFile(self, fileName):
        # hack to prevent ROOT from loading graphic libraries and hence bother
        # our fellow Mac users
        if self.verbose is True:
            print("## importing ROOT...")
        import PyUtils.RootUtils as ru
        ROOT = ru.import_root()
        self.ROOT = ROOT
        if self.verbose is True:
            print("## importing ROOT... [DONE]")
        # prevent ROOT from being too verbose
        rootMsg = ShutUp()
        rootMsg.mute()
        ROOT.gErrorIgnoreLevel = ROOT.kFatal
 
        poolFile = None
        try:
            poolFile = ROOT.TFile.Open( fileName, PoolOpts.READ_MODE )
        except Exception as e:
            rootMsg.unMute()
            print("## Failed to open file [%s] !!" % fileName)
            print("## Reason:")
            print(e)
            print("## Bailing out...")
            raise IOError("Could not open file [%s]" % fileName)
 
        rootMsg.unMute()
 
        if poolFile is None:
            print("## Failed to open file [%s] !!" % fileName)
            msg = "Could not open file [%s]" % fileName
            raise IOError(msg)
 
        self.poolFile = poolFile
        assert self.poolFile.IsOpen() and not self.poolFile.IsZombie(), \
               "Invalid POOL file or a Zombie one" 
        self._fileInfos = {
            'name' : self.poolFile.GetName(),
            'size' : self.poolFile.GetSize(),
            }
        return
 

__processFile()

python.PoolFile.PoolFile.__processFile ( self )

private

Definition at line 548 of file PoolFile.py.

    def __processFile(self):
        
        for name in {PoolOpts.TTreeNames.DataHeader, PoolOpts.RNTupleNames.DataHeader}:
            dhKey = self.poolFile.FindKey( name )
            if dhKey:
                obj = self.poolFile.Get( name )
                if isinstance(obj, self.ROOT.TTree):
                    nEntries = obj.GetEntries()
                elif isRNTuple(obj):
                    try:
                        nEntries = self.ROOT.Experimental.RNTupleReader.Open(obj).GetNEntries()
                    except AttributeError:
                        # ROOT 6.36 and later
                        nEntries = self.ROOT.RNTupleReader.Open(obj).GetNEntries()
                else:
                    raise NotImplementedError(f"Keys of type {type(obj)!r} not supported")
                break
        else:
            nEntries = 0
 
        keys = []
        containers = []
        for k in self.poolFile.GetListOfKeys():
            keyname = k.GetName()
            obj = self.poolFile.Get( keyname )
            if isinstance(obj, self.ROOT.TTree):
                containerName = obj.GetName()
                nEntries = obj.GetEntries()
                dirType = "T"
            elif isRNTuple(obj):
                try:
                    reader = self.ROOT.Experimental.RNTupleReader.Open(obj)
                except AttributeError:
                    # ROOT 6.36 and later
                    reader = self.ROOT.RNTupleReader.Open(obj)
                containerName = reader.GetDescriptor().GetName()
                nEntries = reader.GetNEntries()
                dirType = "N"
            else:
                raise NotImplementedError(f"Keys of type {type(obj)!r} not supported")
            if containerName not in containers:
                keys.append(k)
                containers.append(containerName)
                pass
            if keyname.startswith(PoolOpts.POOL_HEADER) and not keyname.endswith('Form'):
                self.dataHeaderA[PoolOpts.augmentationName(keyname)] = \
                    PoolRecord("DataHeader", 0, 0, 0,
                               nEntries = nEntries,
                               dirType = dirType)
 
        keys.sort (key = lambda x: x.GetName())
        self.keys = keys
        del containers
        
        for k in keys:
            obj = self.poolFile.Get( k.GetName() )
            if isinstance(obj, self.ROOT.TTree):
                name = obj.GetName()
            elif isRNTuple(obj):
                try:
                    inspector = self.ROOT.Experimental.RNTupleInspector.Create(obj)
                except AttributeError:
                    inspector = self.ROOT.RNTupleInspector.Create(obj)
                name = inspector.GetDescriptor().GetName()
 
            if PoolOpts.isDataHeader(name):
                contName     = "DataHeader"
                if isinstance(obj, self.ROOT.TTree):
                    memSize      = obj.GetTotBytes() / Units.kb
                    diskSize     = obj.GetZipBytes() / Units.kb
                    memSizeNoZip = 0.0
                    if diskSize < 0.001:
                        memSizeNoZip = memSize
                    nEntries     = obj.GetEntries()
                    
                    dhBranchNames = [
                        br.GetName() for br in obj.GetListOfBranches()
                        if br.GetName().count("DataHeader_p") > 0
                    ]
                    if len(dhBranchNames) == 1:
                        dhBranch = obj.GetBranch(dhBranchNames[0])
                        typeName = dhBranch.GetClassName()
                        if not typeName and (leaf := dhBranch.GetListOfLeaves().At(0)):
                            typeName = leaf.GetTypeName()
                        poolRecord = retrieveBranchInfos(
                            dhBranch,
                            PoolRecord( contName, 0., 0., 0.,
                                        nEntries,
                                        dirType = "T",
                                        typeName = typeName ),
                            ident = "  "
                        )
                    else:
                        poolRecord = PoolRecord(contName, memSize, diskSize, memSizeNoZip,
                                                nEntries,
                                                dirType = "T")
 
                    self.dataHeader = poolRecord
                elif isRNTuple(obj):
                    diskSize = inspector.GetCompressedSize() / Units.kb
                    memSize = inspector.GetUncompressedSize() / Units.kb
 
                    memSizeNoZip = 0.0
                    if diskSize < 0.001:
                        memSizeNoZip = memSize
                    nEntries     = inspector.GetDescriptor().GetNEntries()
                    poolRecord = PoolRecord(contName, memSize, diskSize, memSizeNoZip,
                                            nEntries,
                                            dirType = "N")
                    self.dataHeader = poolRecord
            elif PoolOpts.isData(name):
                if isinstance(obj, self.ROOT.TTree):
                    if not hasattr(obj, 'GetListOfBranches'):
                        continue
                    branches = obj.GetListOfBranches()
                    dirType = "T"
                    if name in (PoolOpts.EVENT_DATA, PoolOpts.META_DATA):
                        dirType = "B"
                    for branch in branches:
                        poolRecord = retrieveBranchInfos(
                            branch,
                            make_pool_record(branch, dirType),
                            ident = "  "
                        )
                        poolRecord.augName = PoolOpts.augmentationName(name)
                        self.augNames.add(poolRecord.augName)
                        self.data += [ poolRecord ]
                elif isRNTuple(obj):
                    descriptor = inspector.GetDescriptor()
                    dirType = "N"
                    if name in {PoolOpts.RNTupleNames.EventData, PoolOpts.RNTupleNames.MetaData}:
                        dirType = "F"
                    fieldZeroId = descriptor.GetFieldZeroId()
                    for fieldDescriptor in descriptor.GetFieldIterable(fieldZeroId):
                        fieldId = fieldDescriptor.GetId()
                        fieldTreeInspector = inspector.GetFieldTreeInspector(fieldId)
                        diskSize = fieldTreeInspector.GetCompressedSize() / Units.kb
                        memSize = fieldTreeInspector.GetUncompressedSize() / Units.kb
                        typeName = fieldDescriptor.GetTypeName()
                        fieldName = fieldDescriptor.GetFieldName()
                        poolRecord = PoolRecord(fieldName, memSize, diskSize, memSize,
                                                descriptor.GetNEntries(),
                                                dirType=dirType,
                                                typeName=typeName)
                        poolRecord.augName = PoolOpts.augmentationName(name)
                        self.augNames.add(poolRecord.augName)
                        self.data += [ poolRecord ]
        # loop over keys
        
        return
    

checkFile()

python.PoolFile.PoolFile.checkFile	(		self,
			sorting = PoolRecord.Sorter.DiskSize )

Definition at line 708 of file PoolFile.py.

    def checkFile(self, sorting = PoolRecord.Sorter.DiskSize):
        if self.verbose is True:
            print(self.fileInfos())
            if len(self.augNames) > 1:
                for aug in self.augNames:
                    if len(aug) > 0:
                        print( "Nbr %s Events: %i" % (aug, self.dataHeaderA[aug].nEntries) )
 
        
        data = self.data
        if sorting in PoolRecord.Sorter.allowedValues():
            import operator
            data.sort(key = operator.attrgetter(sorting) )
 
        def _get_val(x, dflt=-999.):
            if PoolOpts.FAST_MODE:
                return dflt
            return x
 
        totMemSize  = _get_val(self.dataHeader.memSize, dflt=0.)
        totDiskSize = self.dataHeader.diskSize
        
        def _safe_div(num,den):
            if float(den) == 0.:
                return 0.
            return num/den   
                 
        if self.verbose is True:
            print("")
            print("="*80)
            print(PoolOpts.HDR_FORMAT % ( "Mem Size", "Disk Size","Size/Evt",
                                          "MissZip/Mem","items",
                                          "(X) Container Name (X=Tree|Branch)" ))
            print("="*80)
            
            print(PoolOpts.ROW_FORMAT % (
                _get_val (self.dataHeader.memSize),
                self.dataHeader.diskSize,
                _safe_div(self.dataHeader.diskSize,float(self.dataHeader.nEntries)),
                _get_val (_safe_div(self.dataHeader.memSizeNoZip,
                                    self.dataHeader.memSize)),
                self.dataHeader.nEntries,
                "("+self.dataHeader.dirType+") "+self.dataHeader.name
                ))
            print("-"*80)
 
        totMemSizeA = {}
        totDiskSizeA = {}
        for d in data:
            totMemSize  += 0. if PoolOpts.FAST_MODE else d.memSize
            totDiskSize += d.diskSize
            memSizeNoZip = d.memSizeNoZip/d.memSize if d.memSize != 0. else 0.
            aug = d.augName
            totMemSizeA[aug]  = totMemSizeA.get(aug,0.) + d.memSize
            totDiskSizeA[aug] = totDiskSizeA.get(aug,0.) + d.diskSize
            if self.verbose is True:
                print(PoolOpts.ROW_FORMAT % (
                    _get_val (d.memSize),
                    d.diskSize,
                    _safe_div(d.diskSize, float(self.dataHeader.nEntries)),
                    _get_val (memSizeNoZip),
                    d.nEntries,
                    "("+d.dirType+") "+d.name
                    ))
 
        if self.verbose is True:
            print("="*80)
            if len(self.augNames) > 1:
                augs = sorted(self.augNames)
                for a in augs:
                    print(PoolOpts.ROW_FORMAT % (
                        totMemSizeA[a], totDiskSizeA[a],
                        _safe_div(totDiskSizeA[a], float(self.dataHeaderA[a].nEntries)),
                        0.0,
                        self.dataHeaderA[a].nEntries,
                        "Aug Stream: " + ('MAIN' if a=='' else a)
                        ))
                print("-"*80)
            print(PoolOpts.ROW_FORMAT % (
                totMemSize, totDiskSize,
                _safe_div(totDiskSize, float(self.dataHeader.nEntries)),
                0.0, self.dataHeader.nEntries,
                "TOTAL (POOL containers)"
                ))
            print("="*80)
            if PoolOpts.FAST_MODE:
                print("::: warning: FAST_MODE was enabled: some columns' content ",)
                print("is meaningless...")
        return
 

detailedDump()

python.PoolFile.PoolFile.detailedDump	(		self,
			bufferName = None )

Definition at line 798 of file PoolFile.py.

    def detailedDump(self, bufferName = None ):
        if self.poolFile is None or \
           self.keys is None:
            print("Can't perform a detailedDump with a shelve file as input !")
            return
                  
        if bufferName is None:
            bufferName = "/dev/stdout"
        out = open( bufferName, "w" )
        sys.stdout.flush()
        save_stdout_fileno = os.dup (sys.stdout.fileno())
        os.dup2( out.fileno(),        sys.stdout.fileno() )
 
        out.write( "#" * 80 + os.linesep )
        out.write( "## detailed dump" + os.linesep )
        out.flush()
        
        for key in self.keys:
            tree = key.ReadObj()
            name = tree.GetName()
 
            if PoolOpts.isDataHeader(name) or \
               PoolOpts.isData(name):
                try:
                    print ("=== [%s] ===" % name, file=sys.stderr)
                    tree.Print()
                except Exception as err:
                    print ("Caught:",err, file=sys.stderr)
                    print (sys.exc_info()[0], file=sys.stderr)
                    print (sys.exc_info()[1], file=sys.stderr)
                    pass
                pass
            pass
        out.write( "#" * 80 + os.linesep )
        out.flush()
        out.write( "#" * 80 + os.linesep )

fileInfos()

python.PoolFile.PoolFile.fileInfos

(

self

)

Definition at line 700 of file PoolFile.py.

    def fileInfos(self):
        return os.linesep.join( [
            "File:" + self._fileInfos['name'],
            "Size: %12.3f kb" % (self._fileInfos['size'] / Units.kb),
            "Nbr Events: %i" % self.dataHeader.nEntries
            ] )
 
    

Member Data Documentation

_fileInfos

dict python.PoolFile.PoolFile._fileInfos = None

protected

Definition at line 465 of file PoolFile.py.

augNames

python.PoolFile.PoolFile.augNames = set()

Definition at line 470 of file PoolFile.py.

data

list python.PoolFile.PoolFile.data = []

Definition at line 472 of file PoolFile.py.

dataHeader

python.PoolFile.PoolFile.dataHeader

Initial value:

=  PoolRecord("DataHeader", 0, 0, 0,
                                     nEntries = 0,
                                     dirType = "T")

Definition at line 467 of file PoolFile.py.

dataHeaderA

dict python.PoolFile.PoolFile.dataHeaderA = {}

Definition at line 471 of file PoolFile.py.

keys

python.PoolFile.PoolFile.keys = None

Definition at line 466 of file PoolFile.py.

poolFile

python.PoolFile.PoolFile.poolFile = None

Definition at line 484 of file PoolFile.py.

ROOT

python.PoolFile.PoolFile.ROOT = ROOT

Definition at line 513 of file PoolFile.py.

verbose

python.PoolFile.PoolFile.verbose = verbose

Definition at line 473 of file PoolFile.py.

---

The documentation for this class was generated from the following file:

• PoolFile.py