ATLAS Offline Software
Classes | Functions | Variables
python.PoolFile Namespace Reference

Classes

class  Counter
 
class  DiffFiles
 
class  PoolFile
 
class  PoolFileCatalog
 
class  PoolOpts
 
class  PoolRecord
 
class  Units
 — data ---------------------------------------------------------------— More...
 

Functions

def isRNTuple (obj)
 
def _get_total_size (branch)
 
def file_name (fname)
 
def _setup_ssl (root)
 
def _root_open (fname)
 
def retrieveBranchInfos (branch, poolRecord, ident="")
 
def make_pool_record (branch, dirType)
 
def extract_items (pool_file, verbose=True, items_type='eventdata')
 
def poolRecord (self, name)
 
def saveReport (self, fileName)
 
def _save_shelve_report (self, fileName)
 
def _save_csv_report (self, fileName)
 
def __del__ (self)
 

Variables

 __author__
 
 __all__
 — data ---------------------------------------------------------------— More...
 
 stdout
 out.write( "#### Map ####" + os.linesep ) out.flush() self.poolFile.Map() out.write( "#" * 80 + os.linesep ) More...
 
 poolFile
 

Function Documentation

◆ __del__()

def python.PoolFile.__del__ (   self)

Definition at line 967 of file PoolFile.py.

967  def __del__(self):
968  if self.poolFile and hasattr(self.poolFile, 'Close'):
969  try:
970  self.poolFile.Close()
971  self.poolFile = None
972  except Exception as err:
973  print("WARNING:",err)
974  pass
975 

◆ _get_total_size()

def python.PoolFile._get_total_size (   branch)
private

Definition at line 309 of file PoolFile.py.

309 def _get_total_size (branch):
310  if PoolOpts.FAST_MODE:
311  return -1.
312  if not PoolOpts.SUPER_DETAILED_BRANCH_SZ:
313  return branch.GetTotalSize()
314  brSize = 0
315  branch.LoadBaskets()
316  for bnum in range(0, branch.GetWriteBasket()):
317  basket = branch.GetBasket(bnum)
318  brSize += basket.GetObjlen() - 8
319  return brSize
320 

◆ _root_open()

def python.PoolFile._root_open (   fname)
private

Definition at line 395 of file PoolFile.py.

395 def _root_open(fname):
396  import PyUtils.RootUtils as ru
397  root = ru.import_root()
398  import re
399 
400  with ShutUp(filters=[
401  re.compile('TClass::TClass:0: RuntimeWarning: no dictionary for class.*') ]):
402  root.gSystem.Load('libRootCollection')
403  root_open = root.TFile.Open
404 
405  # we need to get back the protocol b/c of the special
406  # case of secure-http which needs to open TFiles as TWebFiles...
407  protocol, _ = file_name(fname)
408  if protocol == 'https':
409  _setup_ssl(root)
410  root_open = root.TWebFile.Open
411 
412  f = root_open(fname, 'READ')
413  if f is None or not f:
414  import errno
415  raise IOError(errno.ENOENT,
416  'No such file or directory',fname)
417  return f
418  return
419 

◆ _save_csv_report()

def python.PoolFile._save_csv_report (   self,
  fileName 
)
private
Save all the gathered informations into a CSV file

Definition at line 945 of file PoolFile.py.

945  def _save_csv_report(self, fileName):
946  """
947  Save all the gathered informations into a CSV file
948  """
949  import csv, os
950  if os.path.exists (fileName):
951  os.unlink (fileName)
952  args = {'newline' : ''}
953  f = open (fileName, 'w', **args)
954  o = csv.writer (f)
955  o.writerow (['file name', self._fileInfos['name']])
956  o.writerow (['file size', self._fileInfos['size']])
957  o.writerow (['nbr evts', self.dataHeader.nEntries])
958  o.writerow (['mem size', 'disk size', 'mem size nozip', 'items',
959  'container name', 'branch type'])
960 
961  for d in self.data:
962  o.writerow ([d.memSize, d.diskSize, d.memSizeNoZip,
963  d.nEntries, d.name, d.dirType])
964  f.close()
965  return
966 

◆ _save_shelve_report()

def python.PoolFile._save_shelve_report (   self,
  fileName 
)
private
Save all the gathered informations into a python shelve
Data can then be read like so:
 >>> import shelve
 >>> db = shelve.open( 'myfile.dat', 'r' )
 >>> report = db['report']
 >>> print ('fileSize:',report['fileSize'])
 >>> print ('dataHeader/memSize:',report['dataHeader'].memSize)
 >>> for d in report['data']:
 ...   print ('data:',d.name,d.nEntries,d.memSize)

Definition at line 920 of file PoolFile.py.

920  def _save_shelve_report(self, fileName):
921  """
922  Save all the gathered informations into a python shelve
923  Data can then be read like so:
924  >>> import shelve
925  >>> db = shelve.open( 'myfile.dat', 'r' )
926  >>> report = db['report']
927  >>> print ('fileSize:',report['fileSize'])
928  >>> print ('dataHeader/memSize:',report['dataHeader'].memSize)
929  >>> for d in report['data']:
930  ... print ('data:',d.name,d.nEntries,d.memSize)
931  """
932  import shelve, os
933  if os.path.exists (fileName):
934  os.unlink (fileName)
935  db = shelve.open (fileName)
936  db['report'] = {
937  'fileInfos' : self._fileInfos,
938  'nbrEvts' : self.dataHeader.nEntries,
939  'dataHeader' : self.dataHeader,
940  'data' : self.data
941  }
942  db.close()
943  return
944 

◆ _setup_ssl()

def python.PoolFile._setup_ssl (   root)
private

Definition at line 381 of file PoolFile.py.

381 def _setup_ssl(root):
382  x509_proxy = os.environ.get('X509_USER_PROXY', '')
383  if x509_proxy:
384  # setup proper credentials
385  root.TSSLSocket.SetUpSSL(
386  x509_proxy,
387  "/etc/grid-security/certificates",
388  x509_proxy,
389  x509_proxy)
390  else:
391  print("## warning: protocol https is requested but no X509_USER_PROXY was found! (opening the file might fail.)")
392  pass
393  return
394 

◆ extract_items()

def python.PoolFile.extract_items (   pool_file,
  verbose = True,
  items_type = 'eventdata' 
)
Helper function to read a POOL file and extract the item-list from the
DataHeader content.
@params
  `pool_file`  the name of the pool file to inspect
  `verbose`    self-explanatory
  `items_type` what kind of items one is interested in
               allowed values: 'eventdata' 'metadata'
Note: this function is actually executed in a forked sub-process
      if `fork` is True

Definition at line 454 of file PoolFile.py.

454 def extract_items(pool_file, verbose=True, items_type='eventdata'):
455  """Helper function to read a POOL file and extract the item-list from the
456  DataHeader content.
457  @params
458  `pool_file` the name of the pool file to inspect
459  `verbose` self-explanatory
460  `items_type` what kind of items one is interested in
461  allowed values: 'eventdata' 'metadata'
462  Note: this function is actually executed in a forked sub-process
463  if `fork` is True
464  """
465  _allowed_values = ('eventdata',
466  'metadata',)
467  if items_type not in _allowed_values:
468  err = "".join([
469  "invalid argument for 'items_type'. ",
470  "got: [%s] " % items_type,
471  "(allowed values: %r)" % _allowed_values
472  ])
473  raise ValueError(err)
474 
475  key = '%s_items' % items_type
476  f_root = _root_open(pool_file)
477  import PyUtils.FilePeekerTool as fpt
478  fp = fpt.FilePeekerTool(f_root)
479  items = fp.getPeekedData(key)
480 
481  if items is None:
482  items = []
483  return items
484 

◆ file_name()

def python.PoolFile.file_name (   fname)
take a file name, return the pair (protocol, 'real' file name)

Definition at line 321 of file PoolFile.py.

321 def file_name(fname):
322  """take a file name, return the pair (protocol, 'real' file name)
323  """
324  fname = os.path.expanduser(os.path.expandvars(fname))
325 
326  def _normalize_uri(uri):
327  if uri.startswith('/'):
328  return 'file:'+uri
329  return uri
330 
331  from urllib.parse import urlsplit
332  url = urlsplit(_normalize_uri(fname))
333  protocol = url.scheme
334  def _normalize(fname):
335  from posixpath import normpath
336  fname = normpath(fname)
337  if fname.startswith('//'): fname = fname[1:]
338  return fname
339 
340  if protocol in ('', 'file', 'pfn'):
341  protocol = ''
342  fname = _normalize(url.path)
343 
344 
345  if fname.startswith('/castor/'):
346  protocol = 'rfio'
347  fname = protocol + ':' + fname
348 
349  elif protocol in ('rfio', 'castor'):
350  protocol = 'rfio'
351  fname = _normalize(url.path)
352  fname = protocol+':'+fname
353 
354  elif protocol in ('root','dcap', 'dcache', 'http', 'https', 'dav', 'davs'):
355  pass
356 
357  elif protocol in ('gsidcap',):
358  protocol = 'gfal:gsidcap'
359  pass
360 
361  elif protocol in ('lfn','fid',):
362  # percolate through the PoolFileCatalog
363  from PyUtils.PoolFile import PoolFileCatalog as pfc
364  fname = pfc().pfn(protocol+':'+url.path)
365  pass
366 
367  elif protocol in ('ami',):
368  # !! keep order of tokens !
369  for token in ('ami:', '//', '/'):
370  if fname.startswith(token):
371  fname = fname[len(token):]
372  fname = 'ami://' + fname
373  pass
374 
375  else:
376  print(f'## warning: unknown protocol [{protocol}]. we will just return our input')
377  pass
378 
379  return (protocol, fname)
380 

◆ isRNTuple()

def python.PoolFile.isRNTuple (   obj)

Definition at line 36 of file PoolFile.py.

36 def isRNTuple(obj):
37  # MN: remove the "try" after migration to ROOT 6.34
38  try: from ROOT import RNTuple
39  except(ImportError): from ROOT.Experimental import RNTuple
40  return isinstance( obj, RNTuple )
41 
42 

◆ make_pool_record()

def python.PoolFile.make_pool_record (   branch,
  dirType 
)

Definition at line 441 of file PoolFile.py.

441 def make_pool_record (branch, dirType):
442  memSize = _get_total_size (branch) / Units.kb
443  zipBytes = branch.GetZipBytes()
444  memSizeNoZip = memSize if zipBytes < 0.001 else 0.
445  diskSize = branch.GetZipBytes() / Units.kb
446  typeName = branch.GetClassName()
447  if not typeName and (leaf := branch.GetListOfLeaves().At(0)):
448  typeName = leaf.GetTypeName()
449  return PoolRecord(branch.GetName(), memSize, diskSize, memSizeNoZip,
450  branch.GetEntries(),
451  dirType=dirType,
452  typeName=typeName)
453 

◆ poolRecord()

def python.PoolFile.poolRecord (   self,
  name 
)
Return a PoolRecord according to its (branch) name
Raise KeyError if no match is found

Definition at line 900 of file PoolFile.py.

900  def poolRecord(self, name):
901  """
902  Return a PoolRecord according to its (branch) name
903  Raise KeyError if no match is found
904  """
905  for data in self.data:
906  if data.name == name:
907  return data
908  raise KeyError("No PoolRecord with name [%s]" % name)
909 

◆ retrieveBranchInfos()

def python.PoolFile.retrieveBranchInfos (   branch,
  poolRecord,
  ident = "" 
)

Definition at line 420 of file PoolFile.py.

420 def retrieveBranchInfos( branch, poolRecord, ident = "" ):
421  fmt = "%s %3i %8.3f %8.3f %8.3f %s"
422  if 0:
423  out = fmt % ( ident,
424  branch.GetListOfBranches().GetSize(),
425  _get_total_size (branch),
426  branch.GetTotBytes(),
427  branch.GetZipBytes(),
428  branch.GetName() )
429  print(out)
430 
431  branches = branch.GetListOfBranches()
432  for b in branches:
433  poolRecord.memSize += _get_total_size (b) / Units.kb
434  if (b.GetZipBytes() < 0.001):
435  poolRecord.memSizeNoZip += _get_total_size (b) / Units.kb
436  poolRecord.diskSize += b.GetZipBytes() / Units.kb
437  poolRecord = retrieveBranchInfos ( b, poolRecord, ident+" " )
438 
439  return poolRecord
440 

◆ saveReport()

def python.PoolFile.saveReport (   self,
  fileName 
)
Save all the gathered informations into a python shelve or a CSV file
(depending on the @param `fileName` extension)

Definition at line 910 of file PoolFile.py.

910  def saveReport (self, fileName):
911  """
912  Save all the gathered informations into a python shelve or a CSV file
913  (depending on the @param `fileName` extension)
914  """
915  import os
916  if os.path.splitext(fileName)[-1] == '.csv':
917  return self._save_csv_report (fileName)
918  return self._save_shelve_report (fileName)
919 

Variable Documentation

◆ __all__

python.PoolFile.__all__
private

— data ---------------------------------------------------------------—

Definition at line 11 of file PoolFile.py.

◆ __author__

python.PoolFile.__author__
private

Definition at line 8 of file PoolFile.py.

◆ poolFile

python.PoolFile.poolFile

Definition at line 971 of file PoolFile.py.

◆ stdout

python.PoolFile.stdout

out.write( "#### Map ####" + os.linesep ) out.flush() self.poolFile.Map() out.write( "#" * 80 + os.linesep )

Definition at line 897 of file PoolFile.py.

python.PoolFile._save_shelve_report
def _save_shelve_report(self, fileName)
Definition: PoolFile.py:920
python.PoolFile.__del__
def __del__(self)
Definition: PoolFile.py:967
python.PoolFile.poolRecord
def poolRecord(self, name)
Definition: PoolFile.py:900
python.PoolFile.saveReport
def saveReport(self, fileName)
Definition: PoolFile.py:910
python.PoolFile.file_name
def file_name(fname)
Definition: PoolFile.py:321
python.PoolFile.retrieveBranchInfos
def retrieveBranchInfos(branch, poolRecord, ident="")
Definition: PoolFile.py:420
plotBeamSpotVxVal.range
range
Definition: plotBeamSpotVxVal.py:195
python.PoolFile._save_csv_report
def _save_csv_report(self, fileName)
Definition: PoolFile.py:945
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:25
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
python.PoolFile.make_pool_record
def make_pool_record(branch, dirType)
Definition: PoolFile.py:441
python.PoolFile._setup_ssl
def _setup_ssl(root)
Definition: PoolFile.py:381
python.PoolFile.isRNTuple
def isRNTuple(obj)
Definition: PoolFile.py:36
python.PoolFile._root_open
def _root_open(fname)
Definition: PoolFile.py:395
ROOT::Experimental
Definition: RNTCollection.h:28
python.PoolFile.extract_items
def extract_items(pool_file, verbose=True, items_type='eventdata')
Definition: PoolFile.py:454
python.PoolFile._get_total_size
def _get_total_size(branch)
Definition: PoolFile.py:309