python.DiskUtils Namespace Reference

Classes
class	AccessError
class	Backend
class	EOS
class	FileSet
class	FilterError
class	Local

Functions
def	deprecated (message)
def	_rationalise (path)
def	storageManager (name)
def	filelist (files, prefix=None)
def	ls (path, longls=False)
def	cp (src, dest='.')
def	get_lumi_blocks (root_file)
def	make_lumi_block_map_file (file_set, path)

Variables
	StorageManager = namedtuple('StorageManager', ['name', 'prefix', 'cp', 'ls', 'longls'])
	CastorMgr = StorageManager(name='castor', prefix='root://castoratlas/', cp='xrdcp', ls='nsls %s', longls='nsls -l %s')
	RFIOMgr = StorageManager(name='rfio', prefix='rfio:', cp='rfcp', ls='rfdir %s', longls='rfdir %s')
	EOSMgr = StorageManager(name='eos', prefix='root://eosatlas.cern.ch/', cp='xrdcp', ls='/bin/sh -l -c "LD_LIBRARY_PATH=/usr/lib64/ eos ls %s"', longls='/bin/sh -l -c "LD_LIBRARY_PATH=/usr/lib64/ eos ls -l %s"')
	UnixMgr = StorageManager(name='unix', prefix='', cp='cp', ls='ls %s', longls='ls -l %s')

Function Documentation

_rationalise()

def python.DiskUtils._rationalise (   path )

private

Rationalise a path, removing prefix and esuring single leading slash

Definition at line 35 of file DiskUtils.py.

def _rationalise(path):
    """
    Rationalise a path, removing prefix and esuring single leading slash
    """
    for p in ('root://castoratlas/', 'root://eosatlas.cern.ch/', 'rfio:', 'castor:'):
        if path.startswith(p):
            path = path[len(p):]
            if path.startswith('//'):
                path = path[1:]
            if not path.startswith('/'):
                path = '/'+path
            break
 
    return path
 
@deprecated("EOS is mounted on /eos with fuse, so you probably don't need this abstraction")

cp()

def python.DiskUtils.cp	(		src,
			dest = '.'
	)

Definition at line 128 of file DiskUtils.py.

def cp(src, dest='.'):
    src = _rationalise(src)
    dest = _rationalise(dest)
    srcmgr = storageManager(src)
    destmgr = storageManager(dest)
 
    cp = 'cp'
    if srcmgr.cp == 'xrdcp' or destmgr.cp == 'xrdcp': cp = 'xrdcp'
 
    return os.system('%s %s%s %s%s' %(cp, srcmgr.prefix, src, destmgr.prefix, dest))
 

deprecated()

def python.DiskUtils.deprecated

(

message

)

Definition at line 14 of file DiskUtils.py.

def deprecated(message):
    def deco(fn):
        @wraps(fn)
        def wrapper(*args, **kwargs):
            print('WARNING: [InDetBeamSpotExample.DiskUtils]',
                    '{}() is deprecated and will be removed'.format( fn.__name__),
                    file=sys.stderr)
            print('WARNING:                                 ', message,
                    file=sys.stderr)
            return fn(*args, **kwargs)
        return wrapper
    return deco
 

filelist()

def python.DiskUtils.filelist	(		files,
			prefix = None
	)

lists CASTOR/EOS name server directory/file entries.
If path is a directory, filelist lists the entries in the directory;
they are sorted alphabetically.

`files` specifies the CASTOR/EOS pathname.
`prefix` specifies the prefix one wants to prepend to the path found.
         (e.g. prefix='root://castoratlas/' or 'root://eosatlas.cern.ch//')
         if prefix=True it will determin the prefix based on the pathname

ex:
filelist('/castor/cern.ch/atlas/*')
filelist('/castor/cern.ch/atl*/foo?[bar]/*.pool.root.?')
filelist('/eos/atlas/*', prefix='root://eosatlas.cern.ch/')
filelist('/castor/cern.ch/atlas/*', prefix=True)

Definition at line 64 of file DiskUtils.py.

def filelist(files, prefix=None):
    """
    lists CASTOR/EOS name server directory/file entries.
    If path is a directory, filelist lists the entries in the directory;
    they are sorted alphabetically.
 
    `files` specifies the CASTOR/EOS pathname.
    `prefix` specifies the prefix one wants to prepend to the path found.
             (e.g. prefix='root://castoratlas/' or 'root://eosatlas.cern.ch//')
             if prefix=True it will determin the prefix based on the pathname
 
    ex:
    filelist('/castor/cern.ch/atlas/*')
    filelist('/castor/cern.ch/atl*/foo?[bar]/*.pool.root.?')
    filelist('/eos/atlas/*', prefix='root://eosatlas.cern.ch/')
    filelist('/castor/cern.ch/atlas/*', prefix=True)
    """
 
    path, fname = os.path.split(files)
    path = _rationalise(path)
 
    if ( path.count('*') > 0 or path.count('?') > 0 or
         path.count('[') > 0 or path.count(']') > 0 ) :
        paths = ls(path)
        return sum([ls(os.path.join(p,fname))
                    for p in paths], [])
 
    mgr = storageManager(path)
 
    try:
        flist = subprocess.check_output(mgr.ls % path, shell=True).split()
    except subprocess.CalledProcessError as err:
        print(err.output)
        return []
 
    if not (os.path.basename(files) in ['', '*']): # no need to filter
        pattern = fnmatch.translate(os.path.basename(files))
        flist = filter(lambda x: re.search(pattern, x), flist)
 
    if prefix:
        if isinstance(prefix, str):
            return [os.path.join(prefix+path, p) for p in flist]
        else:
            return [os.path.join(mgr.prefix+path, p) for p in flist]
    else:
        return [os.path.join(path, p) for p in flist]
 
@deprecated("EOS is mounted on /eos with fuse, so you probably don't need this abstraction")

get_lumi_blocks()

def python.DiskUtils.get_lumi_blocks

(

root_file

)

Definition at line 143 of file DiskUtils.py.

def get_lumi_blocks(root_file):
    
    try: 
        from PyUtils.RootUtils import import_root
        root = import_root()
        f = root.TFile.Open(root_file, 'READ')
 
        meta = f.Get( 'MetaData' )
        if not meta:
            raise Exception('No metadata')
        
        meta.GetEntry( 0 )
 
        esiName= 'Stream'
        esiTypeName = 'EventStreamInfo'
        for l in meta.GetListOfLeaves():
            if l.GetTypeName().startswith(esiTypeName):
                esiTypeName = l.GetTypeName()
                esiName = l.GetName()
                break
 
        if esiTypeName != 'EventStreamInfo_p3':
            raise Exception("old schema is not supported:", esiTypeName)
 
        import cppyy
 
        esic = cppyy.gbl.EventStreamInfoPTCnv_p3()
        esi = getattr (meta, esiName)
        if esiName.startswith(esiTypeName):
            esiName = esiName[len(esiTypeName)+1:]
 
        return(  list(esic.lumiBlockNumbers(esi)) )
 
    except  Exception as e:
        print( "Failed to read MetaData will fall back to looping ", repr(e))
    finally: 
        f.Close()
    
    try:
        from PyUtils.RootUtils import import_root
        root = import_root()
        f = root.TFile.Open(root_file, 'READ')
        lumiblocks = set()
        metadata= f.Get('CollectionMetadata') if f else None
        if metadata:
            metadata.GetEntry(0)
            import ctypes
            key_name = str(ctypes.c_char_p(metadata.Key).value)
            assert key_name == 'POOLCollectionID' 
        del metadata
        coll_tree = f.Get('POOLCollectionTree') if f else None
        if coll_tree:
            evtmax = coll_tree.GetEntries()
            if evtmax in (-1, None):
                evtmax = 0
            evtmax = int(evtmax)
            for row in range(evtmax):
                if coll_tree.GetEntry(row) < 0:
                    break
                lbn = coll_tree.LumiBlockN
                lumiblocks.add(lbn)
        del coll_tree
        return list( lumiblocks )
    finally:
        f.Close()
 

ls()

def python.DiskUtils.ls	(		path,
			longls = False
	)

Simple list of files

`longls` specifies long listing format

Definition at line 112 of file DiskUtils.py.

def ls(path, longls=False):
    """
    Simple list of files
 
    `longls` specifies long listing format
    """
 
    path = _rationalise(path)
    mgr = storageManager(path)
 
    if longls:
        return subprocess.check_output(mgr.longls % path, shell=True)
    else:
        return subprocess.check_output(mgr.ls % path, shell=True)
 
@deprecated("EOS is mounted on /eos with fuse, so you probably don't need this abstraction")

make_lumi_block_map_file()

def python.DiskUtils.make_lumi_block_map_file	(		file_set,
			path
	)

Definition at line 209 of file DiskUtils.py.

def make_lumi_block_map_file(file_set, path):
    with open(path, 'w') as mapfile:
        for f, lbs in file_set.with_lumi_blocks():
            print('Reading:', f)
            mapfile.write('{} {}\n'.format(
                os.path.basename(f),
                ','.join(str(x) for x in lbs)))
 
 

storageManager()

def python.DiskUtils.storageManager

(

name

)

Return SotrageManager to deal with listing, copying and reading files from various storage systems

Definition at line 51 of file DiskUtils.py.

def storageManager(name):
    """
    Return SotrageManager to deal with listing, copying and reading files from various storage systems
    """
    name = _rationalise(name)
    if name.startswith('/castor/'):
        return CastorMgr
    elif name.startswith('/eos/'):
        return EOSMgr
    else:
        return UnixMgr
 
@deprecated("DiskUtils.FileSet replaces this functionality")

Variable Documentation

CastorMgr

python.DiskUtils.CastorMgr = StorageManager(name='castor', prefix='root://castoratlas/', cp='xrdcp', ls='nsls %s', longls='nsls -l %s')

Definition at line 30 of file DiskUtils.py.

EOSMgr

python.DiskUtils.EOSMgr = StorageManager(name='eos', prefix='root://eosatlas.cern.ch/', cp='xrdcp', ls='/bin/sh -l -c "LD_LIBRARY_PATH=/usr/lib64/ eos ls %s"', longls='/bin/sh -l -c "LD_LIBRARY_PATH=/usr/lib64/ eos ls -l %s"')

Definition at line 32 of file DiskUtils.py.

RFIOMgr

python.DiskUtils.RFIOMgr = StorageManager(name='rfio', prefix='rfio:', cp='rfcp', ls='rfdir %s', longls='rfdir %s')

Definition at line 31 of file DiskUtils.py.

StorageManager

python.DiskUtils.StorageManager = namedtuple('StorageManager', ['name', 'prefix', 'cp', 'ls', 'longls'])

Definition at line 29 of file DiskUtils.py.

UnixMgr

python.DiskUtils.UnixMgr = StorageManager(name='unix', prefix='', cp='cp', ls='ls %s', longls='ls -l %s')

Definition at line 33 of file DiskUtils.py.