Classes
class	AccessType
class	Tee

Functions
	retry_function_time (func, args, retryException, retryMaxTime=defaultRetryMaxTime, retryStartTime=defaultRetryMaxTime)
	retry_file_access (func, args, retryException=OSError, retryMaxTime=defaultRetryMaxTime, retryStartTime=defaultRetryMaxTime)
	get_access_type (filename)
	exists (filename)
	getsize (filename)
	getmtime (filename)
	listdir (filename)
	stat (filename)
	lstat (filename)
	access (filename, mode)
	remove (filename)
	exists_suffix (filename, suffixRE)
	exists_suffix_number (filename)
	mode_string (filename)
	mode_to_string (mode)

Variables
str	__doc__ = """A set of file utilities that can be used for several file systems (local files, rfio, castor)"""
list	__all__ = [ ]
float	defaultRetryMaxTime = 1.0
float	defaultRetryStartTime = 0.1
	IO_LOCAL = AccessType('local' , r'(.*)' , r'\1' , os , os.path)
	IO_RFIO = AccessType('rfio' , r'^rfio:' , r'rfio:' , rfio, rfio)
	IO_CASTOR = AccessType('castor', r'^(?:rfio:)?/castor/', 'rfio:/castor/', rfio, rfio)
	IO_XROOTD = AccessType('xrootd', r'^root:' , r'root:' , dummyaccess, dummyaccess )
	IO_LFN = AccessType('lfn' , r'^LFN:' , r'LFN:' , dummyaccess, dummyaccess )
tuple	_accessTypes = ( IO_LFN, IO_XROOTD, IO_CASTOR, IO_RFIO, IO_LOCAL )

Function Documentation

◆ access()

python.fileutil.access	(		filename,
			mode )

Definition at line 145 of file fileutil.py.

def access(filename,mode):
    at = get_access_type(filename)
    return retry_file_access( at.baseModule.access, (filename,mode) )
 
 

◆ exists()

python.fileutil.exists ( filename )

Definition at line 115 of file fileutil.py.

def exists(filename):
    at = get_access_type(filename)
    return retry_file_access( at.pathModule.exists, filename )
 
 

◆ exists_suffix()

python.fileutil.exists_suffix	(		filename,
			suffixRE )

Test if a file exists in the same directory as <filename>, with the
same name, but with an additional suffix given as a regular expression
in <suffixRE>. It returns a list of all matching suffices, or an empty
list if no matching filename+suffix was found.

Definition at line 160 of file fileutil.py.

def exists_suffix(filename,suffixRE):
    """Test if a file exists in the same directory as <filename>, with the
    same name, but with an additional suffix given as a regular expression
    in <suffixRE>. It returns a list of all matching suffices, or an empty
    list if no matching filename+suffix was found."""
    sufs = []
    dirname  = os.path.dirname(filename) or os.curdir
    # if directory does not exist, then file will never exist (and prevent crash later on)
    if not os.path.isdir(dirname): return []
    filename = os.path.basename(filename)
    pat = re.compile( '^%s(%s)' % (filename,suffixRE) )
    for f in listdir(dirname):
        if pat.search( f ):
            sufs.append( pat.sub( r'\1', f ) )
 
    return sufs
 
 

◆ exists_suffix_number()

python.fileutil.exists_suffix_number ( filename )

Test if a file exists in the same directory as <filename>, with the same name,
but a non-negative integer added at the end of the name. It returns the filename
with the highest number added, or None if no such file exists.

Definition at line 178 of file fileutil.py.

def exists_suffix_number(filename):
    """Test if a file exists in the same directory as <filename>, with the same name,
    but a non-negative integer added at the end of the name. It returns the filename
    with the highest number added, or None if no such file exists."""
    sufs = exists_suffix(filename, r'[0-9]+')
    maxnum = -1
    found = None
    for suf in sufs:
        num = int(suf)
        if num > maxnum:
            maxnum = num
            found = filename + suf
 
    return found
 
 

◆ get_access_type()

python.fileutil.get_access_type ( filename )

Definition at line 109 of file fileutil.py.

def get_access_type(filename):
    for at in _accessTypes:
        if at.matches( filename ): return at
    return IO_LOCAL
 
 

◆ getmtime()

python.fileutil.getmtime ( filename )

Definition at line 125 of file fileutil.py.

def getmtime(filename):
    at = get_access_type(filename)
    return retry_file_access( at.pathModule.getmtime, filename )
 
 

◆ getsize()

python.fileutil.getsize ( filename )

Definition at line 120 of file fileutil.py.

def getsize(filename):
    at = get_access_type(filename)
    return retry_file_access( at.pathModule.getsize, filename )
 
 

◆ listdir()

python.fileutil.listdir ( filename )

Definition at line 130 of file fileutil.py.

def listdir(filename):
    at = get_access_type(filename)
    return retry_file_access( at.baseModule.listdir, filename )
 
 

◆ lstat()

python.fileutil.lstat ( filename )

Definition at line 140 of file fileutil.py.

def lstat(filename):
    at = get_access_type(filename)
    return retry_file_access( at.baseModule.lstat, filename )
 
 

◆ mode_string()

python.fileutil.mode_string ( filename )

Definition at line 194 of file fileutil.py.

def mode_string(filename):
    return mode_to_string( lstat(filename).st_mode )

◆ mode_to_string()

python.fileutil.mode_to_string ( mode )

Return the unix like string corresponding to the file access mode (rwxd etc)

Definition at line 198 of file fileutil.py.

def mode_to_string(mode):
    """Return the unix like string corresponding to the file access mode (rwxd etc)"""
    modeList = ['-']*10
    # first character (dir/symlink)
    if statconsts.S_ISDIR(mode):
        modeList[0] = 'd'
    elif statconsts.S_ISLNK(mode):
        modeList[0] = 'l'
    # user modes
    if mode & statconsts.S_IRUSR: modeList[1] = 'r'
    if mode & statconsts.S_IWUSR: modeList[2] = 'w'
    if   mode & statconsts.S_ISUID: modeList[3] = 's'
    elif mode & statconsts.S_IXUSR: modeList[3] = 'x'
    # group modes
    if mode & statconsts.S_IRGRP: modeList[4] = 'r'
    if mode & statconsts.S_IWGRP: modeList[5] = 'w'
    if   mode & statconsts.S_ISGID: modeList[6] = 's'
    elif mode & statconsts.S_IXGRP: modeList[6] = 'x'
    # other modes
    if mode & statconsts.S_IROTH: modeList[7] = 'r'
    if mode & statconsts.S_IWOTH: modeList[8] = 'w'
    if mode & statconsts.S_IXOTH: modeList[9] = 'x'
 
    return ''.join(modeList)
 

◆ remove()

python.fileutil.remove ( filename )

Remove file <filename> if it exists. Only supported for local files.

Definition at line 150 of file fileutil.py.

def remove(filename):
    """Remove file <filename> if it exists. Only supported for local files."""
    at = get_access_type(filename)
    if at == IO_LOCAL:
        if exists(filename): retry_file_access( os.remove, filename )
    else:
        print ("WARNING: file %s file %s can not be removed" %
               (at.name, filename))
    
 

◆ retry_file_access()

python.fileutil.retry_file_access	(	func,
		args,
		retryException = OSError,
		retryMaxTime = defaultRetryMaxTime,
		retryStartTime = defaultRetryMaxTime )

Definition at line 47 of file fileutil.py.

                       retryStartTime = defaultRetryMaxTime ):
 
    return retry_function_time( func, args, retryException, retryMaxTime, retryStartTime )

◆ retry_function_time()

python.fileutil.retry_function_time	(	func,
		args,
		retryException,
		retryMaxTime = defaultRetryMaxTime,
		retryStartTime = defaultRetryMaxTime )

Call function several times if it throws a <retryException>.
It will wait an increasing amount of time in between tries.
First waiting time is <retryStartTime>, which is increased by
a factor of 2 for each retry. It will give up and raise the
original exception if it still fails after a total retry time of <retryMaxTime>.
<func>: function to be called
<args>: tuple with the function arguments, or the single function argument

Definition at line 14 of file fileutil.py.

                         retryStartTime = defaultRetryMaxTime ):
    """Call function several times if it throws a <retryException>.
    It will wait an increasing amount of time in between tries.
    First waiting time is <retryStartTime>, which is increased by
    a factor of 2 for each retry. It will give up and raise the
    original exception if it still fails after a total retry time of <retryMaxTime>.
    <func>: function to be called
    <args>: tuple with the function arguments, or the single function argument"""
    if type(args) is not tuple: args = (args,)
    retryDelay = retryStartTime
    if retryDelay <= 0: retryDelay = 0.1 # avoid infinite loop
    OK = False
    tStart = time.time()
    while not OK:
        try:
            val = func( *args )
            OK = True
        except retryException:
            OK = False
            dt = time.time() - tStart
            argsStr = ', '.join( [ '%r' % a for a in args ] )
            if dt > retryMaxTime:
                print ("%s(%s) Failed" % (func.__name__,argsStr))
                raise
            time.sleep(retryDelay)
            retryDelay *= 2
            print ("Retrying %s(%s)" % (func.__name__,argsStr))
 
    return val
 
 

◆ stat()

python.fileutil.stat ( filename )

Definition at line 135 of file fileutil.py.

def stat(filename):
    at = get_access_type(filename)
    return retry_file_access( at.baseModule.stat, filename )
 
 

Variable Documentation

◆ all

list python.fileutil.__all__ = [ ]

private

Definition at line 9 of file fileutil.py.

◆ doc

str python.fileutil.__doc__ = """A set of file utilities that can be used for several file systems (local files, rfio, castor)"""

private

Definition at line 7 of file fileutil.py.

◆ _accessTypes

tuple python.fileutil._accessTypes = ( IO_LFN, IO_XROOTD, IO_CASTOR, IO_RFIO, IO_LOCAL )

protected

Definition at line 76 of file fileutil.py.

◆ defaultRetryMaxTime

float python.fileutil.defaultRetryMaxTime = 1.0

Definition at line 11 of file fileutil.py.

◆ defaultRetryStartTime

float python.fileutil.defaultRetryStartTime = 0.1

Definition at line 12 of file fileutil.py.

◆ IO_CASTOR

python.fileutil.IO_CASTOR = AccessType('castor', r'^(?:rfio:)?/castor/', 'rfio:/castor/', rfio, rfio)

Definition at line 73 of file fileutil.py.

◆ IO_LFN

python.fileutil.IO_LFN = AccessType('lfn' , r'^LFN:' , r'LFN:' , dummyaccess, dummyaccess )

Definition at line 75 of file fileutil.py.

◆ IO_LOCAL

python.fileutil.IO_LOCAL = AccessType('local' , r'(.*)' , r'\1' , os , os.path)

Definition at line 71 of file fileutil.py.

◆ IO_RFIO

python.fileutil.IO_RFIO = AccessType('rfio' , r'^rfio:' , r'rfio:' , rfio, rfio)

Definition at line 72 of file fileutil.py.

◆ IO_XROOTD

python.fileutil.IO_XROOTD = AccessType('xrootd', r'^root:' , r'root:' , dummyaccess, dummyaccess )

Definition at line 74 of file fileutil.py.

Classes

Functions

Variables