python.trfUtils Namespace Reference

Classes
class	analytic
	Analytics service class. More...
class	Fit
	Low-level fitting class. More...
class	Job
	Job: a set of pieces of information relevant to a given work function. More...
class	JobGroup
	JobGroup: a set of Job objects and pieces of information relevant to a given set of Job objects. More...
class	math
	some mathematical tools More...
class	memFileToTable
	Extract a table of data from a txt file. More...
class	ParallelJobProcessor
	ParallelJobProcessor: a multiple-process processor of Job objects. More...

Functions
def	findFile (pathvar, fname)
	Find a named file along a colon separated PATH type variable. More...
def	getAncestry (listMyOrphans=False)
	List all processes and parents and form a dictionary where the parent key lists all child PIDs. More...
def	listChildren (psTree=None, parent=os.getpid(), listOrphans=False)
	Find all the children of a particular PID (calls itself recursively to descend into each leaf) More...
def	infanticide (childPIDs=None, sleepTime=3, message=True, listOrphans=False)
	Kill all PIDs. More...
def	call (args, bufsize=0, executable=None, stdin=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, message="", logger=msg, loglevel=None, timeout=None, retry=2, timefactor=1.5, sleeptime=10)
def	asetupReport ()
	Return a string with a report of the current athena setup. More...
def	releaseIsOlderThan (major, minor=None)
	Test (to the best of our knowledge) if the current release is older than a major, minor version number. More...
def	asetupReleaseIsOlderThan (asetup_string, major, minor=None)
	Test (to the best of our knowledge) if the asetup release is older than a major, minor version number. More...
def	shQuoteStrings (strArray=sys.argv)
	Quote a string array so that it can be echoed back on the command line in a cut 'n' paste safe way. More...
def	lineByLine (filename, strip=True, removeTimestamp=True, substepName=None)
	Generator to return lines and line count from a file. More...
def	prettyXML (element, indent=' ', poolFileCatalogFormat=False)
	XML pretty print an ElementTree.ELement object. More...
def	isodate ()
	Return isoformated 'now' string. More...
def	forceToAlphaNum (string)
	Strip a string down to alpha-numeric characters only. More...
def	cmpMetadata (metadata1, metadata2, guidCheck='valid')
	Compare metadata for files, but taking into account that GUID can vary. More...
def	unpackTarFile (filename, directory=".")
	Unpack a given tarfile. More...
def	unpackDBRelease (tarball, dbversion=None)
	Ensure that the DBRelease tarball has been unpacked. More...
def	setupDBRelease (setup)
	Run a DBRelease setup. More...
def	cvmfsDBReleaseCheck (dbrelease)
	Validate a DBRelease exists on cvmfs and return the path to the setup script. More...
def	pickledDump (argdict)
	Dump a list of arguments to the pickle file given in the 'dumpPickle' argument. More...
def	JSONDump (argdict)
	Dump a list of arguments to the JSON file given in the 'dumpJSON' argument. More...
def	convertToStr (in_string)
	Recursively convert unicode to str, useful when we have just loaded something from json (TODO: make the transforms happy with unicode as well as plain str!) More...
def	cliToKey (option)
	Convert a command line option to the dictionary key that will be used by argparse. More...
def	printHR (the_object)
	print in a human-readable way the items of a given object More...
def	uniqueIdentifier ()
	return a URL-safe, base 64-encoded pseudorandom UUID More...
def	units (quantity=None, unitSingular="unit", unitPlural="units")
	return either singular or plural units as appropriate for a given quantity More...
def	isInteractiveEnv ()
def	initialise_processes ()
	initisation procedure for processes of process pool More...
def	ValgrindCommand (defaultOptions=True, extraOptionsList=None, AthenaSerialisedConfigurationFile="athenaConf.pkl", returnFormat="string")
def	VTuneCommand (defaultOptions=True, extraOptionsList=None, AthenaSerialisedConfigurationFile="athenaConf.pkl", returnFormat="string")
	return VTune command @detail This function returns a VTune command for use with Athena. More...
def	calcCpuTime (start, stop)
def	calcWallTime (start, stop)
def	bind_port (host, port)
def	reportEventsPassedSimFilter (log)
	summarize events passed the ISF_SimEventFilter @detail this function sums up all events passed the ISF_SimEventFilter out of all total events. More...

Variables
	msg

Function Documentation

asetupReleaseIsOlderThan()

def python.trfUtils.asetupReleaseIsOlderThan	(		asetup_string,
			major,
			minor = None
	)

Test (to the best of our knowledge) if the asetup release is older than a major, minor version number.

Parameters

asetup_string	asetup string
major	Major release number
minor	Minor release number (if not specified, will not be matched against)

Returns

Boolean if current release is found to be older

Definition at line 303 of file trfUtils.py.

def asetupReleaseIsOlderThan(asetup_string, major, minor=None):
    try:
        relmajor = None
        relminor = None
 
        # First split the asetup_string by comma
        split_string = asetup_string.split(',')
        # master is always the newest
        if 'master' in split_string:
            return False
 
        # First try major.minor.bugfix
        reg_exp = re.compile(r'(?P<major>\d+)\.(?P<minor>\d+)\.(?P<other>.*)')
        for part in split_string:
            part = part.strip()
            match = re.match(reg_exp, part)
            if match:
                relmajor = int(match.group('major'))
                relminor = int(match.group('minor'))
                msg.info('Detected asetup release {0}.{1}(.{2})'.format(relmajor, relminor, match.group('other')))
                break
 
        # Then try major.minor
        if relmajor is None:
            reg_exp = re.compile(r'(?P<major>\d+)\.(?P<minor>\d+)')
            for part in split_string:
                part = part.strip()
                match = re.match(reg_exp, part)
                if match:
                    relmajor = int(match.group('major'))
                    relminor = int(match.group('minor'))
                    msg.info('Detected asetup release {0}.{1}'.format(relmajor, relminor))
                    break
 
        # Bail out
        if relmajor is None:
            raise RuntimeError('asetup version could not be parsed')
 
        # Major beats minor, so test this first
        if relmajor < major:
            return True
        if relmajor > major:
            return False
 
        # First case is major equality and don't care about minor
        if minor is None or relminor >= minor:
            return False
        return True
 
    except Exception as e:
        msg.warning('Exception thrown when attempting to detect asetup athena version ({0}) from {1}. No release check possible'.format(e, asetup_string))
    return False
 
 

asetupReport()

def python.trfUtils.asetupReport

(

)

Return a string with a report of the current athena setup.

Definition at line 223 of file trfUtils.py.

def asetupReport():
    setupMsg = str()
    eVars = ['AtlasBaseDir', 'AtlasProject', 'AtlasVersion', 'AtlasPatch', 'AtlasPatchVersion', 'CMTCONFIG', 'TestArea']
    if "AtlasProject" in os.environ:
        CMake_Platform = "{0}_PLATFORM".format(os.environ["AtlasProject"])
        if CMake_Platform in os.environ:
            eVars.remove("CMTCONFIG")
            eVars.append(CMake_Platform)
    for eVar in eVars:
        if eVar in os.environ:
            setupMsg += '\t%s=%s\n' % (eVar, os.environ[eVar])
    # Look for patches so that the job can be rerun
    if 'WorkDir_DIR' in os.environ and os.access(os.environ['WorkDir_DIR'], os.R_OK):
        pass
        # lstags is obsolete with git releases.
        # setupMsg += "\n\tPatch packages are:\n"
        # try:
        #     cmd = ['lstags']
        #     lstagsOut = Popen(cmd, shell = False, stdout = PIPE, stderr = STDOUT, bufsize = 1).communicate()[0]
        #     setupMsg +=  "\n".join([ "\t\t{0}".format(pkg) for pkg in lstagsOut.decode().split("\n") ])
        # except (CalledProcessError, OSError) as e:
        #     setupMsg += 'Execution of lstags failed: {0}'.format(e)
    else:
        setupMsg+= "No readable patch area found"
 
    return setupMsg.rstrip()
 
 

bind_port()

def python.trfUtils.bind_port	(		host,
			port
	)

Definition at line 1698 of file trfUtils.py.

def bind_port(host, port):
    ret = 0
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    try:
        s.bind((host, port))
    except socket.error as e:
        if e.errno == 98:
            print("Port %s is already in use" %port)
        else:
            # something else raised the socket.error exception
            print(e)
        ret=1
    s.close()
    return ret
 

calcCpuTime()

def python.trfUtils.calcCpuTime	(		start,
			stop
	)

Definition at line 1683 of file trfUtils.py.

def calcCpuTime(start, stop):
    cpuTime = None
    if start and stop:
        cpuTime = reduce(lambda x1, x2: x1+x2, list(map(lambda x1, x2: x2-x1, start[2:4], stop[2:4])))
 
    return cpuTime
 
# calculate wallTime from os.times() times tuple

calcWallTime()

def python.trfUtils.calcWallTime	(		start,
			stop
	)

Definition at line 1691 of file trfUtils.py.

def calcWallTime(start, stop):
    wallTime = None
    if start and stop:
        wallTime = stop[4] - start[4]
 
    return wallTime
 

call()

def python.trfUtils.call	(		args,
			bufsize = 0,
			executable = None,
			stdin = None,
			preexec_fn = None,
			close_fds = False,
			shell = False,
			cwd = None,
			env = None,
			universal_newlines = False,
			startupinfo = None,
			creationflags = 0,
			message = "",
			logger = msg,
			loglevel = None,
			timeout = None,
			retry = 2,
			timefactor = 1.5,
			sleeptime = 10
	)

Definition at line 155 of file trfUtils.py.

def call(args, bufsize=0, executable=None, stdin=None, preexec_fn=None, close_fds=False, shell=False, cwd=None, env=None, universal_newlines=False, startupinfo=None, creationflags=0, message="", logger=msg, loglevel=None, timeout=None, retry=2, timefactor=1.5, sleeptime=10):
 
    def logProc(p):
        line=p.stdout.readline()
        if line:
            line="%s%s" % (message, line.rstrip())
            if logger is None:
                print(line)
            else:
                logger.log(loglevel, line)
 
    def flushProc(p):
        line=p.stdout.readline()
        while line:
            line="%s%s" % (message, line.strip())
            if logger is None:
                print(line)
            else:
                logger.log(loglevel, line)
            line=p.stdout.readline()
 
    if loglevel is None:
        loglevel=logging.DEBUG
 
    if timeout is None or timeout<=0: # no timeout set
        msg.info('Executing %s...', args)
        starttime = time.time()
        p=Popen(args=args, bufsize=bufsize, executable=executable, stdin=stdin, stdout=PIPE, stderr=STDOUT, preexec_fn=preexec_fn, close_fds=close_fds, shell=shell, cwd=cwd, env=env, universal_newlines=universal_newlines, startupinfo=startupinfo, creationflags=creationflags)
        while p.poll() is None:
            logProc(p)
        flushProc(p)
        if timeout is not None:
            msg.info('Executed call within %d s.', time.time()-starttime)
        return p.returncode
 
    else: #timeout set
        n=0
        while n<=retry:
            msg.info('Try %i out of %i (time limit %ss) to call %s.', n+1, retry+1, timeout, args)
            starttime = time.time()
            endtime=starttime+timeout
            p=Popen(args=args, bufsize=bufsize, executable=executable, stdin=stdin, stdout=PIPE, stderr=STDOUT, preexec_fn=preexec_fn, close_fds=close_fds, shell=shell, cwd=cwd, env=env, universal_newlines=universal_newlines, startupinfo=startupinfo, creationflags=creationflags)
            while p.poll() is None and time.time()<endtime:
                logProc(p)
            if p.poll() is None:
                msg.warning('Timeout limit of %d s reached. Kill subprocess and its children.', timeout)
                parent=p.pid
                pids=[parent]
                pids.extend(listChildren(parent=parent))
                infanticide(pids)
                msg.info('Checking if something is left in buffer.')
                flushProc(p)
                if n!=retry:
                    msg.info('Going to sleep for %d s.', sleeptime)
                    time.sleep(sleeptime)
                n+=1
                timeout*=timefactor
                sleeptime*=timefactor
            else:
                flushProc(p)
                msg.info('Executed call within %d s.', time.time()-starttime)
                return p.returncode
 
        msg.warning('All %i tries failed!', n)
        raise Exception
 
 

cliToKey()

def python.trfUtils.cliToKey

(

option

)

Convert a command line option to the dictionary key that will be used by argparse.

Definition at line 652 of file trfUtils.py.

def cliToKey(option):
    return option.lstrip('-').replace('-', '_')
 
 

cmpMetadata()

def python.trfUtils.cmpMetadata	(		metadata1,
			metadata2,
			guidCheck = 'valid'
	)

Compare metadata for files, but taking into account that GUID can vary.

Compare metadata dictionaries, but allowing for differences in the file_guid property as this is generated randomly for file types without an intrinsic GUID

Parameters

metadata1	Filel metadata dictionary
metadata2	File2 metadata dictionary
giudCheck	How to compare GUIDs. Valid values are: equal GUIDs must be the same valid GUIDs must be valid, but don't have to be the same ignore The file_guid key is ignored

Returns

True if metadata is the same, otherwise False

Definition at line 464 of file trfUtils.py.

def cmpMetadata(metadata1, metadata2, guidCheck = 'valid'):
    # First check we have the same files
    allFiles = set(metadata1) | set(metadata2)
    if len(allFiles) > len(metadata1) or len(allFiles) > len(metadata2):
        msg.warning('In metadata comparison file lists are not equal - fails ({0} != {1}'.format(metadata1, metadata2))
        return False
    for fname in allFiles:
        allKeys = set(metadata1[fname]) | set(metadata2[fname])
        if len(allKeys) > len(metadata1[fname]) or len(allFiles) > len(metadata2[fname]):
            msg.warning('In metadata comparison key lists are not equal - fails')
            return False
        for key in allKeys:
            if key == 'file_guid':
                if guidCheck == 'ignore':
                    continue
                elif guidCheck == 'equal':
                    if metadata1[fname]['file_guid'].upper() == metadata2[fname]['file_guid'].upper():
                        continue
                    else:
                        msg.warning('In metadata comparison strict GUID comparison failed.')
                        return False
                elif guidCheck == 'valid':
                    try:
                        uuid.UUID(metadata1[fname]['file_guid'])
                        uuid.UUID(metadata2[fname]['file_guid'])
                        continue
                    except ValueError:
                        msg.warning('In metadata comparison found invalid GUID strings.')
                        return False
            if metadata1[fname][key] != metadata2[fname][key]:
                msg.warning('In metadata comparison found different key values: {0!s} != {1!s}'.format(metadata1[fname][key], metadata2[fname][key]))
    return True
 
 

convertToStr()

def python.trfUtils.convertToStr

(

in_string

)

Recursively convert unicode to str, useful when we have just loaded something from json (TODO: make the transforms happy with unicode as well as plain str!)

Definition at line 636 of file trfUtils.py.

def convertToStr(in_string):
    if isinstance(in_string, dict):
        return dict([(convertToStr(key), convertToStr(value)) for key, value in in_string.items()])
    elif isinstance(in_string, list):
        return [convertToStr(element) for element in in_string]
    # Unicode is always str in Python3, but bytes are not
    # TODO: remove unicode comparison after Python 3 migration
    elif in_string.__class__.__name__ == 'unicode':
        return in_string.encode('utf-8')
    elif in_string.__class__.__name__ == 'bytes':
        return in_string.decode('utf-8')
    else:
        return in_string
 
 

cvmfsDBReleaseCheck()

def python.trfUtils.cvmfsDBReleaseCheck

(

dbrelease

)

Validate a DBRelease exists on cvmfs and return the path to the setup script.

Parameters

dbrelease

The DBRelease number (X.Y.Z[.A]) or "current"

Exceptions

trfExceptions.TransformSetupException

If the DBRelease setup is unreadable or the dbrelease parameter is not understood

Returns

Path to setup.py script for this DBRelease

Definition at line 569 of file trfUtils.py.

def cvmfsDBReleaseCheck(dbrelease):
    dbsetup = None
    dbdMatch = re.match(r'([\d\.]+|current)$', dbrelease)
    msg.debug('Attempting to setup DBRelease {0} from cvmfs'.format(dbrelease))
    if dbdMatch:
        if 'VO_ATLAS_SW_DIR' in os.environ:
            msg.debug('Found site defined path to ATLAS software: {0}'.format(os.environ['VO_ATLAS_SW_DIR']))
            dbsetup = path.join(os.environ['VO_ATLAS_SW_DIR'], 'database', 'DBRelease', dbrelease, 'setup.py')
            if os.access(dbsetup, os.R_OK):
                return dbsetup
            msg.warning('Site defined path to ATLAS software seems invalid (failed to access {0}). Will also try standard cvmfs path.'.format(dbsetup))
        else:
            msg.debug('Using standard CVMFS path to ATLAS software')
 
        dbsetup = path.join('/cvmfs/atlas.cern.ch/repo/sw/database/DBRelease', dbrelease, 'setup.py')
        if not os.access(dbsetup, os.R_OK):
            raise trfExceptions.TransformSetupException(trfExit.nameToCode('TRF_DBRELEASE_PROBLEM'),
                                                                'CVMFS DBRelease setup file {0} was not readable'.format(dbsetup))
        msg.debug('Using cvmfs based dbrelease: {0}'.format(path.dirname(dbsetup)))
    else:
        raise trfExceptions.TransformSetupException(trfExit.nameToCode('TRF_DBRELEASE_PROBLEM'),
                                                    'Unable to interpret DBRelease "{0}" as either a tarball or a CVMFS release directory'.format(dbrelease))
    return dbsetup
 
 

findFile()

def python.trfUtils.findFile	(		pathvar,
			fname
	)

Find a named file along a colon separated PATH type variable.

Note will also work for finding directories

Returns

Full path to file or None is file is not found

Definition at line 37 of file trfUtils.py.

def findFile(pathvar, fname):
    # First see if the file already includes a path.
    msg.debug('Finding full path for {fileName} in path {path}'.format(
        fileName = fname,
        path = pathvar
    ))
    if fname.startswith('/'):
        return(fname)
 
    # Split the path.
    pathElements = pathvar.split(':')
    for pathElement in pathElements:
        if path.exists(path.join(pathElement, fname)):
            return(path.join(pathElement, fname))
 
    return(None)
 
 

forceToAlphaNum()

def python.trfUtils.forceToAlphaNum

(

string

)

Strip a string down to alpha-numeric characters only.

Note

This is used to force executor names and substep aliases to a form that the substep argument parser will recognise. None is still allowed as this is the default for "unset" in some cases.

Definition at line 443 of file trfUtils.py.

def forceToAlphaNum(string):
    if string is None or string.isalnum():
        return string
    newstring = ''
    for piece in string:
        if piece.isalnum():
            newstring += piece
    msg.warning("String {0} was stripped to alphanumeric characters only: {1}".format(string, newstring))
    return newstring
 
 

getAncestry()

def python.trfUtils.getAncestry

(

listMyOrphans = False

)

List all processes and parents and form a dictionary where the parent key lists all child PIDs.

Parameters

listMyOrphans

If this is True, then processes which share the same pgid as this process and have parent PID=1 (i.e., init) get added to this process's children, which allows these orphans to be added to the kill list. N.B. this means that orphans have two entries - as child of init and a child of this process

Definition at line 62 of file trfUtils.py.

def getAncestry(listMyOrphans = False):
    psCmd = ['ps', 'ax', '-o', 'pid,ppid,pgid,args', '-m']
 
    try:
        msg.debug('Executing %s', psCmd)
        p = Popen(psCmd, stdout=PIPE, stderr=PIPE)
        stdout = p.communicate()[0]
        psPID = p.pid
    except OSError as e:
        msg.error('Failed to execute "ps" to get process ancestry: %s', repr(e))
        raise
 
    childDict = {}
    myPgid = os.getpgrp()
    myPid = os.getpid()
    for line in stdout.decode().split('\n'):
        try:
            (pid, ppid, pgid, cmd) = line.split(None, 3)
            pid = int(pid)
            ppid = int(ppid)
            pgid = int(pgid)
            # Ignore the ps process
            if pid == psPID:
                continue
            if ppid in childDict:
                childDict[ppid].append(pid)
            else:
                childDict[ppid] = [pid]
            if listMyOrphans and ppid == 1 and pgid == myPgid:
                msg.info("Adding PID {0} to list of my children as it seems to be orphaned: {1}".format(pid, cmd))
                if myPid in childDict:
                    childDict[myPid].append(pid)
                else:
                    childDict[myPid] = [pid]
 
        except ValueError:
            # Not a nice line
            pass
    return childDict
 

infanticide()

def python.trfUtils.infanticide	(		childPIDs = None,
			sleepTime = 3,
			message = True,
			listOrphans = False
	)

Kill all PIDs.

Note

Even if this function is used, subprocess objects need to join() with the child to prevent it becoming a zombie

Parameters

childPIDs	Explicit list of PIDs to kill; if absent then listChildren() is called
sleepTime	Time between SIGTERM and SIGKILL
message	Boolean if messages should be printed
listOrphans	Parameter value to pass to getAncestry(), if necessary (beware, killing orphans is dangerous, you may kill "upstream" processes; Caveat Emptor)

Definition at line 132 of file trfUtils.py.

def infanticide(childPIDs = None, sleepTime = 3, message = True, listOrphans = False):
    if childPIDs is None:
        childPIDs = listChildren(listOrphans = listOrphans)
 
    if len(childPIDs) > 0 and message:
        msg.info('Killing these child processes: {0}...'.format(childPIDs))
 
    for pid in childPIDs:
        try:
            os.kill(pid, signal.SIGTERM)
        except OSError:
            pass
 
    time.sleep(sleepTime)
 
    for pid in childPIDs:
        try:
            os.kill(pid, signal.SIGKILL)
        except OSError:
            # OSError happens when the process no longer exists - harmless
            pass
 
 

initialise_processes()

def python.trfUtils.initialise_processes

(

)

initisation procedure for processes of process pool

Definition at line 855 of file trfUtils.py.

def initialise_processes():
    # Multiprocessing uses signals to communicate with subprocesses, so the
    # following two lines prevent the transforms signal handlers from
    # interfering:
    from PyJobTransforms.trfSignal import resetTrfSignalHandlers
    resetTrfSignalHandlers()
    signal.signal(signal.SIGINT, signal.SIG_IGN)
 
 

isInteractiveEnv()

def python.trfUtils.isInteractiveEnv

(

)

Definition at line 703 of file trfUtils.py.

def isInteractiveEnv():
    isInteractiveEnv = False
    # PS1 is for sh, bash; prompt is for tcsh and zsh
    if 'PS1' in os.environ or 'prompt' in os.environ:
        isInteractiveEnv = True
    elif os.isatty(sys.stdout.fileno()) or os.isatty(sys.stdin.fileno()):
        isInteractiveEnv = True
 
    return isInteractiveEnv
 
 

isodate()

def python.trfUtils.isodate

(

)

Return isoformated 'now' string.

Uses datetime.isoformat method, but suppressing microseconds

Definition at line 434 of file trfUtils.py.

def isodate():
    return datetime.now().replace(microsecond=0).isoformat()
 
 

JSONDump()

def python.trfUtils.JSONDump

(

argdict

)

Dump a list of arguments to the JSON file given in the 'dumpJSON' argument.

Definition at line 617 of file trfUtils.py.

def JSONDump(argdict):
    if 'dumpJSON' not in argdict:
        return
 
    from PyJobTransforms.trfArgClasses import argument
    theArgumentDictionary = {}
    for k, v in argdict.items():
        if k == 'dumpJSON':
            continue
        if isinstance(v, argument):
            theArgumentDictionary[k] = getattr(v, "dumpvalue", v.value)
        else:
            theArgumentDictionary[k] = v
    with open(argdict['dumpJSON'], 'w') as JSONFile:
        import json
        json.dump(theArgumentDictionary, JSONFile, sort_keys=True, indent=2)
 

lineByLine()

def python.trfUtils.lineByLine	(		filename,
			strip = True,
			removeTimestamp = True,
			substepName = None
	)

Generator to return lines and line count from a file.

Parameters

filename	Filename to open and deliver lines from
strip	If lines get stripped before being returned (default True)
removeTimestamp	Removes timestamp from left.(default True) Since strings are removed only from left, this option requires explicit removal of substepName.
substepName	Removes substepName from left, if it's value is provided. (default None)

Note

This is useful so that multiple parts of code can co-operatively take lines from the file

Definition at line 372 of file trfUtils.py.

def lineByLine(filename, strip=True, removeTimestamp=True, substepName=None):
    linecounter = 0
    encargs = {'encoding' : 'utf8'}
    f = open(filename, 'r', **encargs)
    for line in f:
        linecounter += 1
        if substepName and isinstance(substepName, str):    # Remove substepName only if caller provides that string.
            line = line.lstrip(substepName)
        if removeTimestamp:
            line = line.lstrip('0123456789:-, ')            # Remove timestamps in both serial and MP mode.
        if strip:
            line = line.strip()
        yield line, linecounter
    f.close()
 
 

listChildren()

def python.trfUtils.listChildren	(		psTree = None,
			parent = os.getpid(),
			listOrphans = False
	)

Find all the children of a particular PID (calls itself recursively to descend into each leaf)

Note

The list of child PIDs is reversed, so the grandchildren are listed before the children, etc. so signaling left to right is correct

Parameters

psTree	The process tree returned by trfUtils.listChildren(); if None then trfUtils.listChildren() is called internally.
parent	The parent process for which to return all the child PIDs
listOrphans	Parameter value to pass to getAncestry() if necessary

Returns

children List of child PIDs

Take a psTree dictionary and list all children

Definition at line 109 of file trfUtils.py.

def listChildren(psTree = None, parent = os.getpid(), listOrphans = False):  # noqa: B008 (PID is constant)
    '''Take a psTree dictionary and list all children'''
    if psTree is None:
        psTree = getAncestry(listMyOrphans = listOrphans)
 
    msg.debug("List children of %d (%s)", parent, psTree.get(parent, []))
    children = []
    if parent in psTree:
        children.extend(psTree[parent])
        for child in psTree[parent]:
            children.extend(listChildren(psTree, child))
    children.reverse()
    return children
 
 

pickledDump()

def python.trfUtils.pickledDump

(

argdict

)

Dump a list of arguments to the pickle file given in the 'dumpPickle' argument.

Note

This is a copy of the JSONDump function, but should in fact be deprecated soon so no point in merging the common parts! TODO: Deprecate me!

Definition at line 598 of file trfUtils.py.

def pickledDump(argdict):
    if 'dumpPickle' not in argdict:
        return
 
    from PyJobTransforms.trfArgClasses import argument
    theArgumentDictionary = {}
    for k, v in argdict.items():
        if k == 'dumpPickle':
            continue
        if isinstance(v, argument):
            theArgumentDictionary[k] = getattr(v, "dumpvalue", v.value)
        else:
            theArgumentDictionary[k] = v
    with open(argdict['dumpPickle'], 'wb') as pickleFile:
        import pickle as pickle
        pickle.dump(theArgumentDictionary, pickleFile)
 
 

prettyXML()

def python.trfUtils.prettyXML	(		element,
			indent = ' ',
			poolFileCatalogFormat = False
	)

XML pretty print an ElementTree.ELement object.

Parameters

element	ElementTree.ELement object to print
indent	Indent parameter for minidom toprettyxml method
poolFileCatalogFormat	Whether to reformat the XML as a classic POOLFILECATALOG document

Returns

String with the pretty printed XML version

Note

This is rather a convoluted way to get the correct DOCTYPE set and there's probably a better way to do it, but as this is a deprecated way of delivering metadata upstream it's not worth improving at this stage.

Definition at line 397 of file trfUtils.py.

def prettyXML(element, indent = ' ', poolFileCatalogFormat = False):
    # Use minidom for pretty printing
    # See http://broadcast.oreilly.com/2010/03/pymotw-creating-xml-documents.html
    xmlstring = ElementTree.tostring(element, 'utf-8')
    try:
        metadataDoc = minidom.parseString(xmlstring)
    except ExpatError:
        # Getting weird \x00 NULLs on the end of some GUIDs, which minidom.parsestring does not like (is this APR?)
        msg.warning('Error parsing ElementTree string - will try removing hex literals ({0!r})'.format(xmlstring))
        xmlstring = xmlstring.replace('\x00', '')
        metadataDoc = minidom.parseString(xmlstring)
 
 
    if poolFileCatalogFormat is False:
        return metadataDoc.toprettyxml(indent=indent, encoding='UTF-8')
 
    # Now create a new document with the correct doctype for classic POOLFILECATALOG
    # See http://stackoverflow.com/questions/2337285/set-a-dtd-using-minidom-in-python
    imp = minidom.DOMImplementation()
    doctype = imp.createDocumentType(qualifiedName='POOLFILECATALOG', publicId='', systemId='InMemory')
    doc = imp.createDocument(None, 'POOLFILECATALOG', doctype)
 
    # Cut and paste the parsed document into the new one
    # See http://stackoverflow.com/questions/1980380/how-to-render-a-doctype-with-pythons-xml-dom-minidom
    refel = doc.documentElement
    for child in metadataDoc.childNodes:
        if child.nodeType==child.ELEMENT_NODE:
            doc.replaceChild(doc.importNode(child, True), doc.documentElement)
            refel= None
        elif child.nodeType!=child.DOCUMENT_TYPE_NODE:
            doc.insertBefore(doc.importNode(child, True), refel)
 
    return doc.toprettyxml(indent=indent, encoding='UTF-8')
 
 

printHR()

def python.trfUtils.printHR

(

the_object

)

print in a human-readable way the items of a given object

This function prints in a human-readable way the items of a given object.

Parameters

object

to print

Definition at line 660 of file trfUtils.py.

def printHR(the_object):
    # dictionary
    if isinstance(the_object, dict):
        for key, value in sorted(the_object.items()):
            print(u'{key}: {value}'.format(key = key, value = value))
    # list or tuple
    elif isinstance(the_object, list) or isinstance(the_object, tuple):
        for element in the_object:
            print(element)
    # other
    else:
        print(the_object)
 
 

releaseIsOlderThan()

def python.trfUtils.releaseIsOlderThan	(		major,
			minor = None
	)

Test (to the best of our knowledge) if the current release is older than a major, minor version number.

There's no unambiguous reference to the release that encompasses all of the development builds (dev, devval, migs), but almost everything can be determined from AtlasVersion and AtlasBaseDir. If neither of those contain version information then we assume a development build that is new by definition (so we return False)

Parameters

major	Major release number
minor	Minor release number (if not specified, will not be matched against)

Returns

Boolean if current release is found to be older

Definition at line 261 of file trfUtils.py.

def releaseIsOlderThan(major, minor=None):
    if 'AtlasVersion' not in os.environ or 'AtlasBaseDir' not in os.environ:
        msg.warning("Could not find 'AtlasVersion' and 'AtlasBaseDir' in the environment - no release match possible")
        return False
    try:
        # First try AtlasVersion, which is clean
        relRegExp = re.compile(r'(?P<major>\d+)\.(?P<minor>\d+)\.(?P<other>.*)')
        relMatch = re.match(relRegExp, os.environ['AtlasVersion'])
        if not relMatch:
            # Now try the final part of AtlasBaseDir
            leafDir = path.basename(os.environ['AtlasBaseDir'])
            relMatch = re.match(relRegExp, leafDir)
            if not relMatch:
                msg.info('No identifiable numbered release found from AtlasVersion or AtlasBaseDir - assuming dev/devval/mig')
                return False
 
        relmajor = int(relMatch.group('major'))
        relminor = int(relMatch.group('minor'))
        msg.info('Detected release major {0}, minor {1} (.{2}) from environment'.format(relmajor, relminor, relMatch.group('other')))
 
        # Major beats minor, so test this first
        if relmajor < major:
            return True
        if relmajor > major:
            return False
 
        # First case is major equality and don't care about minor
        if minor is None or relminor >= minor:
            return False
        return True
 
    except Exception as e:
        msg.warning('Exception thrown when attempting to detect athena version ({0}). No release check possible'.format(e))
    return False
 
 

reportEventsPassedSimFilter()

def python.trfUtils.reportEventsPassedSimFilter

(

log

)

summarize events passed the ISF_SimEventFilter @detail this function sums up all events passed the ISF_SimEventFilter out of all total events.

All this inforamation is extracted from log.ReSim

Definition at line 1716 of file trfUtils.py.

def reportEventsPassedSimFilter(log):
 
    # Currently the pattern which contains the information for passed events is for example like:
    #  ISF_SimEventFilter   INFO accepted 1 out of 10 events for filter ISF_SimEventFilter (SimEventFilter)
    # In case the filter name truncated by ... due to long timestamps, the pattern could still match
    # e.g. ISF_SimEventFi... or ISF_SimEventFil...
    regExp = re.compile(r'ISF_SimEventFi[lter|...]+\s.*INFO.*accepted\s*(?P<events>[0-9]*)\s*out of\s*(?P<total>[0-9]*).*')
    try:
        myGen = lineByLine(log)
    except IOError as e:
        msg.warning('Failed to open transform logfile {0}: {1:s}'.format(log, e))
 
    resimevents = None
    passed_events = 0
    total_events = 0
    for line, lineCounter in myGen:
        m = regExp.match(line) 
        if m:
            passed_events += int(m.group('events'))
            total_events += int(m.group('total'))
            resimevents = passed_events
 
    if resimevents is not None:
        msg.info("Summary of events passed the ISF_SimEventFilter: {0} events of total {1}".format(passed_events, total_events) )
    else:
        msg.warning("Returning null value for the resimevents. No line matched with the regExp for extracting events passed the ISF_SimEventFilter")
 
    return resimevents

setupDBRelease()

def python.trfUtils.setupDBRelease

(

setup

)

Run a DBRelease setup.

Parameters

setup

DMRelease setup script location (absolute or relative path)

Returns

: None

Definition at line 543 of file trfUtils.py.

def setupDBRelease(setup):
    try:
        dbdir=path.abspath(path.dirname(setup))
        msg.debug('Will add {0} to sys.path to load DBRelease setup module'.format(dbdir))
        # N.B. We cannot use __import__ because the X.Y.Z directory name is illegal for a python module path
        opath = sys.path
        sys.path.insert(0, dbdir)
        from setup import Setup
        # Instansiate the Setup module, which activates the customisation
        Setup(dbdir)
        sys.path = opath
        msg.debug('DBRelease setup module was initialised successfully')
    except ImportError as e:
        errMsg = 'Import error while trying to load DB Setup module: {0}'.format(e)
        msg.error(errMsg)
        raise trfExceptions.TransformSetupException(trfExit.nameToCode('TRF_DBRELEASE_PROBLEM'), errMsg)
    except Exception as e:
        errMsg = 'Unexpected error while trying to load DB Setup module: {0}'.format(e)
        msg.error(errMsg)
        raise trfExceptions.TransformSetupException(trfExit.nameToCode('TRF_DBRELEASE_PROBLEM'), errMsg)
 
 

shQuoteStrings()

def python.trfUtils.shQuoteStrings

(

strArray = sys.argv

)

Quote a string array so that it can be echoed back on the command line in a cut 'n' paste safe way.

Parameters

strArray

Array of strings to quote

Technique is to first quote any pre-existing single quotes, then single quote all of the array elements so that the shell sees them as a single argument

Definition at line 361 of file trfUtils.py.

def shQuoteStrings(strArray = sys.argv):
    return [ "'" + qstring.replace("'", "\\'") + "'" for qstring in strArray ]
 
 

uniqueIdentifier()

def python.trfUtils.uniqueIdentifier

(

)

return a URL-safe, base 64-encoded pseudorandom UUID

This function returns a URL-safe, base 64-encoded pseudorandom Universally Unique IDentifier (UUID).

Returns

string of URL-safe, base 64-encoded pseudorandom UUID

Definition at line 678 of file trfUtils.py.

def uniqueIdentifier():
    return str(base64.urlsafe_b64encode(uuid.uuid4().bytes).strip("="))
 
 

units()

def python.trfUtils.units	(		quantity = None,
			unitSingular = "unit",
			unitPlural = "units"
	)

return either singular or plural units as appropriate for a given quantity

This function returns either singular or plural units as appropriate for a given quantity. So, a quantity of 1 would cause the return of singular units and a quantity of 2 would cause the return of plural units.

Parameters

quantity	the numerical quantity
unitSingular	the string for singular units
unitSingular	the string for plural units

Returns

string of singular or plural units

Definition at line 691 of file trfUtils.py.

def units(
    quantity = None,
    unitSingular = "unit",
    unitPlural = "units"
    ):
    if quantity == 1:
        return unitSingular
    else:
        return unitPlural
 
 
# @brief returns if the current job is running in interactive environment.

unpackDBRelease()

def python.trfUtils.unpackDBRelease	(		tarball,
			dbversion = None
	)

Ensure that the DBRelease tarball has been unpacked.

Extract the dbversion number and look for an unpacked directory. If found then this release is already setup. If not then try to unpack the tarball.

Parameters

tarball	The tarball file
dbversion	The version number (if not given the look at the tarball name to get it)

Exceptions

trfExceptions.TransformSetupException

If the DBRelease tarball is unreadable or the version is not understood

Returns

Two element tuple: (True if release was unpacked or False if release was already unpacked, dbsetup path)

Definition at line 520 of file trfUtils.py.

def unpackDBRelease(tarball, dbversion=None):
    if dbversion is None:
        dbdMatch = re.match(r'DBRelease-([\d\.]+)\.tar\.gz', path.basename(tarball))
        if dbdMatch is None:
            raise trfExceptions.TransformSetupException(trfExit.nameToCode('TRF_DBRELEASE_PROBLEM'),
                                                        'Could not find a valid version in the DBRelease tarball: {0}'.format(tarball))
        dbversion = dbdMatch.group(1)
    dbsetup = path.abspath(path.join("DBRelease", dbversion, "setup.py"))
    if os.access(dbsetup, os.R_OK):
        msg.debug('DBRelease {0} is already unpacked, found {1}'.format(tarball, dbsetup))
        return False, dbsetup
    else:
        msg.debug('Will attempt to unpack DBRelease {0}'.format(tarball))
        unpackTarFile(tarball)
        msg.info('DBRelease {0} was unpacked'.format(tarball))
        if not os.access(dbsetup, os.R_OK):
            raise trfExceptions.TransformSetupException(trfExit.nameToCode('TRF_DBRELEASE_PROBLEM'),
                                                        'DBRelease setup file {0} was not readable, even after unpacking {1}'.format(dbsetup, tarball))
        return True, dbsetup
 

unpackTarFile()

def python.trfUtils.unpackTarFile	(		filename,
			directory = "."
	)

Unpack a given tarfile.

Parameters

filename	Tar file to unpck
directory	Directory target for the unpacking

Definition at line 501 of file trfUtils.py.

def unpackTarFile(filename, directory="."):
    try:
        tar = tarfile.open(filename)
        tar.extractall(path=directory)
        tar.close()
    except Exception as e:
        errMsg = 'Error encountered while unpacking {0} to {1}: {2}'.format(filename, directory, e)
        msg.error(errMsg)
        raise trfExceptions.TransformSetupException(trfExit.nameToCode('TRF_SETUP'), errMsg)
 
 

ValgrindCommand()

def python.trfUtils.ValgrindCommand	(		defaultOptions = True,
			extraOptionsList = None,
			AthenaSerialisedConfigurationFile = "athenaConf.pkl",
			returnFormat = "string"
	)

Definition at line 1576 of file trfUtils.py.

def ValgrindCommand(
    defaultOptions                    = True,
    extraOptionsList                  = None,
    AthenaSerialisedConfigurationFile = "athenaConf.pkl",
    returnFormat                      = "string"
    ):
 
    # Access Valgrind suppressions files by finding the paths from
    # environment variables. Append the files to the Valgrind suppressions
    # options.
    suppressionFilesAndCorrespondingPathEnvironmentVariables = {
        "etc/valgrind-root.supp": "ROOTSYS",
        "Gaudi.supp":             "DATAPATH",
        "oracleDB.supp":          "DATAPATH",
        "valgrindRTT.supp":       "DATAPATH",
        "root.supp":              "DATAPATH"
    }
    optionsList = ["valgrind"]
    # If default options are not suppressed, use them.
    if defaultOptions:
        optionsList.append("--num-callers=30")
        optionsList.append("--tool=memcheck")
        optionsList.append("--leak-check=full")
        optionsList.append("--smc-check=all")
    # If extra options are specified, append them to the existing options.
    if extraOptionsList:
        for option in extraOptionsList:
            optionsList.append(option)
    # Add suppression files and athena commands
    for suppressionFile, pathEnvironmentVariable in suppressionFilesAndCorrespondingPathEnvironmentVariables.items():
        suppFile = findFile(os.environ[pathEnvironmentVariable], suppressionFile)
        if suppFile:
            optionsList.append("--suppressions=" + suppFile)
        else:
            msg.warning("Bad path to suppression file: {sfile}, {path} not defined".format(
                sfile = suppressionFile, path = pathEnvironmentVariable)
            )
    optionsList.append("$(which python)")
    optionsList.append("$(which athena.py)")
    optionsList.append(AthenaSerialisedConfigurationFile)
    # Return the command in the requested format, string (by default) or list.
    if returnFormat is None or returnFormat == "string":
        return(" ".join(optionsList))
    elif returnFormat == "list":
        return(optionsList)
    else:
        print(
            "error: invalid Valgrind command format request (requested " +
            "format: {format}; valid formats: string, list)".format(
            format = returnFormat
        ))
        raise(Exception)
 

VTuneCommand()

def python.trfUtils.VTuneCommand	(		defaultOptions = True,
			extraOptionsList = None,
			AthenaSerialisedConfigurationFile = "athenaConf.pkl",
			returnFormat = "string"
	)

return VTune command @detail This function returns a VTune command for use with Athena.

The command is returned as a string (by default) or a list, as requested using the argument returnFormat. The function will return a default VTune command specification, unless the user suppress them through an option. To append additional options to the command specification the argument extraOptionsList is used. This causes the list of extra specified command options to be appended to the command specification, which will contain the default options unless these are suppressed. The Athena serialised configuration file is specified using the argument AthenaSerialisedConfigurationFile.

Returns

command as string

Definition at line 1642 of file trfUtils.py.

def VTuneCommand(
    defaultOptions                    = True,
    extraOptionsList                  = None,
    AthenaSerialisedConfigurationFile = "athenaConf.pkl",
    returnFormat                      = "string"
    ):
 
    # Access VTune suppressions files by finding the paths from
    # environment variables. Append the files to the VTune suppressions
    # options.
    optionsList = ["vtune"]
    # If default options are not suppressed, use them.
    if defaultOptions:
        optionsList.append("-run-pass-thru=--no-altstack")
        optionsList.append("-mrte-mode=native")
    # If extra options are specified, append them to the existing options.
    isCollectSpecified=False
    if extraOptionsList:
        for option in extraOptionsList:
            optionsList.append(option)
            if option.startswith("-collect="):
                isCollectSpecified=True
    if not isCollectSpecified:
        optionsList.append("-collect=hotspots")
    optionsList.append("-- $(which python)")
    optionsList.append("$(which athena.py)")
    optionsList.append(AthenaSerialisedConfigurationFile)
    # Return the command in the requested format, string (by default) or list.
    if returnFormat is None or returnFormat == "string":
        return(" ".join(optionsList))
    elif returnFormat == "list":
        return(optionsList)
    else:
        print(
            "error: invalid VTune command format request (requested " +
            "format: {format}; valid formats: string, list)".format(
            format = returnFormat
        ))
        raise(Exception)
 
# calculate cpuTime from os.times() times tuple

Variable Documentation

msg

python.trfUtils.msg

Definition at line 31 of file trfUtils.py.