python.trfAMI Namespace Reference

Classes
class	TagInfo
	Stores the information about a given tag. More...
class	TrfConfig
	Stores the configuration of a transform. More...

Functions
def	_parseExecDict (substep, value)
	Back convert a pre/postExec dictionary into a set of command line compatible strings. More...
def	_parseIncludeDict (substep, value, joinWithChar=",")
	Back convert a pre/postInclude dictionary into a set of command line compatible strings By default use a comma for joining up the list values. More...
def	isNewAMITag (tag)
def	getAMIClient (endpoints=['atlas-replica', 'atlas'])
	Get an AMI client. More...
def	getProdSysTagsCharacters ()
	Get list of characters of ProdSys tags. More...
def	getPANDAClient ()
	Get PANDA client. More...
def	ReadablePANDA (s)
	Un-escape information from PANDA. More...
def	getTrfConfigFromPANDA (tag)
	Get information about a ProdSys tag from PANDA. More...
def	get_ami_tag (client, tag, suppressNonJobOptions=True)
def	remove_enclosing_quotes (s)
def	getTrfConfigFromAMI (tag, suppressNonJobOptions=True)
	Get information about a T0 tag from AMI. More...
def	deserialiseFromAMIString (amistring)
	Convert from a string to a python object. More...

Variables
	msg
	AMIerrorCode

Function Documentation

_parseExecDict()

def python.trfAMI._parseExecDict (   substep,   value  )

private

Back convert a pre/postExec dictionary into a set of command line compatible strings.

Definition at line 117 of file trfAMI.py.

def _parseExecDict(substep, value):
    string = ""
    for bit in value:
        string += " '" + substep + ":" + str(bit).replace("'", "\\'")+"'"
    return string
 

_parseIncludeDict()

def python.trfAMI._parseIncludeDict (   substep,   value,   joinWithChar = ","  )

private

Back convert a pre/postInclude dictionary into a set of command line compatible strings By default use a comma for joining up the list values.

Definition at line 126 of file trfAMI.py.

def _parseIncludeDict(substep, value, joinWithChar = ","):
    string = "'"
    string += substep + ":" + joinWithChar.join(value).replace("'", "\\'")+"'"
    return string
 

deserialiseFromAMIString()

def python.trfAMI.deserialiseFromAMIString

(

amistring

)

Convert from a string to a python object.

As we don't know how the string is encoded we try JSON then a safe eval. This is a bit ugly :-(

Definition at line 534 of file trfAMI.py.

def deserialiseFromAMIString(amistring):
    try:
        result = json.loads(amistring)
    except ValueError as e_json:
        msg.debug("Failed to decode {0} as JSON: {1}".format(amistring, e_json))
        try:
            result = ast.literal_eval(amistring)
        except SyntaxError:
            errMsg = "Failed to deserialise AMI string '{0}' using JSON or eval".format(amistring)
            msg.error(errMsg)
            raise TransformAMIException(AMIerrorCode, errMsg)
    return result

get_ami_tag()

def python.trfAMI.get_ami_tag	(		client,
			tag,
			suppressNonJobOptions = True
	)

Get AMI-tag information.

Args:
    :client: the pyAMI client [ pyAMI.client.Client ]
    :tag: the AMI-tag [ str ]

Returns:
    an array of python dictionnaries.

Definition at line 381 of file trfAMI.py.

def get_ami_tag(client, tag, suppressNonJobOptions = True):
        '''Get AMI-tag information.
 
        Args:
            :client: the pyAMI client [ pyAMI.client.Client ]
            :tag: the AMI-tag [ str ]
 
        Returns:
            an array of python dictionnaries.
        '''
 
        command = [
                'AMIGetAMITagInfo',
                '-amiTag="%s"' % tag,
                '-cached',
        ]
 
        if suppressNonJobOptions:
            command += ['-suppressNonJobOptions']
 
        msg.debug(command)
 
        return client.execute(command, format = 'dict_object').get_rows('amiTagInfo')
 

getAMIClient()

def python.trfAMI.getAMIClient

(

endpoints = ['atlas-replica','atlas']

)

Get an AMI client.

Note

Always return a client to the primary replica. The caller is allowed to update the replica via the config.endpoints value.

Returns

pyAMI.client.Client instance

Definition at line 222 of file trfAMI.py.

def getAMIClient(endpoints = ['atlas-replica','atlas']):
    msg.debug('Getting AMI client...')
    
    try:
        from pyAMI.client import Client
    except ImportError:
        raise TransformAMIException(AMIerrorCode, 'Import of pyAMI modules failed.')
        
    msg.debug("Attempting to get AMI client for endpoints {0}".format(endpoints))
    amiclient = Client(endpoints, ignore_proxy = True)
    return amiclient
 

getPANDAClient()

def python.trfAMI.getPANDAClient

(

)

Get PANDA client.

Returns

cx_Oracle cursor instance

Definition at line 248 of file trfAMI.py.

def getPANDAClient():
    msg.debug('Getting PANDA client...')
    try:
        import cx_Oracle
    except ImportError:
        raise TransformAMIException(AMIerrorCode, 'Import of cx_Oracle failed (is Oracle setup on this machine?).')
        
    try:
        cur = cx_Oracle.connect('atlas_grisli_r/panda_c10@adcr_panda').cursor()
    except Exception: 
        msg.debug('An exception occurred while connecting to PANDA database: %s', traceback.format_exc())
        raise TransformAMIException(AMIerrorCode, 'Failed to get PANDA client connection (N.B. this does not work from outside CERN).')
        
    return cur
 

getProdSysTagsCharacters()

def python.trfAMI.getProdSysTagsCharacters

(

)

Get list of characters of ProdSys tags.

Returns

list of characters

Definition at line 236 of file trfAMI.py.

def getProdSysTagsCharacters():
    # Due to move to uniform tag definition in AMI this list is now frozen
    # So just hard code it
    msg.debug('Getting list of ProdSys tag characters...')
 
    defaultList=['z', 'p', 'e', 's', 'd', 'r', 't', 'a', 'b', 'w']
 
    return defaultList
 
 

getTrfConfigFromAMI()

def python.trfAMI.getTrfConfigFromAMI	(		tag,
			suppressNonJobOptions = True
	)

Get information about a T0 tag from AMI.

Parameters

tag	Tag for which information is requested

Returns

list of PyJoCbTransforms.trfAMI.TRFConfig instances

Definition at line 416 of file trfAMI.py.

def getTrfConfigFromAMI(tag, suppressNonJobOptions = True):
    msg.debug('Using AMI to get info about tag %s', tag)
 
    try:
#        import pyAMI.atlas.api
        import pyAMI.exception
    except ImportError as e:
        raise TransformAMIException(AMIerrorCode, 'Import of pyAMI modules failed ({0})'.format(e))
        
    try:
        amiclient=getAMIClient()
#        result = pyAMI.atlas.api.get_ami_tag(amiclient, tag)
        result = get_ami_tag(amiclient, tag, suppressNonJobOptions)
    except pyAMI.exception.Error as e:
        msg.warning('An exception occured when connecting to primary AMI: {0}'.format(e))
        msg.debug('Exception: {0}'.format(e))
        if 'please login' in e.message or 'certificate expired' in e.message:
            raise TransformAMIException(AMIerrorCode, 'Getting tag info from AMI failed with credential problem. '
                                        'Please check your AMI account status.')
        if 'Invalid amiTag' in e.message:
            raise TransformAMIException(AMIerrorCode, 'Invalid AMI tag ({0}).'.format(tag))
            
        msg.debug("Error may not be fatal - will try AMI replica catalog")
 
    try:
        trf = TrfConfig()
        trf.name = result[0]['transformation']
        trf.inputs=result[0].get('inputs', {})
        trf.outputs=result[0].get('outputs', {})
        trf.release = result[0]['SWReleaseCache'].replace('_', ',')
 
        if 'phconfig' in result[0]:
            trf.physics=deserialiseFromAMIString(result[0]['phconfig'])
        else:
            physics = {}
            for k, v in result[0].items():
                if 'Exec' in k:
                    execStrList = [execStr for execStr in convertToStr(v).replace('" "', '"" ""').split('" "')]
                    physics[convertToStr(k)] = [remove_enclosing_quotes(execStr).replace('\\"', '"') for execStr in execStrList]
                elif '" "' in v:
                    msg.info('found a quoted space (" ") in parameter value for %s, converting to list', k)
                    subStrList = [subStr for subStr in convertToStr(v).replace('" "', '"" ""').split('" "')]
                    physics[convertToStr(k)] = [remove_enclosing_quotes(subStr).replace('\\"', '"') for subStr in subStrList]
                else:
                    physics[convertToStr(k)] = convertToStr(remove_enclosing_quotes(v))
 
            msg.debug('Result from AMI after string cleaning:')
            msg.debug('%s', dumps(physics, indent = 4))
 
            if suppressNonJobOptions:
                for k in list(physics):
                    if k in ['inputs', 'outputs', 'productionStep', 'transformation', 'SWReleaseCache']:
                        physics.pop(k)
 
            for k, v in physics.items():
                if 'triggerConfig' in k or 'triggerConfigByRun' in k:
                    if ' ' in v:
                        physics[k] = v.replace(' ', ',')
                        msg.warning('Attempted to correct illegal trigger configuration string: {0} -> {1}'.format(v, physics[k]))
 
            msg.debug("Checking for pseudo-argument internal to ProdSys...")
            if 'extraParameter' in physics:
                val = physics.pop('extraParameter')
                msg.debug("Removed extraParamater=%s from arguments.", val)
 
            msg.debug("Checking for input/output file arguments...")
            for arg in list(physics):
                if arg.lstrip('-').startswith('input') and arg.endswith('File'):
                    value = physics.pop(arg)
                    msg.debug("Found input file argument %s=%s.", arg, value)
                    trf.inFiles[arg] = value
                elif arg.lstrip('-').startswith('output') and arg.endswith('File'):
                    value = physics.pop(arg)
                    msg.debug("Found output file argument %s=%s.", arg, value)
                    trf.outFiles[arg] = value
 
            msg.debug("Checking for not set arguments...")
            for arg, value in physics.items():
                if value == "NONE" or value == "none" or value == ["NONE"]:
                    val = physics.pop(arg)
                    msg.debug("Removed %s=%s from arguments.", arg, val)
 
            trf.physics = physics
 
        if not isinstance(trf.physics, dict):
            raise TransformAMIException(AMIerrorCode, "Bad result for tag's phconfig: {0}".format(trf.physics))
 
        if trf.inFiles == {}:
            if 'inputs' in result[0]:
                trf.inFiles=deserialiseFromAMIString(result[0]['inputs'])
                for inFileType, inFileName in trf.inFiles.items():
                    # Not all AMI tags actually have a working filename, so fallback to trfDefaultFiles
                    # if necessary
                    if inFileName == '' or inFileName =={} or inFileName == [] or inFileName == '{}':
                        trf.inFiles[inFileType] = getInputFileName(inFileType, tag)
 
        if 'outputs' in result[0]:
            outputs=deserialiseFromAMIString(result[0]['outputs'])
            trf.outFiles=dict( (k, getOutputFileName(k.removeprefix('output').removesuffix('File')) ) for k in outputs )
            trf.outfmts=[ outputs[k]['dstype'] for k in outputs ]
    except KeyError as e:
        raise TransformAMIException(AMIerrorCode, "Missing key in AMI data: {0}".format(e))
    except Exception as e:
        raise TransformAMIException(AMIerrorCode, "Got a very unexpected exception while parsing AMI outputs!"
                                    " Please report.\nParsing:\n{0}\nRaised:\n{1}".format(result, e))
 
    # Now fix up for command line in the case of a new transform:
    if '_tf.py' in trf.name:
        trf.newTransform=True
    else:
        trf.newTransform=False
        
    return [ trf ]
 
 

getTrfConfigFromPANDA()

def python.trfAMI.getTrfConfigFromPANDA

(

tag

)

Get information about a ProdSys tag from PANDA.

Parameters

tag	Tag for which information is requested

Returns

list of PyJoCbTransforms.trfAMI.TRFConfig instances

Definition at line 272 of file trfAMI.py.

def getTrfConfigFromPANDA(tag):
    
    msg.debug('Using PANDA to get info about tag %s', tag)
            
    try:
        pandaclient=getPANDAClient()
        pandaclient.execute("select trf,trfv,lparams,vparams,formats,cache from t_trf_config where tag='%s' and cid=%d" %(tag[:1],int(tag[1:]) ) )
        result=pandaclient.fetchone()
    except Exception:
        msg.info('An exception occurred: %s', traceback.format_exc())
        raise TransformAMIException(AMIerrorCode, 'Getting tag info from PANDA failed.')
 
    if result is None:
        raise TransformAMIException(AMIerrorCode, 'Tag %s not found in PANDA database' % tag)
    
    msg.debug('Raw data returned from panda DB is:' + os.linesep + str(result))
    
    trfn=result[0].split(',')
    msg.debug('List of transforms: %s', trfn)
    trfv=result[1].split(',')
    msg.debug('List of releases: %s', trfv)
    lparams=result[2].split(';')
    msg.debug('List of arguments: %s', lparams)
    vparams=result[3].split(';')
    msg.debug('List of argument values: %s', vparams)
    formats=result[4].split('.')
    msg.debug('List of formats: %s', formats)
    cache=result[5].split(',')
    msg.debug('List of caches: %s', formats)
 
 
    if not ( len(trfn) == len(trfv) == len(lparams) == len(vparams) ):
        raise TransformAMIException(AMIerrorCode, 'Inconsistency in number of trfs.')
    
    # Cache can be a single value, in which case it needs replicated for other
    # transform steps, or it can be multivalued - great schema design guys :-(
    if len(cache) != len(trfv):
        if len(cache) == 1:
            cache = cache * len(trfv)
        else:
            raise TransformAMIException(AMIerrorCode, 'Inconsistency in number of caches entries vs. release numbers ({0}; {1}).'.format(cache, trfv))
 
    listOfTrfs=[]
 
    for iTrf in range(len(trfn)):
 
        trf = TrfConfig()
        trf.name =trfn[iTrf]
        if '_tf.py' in trf.name:
            trf.newTransform=True
        else:
            trf.newTransform=False
        trf.release=trfv[iTrf] + "," + cache[iTrf]
 
        keys=lparams[iTrf].split(',')
        values=vparams[iTrf].split(',')
 
        if ( len(keys) != len(values) ):
            raise TransformAMIException(AMIerrorCode, 'Inconsistency in number of arguments.')
 
        physics = dict( (k, ReadablePANDA(v) ) for (k,v) in zip(keys, values))
        # Hack to correct trigger keys being stored with spaces in panda  
        for k, v in physics.items():
            if 'triggerConfig' in k or 'triggerConfigByRun' in k:
                if ' ' in v:
                    physics[k] = v.replace(' ', ',')
                    msg.warning('Attempted to correct illegal trigger configuration string: {0} -> {1}'.format(v, physics[k]))
            if 'Exec' in k:
                # Mash up to a list, where %8C is used as the quote delimitation character
                physics[k] = [ execElement.replace("%8C", "") for execElement in v.split("%8C %8C") ]
 
        msg.debug("Checking for pseudo-argument internal to ProdSys...")
        if 'extraParameter' in physics:
            val=physics.pop('extraParameter')
            msg.debug("Removed extraParamater=%s from arguments.", val)
 
        msg.debug("Checking for input/output file arguments...")
        for arg in list(physics):
            if arg.lstrip('-').startswith('input') and arg.endswith('File'):
                value=physics.pop(arg)
                msg.debug("Found input file argument %s=%s.", arg, value ) 
                trf.inFiles[arg]=value
            elif arg.lstrip('-').startswith('output') and arg.endswith('File'):
                value=physics.pop(arg)
                msg.debug("Found output file argument %s=%s.", arg, value )
                trf.outFiles[arg]=value
 
        msg.debug("Checking for not set arguments...")
        for arg,value in physics.items():
            if value=="NONE" or value=="none" or value==["NONE"]:
                val=physics.pop(arg)
                msg.debug("Removed %s=%s from arguments.", arg, val )
 
        trf.physics=physics
 
        listOfTrfs.append(trf)
        
        
 
    listOfTrfs[0].inDS=None # not yet implemented
    listOfTrfs[-1].outfmts=formats
    
    return listOfTrfs
 
 
'''
directly copied from pyAMI.atlas 5 API
should be removed once suppressNonJobOptions is in official release
'''

isNewAMITag()

def python.trfAMI.isNewAMITag

(

tag

)

Definition at line 131 of file trfAMI.py.

def isNewAMITag(tag):
    newTagDict = {
        'a' : 764,
        'b' : 545,
        'c' : 864,
        'd' : 1351,
        'e' : 3764,
        'f' : 557,
        'g' : 46,
        'h' : 32,
        'j' : 46,
        'k' : 34,
        'm' : 1377,
        'o' : 4741,
        'p' : 2295,
        'q' : 430,
        'r' : 6382,
        's' : 2559,
        't' : 597,
        'u' : 51,
        'v' : 139,
        'w' : 501,
        'x' : 302,
    }
 
    if tag[0] in newTagDict:
        if int(tag[1:]) > newTagDict[tag[0]]:
            msg.debug('it is a new tag')
            return True
 
    msg.debug('it is NOT a new tag')
    return False
 
 

ReadablePANDA()

def python.trfAMI.ReadablePANDA

(

s

)

Un-escape information from PANDA.

Provided by Pavel.

Definition at line 265 of file trfAMI.py.

def ReadablePANDA(s):
    return s.replace('%0B',' ').replace('%9B','; ').replace('%8B','"').replace('%3B',';').replace('%2C',',').replace('%2B','+')
 
 

remove_enclosing_quotes()

def python.trfAMI.remove_enclosing_quotes

(

s

)

Definition at line 405 of file trfAMI.py.

def remove_enclosing_quotes(s):
    try:
        if s[0] == s[-1] and s[0] in ('"', "'"):
            s = s[1:-1]
    except Exception:
        pass
    return s
 

Variable Documentation

AMIerrorCode

python.trfAMI.AMIerrorCode

Definition at line 24 of file trfAMI.py.

msg

python.trfAMI.msg

Definition at line 17 of file trfAMI.py.