trfArgClasses.py

Go to the documentation of this file.

# Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
 

 
import argparse
import bz2
import copy
import glob
import os
import re
import subprocess
import uuid
 
import logging
msg = logging.getLogger(__name__)
 
import PyJobTransforms.trfExceptions as trfExceptions
 
from PyJobTransforms.trfFileUtils import athFileInterestingKeys, AthenaLiteFileInfo, NTUPEntries, HISTEntries, PHYSVALEntries, PRWEntries, urlType, ROOTGetSize
from PyJobTransforms.trfUtils import call
from PyJobTransforms.trfExeStepTools import commonExecutorStepName
from PyJobTransforms.trfExitCodes import trfExit as trfExit
from PyJobTransforms.trfDecorators import timelimited
from PyJobTransforms.trfAMI import getAMIClient
 
 

class argFactory(object):
    def __init__(self, genclass, *args, **kwargs):
        msg.debug('Initialised class %s with args=%s; kwargs=%s', genclass, args, kwargs)
        self._genclass = genclass
        self._args = args
        self._kwargs = kwargs
    
    def __call__(self, valueString=None):
        msg.debug('Called class %s with value=%s; args=%s; kwargs=%s', self._genclass, valueString, self._args, self._kwargs)
        
        # Wrap this step in our own try/except because if this goes wrong we want to see the exception
        # instead of having it masked by the argparse module
        try:
            # Passing None suppresses the value passed to the constructor, thus the constructor's own
            # default value is used - generally this will match the default value for the underlying
            # python object
            if valueString is None:
                obj = self._genclass(*self._args, **self._kwargs)
            else:
                obj = self._genclass(valueString, *self._args, **self._kwargs)
        except Exception as e:
            msg.fatal('Got this exception raised when calling object factory: {0}'.format(e))
            raise
        return obj
    
    def __str__(self):
        return 'argFactory for {0}, args {1}, kwargs {2}'.format(self._genclass, self._args, self._kwargs)
 
 

class argAction(argparse.Action):
    def __init__(self, factory, option_strings, dest, **kwargs):
        self._factory = factory
        super().__init__(option_strings, dest, **kwargs)
 
    def __call__(self, parser, namespace, values, option_string=None):
        msg.debug('Called action for factory=%s; values=%s', self._factory, values)
 
        # call the factory for each value
        if isinstance(values, list):
            if not values:
                # in case of empty list, run factory on None to get the default
                setattr(namespace, self.dest, [self._factory(None)])
            else:
                setattr(namespace, self.dest, [self._factory(v) for v in values])
        else:
            setattr(namespace, self.dest, self._factory(values))
 
 

class argActionFactory(object):
    def __init__(self, genclass, *args, **kwargs):
        msg.debug('Initialised action class %s with args=%s; kwargs=%s', genclass, args, kwargs)
        self._factory = argFactory(genclass, *args, **kwargs)
 
    @property
    def factory(self):
        return self._factory
 
    def __call__(self, option_strings, dest, **kwargs):
        return argAction(self._factory, option_strings, dest, **kwargs)
 
    def __str__(self):
        return 'argActionFactory for {0}'.format(self._factory)
 
 

class argument(object):
    
    
    def __init__(self, value = None, runarg = True, name = None):
        self._runarg = runarg
        self._name = name
                
        
        self.value = value
 
    
    @property
    def value(self):
        return self._value
 
    
    @value.setter
    def value(self, value):
        self._value = value
    
    
    @property
    def isRunarg(self):
        return self._runarg
    
    
    @property
    def name(self):
        return self._name
    
    
    @name.setter
    def name(self, value):
        self._name = value
 
    @property
    def prodsysDescription(self):
        desc = {'type' : None}
        return desc
        
    
    def __str__(self):
        return '{0}: Value {1} (isRunArg={2})'.format(self.__class__.__name__, self._value, self._runarg)
    
    
    def __repr__(self):
        return repr(self.value)
    
    
    def __eq__(self,other):
        return self.value == other.value
 
    def __nq__(self, other):
        return self.value != other.value
 
    def __lt__(self, other):
        return self.value < other.value
 
    def __gt__(self, other):
        return self.value > other.value
 

class argString(argument):
 
    
    def __init__(self, value = None, runarg = True, name = None, choices = None):
        self._choices = choices
        super(argString, self).__init__(value = value, runarg = runarg, name=name)
 
    
    @property
    def value(self):
        return self._value
 
    
    @value.setter
    def value(self, value):
        if value is None:
            # For strings, None maps to ''
            self._value = ''
        else:
            # Call string converter - should work for everything...
            self._value = str(value)
        if self._choices:
            if self._value not in self._choices:
                raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CHOICES_FAIL'), 'Converted argument {0} for {1} did not match any valid choice: {2}'.format(value, self._name, self._choices))
 
    
    def choices(self):
        return self._choices
 
    # prodsysDescription: human readable from of type plus possible values
    @property
    def prodsysDescription(self):
        desc = {'type' : 'str'}
        if self._choices:
            desc['choices'] = self._choices
        return desc
    
    
    def __str__(self):
        return self.value
 
 

class argInt(argument):
    
    
    @property
    def value(self):
        return self._value
 
    
    @value.setter
    def value(self, value):
        if value is None:
            # For ints None maps to 0
            self._value = 0
        else:
            if isinstance(value, int):
                self._value = value
            else:
                
                try:
                    self._value = int(value)
                except ValueError as e:
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 
                                                              'Failed to convert value {0} to int: {1}'.format(value, e))
 
    # prodsysDescription: human readable from of type plus possible values
    @property
    def prodsysDescription(self):
        desc = {'type' : 'int'}
        return desc
 
                
 

class argFloat(argument):
    
    
    def __init__(self, value=None, min=None, max=None, runarg=True, name=None):
        self._min = min
        self._max = max
        super(argFloat, self).__init__(value = value, runarg = runarg, name=name)
 
    
    @property
    def value(self):
        return self._value
 
    @property
    def prodsysDescription(self):
        desc = {'type' : 'float'}
        if self._min:
            desc['min'] = self._min
        if self._max:
            desc['max'] = self._max
        return desc
    
    
    @value.setter
    def value(self, value=None):
        # Default value will be 0.0 or self._min (if defined)
        if value is None:
            if self._min is not None:
                self._value = self._min
            else:
                self._value = 0.0
        else:
            try:
                if isinstance(value, float):
                    self._value = value
                else:
                    self._value = float(value)
            except ValueError:
                raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'),
                                                          'Failed to convert %s to a float' % str(value))
            
            if (self._min is not None and self.value < self._min) or (self._max is not None and self._value > self._max):
                raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_OUT_OF_RANGE'),
                                                          'argFloat value out of range: %g is not between %s and %s' % 
                                                          (self.value, self._min, self._max))
 
 

class argBool(argument):
    
    
    @property
    def value(self):
        return self._value
 
    
    @value.setter
    def value(self, value):
        # Default value matches the python bool() constructor
        if value is None:
            self._value = False
        else:
            if isinstance(value, bool):
                self._value = value
            else:
                self._value = strToBool(value)
 
    # prodsysDescription: human readable from of type plus possible values
    @property
    def prodsysDescription(self):
        desc = {'type' : 'bool'}
        return desc
 

class argList(argument):
    
    
    def __init__(self, value = [], supressEmptyStrings = True, splitter=',', runarg=True, name=None):
        self._splitter = splitter
        self._supressEmptyStrings = supressEmptyStrings
 
        super(argList, self).__init__(value = value, runarg = runarg, name=name)
 
    
    @property
    def value(self):
        return self._value
 
    # prodsysDescription: human readable from of type plus possible values
    @property
    def prodsysDescription(self):
        desc = {'type' : 'list', 'listtype': 'str'}
        if self._supressEmptyStrings:
            desc['supress Empty Strings'] = self._supressEmptyStrings
        return desc
 
 
    
    @value.setter
    def value(self, value):
        if isinstance(value, (list, tuple)):
            self._value = list(value)
        elif value is None:
            self._value = []
            return
        else:
            try:
                if self._supressEmptyStrings:
                    self._value = [ v for v in value.split(self._splitter) if v != '' ]
                else:
                    self._value = value.split(self._splitter)
            except AttributeError:
                raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'),
                                                          'Failed to convert %s to a list' % str(value))
 
    
    def append(self, addme):
        self._value.append(addme)
    
    
    def __str__(self):
        return " ".join(self._value)
    
    
    def __repr__(self):
        return '[' + ','.join([ repr(s) for s in self._value ]) + ']'
 
 

class argIntList(argList):
    
    @property
    def value(self):
        return self._value
    
 
    
    @value.setter
    def value(self, value):
        if isinstance(value, list):
            for v in value:
                if not isinstance(v, int):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_ERROR'),
                                                              'Illegal value {0} in list of ints'.format(v))
            self._value = value
        elif value is None:
            self._value = []
            return
        else:
            try:
                if self._supressEmptyStrings:
                    self._value = [ v for v in value.split(self._splitter) if v != '' ]
                else:
                    self._value = value.split(self._splitter)
                self._value = [ int(el) for el in self._value ]
            except (AttributeError, ValueError):
                raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'),
                                                          'Failed to convert %s to a list of ints' % str(value))
                    
    def __str__(self):
        return " ".join([ str(el) for el in self._value ])
 
    @property
    def prodsysDescription(self):
        desc = {'type' : 'list', 'listtype' : 'int'}
        return desc
                    
        
# Special list which stores k:v pairs, where the value is an float (used for AthenaMP merge target size)
class argKeyFloatValueList(argList):
    
    def __init__(self, value = {}, supressEmptyStrings = True, splitter=',', kvsplitter=":", runarg=True, name=None):
        self._splitter = splitter
        self._kvsplitter = kvsplitter
        self._supressEmptyStrings = supressEmptyStrings
 
        super(argList, self).__init__(value = value, runarg = runarg, name=name)
    
    
    @property
    def value(self):
        return self._value
 
    
    @value.setter
    def value(self, value):
        if isinstance(value, dict):
            for k, v in value.items():
                if not isinstance(k, str):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_ERROR'),
                                                              'Illegal key argument type {0} in dictionary for argKeyFloatValueList'.format(k))
                if not isinstance(v, float):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_ERROR'),
                                                              'Illegal value argument type {0} in dictionary for argKeyFloatValueList'.format(v))
            self._value = value
        elif value is None:
            self._value = {}
            return
        else:
            self._value = {}
            try:
                if self._supressEmptyStrings:
                    kvlist = [ v for v in value.split(self._splitter) if v != '' ]
                else:
                    kvlist = value.split(self._splitter)
                for item in kvlist:
                    k, v = item.split(self._kvsplitter, 1)
                    self._value[k] = float(v)
            except (AttributeError, ValueError):
                raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'),
                                                          'Failed to convert {0} to a dictionary of string:int values'.format(value))
 
    def __str__(self):
        return str(self._value)
 
    @property
    def prodsysDescription(self):
        desc = {'type' : 'list', 'listtype' : 'str:float'}
        return desc
        
 

class argFile(argList):    
 
    
    def __init__(self, value=list(), type=None, subtype=None, io = 'output', splitter=',', runarg=True, guid=None, 
                 multipleOK = None, name=None, executor=list(), mergeTargetSize=-1, auxiliaryFile=False):
        # Set these values before invoking super().__init__ to make sure they can be
        # accessed in our setter 
        self._dataset = None
        self._urlType = None
        self._type = type
        self._subtype = subtype
        self._guid = guid
        self._mergeTargetSize = mergeTargetSize
        self._auxiliaryFile = auxiliaryFile
        self._originalName = None
 
        # User setter to get valid value check
        self.io = io
 
        self._exe = executor
 
        
 
        self._metadataKeys = {'file_size': self._getSize,
                              'integrity': self._getIntegrity,
                              'file_guid': self._generateGUID,
                              '_exists': self._exists,
                              }
        self._fileMetadata = {}
        if multipleOK is None:
            if self._io == 'input':
                self._multipleOK = True
            else:
                self._multipleOK = False
        else:
            self._multipleOK = multipleOK
            
        
        super(argFile, self).__init__(value=value, splitter=splitter, runarg=runarg, name=name)
    
 
    
    @property
    def value(self):
        return self._value
 
    
    @value.setter
    def value(self, value):
        self.valueSetter(value)
        
    
    @property
    def multipleOK(self):
        return self._multipleOK
    
    
    @multipleOK.setter
    def multipleOK(self, value):
        self._multipleOK = value
        
    
    @property
    def mergeTargetSize(self):
        return self._mergeTargetSize
 
    
    @mergeTargetSize.setter
    def mergeTargetSize(self, value):
        if value is None:
            self._mergeTargetSize = 0
        else:
            self._mergeTargetSize = value
 
    @property
    def prodsysDescription(self):
        if isinstance(self._type, dict):
            if self._type=={}:
                desc = {'type' : 'file', 'subtype' : "NONE" }
            else:
                desc = {'type' : 'file', 'subtype' : dict((str(k).upper(), str(v).upper()) for (k,v) in self._type.items())}
        else:
            desc = {'type' : 'file', 'subtype' : str(self._type).upper()}
            desc['multiple'] = self._multipleOK
        return desc 
    
    
    @property
    def executor(self):
        return self._exe
        
    
    def valueSetter(self, value):
        
        if isinstance(value, (list, tuple)):
            if len(value) > 0 and isinstance(value[0], dict): # Tier-0 style expanded argument with metadata
                self._value=[]
                for myfile in value:
                    try:
                        self._value.append(myfile['lfn'])
                        self._resetMetadata(files = [myfile['lfn']])
                    except KeyError:
                        raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'),
                                                                  'Filename (key "lfn") not found in Tier-0 file dictionary: {0}'.format(myfile))
                    for k, v in myfile.items():
                        if k == 'guid':
                            self._setMetadata([myfile['lfn']], {'file_guid': v})
                        elif k == 'events':
                            self._setMetadata([myfile['lfn']], {'nentries': v})
                        elif k == 'checksum':
                            self._setMetadata([myfile['lfn']], {'checksum': v})
                        elif k == 'dsn':
                            if not self._dataset:
                                self.dataset = v
                            elif self.dataset != v:
                                raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_DATASET'),
                                                                          'Inconsistent dataset names in Tier-0 dictionary: {0} != {1}'.format(self.dataset, v))
            else:
                self._value = list(value)
                self._getDatasetFromFilename(reset = False)
                self._resetMetadata()
        elif value is None:
            self._value = []
            return
        else:
            try:
                if value.lower().startswith('lfn'):
                    # Resolve physical filename using pool file catalog.
                    from PyUtils.PoolFile import file_name
                    protocol, pfn = file_name(value)
                    self._value = [pfn]
                    self._getDatasetFromFilename(reset = False)
                    self._resetMetadata()
                else:
                    # Don't split output filename if it contains a list in square brackets
                    if self._io == 'output' and ('[' in value) and (']' in value):
                        self._value = [value]
                    else:
                        self._value = value.split(self._splitter)
                    self._getDatasetFromFilename(reset = False)
                    self._resetMetadata()
            except (AttributeError, TypeError):
                raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'),
                                                          'Failed to convert %s to a list' % str(value))
 
        
        deDuplicatedValue = []
        for fname in self._value:
            if fname not in deDuplicatedValue:
                deDuplicatedValue.append(fname)
            else:
                msg.warning("Removing duplicated file {0} from file list".format(fname))
        if len(self._value) != len(deDuplicatedValue):
            self._value = deDuplicatedValue
            msg.warning('File list after duplicate removal: {0}'.format(self._value))
        
        # Find our URL type (if we actually have files!)
        # At the moment this is assumed to be the same for all files in this instance
        # although in principle one could mix different access methods in the one input file type
        if len(self._value) > 0:
            self._urlType = urlType(self._value[0])
        else:
            self._urlType = None
 
        
        if self._io == 'input':
            
            if self._urlType == 'posix':
                msg.debug('Found POSIX filesystem input - activating globbing')
                newValue = []
                for filename in self._value:
                    # Simple case
                    globbedFiles = glob.glob(filename)
                    if len(globbedFiles) == 0:          # No files globbed for this 'filename' argument.
                        raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_INPUT_FILE_ERROR'),
                        'Input file argument {0} globbed to NO input files - probably the file(s) are missing'.format(filename))
 
                    globbedFiles.sort()
                    newValue.extend(globbedFiles)
 
                self._value = newValue
                msg.debug ('File input is globbed to %s' % self._value)
 
            elif self._urlType == 'root':
                msg.debug('Found root filesystem input - activating globbing')
                newValue = []
                for filename in self._value:
                    if str(filename).startswith("root"):
                        msg.debug('Found input file name starting with "root," setting XRD_RUNFORKHANDLER=1, which enables fork handlers for xrootd in direct I/O')
                        os.environ["XRD_RUNFORKHANDLER"] = "1"
                    if str(filename).startswith("https") or str(filename).startswith("davs") or not(str(filename).endswith('/')) and '*' not in filename and '?' not in filename:
                        msg.debug('Seems that only one file was given: {0}'.format(filename))
                        newValue.extend(([filename]))
                    else:
                        # Hopefully this recognised wildcards...
                        path = filename
                        fileMask = ''
                        if '*' in filename or '?' in filename:
                            msg.debug('Split input into path for listdir() and a filemask to select available files.')
                            path = filename[0:filename.rfind('/')+1]
                            msg.debug('path: {0}'.format(path))
                            fileMask = filename[filename.rfind('/')+1:len(filename)]
                            msg.debug('Will select according to: {0}'.format(fileMask))
 
                        cmd = ['/afs/cern.ch/project/eos/installation/atlas/bin/eos.select' ]
                        if not os.access ('/afs/cern.ch/project/eos/installation/atlas/bin/eos.select', os.X_OK ):
                            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_INPUT_FILE_ERROR'),
                            'No execute access to "eos.select" - could not glob EOS input files.')
 
                        cmd.extend(['ls'])
                        cmd.extend([path])
 
                        myFiles = []
                        try:
                            proc = subprocess.Popen(args = cmd,bufsize = 1, shell = False, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
                            rc = proc.wait()
                            output = proc.stdout.readlines()
                            if rc!=0:
                                raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_INPUT_FILE_ERROR'),
                                                            'EOS list command ("{0!s}") failed: rc {1}, output {2}'.format(cmd, rc, output))
                            msg.debug("eos returned: {0}".format(output))
                            for line in output:
                                if "root" in line:
                                    myFiles += [str(path)+str(line.rstrip('\n'))]
 
                            patt = re.compile(fileMask.replace('*','.*').replace('?','.'))
                            for srmFile in myFiles:
                                if fileMask != '':
                                    if(patt.search(srmFile)) is not None:
                                    #if fnmatch.fnmatch(srmFile, fileMask):
                                        msg.debug('match: %s',srmFile)
                                        newValue.extend(([srmFile]))
                                else:
                                    newValue.extend(([srmFile]))
                                
                            msg.debug('Selected files: %s', newValue)
                        except (AttributeError, TypeError, OSError):
                            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_RUNTIME_ERROR'),
                                                                      'Failed to convert %s to a list' % str(value))
                if len(self._value) > 0 and len(newValue) == 0:
                    # Woops - no files!
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_INPUT_FILE_ERROR'),
                                                              'Input file argument(s) {0!s} globbed to NO input files - ls command failed')
                self._value = newValue
                msg.debug ('File input is globbed to %s' % self._value)
        # Check if multiple outputs are ok for this object
        elif self._multipleOK is False and len(self._value) > 1:
            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_OUTPUT_FILE_ERROR'), 
                                                      'Multiple file arguments are not supported for {0} (was given: {1}'.format(self, self._value))
            
    @property
    def io(self):
        return (self._io)
 
    @io.setter
    def io(self, value):
        if value not in ('input', 'output', 'temporary'):
            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_RUNTIME_ERROR'), 
                                                      'File arguments must be specified as input, output or temporary - got {0}'.format(value))
        self._io = value
    
    @property
    def dataset(self):
        return self._dataset
    
    @dataset.setter
    def dataset(self, value):
        self._dataset = value
    
    @property
    def orignalName(self):
        return self._originalName
    
    @orignalName.setter
    def originalName(self, value):
        self._originalName = value
    
    @property
    def type(self):
        return self._type
    
    @type.setter
    def type(self, value):
        self._type = value
 
    @property
    def subtype(self):
        return self._subtype
    
    @subtype.setter
    def subtype(self, value):
        self._subtype = value
 
    
    @property
    def name(self):
        return self._name
    
    
    @name.setter
    def name(self, value):
        self._name = value
        m = re.match(r'(input|output|tmp.)([A-Za-z0-9_]+?)(File)?$', value)
        if m:
            msg.debug("ArgFile name setter matched this: {0}".format(m.groups()))
            if self._type is None:
                dtype = m.group(2).split('_', 1)[0]
                # But DRAW/DESD/DAOD are really just RAW, ESD, AOD in  format
                if re.match(r'D(RAW|ESD|AOD)', dtype):
                    dtype = dtype[1:]
                msg.debug("Autoset data type to {0}".format(dtype))
                self._type = dtype
            if self._subtype is None:
                msg.debug("Autoset data subtype to {0}".format(m.group(2)))
                self._subtype = m.group(2)
        else:
            msg.debug("ArgFile name setter did not match against '{0}'".format(value))
 
    @property
    def auxiliaryFile(self):
        return self._auxiliaryFile
    
    
    @property
    def metadata(self):
        self.getMetadata()
        return self._fileMetadata
    
    
    @property
    def nentries(self):
        return self.getnentries()
        
    
    def getnentries(self, fast=False):
        totalEvents = 0
        for fname in self._value:
            events = self.getSingleMetadata(fname=fname, metadataKey='nentries', populate = not fast)
            if events is None:
                msg.debug('Got events=None for file {0} - returning None for this instance'.format(fname))
                return None
            if events == 'UNDEFINED':
                msg.debug('Got events=UNDEFINED for file {0} - returning UNDEFINED for this instance'.format(fname))
                return 'UNDEFINED'
            if not isinstance(events, int):
                msg.warning('Got unexpected events metadata for file {0}: {1!s} - returning None for this instance'.format(fname, events))
                return None
            totalEvents += events
        
        return totalEvents
                
    
    
    def _resetMetadata(self, files=[]):
        if files == [] or '_fileMetadata' not in dir(self):
            self._fileMetadata = {}
            for fname in self.value:
                self._fileMetadata[fname] = {}
        else:
            for fname in files:
                if fname in self.value:
                    self._fileMetadata[fname] = {}
                elif fname in self._fileMetadata:
                    del self._fileMetadata[fname]
        msg.debug('Metadata dictionary now {0}'.format(self._fileMetadata))
        
        # If we have the special guid option, then manually try to set GUIDs we find
        if self._guid is not None:
            msg.debug('Now trying to set file GUID metadata using {0}'.format(self._guid))
            for fname, guid in self._guid.items():
                if fname in self._value:
                    self._fileMetadata[fname]['file_guid'] = guid
                else:
                    msg.warning('Explicit GUID {0} was passed for file {1}, but this file is not a member of this instance'.format(guid, fname))
    
    
    def getMetadata(self, files = None, metadataKeys = None, maskMetadataKeys = None, populate = True, flush = False):
        # Normalise the files and keys parameter
        if files is None:
            files = self._value
        elif isinstance(files, str):
            files = (files,)
        msg.debug('getMetadata will examine these files: {0!s}'.format(files))
 
        if metadataKeys is None:
            metadataKeys = list(self._metadataKeys)
        elif isinstance(metadataKeys, str):
            metadataKeys = [metadataKeys,]
        if maskMetadataKeys is not None:
            metadataKeys = [k for k in metadataKeys if k not in maskMetadataKeys]
        msg.debug('getMetadata will retrieve these keys: {0!s}'.format(metadataKeys))
        
        if flush is True:
            msg.debug('Flushing cached metadata values')
            self._resetMetadata()
            
        if populate is True:
            msg.debug('Checking metadata values')
            self._readMetadata(files, metadataKeys)
 
        metadata = {}
        for fname in files:
            metadata[fname] = {}
            for mdkey in metadataKeys:
                try:
                    metadata[fname][mdkey] = self._fileMetadata[fname][mdkey]
                except KeyError:
                    # This should not happen, unless we skipped populating
                    if populate:
                        msg.error('Did not find metadata key {0!s} for file {1!s} - setting to None'.format(mdkey, fname))
                    metadata[fname][mdkey] = None
        return metadata
    
    
    def getSingleMetadata(self, fname, metadataKey, populate = True, flush = False):
        if not (isinstance(fname, str) and isinstance(metadataKey, str)):
            raise trfExceptions.TransformInternalException(trfExit.nameToCode('TRF_INTERNAL'), 
                                                           'Illegal call to getSingleMetadata function: {0!s} {1!s}'.format(fname, metadataKey))
        md = self.getMetadata(files = fname, metadataKeys = metadataKey, populate = populate, flush = flush)
        return md[fname][metadataKey]
    
    
    
    def _readMetadata(self, files, metadataKeys):
        msg.debug('Retrieving metadata keys {1!s} for files {0!s}'.format(files, metadataKeys))
        for fname in files:
            if fname not in self._fileMetadata:
                self._fileMetadata[fname] = {}
        for fname in files:
            # Always try for a simple existence test first before producing misleading error messages
            # from metadata populator functions
            if '_exists' not in self._fileMetadata[fname]:
                self._metadataKeys['_exists'](files)
            if self._fileMetadata[fname]['_exists'] is False:
                # N.B. A log ERROR message has printed by the existence test, so do not repeat that news here
                for key in metadataKeys:
                    if key != '_exists':
                        self._fileMetadata[fname][key] = None
            else:
                # OK, file seems to exist at least... 
                for key in metadataKeys:
                    if key not in self._metadataKeys:
                        msg.debug('Metadata key {0} is unknown for {1}'.format(key, self.__class__.__name__))
                        self._fileMetadata[fname][key] = 'UNDEFINED'
                    else:
                        if key in self._fileMetadata[fname]:
                            msg.debug('Found cached value for {0}:{1} = {2!s}'.format(fname, key, self._fileMetadata[fname][key]))
                        else:
                            msg.debug('No cached value for {0}:{1}. Calling generator function {2} ({3})'.format(fname, key, self._metadataKeys[key].__name__, self._metadataKeys[key]))
                            try:
                                # For efficiency call this routine with all files we have
                                msg.info("Metadata generator called to obtain {0} for {1}".format(key, files))
                                self._metadataKeys[key](files)
                            except trfExceptions.TransformMetadataException as e:
                                msg.error('Calling {0!s} raised an exception: {1!s}'.format(self._metadataKeys[key].__name__, e))
                            if key not in self._fileMetadata[fname]:
                                msg.warning('Call to function {0} for {1} file {2} failed to populate metadata key {3}'.format(self._metadataKeys[key].__name__, self.__class__.__name__, fname, key))
                                self._fileMetadata[fname][key] = None
                            msg.debug('Now have {0}:{1} = {2!s}'.format(fname, key, self._fileMetadata[fname][key]))
    
    
    
    def _setMetadata(self, files=None, metadataKeys={}):
        if files is None:
            files = self._value
        for fname in files:
            if fname not in self._fileMetadata:
                self._fileMetadata[fname] = {}
            for k, v in metadataKeys.items():
                msg.debug('Manualy setting {0} for file {1} to {2}'.format(k, fname, v))
                self._fileMetadata[fname][k] = v
    
    
    
    def isCached(self, files = None, metadataKeys = None):
        msg.debug('Testing for cached values for files {0} and keys {1}'.format(files, metadataKeys))
        if files is None:
            files = self._value
        elif isinstance(files, str):
            files = (files,)
        if metadataKeys is None:
            metadataKeys = list(self._metadataKeys)
        elif isinstance(metadataKeys, str):
            metadataKeys = (metadataKeys,)
            
        isCachedFlag = True
        for fname in files:
            for key in metadataKeys:
                if key not in self._fileMetadata[fname]:
                    isCachedFlag = False
                    break
            if isCachedFlag is False:
                break
            
        return isCachedFlag
    
    
    def _getDatasetFromFilename(self, reset = False):
        if reset:
            self._dataset = None
        newValue = []
        for filename in self._value:
            if filename.find('#') > -1:
                (dataset, fname) = filename.split('#', 1)
                newValue.append(fname)
                msg.debug('Current dataset: {0}; New dataset {1}'.format(self._dataset, dataset))
                if self._dataset and (self._dataset != dataset):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_DATASET'), 
                                                              'Found inconsistent dataset assignment in argFile setup: %s != %s' % (self._dataset, dataset))
                self._dataset = dataset
        if len(newValue) == 0:
            return
        elif len(newValue) != len (self._value):
            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_DATASET'), 
                                                      'Found partial dataset assignment in argFile setup from {0} (dsn#lfn notation must be uniform for all inputs)'.format(self._value))
        self._value = newValue
    
    
    def _getSize(self, files):
        for fname in files:
            if self._urlType == 'posix':
                try:
                    self._fileMetadata[fname]['size'] = os.stat(fname).st_size
                except OSError as e:
                    msg.error('Got exception {0!s} raised while stating file {1}'.format(e, fname))
                    self._fileMetadata[fname]['size'] = None
            else:
                # OK, let's see if ROOT can do it...
                msg.debug('Calling ROOT TFile.GetSize({0})'.format(fname))
                self._fileMetadata[fname]['size'] = ROOTGetSize(fname)
 
 
    
    def _getIntegrity(self, files):
        for fname in files:
            is_binary = False
            with open(fname) as f: 
                try:
                    while True:
                        chunk = len(f.read(1024*1024))
                        msg.debug('Read {0} bytes from {1}'.format(chunk, fname)) 
                        if chunk == 0:
                            break
                    self._fileMetadata[fname]['integrity'] = True
                except OSError as e:
                    msg.error('Got exception {0!s} raised while checking integrity of file {1}'.format(e, fname))
                    self._fileMetadata[fname]['integrity'] = False
                except UnicodeDecodeError:
                    msg.debug('Problem reading file as unicode, attempting with binary')
                    is_binary = True
            if is_binary:
                with open(fname,'rb') as f:
                    try:
                        while True:
                            chunk = len(f.read(1024*1024))
                            msg.debug('Read {0} bytes from {1}'.format(chunk, fname))
                            if chunk == 0:
                                break
                        self._fileMetadata[fname]['integrity'] = True
                    except OSError as e:
                        msg.error('Got exception {0!s} raised while checking integrity of file {1}'.format(e, fname))
                        self._fileMetadata[fname]['integrity'] = False
 
    
    def _generateGUID(self, files):
        for fname in files:
            msg.debug('Generating a GUID for file {0}'.format(fname))
            self._fileMetadata[fname]['file_guid'] = str(uuid.uuid4()).upper()
            
            
    
    def _exists(self, files):
        import re
        msg.debug('Testing existance for {0}'.format(files))
        def split_filelist(fn):
            if self.io != 'output':
                return [fn]
            file_split_regex = re.compile(r"(.+)\[(.+)](.+)")
            if ('[' in fn) and (']' in fn):
                match = file_split_regex.match(fn)
                return [f"{match.group(1)}{it}{match.group(3)}" for it in match.group(2).split(',')]
            else:
                return [fn]
        for fname in files:
            file_list = split_filelist(fname)
            if self._urlType == 'posix':
                try:
                    size = map(lambda fn: os.stat(fn).st_size, file_list)
                    self._fileMetadata[fname]['file_size'] = sum(size)
                    self._fileMetadata[fname]['_exists'] = True
                    msg.debug('POSIX file {0} exists (or all elements of list)'.format(fname))
                except OSError as e:
                    msg.error('Got exception {0!s} raised while stating file {1} (or some element of list)  - probably it does not exist'.format(e, fname))
                    self._fileMetadata[fname]['_exists'] = False
            else:
                # OK, let's see if ROOT can do it...
                msg.debug('Calling ROOT TFile.GetSize on {0} (or elements of list)'.format(fname))
                size = map(ROOTGetSize, file_list)
                if None in size:
                    self._fileMetadata[fname]['_exists'] = False
                    msg.error('Non-POSIX file {0} (or element of list) could not be opened - probably it does not exist'.format(fname))
                else:
                    msg.debug('Non-POSIX file {0} (or all elements of list) exists'.format(fname))
                    self._fileMetadata[fname]['file_size'] = sum(size)
                    self._fileMetadata[fname]['_exists'] = True
 
    
    def __str__(self):
        return "{0}={1} (Type {2}, Dataset {3}, IO {4})".format(self.name, self.value, self.type, self.dataset, self.io)
    
    
    
    def _mergeArgs(self, argdict, copyArgs=None):
        if copyArgs:
            myargdict = {}
            for arg in copyArgs:
                if arg in argdict:
                    myargdict[arg] = copy.copy(argdict[arg])
            
        else:
            myargdict = copy.copy(argdict)
        # Never do event count checks for self merging
        myargdict['checkEventCount'] = argSubstepBool('False', runarg=False) 
        newopts = []
        if 'athenaopts' in myargdict:
            # Need to ensure that "nprocs" is not passed to merger
            # and prevent multiple '--threads' options when there are multiple sub-steps in 'athenopts'
            for subStep in myargdict['athenaopts'].value:
                hasNprocs = False
                hasNthreads = False
                for opt in myargdict['athenaopts'].value[subStep]:
                    if opt.startswith('--nprocs'):
                        hasNprocs = True
                        continue
                    # Keep at least one '--threads'
                    elif opt.startswith('--threads'):
                        hasNthreads = True
                        if opt in newopts:
                            continue
                    newopts.append(opt)
                # If we have hybrid MP+MT job make sure --threads is not passed to merger
                if hasNprocs and hasNthreads:
                    tmpopts = []
                    for opt in newopts:
                        if opt.startswith('--threads'):
                            continue
                        tmpopts.append(opt)
                    newopts = tmpopts
            myargdict['athenaopts'] = argSubstepList(newopts, runarg=False)
        return myargdict
 
 
class argYODAFile(argFile):
    def __init__(self, value=list(), io = 'output', type=None, splitter=',', runarg=True, multipleOK=None, name=None):
        super(argYODAFile, self).__init__(value=value, io=io, type=type, splitter=splitter, runarg=runarg, multipleOK=multipleOK,
                                           name=name)
 
        self._metadataKeys.update({
                'nentries': self._getNumberOfEvents,
                'lheSumOfPosWeights': self._getWeightedEvents,
                'lheSumOfNegWeights': 0,
                })
 
    def _getNumberOfEvents(self, files):
        msg.debug('Retrieving event count for LHE file {0}'.format(files))
        import tarfile
        for fname in files:
            # Attempt to treat this as a pileup reweighting file
            try :
                tar = tarfile.open(fname, "r:gz")
                lhecount = 0
                for untar in tar.getmembers():
                    fileTXT = tar.extractfile(untar)
                    if fileTXT is not None :
                        lines = fileTXT.read().decode("utf-8")
                        lhecount = lines.count('/event')
 
                self._fileMetadata[fname]['nentries'] = lhecount
            except Exception:
                msg.debug('Entries is set to None - event count undefined for this LHE')
                self._fileMetadata[fname]['nentries'] = -1
 
    def _getWeightedEvents(self, files):
        msg.debug('Retrieving weight count for LHE file {0}'.format(files))
        import tarfile
        import re
 
        for fname in files:
            weightPos = 0
            weightNeg = 0
            try :
                tar = tarfile.open(fname, "r:gz")
                for untar in tar.getmembers():
                    fileTXT = tar.extractfile(untar)
                    next = False
                    if fileTXT is not None :
                        for line in fileTXT :
                            line = line.decode("utf-8")
                            if next :
                                try :
                                    w = float(re.sub(' +',' ',line).split(" ")[2])
                                    if w > 0 : weightPos += w
                                    else : weightNeg += abs(w)
                                except Exception:
                                    pass
                                next = False
                            if "<event" in line :
                                next = True
 
                self._fileMetadata[fname]['lheSumOfPosWeights'] = weightPos
                self._fileMetadata[fname]['lheSumOfNegWeights'] = weightNeg
            except Exception:
                msg.debug('Entries is set to None - negative fraction count undefined for this LHE')
                self._fileMetadata[fname]['lheSumOfPosWeights'] = -1
                self._fileMetadata[fname]['lheSumOfNegWeights'] = -1
 

class argAthenaFile(argFile):
    def __init__(self, value = list(), type=None, subtype=None, io = 'output', splitter=',', runarg=True, multipleOK = None, 
                 name=None, executor=list(), mergeTargetSize=-1, auxiliaryFile=False):
        super(argAthenaFile, self).__init__(value=value, subtype=subtype, io=io, type=type, splitter=splitter, runarg=runarg, 
                                            multipleOK=multipleOK, name=name, executor=executor, mergeTargetSize=mergeTargetSize,
                                            auxiliaryFile=auxiliaryFile)
 
        # Extra metadata known for athena files:
        for key in athFileInterestingKeys:
            self._metadataKeys[key] = self._getAthInfo
            
    
    def _callAthInfo(self, files, doAllFiles, retrieveKeys):
        if doAllFiles:
            myFiles = self._value
        else:
            myFiles = files
        msg.debug('Will retrieve metadata info for {0!s}'.format(myFiles))
        aftype = 'POOL'
        if self._type.upper() in ('BS', 'RAW'):
            aftype = 'BS'
        elif self._type.upper() in ('TAG'):
            aftype = 'TAG'
 
        # N.B. Could parallelise here            
        for fname in myFiles:
            athFileMetadata = AthenaLiteFileInfo(fname, aftype, retrieveKeys=retrieveKeys)
            if athFileMetadata is None:
                raise trfExceptions.TransformMetadataException(trfExit.nameToCode('TRF_METADATA_CALL_FAIL'), 'Call to AthenaLiteFileInfo failed')
            msg.debug('Setting metadata for file {0} to {1}'.format(fname, athFileMetadata[fname]))
            self._fileMetadata[fname].update(athFileMetadata[fname])
 
    
    def _getAthInfo(self, files):
        self._callAthInfo(files, doAllFiles = True, retrieveKeys=athFileInterestingKeys)
 
    @property
    def prodsysDescription(self):
        desc=super(argAthenaFile, self).prodsysDescription
        return desc
 
 

class argBSFile(argAthenaFile):    
 
    integrityFunction = "returnIntegrityOfBSFile"
 
    def _getIntegrity(self, files):
        for fname in files:
            try:
                rc=call(["AtlListBSEvents", "-c", fname], logger=msg, message="Report by AtlListBSEvents: ", timeout=600)
            except trfExceptions.TransformTimeoutException:
                return False
            if rc==0:
                self._fileMetadata[fname]['integrity'] = True
            else:
                self._fileMetadata[fname]['integrity'] = False
 
    @property
    def prodsysDescription(self):
        desc=super(argBSFile, self).prodsysDescription
        return desc
    
    
    def selfMerge(self, output, inputs, counter=0, argdict={}):
        msg.debug('selfMerge attempted for {0} -> {1} with {2} (index {3})'.format(inputs, output, argdict, counter))
        
        # First do a little sanity check
        for fname in inputs:
            if fname not in self._value:
                raise trfExceptions.TransformMergeException(trfExit.nameToCode('TRF_FILEMERGE_PROBLEM'), 
                                                            "File {0} is not part of this agument: {1}".format(fname, self))
        
        from PyJobTransforms.trfExe import bsMergeExecutor, executorConfig
        
        
        myargdict = self._mergeArgs(argdict)
        myargdict['maskEmptyInputs'] = argBool(True)
        myargdict['allowRename'] = argBool(True)
        myargdict['emptyStubFile'] = argString(inputs[0])
        
        # We need a athenaExecutor to do the merge
        # N.B. We never hybrid merge AthenaMP outputs as this would prevent further merging in another
        # task (hybrid merged files cannot be further bybrid merged) 
        myDataDictionary = {'BS_MRG_INPUT' : argBSFile(inputs, type=self.type, io='input'),
                            'BS_MRG_OUTPUT' : argBSFile(output, type=self.type, io='output')}
        myMergeConf = executorConfig(myargdict, myDataDictionary)
        myMerger = bsMergeExecutor(name='BSMergeAthenaMP{0}{1}'.format(self._subtype, counter), conf=myMergeConf, exe = 'file_merging',
                                  inData=set(['BS_MRG_INPUT']), outData=set(['BS_MRG_OUTPUT']))
        myMerger.doAll(input=set(['BS_MRG_INPUT']), output=set(['BS_MRG_OUTPUT']))
        
        # OK, if we got to here with no exceptions, we're good shape
        # Now update our own list of files to reflect the merge
        for fname in inputs:
            self._value.remove(fname)
        self._value.append(output)
 
        msg.debug('Post self-merge files are: {0}'.format(self._value))
        self._resetMetadata(inputs + [output])
        return myMerger
    
 

class argPOOLFile(argAthenaFile):
  
    integrityFunction = "returnIntegrityOfPOOLFile"
    
    # trfValidateRootFile is written in an odd way, so timelimit it here.
    @timelimited()
    def _getIntegrity(self, files):
        for fname in files:
            from PyJobTransforms.trfValidateRootFile import checkFile
            rc=checkFile(fileName=fname, the_type='event', requireTree=False)
            if rc==0:
                self._fileMetadata[fname]['integrity'] = True
            else:
                self._fileMetadata[fname]['integrity'] = False
 
    @property
    def prodsysDescription(self):
        desc=super(argPOOLFile, self).prodsysDescription
        return desc
 
    
    def selfMerge(self, output, inputs, counter=0, argdict={}):
        msg.debug('selfMerge attempted for {0} -> {1} with {2}'.format(inputs, output, argdict))
        
        # First do a little sanity check
        for fname in inputs:
            if fname not in self._value:
                raise trfExceptions.TransformMergeException(trfExit.nameToCode('TRF_FILEMERGE_PROBLEM'), 
                                                            "File {0} is not part of this agument: {1}".format(fname, self))
        
        from PyJobTransforms.trfExe import athenaExecutor, executorConfig
        
        
        myargdict = self._mergeArgs(argdict) 
        
        # We need a athenaExecutor to do the merge
        # N.B. We never hybrid merge AthenaMP outputs as this would prevent further merging in another
        # task (hybrid merged files cannot be further bybrid merged) 
        myDataDictionary = {'POOL_MRG_INPUT' : argPOOLFile(inputs, type=self.type, io='input'),
                            'POOL_MRG_OUTPUT' : argPOOLFile(output, type=self.type, io='output')}
        myMergeConf = executorConfig(myargdict, myDataDictionary)
        myMerger = athenaExecutor(name='POOLMergeAthenaMP{0}{1}'.format(self._subtype, counter), conf=myMergeConf, 
                                  skeletonCA = 'RecJobTransforms.MergePool_Skeleton',
                                  inData=set(['POOL_MRG_INPUT']), outData=set(['POOL_MRG_OUTPUT']),
                                  disableMT=True, disableMP=True)
        myMerger.doAll(input=set(['POOL_MRG_INPUT']), output=set(['POOL_MRG_OUTPUT']))
        
        # OK, if we got to here with no exceptions, we're good shape
        # Now update our own list of files to reflect the merge
        for fname in inputs:
            self._value.remove(fname)
        self._value.append(output)
 
        msg.debug('Post self-merge files are: {0}'.format(self._value))
        self._resetMetadata(inputs + [output])
        return myMerger
 
class argHITSFile(argPOOLFile):
 
    integrityFunction = "returnIntegrityOfPOOLFile"
 
    
    def selfMerge(self, output, inputs, counter=0, argdict={}):
        msg.debug('selfMerge attempted for {0} -> {1} with {2}'.format(inputs, output, argdict))
        
        # First do a little sanity check
        for fname in inputs:
            if fname not in self._value:
                raise trfExceptions.TransformMergeException(trfExit.nameToCode('TRF_FILEMERGE_PROBLEM'), 
                                                            "File {0} is not part of this agument: {1}".format(fname, self))
        
        
        mySubstepName = 'HITSMergeAthenaMP{0}'.format(counter)
        myargdict = self._mergeArgs(argdict)
        
        from PyJobTransforms.trfExe import athenaExecutor, executorConfig
        myDataDictionary = {'HITS' : argHITSFile(inputs, type=self.type, io='input'),
                            'HITS_MRG' : argHITSFile(output, type=self.type, io='output')}
        myMergeConf = executorConfig(myargdict, myDataDictionary)
        myMerger = athenaExecutor(name = mySubstepName,
                                  skeletonCA = 'SimuJobTransforms.HITSMerge_Skeleton',
                                  conf=myMergeConf, 
                                  inData=set(['HITS']), outData=set(['HITS_MRG']),
                                  disableMT=False, disableMP=True)
        myMerger.doAll(input=set(['HITS']), output=set(['HITS_MRG']))
        
        # OK, if we got to here with no exceptions, we're good shape
        # Now update our own list of files to reflect the merge
        for fname in inputs:
            self._value.remove(fname)
        self._value.append(output)
 
        msg.debug('Post self-merge files are: {0}'.format(self._value))
        self._resetMetadata(inputs + [output])
        return myMerger
 
 
class argEVNT_TRFile(argPOOLFile):
 
    integrityFunction = "returnIntegrityOfPOOLFile"
 
    
    def selfMerge(self, output, inputs, counter=0, argdict={}):
        msg.debug('selfMerge attempted for {0} -> {1} with {2}'.format(inputs, output, argdict))
        
        # First do a little sanity check
        for fname in inputs:
            if fname not in self._value:
                raise trfExceptions.TransformMergeException(trfExit.nameToCode('TRF_FILEMERGE_PROBLEM'), 
                                                            "File {0} is not part of this agument: {1}".format(fname, self))
        
        
        mySubstepName = 'EVNT_TRMergeAthenaMP{0}'.format(counter)
        myargdict = self._mergeArgs(argdict)
        
        from PyJobTransforms.trfExe import athenaExecutor, executorConfig
        myDataDictionary = {'EVNT_TR' : argEVNT_TRFile(inputs, type=self.type, io='input'),
                            'EVNT_TR_MRG' : argEVNT_TRFile(output, type=self.type, io='output')}
        myMergeConf = executorConfig(myargdict, myDataDictionary)
        myMerger = athenaExecutor(name = mySubstepName, skeletonFile = 'SimuJobTransforms/skeleton.EVNT_TRMerge.py',
                                  conf=myMergeConf, 
                                  inData=set(['EVNT_TR']), outData=set(['EVNT_TR_MRG']),
                                  disableMT=False, disableMP=True)
        myMerger.doAll(input=set(['EVNT_TR']), output=set(['EVNT_TR_MRG']))
        
        # OK, if we got to here with no exceptions, we're good shape
        # Now update our own list of files to reflect the merge
        for fname in inputs:
            self._value.remove(fname)
        self._value.append(output)
 
        msg.debug('Post self-merge files are: {0}'.format(self._value))
        self._resetMetadata(inputs + [output])
        return myMerger
 
 
class argRDOFile(argPOOLFile):
 
    integrityFunction = "returnIntegrityOfPOOLFile"
 
    
    def selfMerge(self, output, inputs, counter=0, argdict={}):
        msg.debug('selfMerge attempted for {0} -> {1} with {2}'.format(inputs, output, argdict))
        
        # First do a little sanity check
        for fname in inputs:
            if fname not in self._value:
                raise trfExceptions.TransformMergeException(trfExit.nameToCode('TRF_FILEMERGE_PROBLEM'), 
                                                            "File {0} is not part of this agument: {1}".format(fname, self))
 
        
        myargdict = self._mergeArgs(argdict) 
        
        from PyJobTransforms.trfExe import athenaExecutor, executorConfig
        myDataDictionary = {'RDO' : argHITSFile(inputs, type=self.type, io='input'),
                            'RDO_MRG' : argHITSFile(output, type=self.type, io='output')}
        myMergeConf = executorConfig(myargdict, myDataDictionary)
        myMerger = athenaExecutor(name = 'RDOMergeAthenaMP{0}'.format(counter),
                                  skeletonCA = 'SimuJobTransforms.RDOMerge_Skeleton',
                                  conf=myMergeConf, 
                                  inData=set(['RDO']), outData=set(['RDO_MRG']),
                                  disableMT=False, disableMP=True)
        myMerger.doAll(input=set(['RDO']), output=set(['RDO_MRG']))
        
        # OK, if we got to here with no exceptions, we're good shape
        # Now update our own list of files to reflect the merge
        for fname in inputs:
            self._value.remove(fname)
        self._value.append(output)
 
        msg.debug('Post self-merge files are: {0}'.format(self._value))
        self._resetMetadata(inputs + [output])
        return myMerger
    
class argEVNTFile(argPOOLFile):
 
    integrityFunction = "returnIntegrityOfPOOLFile"
 
    
    def selfMerge(self, output, inputs, counter=0, argdict={}):
        msg.debug('selfMerge attempted for {0} -> {1} with {2}'.format(inputs, output, argdict))
        
        # First do a little sanity check
        for fname in inputs:
            if fname not in self._value:
                raise trfExceptions.TransformMergeException(trfExit.nameToCode('TRF_FILEMERGE_PROBLEM'), 
                                                            "File {0} is not part of this agument: {1}".format(fname, self))
        
        
        mySubstepName = 'EVNTMergeAthenaMP{0}'.format(counter)
        myargdict = self._mergeArgs(argdict)
        
        from PyJobTransforms.trfExe import athenaExecutor, executorConfig
        myDataDictionary = {'EVNT' : argEVNTFile(inputs, type=self.type, io='input'),
                            'EVNT_MRG' : argEVNTFile(output, type=self.type, io='output')}
        myMergeConf = executorConfig(myargdict, myDataDictionary)
        myMerger = athenaExecutor(name = mySubstepName, skeletonCA = 'EvgenJobTransforms.EVNTMerge_Skeleton',
                                  conf=myMergeConf,
                                  inData=set(['EVNT']), outData=set(['EVNT_MRG']),
                                  disableMT=False, disableMP=True)
        myMerger.doAll(input=set(['EVNT']), output=set(['EVNT_MRG']))
        
        # OK, if we got to here with no exceptions, we're good shape
        # Now update our own list of files to reflect the merge
        for fname in inputs:
            self._value.remove(fname)
        self._value.append(output)
 
        msg.debug('Post self-merge files are: {0}'.format(self._value))
        self._resetMetadata(inputs + [output])
        return myMerger
    
 

class argHISTFile(argFile):
 
    integrityFunction = "returnIntegrityOfHISTFile"
 
    def __init__(self, value=list(), io = 'output', type=None, subtype=None, splitter=',', runarg=True, countable=True, multipleOK = None,
                 name=None, auxiliaryFile=False):
        super(argHISTFile, self).__init__(value=value, io=io, type=type, subtype=subtype, splitter=splitter, runarg=runarg, multipleOK=multipleOK,
                                          name=name, auxiliaryFile=auxiliaryFile)
 
        # Make events optional for HISTs (can be useful for HIST_AOD, HIST_ESD before hist merging)
        if countable:
            self._metadataKeys.update({
                                       'nentries': self._getNumberOfEvents
                                       })
        
    
    def _getIntegrity(self, files):
        for fname in files:
            self._fileMetadata[fname]['integrity'] = 'UNDEFINED'
        
        
    def _getNumberOfEvents(self, files):
        for fname in files:
            try:
                self._fileMetadata[fname]['nentries'] = HISTEntries(fname)
            except trfExceptions.TransformTimeoutException:
                msg.error('Timeout counting events for {0}'.format(fname))
 
    @property
    def prodsysDescription(self):
        desc=super(argHISTFile, self).prodsysDescription
        return desc
    
 

class argNTUPFile(argFile):
 
    integrityFunction = "returnIntegrityOfNTUPFile"
 
    
    def __init__(self, value=list(), io = 'output', type=None, subtype=None, splitter=',', treeNames=None, runarg=True, multipleOK = None, 
                 name=None, mergeTargetSize=-1, auxiliaryFile=False):
        super(argNTUPFile, self).__init__(value=value, io=io, type=type, subtype=subtype, splitter=splitter, runarg=runarg, multipleOK=multipleOK, 
                                          name=name, mergeTargetSize=mergeTargetSize, auxiliaryFile=auxiliaryFile)
        self._treeNames=treeNames
 
        self._metadataKeys.update({
                                   'nentries': self._getNumberOfEvents,
                                   'file_guid': self._generateGUID,
                                   'integrity': self._getIntegrity,
                                   })
 
        if name and 'NTUP_PILEUP' in name:
            self._metadataKeys.update({
                                       'sumOfWeights': self._getNumberOfEvents,
                                       })
 
    def _getNumberOfEvents(self, files):
        msg.debug('Retrieving event count for NTUP files {0}'.format(files))
        if self._treeNames is None:
            for fname in files:
                # Attempt to treat this as a pileup reweighting file
                myPRWEntries = PRWEntries(fileName=fname)
                if myPRWEntries is not None:
                    self._fileMetadata[fname]['nentries'] = myPRWEntries
                    if self.name and 'NTUP_PILEUP' in self.name:
                        myPRWEntries = PRWEntries(fileName=fname, integral=True)
                        self._fileMetadata[fname]['sumOfWeights'] = myPRWEntries
                else:
                    # Attempt to treat this as a PHYSVAL file
                    myPHYSVALEntries = PHYSVALEntries(fileName=fname)
                    if myPHYSVALEntries is not None:
                        self._fileMetadata[fname]['nentries'] = myPHYSVALEntries
                        if self.name and 'NTUP_PHYSVAL' in self.name:
                            myPHYSVALEntries = PHYSVALEntries(fileName=fname, integral=True)
                            self._fileMetadata[fname]['sumOfWeights'] = myPHYSVALEntries
                    else:
                        msg.debug('treeNames is set to None - event count undefined for this NTUP')
                        self._fileMetadata[fname]['nentries'] = 'UNDEFINED'
        else:
            for fname in files:
                try:
                    self._fileMetadata[fname]['nentries'] = NTUPEntries(fileName=fname, treeNames=self._treeNames)
                except trfExceptions.TransformTimeoutException:
                    msg.error('Timeout counting events for {0}'.format(fname))
 
 
    def _getIntegrity(self, files):
        for fname in files:
            from PyJobTransforms.trfValidateRootFile import checkFile
            rc=checkFile(fileName=fname, the_type='basket', requireTree=False)
            if rc==0:
                self._fileMetadata[fname]['integrity'] = True
            else:
                self._fileMetadata[fname]['integrity'] = False
                
                
    def selfMerge(self, output, inputs, counter=0, argdict={}):
        msg.debug('selfMerge attempted for {0} -> {1} with {2}'.format(inputs, output, argdict))
        
        # First do a little sanity check
        for fname in inputs:
            if fname not in self._value:
                raise trfExceptions.TransformMergeException(trfExit.nameToCode('TRF_FILEMERGE_PROBLEM'), 
                                                            "File {0} is not part of this agument: {1}".format(fname, self))
        
        from PyJobTransforms.trfExe import NTUPMergeExecutor, executorConfig
        
        
        myargdict = self._mergeArgs(argdict) 
        
        # We need a NTUPMergeExecutor to do the merge
        myDataDictionary = {'NTUP_MRG_INPUT' : argNTUPFile(inputs, type=self.type, io='input'),
                            'NYUP_MRG_OUTPUT' : argNTUPFile(output, type=self.type, io='output')}
        myMergeConf = executorConfig(myargdict, myDataDictionary)
        myMerger = NTUPMergeExecutor(name='NTUPMergeAthenaMP{0}{1}'.format(self._subtype, counter), conf=myMergeConf, 
                                     inData=set(['NTUP_MRG_INPUT']), outData=set(['NTUP_MRG_OUTPUT']))
        myMerger.doAll(input=set(['NTUP_MRG_INPUT']), output=set(['NYUP_MRG_OUTPUT']))
        
        # OK, if we got to here with no exceptions, we're good shape
        # Now update our own list of files to reflect the merge
        for fname in inputs:
            self._value.remove(fname)
        self._value.append(output)
 
        msg.debug('Post self-merge files are: {0}'.format(self._value))
        self._resetMetadata(inputs + [output])
        return myMerger
                
    @property
    def prodsysDescription(self):
        desc=super(argNTUPFile, self).prodsysDescription
        return desc  
 
 
 

class argBZ2File(argFile):
    def _getIntegrity(self, files):
        for fname in files:
            # bz2 only supports 'with' from python 2.7
            try:
                f = bz2.BZ2File(fname, 'r')
                while True:
                    chunk = len(f.read(1024*1024))
                    msg.debug('Read {0} bytes from {1}'.format(chunk, fname)) 
                    if chunk == 0:
                        break
                self._fileMetadata[fname]['integrity'] = True
                f.close()
            except OSError as e:
                msg.error('Got exception {0!s} raised while checking integrity of file {1}'.format(e, fname))
                self._fileMetadata[fname]['integrity'] = False
                
 
    @property
    def prodsysDescription(self):
        desc=super(argBZ2File, self).prodsysDescription
        return desc  
 
 

class argFTKIPFile(argBZ2File):
    def __init__(self, value=list(), io = 'output', type=None, splitter=',', runarg=True, multipleOK=None, name=None):
        super(argFTKIPFile, self).__init__(value=value, io=io, type=type, splitter=splitter, runarg=runarg, multipleOK=multipleOK,
                                           name=name)
        self._metadataKeys.update({
                                   'nentries': self._getNumberOfEvents
                                   })
 
    def _getNumberOfEvents(self, files):
        for fname in files:
            try:
                eventCount = 0
                f = bz2.BZ2File(fname, 'r')
                for line in f:
                    if line.startswith('F'):
                        eventCount += 1
                self._fileMetadata[fname]['nentries'] = eventCount
            except OSError as e:
                msg.error('Event count for file {0} failed: {1!s}'.format(fname, e))
                self._fileMetadata[fname]['nentries'] = None
 
    @property
    def prodsysDescription(self):
        desc=super(argFTKIPFile, self).prodsysDescription
        return desc
    

class argHepEvtAsciiFile(argFile):
    def __init__(self, value=list(), io = 'output', type='txt_evt', splitter=',', runarg=True, multipleOK=None, name=None):
        super(argHepEvtAsciiFile, self).__init__(value=value, io=io, type=type, splitter=splitter, runarg=runarg, 
                                                 multipleOK=multipleOK, name=name)
        self._metadataKeys.update({
                                   'nentries': self._getNumberOfEvents
                                   })
        
    def _getNumberOfEvents(self, files):
        for fname in files:
            try:
                eventCount = 0
 
                if '.tar.gz' in fname or '.tgz' in fname:
                    import tarfile
                    with tarfile.open(fname, "r:gz") as tar:
                        for untar in tar.getmembers():
                            fileTXT = tar.extractfile(untar)
                            if fileTXT is not None:
                                # Iterate line-by-line to avoid memory explosion
                                for aline in fileTXT:
                                    if aline.startswith(b'E '):
                                        eventCount += 1
                elif '.gz' in fname:
                    import gzip
                    with gzip.open(fname,'rb') as gzin:
                        for aline in gzin:
                            if aline.startswith(b'E '):
                                eventCount += 1
                else:
                    # Assume uncompressed
                    with open(fname,'r') as infile:
                        for aline in infile:
                            if aline.startswith('E '):
                                eventCount += 1
                self._fileMetadata[fname]['nentries'] = eventCount
            except OSError as e:
                msg.error('Event count for file {0} failed: {1!s}'.format(fname, e))
                self._fileMetadata[fname]['nentries'] = 'UNDEFINED'
 

class argLHEFile(argFile):
    def __init__(self, value=list(), io = 'output', type=None, splitter=',', runarg=True, multipleOK=None, name=None):
        super(argLHEFile, self).__init__(value=value, io=io, type=type, splitter=splitter, runarg=runarg, multipleOK=multipleOK,
                                           name=name)
 
        self._metadataKeys.update({
                'nentries': self._getNumberOfEvents,
                'lheSumOfPosWeights': self._getWeightedEvents,
                'lheSumOfNegWeights': 0,
                })
 
    def _getNumberOfEvents(self, files):
        msg.debug('Retrieving event count for LHE file {0}'.format(files))
        import tarfile
        for fname in files:
            # Decompress this as we read
            try :
                tar = tarfile.open(fname, "r:gz")
                lhecount = 0
                for untar in tar.getmembers():
                    fileTXT = tar.extractfile(untar)
                    if fileTXT is not None :
                        lines = fileTXT.read().decode("utf-8")
                        lhecount = lines.count('/event')
 
                self._fileMetadata[fname]['nentries'] = lhecount
            except Exception:
                msg.debug('Entries is set to None - event count undefined for this LHE')
                self._fileMetadata[fname]['nentries'] = 'UNDEFINED'
 
    def _getWeightedEvents(self, files):
        msg.debug('Retrieving weight count for LHE file {0}'.format(files))
        import tarfile
        import re
 
        for fname in files:
            weightPos = 0
            weightNeg = 0
            try :
                tar = tarfile.open(fname, "r:gz")
                for untar in tar.getmembers():
                    fileTXT = tar.extractfile(untar)
                    next = False
                    if fileTXT is not None :
                        lines = fileTXT.readlines()
                        for line in lines :
                            if next :
                                try :
                                    w = float(re.sub(' +',' ',line).split(" ")[2])
                                    if w > 0 : weightPos += w
                                    else : weightNeg += abs(w)
                                except Exception:
                                    pass
                                next = False
                            if "<event" in line :
                                next = True
 
                self._fileMetadata[fname]['lheSumOfPosWeights'] = weightPos
                self._fileMetadata[fname]['lheSumOfNegWeights'] = weightNeg
            except Exception:
                msg.debug('Entries is set to None - negative fraction count undefined for this LHE')
                self._fileMetadata[fname]['lheSumOfPosWeights'] = 'UNDEFINED'
                self._fileMetadata[fname]['lheSumOfNegWeights'] = 'UNDEFINED'
 

class argSubstep(argument):
    
    
    def __init__(self, value = None, runarg = True, name = None, defaultSubstep = 'all', separator = ':'):
        self._defaultSubstep = defaultSubstep
        self._separator = separator
        super(argSubstep, self).__init__(value, runarg, name)
    
    # Reset getter
    @property
    def value(self):
        return self._value
 
    # The default setter for sustep class
    @value.setter
    def value(self, value):
        msg.debug('Attempting to set argSubstep from {0!s} (type {1}'.format(value, type(value)))
        if value is None:
            self._value = {}
        elif isinstance(value, str):
            self._value = dict(self._parseStringAsSubstep(value))
        elif isinstance(value, (list, tuple)):
            # This is a list of strings to parse, so we go through them one by one
            self._value = {}
            for item in value:
                if not isinstance(item, str):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Failed to convert list item {0!s} to substep (should be a string)'.format(item))
                self._value.update(dict(self._parseStringAsSubstep(item)))
        elif isinstance(value, dict):
            self._value = value
        else:
            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Setter value {0!s} (type {1}) for substep argument cannot be parsed'.format(value, type(value)))
            
    
    
    def _parseStringAsSubstep(self, string):
        subStepMatch = re.match(r'([a-zA-Z0-9,]+)' + self._separator + r'(.*)', string)
        subStepList = []
        if subStepMatch:
            subStep = subStepMatch.group(1).split(',')
            subStepValue = subStepMatch.group(2)
        else:
            subStep = [self._defaultSubstep]
            subStepValue = string
        msg.debug('Parsed {0} as substep {1}, argument {2}'.format(string, subStep, subStepValue))
        for step in subStep:
            subStepList.append((step, subStepValue))
        return subStepList
    
    
    
    def returnMyValue(self, name=None, substep=None, first=False, exe=None):
        if exe:
            name = exe.name
            substep = exe.substep
            first = exe.conf.firstExecutor
 
        name = commonExecutorStepName(name)
 
        value = None
        
        if name in self._value:
            value = self._value[name]
        elif substep in self._value:
            value = self._value[substep]  
        elif first and 'first' in self._value:
            value = self._value['first']
        elif 'default' in self._value:
            value = self._value['default']
            
        
        if 'all' in self._value:
            if value is None:
                value = self._value['all']
            elif isinstance(value, list):
                value = self._value['all'] + value
                
        msg.debug('From substep argument {myvalue} picked value "{value}" for {name}, {substep}, first={first}'.format(myvalue=self._value, value=value, name=name, substep=substep, first=first))
        
        return value
 
    @property
    def prodsysDescription(self):
        desc = {'type': 'substep', 'substeptype': 'str', 'separator': self._separator,
                'default': self._defaultSubstep}
        return desc
    

class argSubstepList(argSubstep):
    
    
    def __init__(self, value = None, runarg = True, name = None, defaultSubstep = 'all', splitter = None, separator=':'):
        self._splitter = splitter
        super(argSubstepList, self).__init__(value, runarg, name, defaultSubstep, separator)
 
        
    # Reset getter
    @property
    def value(self):
        return self._value
 
    @property
    def prodsysDescription(self):
        desc = {'type': 'substep', 'substeptype': 'list', 'listtype': 'str',
                'separator': self._separator,
                'default': self._defaultSubstep}
        return desc
    @value.setter
    def value(self, value):
        msg.debug('Attempting to set argSubstep from {0!s} (type {1}'.format(value, type(value)))
        if value is None:
            self._value = {}
        elif isinstance(value, str):
            self._value = dict(self._parseStringAsSubstep(value))
        elif isinstance(value, (list, tuple)):
            # This is a list of strings to parse
            self._value = {}
            for item in value:
                if not isinstance(item, str):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Failed to convert list item {0!s} to substep (should be a string)'.format(item))
                subStepList = self._parseStringAsSubstep(item)
                for subStep in subStepList:
                    if subStep[0] in self._value:
                        self._value[subStep[0]].extend(subStep[1])
                    else:
                        self._value[subStep[0]] = subStep[1]
        elif isinstance(value, dict):
            for k, v in value.items():
                if not isinstance(k, str):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Dictionary key {0!s} for substep is not a string'.format(k))
                if not isinstance(v, list):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Dictionary value {0!s} for substep is not a list'.format(v))
            self._value = value
        else:
            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Setter value {0!s} (type {1}) for substep argument cannot be parsed'.format(value, type(value)))
 
    
    def _parseStringAsSubstep(self, value):
        subStepList = super(argSubstepList, self)._parseStringAsSubstep(value)
        if self._splitter:
            subStepList = [(s[0], s[1].split(self._splitter)) for s in subStepList]
        else:
            subStepList = [(s[0], [s[1]]) for s in subStepList]
        return subStepList
 

class argSubstepString(argSubstep):
 
    # Reset getter
    @property
    def value(self):
        return self._value
 
    @property
    def prodsysDescription(self):
        desc = {'type': 'substep', 'substeptype': 'str', 'separator': self._separator,
                'default': self._defaultSubstep}
        return desc
 
    @value.setter
    def value(self, value):
        msg.debug('Attempting to set argSubstep from {0!s} (type {1}'.format(value, type(value)))
        if value is None:
            self._value = {}
        elif isinstance(value, str):
            subStepList = self._parseStringAsSubstep(value)
            self._value = dict([(subStep[0], subStep[1]) for subStep in subStepList])
        elif isinstance(value, (list, tuple)):
            # This is a list of strings to parse
            self._value = {}
            for item in value:
                if not isinstance(item, str):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Failed to convert list item {0!s} to substep (should be a string)'.format(item))
                subStepList = self._parseStringAsSubstep(item)
                for subStep in subStepList:
                    self._value[subStep[0]] = subStep[1]
        elif isinstance(value, dict):
            for k, v in value.items():
                if not isinstance(k, str):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Dictionary key {0!s} for substep is not a string'.format(k))
                if not isinstance(v, str):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Dictionary value {0!s} for substep is not a string'.format(v))
            self._value = value
        else:
            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Setter value {0!s} (type {1}) for substep argument cannot be parsed'.format(value, type(value)))
 

class argSubstepBool(argSubstep):
    
    # Reset getter
    @property
    def value(self):
        return self._value
 
    @property
    def prodsysDescription(self):
        desc = {'type': 'substep', 'substeptype': 'bool', 'separator': self._separator,
                'default': self._defaultSubstep}
        return desc
    
    @value.setter
    def value(self, value):
        msg.debug('Attempting to set argSubstep from {0!s} (type {1})'.format(value, type(value)))
        if value is None:
            self._value = {self._defaultSubstep: True}
        elif isinstance(value, bool):
            self._value = {self._defaultSubstep: value}
        elif isinstance(value, str):
            subStepList = self._parseStringAsSubstep(value)
            self._value = dict([(subStep[0], strToBool(subStep[1])) for subStep in subStepList])
        elif isinstance(value, (list, tuple)):
            # This is a list of strings to parse
            self._value = {}
            for item in value:
                if not isinstance(item, str):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Failed to convert list item {0!s} to substep (should be a string)'.format(item))
                subStepList = self._parseStringAsSubstep(item)
                for subStep in subStepList:
                    self._value[subStep[0]] = strToBool(subStep[1])
        elif isinstance(value, dict):
            for k, v in value.items():
                if not isinstance(k, str):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Dictionary key {0!s} for substep is not a string'.format(k))
                if not isinstance(v, bool):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Dictionary value {0!s} for substep is not a bool'.format(v))
            self._value = value
        else:
            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Setter value {0!s} (type {1}) for substep argument cannot be parsed'.format(value, type(value)))
 
 

class argSubstepInt(argSubstep):
    
    # Reset getter
    @property
    def value(self):
        return self._value
 
    @property
    def prodsysDescription(self):
        desc = {'type': 'substep', 'substeptype': 'int', 'separator': self._separator,
                'default': self._defaultSubstep}
        return desc
   
    @value.setter
    def value(self, value):
        msg.debug('Attempting to set argSubstep from {0!s} (type {1}'.format(value, type(value)))
        try:
            if value is None:
                self._value = {}
            elif isinstance(value, int):
                self._value = {self._defaultSubstep: value}
            elif isinstance(value, str):
                subStepList = self._parseStringAsSubstep(value)
                self._value = dict([(subStep[0], int(subStep[1])) for subStep in subStepList])
            elif isinstance(value, (list, tuple)):
                # This is a list of strings to parse
                self._value = {}
                for item in value:
                    if not isinstance(item, str):
                        raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Failed to convert list item {0!s} to substep (should be a string)'.format(item))
                    subStepList = self._parseStringAsSubstep(item)
                    for subStep in subStepList:
                        self._value[subStep[0]] = int(subStep[1])
            elif isinstance(value, dict):
                for k, v in value.items():
                    if not isinstance(k, str):
                        raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Dictionary key {0!s} for substep is not a string'.format(k))
                    if not isinstance(v, int):
                        raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Dictionary value {0!s} for substep is not an int'.format(v))
                self._value = value
            else:
                raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Setter value {0!s} (type {1}) for substep argument cannot be parsed'.format(value, type(value)))
        except ValueError:
            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Failed to convert substep value {0} to int'.format(value))
 
 
 

class argSubstepFloat(argSubstep):
    
    def __init__(self, value=None, min=None, max=None, runarg=True, name=None):
        self._min = min
        self._max = max
        super(argSubstepFloat, self).__init__(value = value, runarg = runarg, name=name)
        
    @property
    def prodsysDescription(self):
        desc = {'type': 'substep', 'substeptype': 'float', 'separator': self._separator,
                'default': self._defaultSubstep}     
        if self._min:
            desc['min'] = self._min
        if self._max:
            desc['max'] = self._max
        return desc
    
        
    # Reset getter
    @property
    def value(self):
        return self._value
    
    @value.setter
    def value(self, value):
        msg.debug('Attempting to set argSubstep from {0!s} (type {1}'.format(value, type(value)))
        try:
            if value is None:
                self._value = {}
            elif isinstance(value, float):
                self._value = {self._defaultSubstep: value}              
            elif isinstance(value, str):
                subStepList = self._parseStringAsSubstep(value)
                self._value = dict([(subStep[0], float(subStep[1])) for subStep in subStepList])
            elif isinstance(value, (list, tuple)):
                # This is a list of strings to parse
                self._value = {}
                for item in value:
                    if not isinstance(item, str):
                        raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 
                                                                  'Failed to convert list item {0!s} to substep (should be a string)'.format(item))
                    subStepList = self._parseStringAsSubstep(item)
                    for subStep in subStepList:
                        self._value[subStep[0]] = float(subStep[1])
            elif isinstance(value, dict):
                for k, v in value.items():
                    if not isinstance(k, str):
                        raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 
                                                                  'Dictionary key {0!s} for substep is not a string'.format(k))
                    if not isinstance(v, float):
                        raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 
                                                                  'Dictionary value {0!s} for substep is not an float'.format(v))
                self._value = value
            else:
                raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 
                                                          'Setter value {0!s} (type {1}) for substep argument cannot be parsed'.format(value, type(value)))
            # Now do min/max checks
            for my_float in self._value.values():
                if (self._min is not None and my_float < self._min) or (self._max is not None and my_float > self._max):
                        raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_OUT_OF_RANGE'),
                                                              'argFloat value out of range: {0} is not between {1} and {2}'.format(my_float, self._min, self._max))
        except ValueError as e:
            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 
                                                      'Failed to convert substep value {0} to float: {1}'.format(value, e))
 
 

class argSubstepSteering(argSubstep):
    # This singleton is where we define some aliases for common production
    # usecases of steering. 
    # "no" - a convenience null option for production managers, does nothing
    # "doRDO_TRIG" - run split trigger for Reco_tf and friends
    # "doOverlay" - run event overlay on presampled RDOs instead of standard HITtoRDO digitization
    # "doFCtoDAOD" - run the FastChain transform, including the Derivation step, to process data from EVNT to DAOD
    # "afterburn" - run the B decay afterburner for event generation
    # "doRAWtoALL" - (deprecated) produce all DESDs and AODs directly from bytestream
    # "doTRIGtoALL" - (deprecated) produce AODs directly from trigger RDOs
    steeringAlises = {
                      'no': {},
                      'doRDO_TRIG': {'RAWtoALL': [('in', '-', 'RDO'), ('in', '+', 'RDO_TRIG'), ('in', '-', 'BS')]},
                      'doOverlay': {'HITtoRDO': [('in', '-', 'HITS'), ('out', '-', 'RDO'), ('out', '-', 'RDO_FILT')],
                                    'Overlay': [('in', '+', ('HITS', 'RDO_BKG')), ('out', '+', 'RDO')]},
                      'doFCtoDAOD': {'Derivation': [('in', '-', 'EVNT')]},
                      'afterburn': {'generate': [('out', '-', 'EVNT')]},
                      'doRAWtoALL': {},
                      'doTRIGtoALL': {}
                      }
    
    # Reset getter
    @property
    def value(self):
        return self._value
    
    # This argument gets dumped in a special way, using an alias directly
    # instead of the expanded value
    @property
    def dumpvalue(self):
        return self._dumpvalue
 
    @property
    def prodsysDescription(self):
        desc = {'type': 'substep', 'substeptype': 'steering', 'listtype': 'str', 'separator': self._separator,
                'default': self._defaultSubstep}
        return desc
    
    
    @value.setter
    def value(self, value):
        msg.debug('Attempting to set argSubstepSteering from {0!s} (type {1})'.format(value, type(value)))
        if value is None:
            self._value = {}
            self._dumpvalue = [""]
        elif isinstance(value, dict):
            # OK, this should be the direct setable dictionary - but do a check of that
            for k, v in value.items():
                if not isinstance(k, str) or not isinstance(v, list):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 
                                                              'Failed to convert dict {0!s} to argSubstepSteering'.format(value))
                for subv in v:
                    if not isinstance(subv, (list, tuple)) or len(subv) != 3 or subv[0] not in ('in', 'out') or subv[1] not in ('+', '-'):                    
                        raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 
                                                                  'Failed to convert dict {0!s} to argSubstepSteering'.format(value))                    
            self._value = value
            # Note we are a little careful here to never reset the dumpvalue - this is 
            # because when processing the _list_ of steering arguments down to a single
            # multi-valued argument we re-call value() with an expanded diectionary and
            # one can nievely reset dumpvalue by mistake
            self._dumpvalue = getattr(self, "_dumpvalue", value)
        elif isinstance(value, (str, list, tuple)):
            if isinstance(value, str):
                value = [value,]
            self._dumpvalue = getattr(self, "_dumpvalue", value)
            # Now we have a list of strings to parse
            self._value = {}
            for item in value:
                if not isinstance(item, str):
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 
                                                              'Failed to convert list item {0!s} to substep (should be a string)'.format(item))
                if item in argSubstepSteering.steeringAlises:
                    msg.debug("Found value {0} in steeringAlises ({1})".format(item, argSubstepSteering.steeringAlises[item]))
                    for substep, steerlist in argSubstepSteering.steeringAlises[item].items():
                        if substep in self._value:
                            self._value[substep].extend(steerlist)
                        else:
                            self._value[substep] = steerlist
                else:
                    subStepList = self._parseStringAsSubstep(item)
                    self._value.update(dict([(subStep[0], self._parseSteeringString(subStep[1])) for subStep in subStepList]))
        else:
            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 
                                                      'Setter value {0!s} (type {1}) for substep argument cannot be parsed'.format(value, type(value)))
 
    def _parseSetterString(self, string):
        if string in argSubstepSteering.steeringAlises:
            return argSubstepSteering.steeringAlises[string]
 
    def _parseSteeringString(self, ivalue):
        retvalue = []
        for subvalue in ivalue.split(','):
            matchedParts = re.match(r'(in|out)(\+|\-)([A-Z_]+)$', subvalue)
            if not matchedParts:
                raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 
                                                          'Failed to convert string {0!s} to argSubstepSteering'.format(subvalue))
            retvalue.append((matchedParts.group(1), matchedParts.group(2), matchedParts.group(3)))
        return retvalue
 
 

class argSubstepConditions(argSubstep):
    @property
    def value(self):
        return self._value
    
    @value.setter
    def value(self, value):
        msg.debug('Attempting to set argSubstepConditions from {0!s} (type {1}'.format(value, type(value)))
        # super().value = value workaround:
        super(self.__class__, self.__class__).value.fset(self, value)
        
        current = None
        for k, v in self._value.items():
            if "CurrentMC" == v:
                if current is None:
                    current = self._amiLookUp(getAMIClient())
                self._value[k] = current
 
    def _amiLookUp(self, client):
        cmd = "COMAGetGlobalTagNameByCurrentState --state=CurrentMC"
        return str(client.execute(cmd, format = 'dom_object').get_rows().pop()['globalTag'])
    
    @property
    def prodsysDescription(self):
        desc = {'type': 'substep', 'substeptype': 'str', 'separator': self._separator,
                'default': self._defaultSubstep}
        return desc
 
        
class trfArgParser(argparse.ArgumentParser):
 
    
    def __init__(self, *args, **kwargs):
        self._helpString = {}
        self._argClass = {}
        self._argGroups = {}
        self._argKeyGroups = {}
        self._argAlias = {}
        super(trfArgParser, self).__init__(*args, **kwargs)
 
    def add_argument(self, *args, **kwargs):
        argName = args[0].lstrip('-')
        msg.debug('Found arg name {0}'.format(argName))
 
        # Ban arguments with hyphens as they cause trouble in signature files and then
        # AMI tag definitions because of the auto-translation to underscores in argparse
        if '-' in argName:
            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_ERROR'),
                                                      'Transform arguments may not use hyphens (use camelCase or underscore')
        
        # Prevent a crash if this argument already exists (there are valid use cases for 'grabbing' an
        # argument, so this is DEBUG, not WARNING)
        if argName in self._argClass:
            msg.debug('Double definition of argument {0} - ignored'.format(argName))
            return
        
        # if there is a help function defined for the argument then populate the helpString dict
        if 'help' in kwargs: 
            self._helpString[argName] = kwargs['help'] # if the help option is present for the argument then put it into the helpString dict key = argument name, value = help 
        else:
            self._helpString[argName] = None
        if 'action' in kwargs and 'factory' in dir(kwargs['action']):
            self._argClass[argName] = kwargs['action'].factory
        elif 'type' in kwargs:
            self._argClass[argName] = kwargs['type']
        else:
            self._argClass[argName] = None
 
        # Remove kwargs which are not understood by ArgumentParser.add_argument()
        strippedArgs = {}
        for arg in ('group',):
            if arg in kwargs:
                strippedArgs[arg] = kwargs.pop(arg)
 
        # Setup aliases
        if len(args) > 1:
            for i in range(1, len(args)):
                argAlias = args[i].lstrip('-')
                msg.debug('Adding an alias of {0}: {1}'.format(argName, argAlias))
                self._argAlias[argAlias] = argName
 
        # Optinally add an argument to an argparse argument group
        if 'group' in strippedArgs:
            if strippedArgs['group'] in self._argGroups:
                msg.debug('Adding argument to group {0}: ({1}; {2})'.format(strippedArgs['group'], args, kwargs))
                self._argGroups[strippedArgs['group']].add_argument(*args, **kwargs)
                self._argKeyGroups[argName] = strippedArgs['group']
            else:
                msg.warning('Argument group {0} not defined - adding argument to main parser'.format(strippedArgs['group']))
                msg.debug('Adding argument: ({0}; {1})'.format(args, kwargs))
                super(trfArgParser, self).add_argument(*args, **kwargs)
        else:
            msg.debug('Adding argument: ({0}; {1})'.format(args, kwargs))
            super(trfArgParser, self).add_argument(*args, **kwargs)
 
    @property
    def getProdsysDesc(self):
        desc = {}
        for name, argClass in self._argClass.items():
            msg.debug('Detected the local variable {0}'.format(name))
            if argClass is not None:
                desc[name] = argClass().prodsysDescription
                if name in self._helpString:
                    desc[name].update({'help': self._helpString[name]})
                if name in self._argKeyGroups:
                    desc[name].update({'group':self._argKeyGroups[name]})
        return desc
            
    
    def defineArgGroup(self, *args):
        # Get an argparse group
        if args[0] in self._argGroups:
            msg.warning('Argument group %s already exists', args[0])
            return
        self._argGroups[args[0]] = self.add_argument_group(*args)
        
    
    def getHelpString(self, argument):
        try:
            return(self._helpString[argument])
        except KeyError:
            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_KEY_ERROR'), 'no help string available for argument %s' %argument)
        return None
 
    
    def dumpArgs(self):
        keyArray = [ '--' + str(key) for key in self._helpString if key not in ('h', 'verbose', 'loglevel', 'dumpargs', 'argdict') ]
        keyArray.sort()
        print('ListOfDefaultPositionalKeys={0}'.format(keyArray))
        
    
    @property
    def allArgs(self):
        return list(self._helpString)
 
    # @brief parsing helper
    def _parse_list_helper(self, value):
        # We build on the value[0] instance as this contains the correct metadata
        # and object references for this instance (shallow copying can
        # mess up object references and deepcopy thows exceptions!)
        newValueObj = value[0]
        msg.debug('Started with: %s = %s', type(newValueObj), newValueObj)
        if isinstance(value[0], argSubstep):
            # Make sure you do not have a reference to the original value - this is a deeper copy
            newValues = dictSubstepMerge(value[0].value, {})
        elif isinstance(value[0], list):
            if len(value) == 1:
                return self._parse_list_helper(value[0])
            msg.debug('Handling a list of arguments for key')
            newValues = []
            for v in value:
                processedValueObj, processedValues = self._parse_list_helper(v)
                processedValueObj.value = processedValues
                newValues.append(processedValueObj)
            newValueObj = newValues
            return newValueObj, newValues
        elif isinstance(value[0].value, list):
            newValues = value[0].value
        elif isinstance(value[0].value, dict):
            newValues = value[0].value
        else:
            newValues = [value[0].value,]
        for valueObj in value[1:]:
            msg.debug('Value Object: %s = %s', type(valueObj), valueObj)
            if isinstance(value[0], argSubstep):
                # Special merger for lists attached to substeps
                newValues = dictSubstepMerge(newValues, valueObj.value)
            elif isinstance(valueObj.value, list):
                # General lists are concatenated
                newValues.extend(valueObj.value)
            elif isinstance(valueObj.value, dict):
                # General dictionaries are merged
                newValues.update(valueObj.value)
            else:
                newValues.append(valueObj.value)
        return newValueObj, newValues
 
    
    def parse_args(self, args = None, namespace = None):
        if namespace:
            super(trfArgParser, self).parse_args(args = args, namespace = namespace)
        else:
            namespace = super(trfArgParser, self).parse_args(args = args)
        for k, v in namespace.__dict__.items():
            msg.debug('Treating key %s (%s)', k, v)
            if isinstance(v, list):
                newValueObj, newValues = self._parse_list_helper(v)
                if not isinstance(newValueObj, list):
                    newValueObj.value = newValues
                namespace.__dict__[k] = newValueObj
                msg.debug('Set to %s', newValues)
 
        return namespace
 
 

def strToBool(string):
    try:
        msg.debug("converting string {string} to boolean".format(string = string))
        if string.lower() == 'false':
            return False
        elif string.lower() == 'true':
            return True
        else:
            raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Failed to convert value {0} to bool'.format(string))
    except AttributeError:
        raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 'Failed to convert value {0} to bool'.format(string))
 

def dictSubstepMerge(dict1, dict2):
    mergeDict = {}
    allKeys = set(dict1) | set(dict2)
    # Find the value type - lists are special...
    listType = False
    if len(dict1) > 0:
        if isinstance(list(dict1.values())[0], list):
            listType = True
    elif len(dict2) > 0:
        if isinstance(list(dict2.values())[0], list):
            listType = True
    if listType:
        for key in allKeys:
            mergeDict[key] = dict1.get(key, []) + dict2.get(key, [])
    else:
        for key in allKeys:
            if key in dict1 and key in dict2:
                # Don't really know what to do if these clash...
                if dict1[key] != dict2[key]:
                    raise trfExceptions.TransformArgException(trfExit.nameToCode('TRF_ARG_CONV_FAIL'), 
                                                              'Merging substep arguments found clashing values for substep {0}: {1}!={2}'.format(key, dict1[key], dict2[key]))
                mergeDict[key] = dict1[key]
            elif key in dict1:
                mergeDict[key] = dict1[key]
            else:
                mergeDict[key] = dict2[key]
                
    return mergeDict