python.iconfTool.models.loaders Namespace Reference

Classes
class	ComponentsDiffFileLoader
class	ComponentsFileLoader

Functions
	__flatten_list (l)
	types_in_properties (comp_name, value, dict_to_update)
	collect_types (conf)
Dict	excludeIncludeComps (dic, args, depth, compsToFollow=[])
Dict	ignoreIrrelevant (dic, args)
Dict	renameComps (dic, args)
Dict	ignoreDefaults (allconf, args, known)
Dict	shortenDefaultComponents (dic, args)
list	isReference (value, compname, conf, svcCache={})
Dict	skipProperties (conf, args)
Dict	loadConfigFile (fname, args)
Dict	loadDifferencesFile (fname)

Variables
	logger = logging.getLogger("confTool")
	level
dict	componentRenamingDict = {}
	baseParser = argparse.ArgumentParser()
	nargs
	default
	help
	action
	type

Function Documentation

__flatten_list()

python.iconfTool.models.loaders.__flatten_list ( l )

private

Definition at line 128 of file loaders.py.

def __flatten_list(l):
    return [item for elem in l for item in elem] if l else []
 
 

collect_types()

python.iconfTool.models.loaders.collect_types

(

conf

)

Definition at line 161 of file loaders.py.

def collect_types(conf):
    name_to_type = {}
    for (comp_name, comp_settings) in conf.items():
        types_in_properties(comp_name, comp_settings, name_to_type)
    return name_to_type
 
 

excludeIncludeComps()

Dict python.iconfTool.models.loaders.excludeIncludeComps	(		dic,
			args,
			depth,
			compsToFollow = [] )

Definition at line 168 of file loaders.py.

def excludeIncludeComps(dic, args, depth, compsToFollow=[]) -> Dict:
    conf = {}
    if depth == 0:
        return conf
    compsToInclude = __flatten_list(args.includeComps)
    compsToExclude = __flatten_list(args.excludeComps)
 
    def eligible(component):
        exclude = any(re.match(s, component) for s in compsToExclude)
        if (component in compsToFollow or component.removeprefix("ToolSvc.") in compsToFollow) and not (exclude or component in args.ignore):
            logger.debug("Considering this component: %s because some other one depends on it", component)
            return True
        include = any(re.match(s, component) for s in compsToInclude)
        if args.includeComps and args.excludeComps:
            return include and not exclude
        elif args.includeComps:
            return include
        elif args.excludeComps:
            return not exclude
 
    for (comp_name, comp_attributes) in dic.items():
        if eligible(comp_name):
            conf[comp_name] = comp_attributes
            if depth > 0:
                types = {}
                types_in_properties(comp_attributes, types, compsToFollow)
                logger.debug("Following up for types included in here %s whole set of components to follow %s ", types, compsToFollow)
                compsToFollow += types.keys()         
            logger.debug("Included component %s", comp_name)
        else:
            logger.debug("Ignored component %s", comp_name)
    if depth > 0:
        conf.update(excludeIncludeComps(dic, args, depth-1, compsToFollow))
    return conf
 

ignoreDefaults()

Dict python.iconfTool.models.loaders.ignoreDefaults	(		allconf,
			args,
			known )

Definition at line 263 of file loaders.py.

def ignoreDefaults(allconf, args, known) -> Dict:
    conf = {}
    def drop_defaults(component_name, val_dict):
        # try picking the name from the dict, if missing use last part of the name, if that fails use the component_name (heuristic)
        component_name_last_part = component_name.split(".")[-1]
        component_type = known.get(component_name, known.get(component_name_last_part, component_name_last_part))
        comp_cls = None
        try:
            from AthenaConfiguration.ComponentFactory import CompFactory
            comp_cls = CompFactory.getComp(component_type)
            logger.debug("Loaded the configuration class %s/%s for defaults elimination", component_type, component_name)
        except Exception:
            logger.debug("Could not find the configuration class %s/%s, no defaults for it can be eliminated", component_type, component_name)
            return val_dict
        c = {}
 
        for k,v in val_dict.items():
            if not hasattr(comp_cls,'_descriptors'):
                logger.debug('No \'_descriptors\' attibute for %s', comp_cls)
                continue
            if k not in comp_cls._descriptors: # property not in descriptors (for instance, removed from component now)
                c[k] = v
            else:    
                default = str(comp_cls._descriptors[k].default)
                sv = str(v)
                if (default == sv or
                    default.replace("StoreGateSvc+", "") == sv.replace("StoreGateSvc+", "") or
                    default.replace("ConditionStore+", "") == sv.replace("ConditionStore+", "")):
                    logger.debug("Dropped default value \'%s\' of property %s in %s because the default is \'%s\'", sv, k, component_name, str(default))
                elif args.ignoreDefaultNamedComps and isinstance(v, str) and sv.endswith(f"/{default}"):
                    logger.debug("Dropped speculatively value %s of property %s in %s because the default it ends with %s", sv, k, component_name, str(default))
                else:
                    c[k] = v
                    logger.debug("Keep value %s of property %s in %s because it is different from default %s", str(v), str(k), component_name, str(comp_cls._descriptors[k].default))
        return c
 
     # collect types for all components (we look for A/B or lost of A/B strings)
    for (comp_name, comp_settings) in allconf.items():
        remaining = drop_defaults(comp_name, comp_settings)
        if len(remaining) != 0: # ignore components that have only default settings
            conf[comp_name] = remaining
    return conf
 

ignoreIrrelevant()

Dict python.iconfTool.models.loaders.ignoreIrrelevant	(		dic,
			args )

Definition at line 203 of file loaders.py.

def ignoreIrrelevant(dic, args) -> Dict:
    def remove_irrelevant(val_dict):
        return (
            { key: val for key, val in val_dict.items() if key not in args.ignore }
            if isinstance(val_dict, dict)
            else val_dict
        )
    conf = {}
    for (key, value) in dic.items():
        conf[key] = remove_irrelevant(value)
    return conf
 

isReference()

list python.iconfTool.models.loaders.isReference	(		value,
			compname,
			conf,
			svcCache = {} )

Returns a list of (component,class) if value stores reference to other components
   value - the value to check
   compname - full component name
   conf - complete config dict

Definition at line 336 of file loaders.py.

def isReference(value, compname, conf, svcCache={}) -> list:
    """Returns a list of (component,class) if value stores reference to other components
       value - the value to check
       compname - full component name
       conf - complete config dict
    """
 
    def _getSvcClass(instance):
        """Find instance in service lists to get class.
        Keeps a cache of the service classes in the svcCache default value.
        That's fine, unless we are dealing with more than one conf in the program.
        In that case, initialise svcCache to {} and specify in the caller."""
        if not svcCache:   # only scan ApplicationMgr once
            props = conf.get('ApplicationMgr',{"":None})
            if isinstance(props,dict):
                for prop,val in props.items():
                    if 'Svc' in prop:
                        try:
                            val = ast.literal_eval(str(val))
                        except Exception:
                            pass
                        if isinstance(val,list):
                            for v in val:
                                if isinstance(v,str):
                                    vv = v.split('/')
                                    if len(vv) == 2:
                                        if svcCache.setdefault(vv[1], vv[0]) != vv[0]:
                                            svcCache[vv[1]] = None # fail if same instance, different class
        return svcCache.get(instance)
 
    try:
        value = ast.literal_eval(str(value))
    except Exception:
        pass
 
    if isinstance(value, str):
        ctype_name = value.split('/')
        cls = ctype_name[0] if len(ctype_name) == 2 else None
        instance = ctype_name[-1]
        ref = None
        if instance:
            if compname and f"{compname}.{instance}" in conf: # private tool
                ref = f"{compname}.{instance}"
            elif f"ToolSvc.{instance}" in conf: # public tool
                ref = f"ToolSvc.{instance}"
            elif cls is not None or instance in conf: # service or other component
                ref = instance
                if cls is None:
                    cls = _getSvcClass(instance)
        if ref is not None:
                return [(ref, cls)]
 
    elif isinstance(value, list):
        refs = [isReference(el, compname, conf) for el in value]
        if any(refs):
            flattened = []
            [flattened.extend(el) for el in refs if el]
            return flattened
    return []
 
 

loadConfigFile()

Dict python.iconfTool.models.loaders.loadConfigFile	(		fname,
			args )

loads config file into a dictionary, supports several modifications of the input switched on via additional arguments
Supports reading: Pickled file with the CA or properties & JSON

Definition at line 409 of file loaders.py.

def loadConfigFile(fname, args) -> Dict:
    """loads config file into a dictionary, supports several modifications of the input switched on via additional arguments
    Supports reading: Pickled file with the CA or properties & JSON
    """
    if args.debug:
        print("Debugging info from reading ", fname, " in ", logger.handlers[0].baseFilename)
        logger.setLevel(logging.DEBUG)
 
    conf = {}
    if fname.endswith(".pkl"):
        with open(fname, "rb") as input_file:
            # determine if there is a old or new configuration pickled
            cfg = pickle.load(input_file)
            logger.info("... Read %s from %s", cfg.__class__.__name__, fname)
            from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator
            if isinstance(cfg, ComponentAccumulator):  # new configuration
                props = cfg.gatherProps()
                # to make json compatible with old configuration
                jos_props = props[2]
                to_json = {}
                for comp, name, value in jos_props:
                    to_json.setdefault(comp, {})[name] = value
                    to_json[comp][name] = value
                conf.update(to_json)
                conf['ApplicationMgr'] = props[0]
                conf['MessageSvc'] = props[1]
 
            elif isinstance(
                cfg, (collections.defaultdict, dict)
            ):  # old configuration
                conf.update(cfg)
                conf.update(pickle.load(input_file)) # special services
                # FIXME: there's a third pickle object with python components
            elif isinstance(cfg, (collections.Sequence)):
                for c in cfg:
                    conf.update(c)
        logger.info("... Read %d items from python pickle file: %s", len(conf), fname)
 
    elif fname.endswith(".json"):
 
        def __keepPlainStrings(element):
            if isinstance(element, str):
                return str(element)
            if isinstance(element, list):
                return [__keepPlainStrings(x) for x in element]
            if isinstance(element, dict):
                return {
                    __keepPlainStrings(key): __keepPlainStrings(value)
                    for key, value in element.items()
                }
            return element
 
        with open(fname, "r") as input_file:
            cfg = json.load(input_file, object_hook=__keepPlainStrings)
            for c in cfg:
                conf.update(cfg)
 
        # For compatibility with HLTJobOptions json, which produces structure:
        # {
        #   "filetype": "joboptions",
        #   "properties": { the thing we are interested in}
        # }
        if 'properties' in conf:
            conf = conf['properties']
 
        logger.info("... Read %d items from json file: %s", len(conf), fname)
 
    else:
        sys.exit("File format not supported.")
 
    if conf is None:
        sys.exit("Unable to load %s file" % fname)
 
    if args.includeComps or args.excludeComps or args.includeClasses or args.excludeClasses:
        logger.info(f"include/exclude comps like {args.includeComps}/{args.excludeComps}")
        conf = excludeIncludeComps(conf, args, args.follow)
 
    if args.ignoreIrrelevant:
        conf = ignoreIrrelevant(conf, args)
 
    if args.renameComps or args.renameCompsFile:
        conf = renameComps(conf, args)
 
    if args.ignoreDefaults:
        known_types = collect_types(conf)
        conf = ignoreDefaults(conf, args, known_types)
 
    if args.shortenDefaultComponents:
        conf = shortenDefaultComponents(conf, args)
 
    if args.skipProperties:
        conf = skipProperties(conf, args)
    return conf
 

loadDifferencesFile()

Dict python.iconfTool.models.loaders.loadDifferencesFile

(

fname

)

Read differences file
Format:
full_component_name.property oldvalue=newvalue
example:
AlgX.ToolA.SubToolB.SettingZ 45=46
It is possible to specify missing values, e.g:
AlgX.ToolA.SubToolB.SettingZ 45=    means that now the old value should be ignored
AlgX.ToolA.SubToolB.SettingZ =46    means that now the new value should be ignored
AlgX.ToolA.SubToolB.SettingZ =      means that any change of the value should be ignored 

Definition at line 598 of file loaders.py.

def loadDifferencesFile(fname) -> Dict:
    """
    Read differences file
    Format:
    full_component_name.property oldvalue=newvalue
    example:
    AlgX.ToolA.SubToolB.SettingZ 45=46
    It is possible to specify missing values, e.g:
    AlgX.ToolA.SubToolB.SettingZ 45=    means that now the old value should be ignored
    AlgX.ToolA.SubToolB.SettingZ =46    means that now the new value should be ignored
    AlgX.ToolA.SubToolB.SettingZ =      means that any change of the value should be ignored 
 
    """
    from collections import defaultdict
    differences = defaultdict(dict)
    count=0
    with open(fname, "r") as f:
        for line in f:
            if line[0] == "#" or line == "\n":
                continue
            line = line.strip()
            compAndProp, values = line.split(" ")           
            comp, prop = compAndProp.rsplit(".", 1)
            o,n = values.split("=")
            oldval,newval = o if o else None, n if n else None
                
            differences[comp][prop] = (oldval,newval)
            count+=1
    logger.info("... Read %d known differences from file: %s", count, fname)
    logger.info("..... %s", str(differences))
 
    return differences
 

renameComps()

Dict python.iconfTool.models.loaders.renameComps	(		dic,
			args )

Definition at line 215 of file loaders.py.

def renameComps(dic, args) -> Dict:
    compsToRename = __flatten_list(args.renameComps)
    if args.renameCompsFile:
        with open( args.renameCompsFile, "r") as refile:
            for line in refile:
                if not (line.startswith("#") or line.isspace() ):
                    compsToRename.append( line.rstrip('\n') )
    componentRenamingDict.update({
        old_name: new_name
        for old_name, new_name in [
            [e.strip() for e in element.split("=")] for element in compsToRename
        ]
    })
    for f,t in componentRenamingDict.items():
        logger.info("Renaming from: %s to %s", f, t)
 
    def rename_comps(comp_name):
        """Renames component if it is in the dict or, when name fragment is in the dict
        The later is for cases like: ToolSvc.ToolA.X.Y is renamed to ToolSvc.ToolB.X.Y
        """
        logger.debug("Trying renaming on, %s", comp_name)
        for k,v in componentRenamingDict.items():
            if k == comp_name:
#                logger.debug("Renamed comp %s to %s", k, v)
                return v
 
            old = f".{k}."
            if old in comp_name:
                return comp_name.replace(old, f".{v}.")
 
            old = f"{k}."
            if comp_name.startswith(old):
                return comp_name.replace(old, f"{v}.")
 
 
            old = f".{k}"
            if comp_name.endswith(old):
                return comp_name.replace(old, f".{k}")
        return comp_name
 
    conf = {}
    for (key, value) in dic.items():        
        renamed = rename_comps(key)
        if renamed != key:
             logger.debug("Renamed comp %s to %s", key, renamed)
        conf[renamed] = value
    return conf
 

shortenDefaultComponents()

Dict python.iconfTool.models.loaders.shortenDefaultComponents	(		dic,
			args )

Definition at line 306 of file loaders.py.

def shortenDefaultComponents(dic, args) -> Dict:
    conf = {}
    def shorten(val):
        value = val
        # the value can possibly be a serialized object (like a list)
        try:
            value = ast.literal_eval(str(value))
        except Exception:
            pass
 
        if isinstance(value, str):
            svalue = value.split("/")
            if len(svalue) == 2 and svalue[0] == svalue[1]:
                logger.debug("Shortened %s", svalue)
                return svalue[0]
        if isinstance(value, list):
            return [shorten(el) for el in value]
        if isinstance(value, dict):
            return shorten_defaults(value)
 
        return value
 
    def shorten_defaults(val_dict):
        if isinstance(val_dict, dict):
            return { key: shorten(val) for key,val in val_dict.items() }
 
    for (key, value) in dic.items():
        conf[key] = shorten_defaults(value)
    return conf
 

skipProperties()

Dict python.iconfTool.models.loaders.skipProperties	(		conf,
			args )

Definition at line 397 of file loaders.py.

def skipProperties(conf, args) -> Dict:
    updated = {}
    for (name, properties) in conf.items():
        updated[name] = {}
        if not isinstance(properties, dict): # keep it
            updated[name] = properties
        else:
            for property_name, value in properties.items():
                if isReference( value, name, conf) or property_name == 'Members': # later for sequences structure
                    updated[name][property_name] = value
    return updated
 

types_in_properties()

python.iconfTool.models.loaders.types_in_properties	(		comp_name,
			value,
			dict_to_update )

Updates the dictionary with (potentially) component name -> component type

Definition at line 132 of file loaders.py.

def types_in_properties(comp_name, value, dict_to_update):
    """Updates the dictionary with (potentially) component name -> component type"""
    parsable = False
    try:
        s = ast.literal_eval(str(value))
        parsable = True
        if isinstance(s, list):
            for el in s:
                types_in_properties(comp_name, el, dict_to_update)
    except Exception:
        pass
    # Exclude non-strings, or strings that look like paths rather than type/name pairs
    if isinstance(value,str):
        slash_startend = value.startswith("/") or value.endswith("/")
        json_dict = value.startswith("{") and value.endswith("}")
        if value.count("/")==1 and not parsable and not slash_startend and not json_dict:
            comp = value.split("/")
            if len(comp) == 2:
                # Record with and without parent
                dict_to_update[f'{comp_name}.{comp[1]}'] = comp[0]
                dict_to_update[f'{comp[1]}'] = comp[0]
                logger.debug("Parsing %s, found type of %s.%s to be %s", value, comp_name, comp[1], comp[0])
            else:
                logger.debug("What is typeless comp? %s", value)
    if isinstance(value, dict):
        for v in value.values():
            types_in_properties(comp_name, v, dict_to_update)
 
 

Variable Documentation

action

python.iconfTool.models.loaders.action

Definition at line 31 of file loaders.py.

baseParser

python.iconfTool.models.loaders.baseParser = argparse.ArgumentParser()

Definition at line 25 of file loaders.py.

componentRenamingDict

dict python.iconfTool.models.loaders.componentRenamingDict = {}

Definition at line 23 of file loaders.py.

default

python.iconfTool.models.loaders.default

Definition at line 29 of file loaders.py.

help

python.iconfTool.models.loaders.help

Definition at line 30 of file loaders.py.

level

python.iconfTool.models.loaders.level

Definition at line 20 of file loaders.py.

logger

python.iconfTool.models.loaders.logger = logging.getLogger("confTool")

Definition at line 19 of file loaders.py.

nargs

python.iconfTool.models.loaders.nargs

Definition at line 28 of file loaders.py.

type

python.iconfTool.models.loaders.type

Definition at line 118 of file loaders.py.