3 from copy
import copy, deepcopy
4 from difflib
import get_close_matches
5 from enum
import EnumMeta
8 from AthenaCommon.Logging
import logging
9 from PyUtils.moduleExists
import moduleExists
10 from PyUtils.Decorators
import deprecate
12 _msg = logging.getLogger(
'AthConfigFlags')
15 """Return whether or not this is a gaudi-based (athena) environment"""
20 """The base flag object.
22 A flag can be set to either a fixed value or a callable, which computes
23 the value based on other flags.
26 __slots__ = [
'_value',
'_setDef',
'_type',
'_help']
32 def __init__(self, default, type=None, help=None):
33 """Initialise the flag with the default value.
35 Optionally set the type of the flag value and the help string.
38 raise ValueError(
"Default value of a flag must not be None")
45 """Set the value of the flag.
47 Can be a constant value or a callable.
58 def get(self, flagdict=None):
59 """Get the value of the flag.
61 If the currently set value is a callable, a dictionary of all available
62 flags needs to be provided.
65 if self.
_value is not None:
66 return deepcopy(self.
_value)
74 raise RuntimeError(
"Flag is using a callable but all flags are not available.")
87 return deepcopy(value)
90 if self.
_value is not None:
96 if (self.
_type is None or value
is None or
97 isinstance(value, self.
_type)
or
101 raise TypeError(f
"Flag is of type '{self._type.__name__}', "
102 f
"but value '{value}' of type '{type(value).__name__}' set.")
106 """Flags to dict converter
108 Used by both FlagAddress and AthConfigFlags. The input must be an
109 iterator over flags to be included in the dict.
113 for key, item
in iterator:
115 subkeys = key.split(
'.')
116 for subkey
in subkeys[:-1]:
117 x = x.setdefault(subkey,{})
118 x[subkeys[-1]] = item
123 if isinstance(f, AthConfigFlags):
125 rname = self.
_flags._renames.get(name, name)
128 elif isinstance(f, FlagAddress):
130 name = f._name+
"."+name
131 rname = self.
_flags._renames.get(name, name)
135 return getattr(self.
_flags, self.
_name +
"." + name)
138 if name.startswith(
"_"):
139 return object.__setattr__(self, name, value)
140 merged = self.
_name +
"." + name
142 if not self.
_flags.hasFlag( merged ):
143 self.
_flags._loadDynaFlags( merged )
145 if not self.
_flags.hasFlag( merged ):
146 raise RuntimeError(
"No such flag: {} The name is likely incomplete.".
format(merged) )
147 return self.
_flags._set( merged, value )
153 raise RuntimeError(
"No such flag: "+ self.
_name+
". The name is likely incomplete." )
162 raise RuntimeError(
"No such flag: "+ self.
_name+
". The name is likely incomplete." )
165 return getattr(self, name)
168 setattr(self, name, value)
171 merged = self.
_name +
"." + name
175 self.
_flags.loadAllDynamicFlags()
176 rmap = self.
_flags._renamed_map()
178 for flag
in self.
_flags._flagdict.keys():
179 if flag.startswith(self.
_name.rstrip(
'.') +
'.'):
180 for newflag
in rmap[flag]:
181 ntrim = len(self.
_name) + 1
182 n_dots_in = flag[:ntrim].
count(
'.')
183 remaining = newflag.split(
'.')[n_dots_in]
184 if remaining
not in used:
189 """Subflag iterator specialized for this address
192 self.
_flags.loadAllDynamicFlags()
194 rename = self.
_flags._renamed_map()
195 for key
in self.
_flags._flagdict.keys():
196 if key.startswith(address.rstrip(
'.') +
'.'):
197 ntrim = len(address) + 1
198 remaining = key[ntrim:]
199 for r
in rename[key]:
200 yield r, getattr(self, remaining)
203 """Convert to a python dictionary
205 Recursively convert this flag and all subflags into a
206 structure of nested dictionaries. All dynamic flags are
207 resolved in the process.
209 The resulting data structure should be easy to serialize as
239 raise RuntimeError(
"Cannot calculate hash of unlocked flag container")
240 elif self.
_hash is None:
245 raise DeprecationWarning(
"__hash__ method in AthConfigFlags is deprecated. Probably called from function decorator, use AccumulatorCache decorator instead.")
254 AthConfigFlags._hashedFlags.append (self)
259 _flagdict = object.__getattribute__(self,
"_flagdict")
262 if name
in _flagdict:
263 return self.
_get(name)
272 if name
in _flagdict:
273 return self.
_get(name)
279 raise AttributeError(f
"No such flag: {name}")
282 if name.startswith(
"_"):
283 return object.__setattr__(self, name, value)
286 return self.
_set(name, value)
287 raise RuntimeError(
"No such flag: "+ name+
". The name is likely incomplete." )
293 return getattr(self, name)
296 setattr(self, name, value)
302 if key.startswith(name):
312 first = r.split(
'.',1)[0]
313 if first
not in used:
318 """Convert to a python dictionary
320 This is identical to the `asdict` in FlagAddress, but for all
328 """mapping from the old names to the new names
330 This is the inverse of _renamed, which maps new names to old
333 Returns a list of the new names corresponding to the old names
334 (since cloneAndReplace may or may not disable access to the old name,
335 it is possible that an old name renames to multiple new names)
340 if old
not in revmap:
341 revmap[old] = [ new ]
343 revmap[old] += [ new ]
346 for old, newlist
in revmap.items():
347 if key.startswith(old +
'.'):
348 stem = key.removeprefix(old)
349 return [ f
'{new}{stem}' if new
else '' for new
in newlist ]
355 """Subflag iterator for all flags
357 This is used by the asdict() function.
370 yield new, getattr(self, old)
371 except ModuleNotFoundError
as err:
372 _msg.debug(f
'missing module: {err}')
375 def addFlag(self, name, setDef, type=None, help=None):
378 raise KeyError(
"Duplicated flag name: {}".
format( name ))
384 The path is the beginning of the flag name (e.g. "X" for flags generated with name "X.*").
385 The generator is a function that returns a flags container, the flags have to start with the same path.
386 When the prefix is True the flags created by the generator are prefixed by "path".
388 Supported calls are then:
389 addFlagsCategory("A", g) - where g is function creating flags is f.addFlag("A.x", someValue)
390 addFlagsCategory("A", g, True) - when flags are defined in g like this: f.addFalg("x", somevalue),
391 The latter option allows to share one generator among flags that are later loaded in different paths.
394 _msg.debug(
"Adding flag category %s", path)
398 """ public interface for _loadDynaFlags """
403 loads the flags of the form "A.B.C" first attempting the path "A" then "A.B" and then "A.B.C"
406 def __load_impl( flagBaseName ):
407 if flagBaseName
in self.
_loaded:
408 _msg.debug(
"Flags %s already loaded",flagBaseName )
411 _msg.debug(
"Dynamically loading the flags under %s", flagBaseName )
416 generator, prefix = self.
_dynaflags[flagBaseName]
423 pathfrags = name.split(
'.')
424 for maxf
in range(1, len(pathfrags)+1):
425 __load_impl(
'.'.
join(pathfrags[:maxf]) )
428 """Force load all the dynamic flags """
447 if f.startswith(name+
'.'):
451 if c.startswith(name):
465 closestMatch = get_close_matches(name,self.
_flagdict.
keys(),1)
466 raise KeyError(f
"No flag with name '{name}' found" +
467 (f
". Did you mean '{closestMatch[0]}'?" if closestMatch
else ""))
473 closestMatch = get_close_matches(name,self.
_flagdict.
keys(),1)
474 raise KeyError(f
"No flag with name '{name}' found" +
475 (f
". Did you mean '{closestMatch[0]}'?" if closestMatch
else ""))
477 @
deprecate(
"Use '[...]' rather than '(...)' to access flags", print_context=
True)
479 return self.
_get(name)
493 raise RuntimeError(
"Attempt to modify locked flag container")
499 """Return an unlocked copy of self (dynamic flags are not loaded)"""
503 cln._renames = deepcopy(self.
_renames)
509 This is to replace subsets of configuration flags like
512 newflags = flags.cloneAndReplace('Muon', 'Trigger.Offline.Muon')
515 _msg.debug(
"cloning flags and replacing %s by %s", subsetToReplace, replacementSubset)
520 subsetToReplace = subsetToReplace.strip(
".")
521 replacementSubset = replacementSubset.strip(
".")
524 if (subsetToReplace == replacementSubset):
525 raise RuntimeError(f
'Can not replace flags {subsetToReplace} with themselves')
529 if src ==
"":
continue
530 if src+
"." in subsetToReplace:
531 raise RuntimeError(f
'Can not replace flags {subsetToReplace} by {replacementSubset} because of already present replacement of {alias} by {src}')
534 newFlags =
copy(self)
535 newFlags._renames = deepcopy(self.
_renames)
537 if replacementSubset
in newFlags._renames:
538 newFlags._renames[subsetToReplace] = newFlags._renames[replacementSubset]
540 newFlags._renames[subsetToReplace] = replacementSubset
543 if replacementSubset
not in newFlags._renames
or newFlags._renames[replacementSubset] == replacementSubset:
544 newFlags._renames[replacementSubset] =
""
546 del newFlags._renames[replacementSubset]
551 if replacementSubset
not in newFlags._renames:
552 newFlags._renames[replacementSubset] = replacementSubset
554 newFlags._hash =
None
558 def join(self, other, prefix=''):
560 Merges two flag containers
561 When the prefix is passed each flag from the "other" is prefixed by "prefix."
565 for (name,flag)
in other._flagdict.items():
566 fullName = prefix+
"."+name
if prefix !=
"" else name
568 raise KeyError(
"Duplicated flag name: {}".
format( fullName ) )
571 for (name,loader)
in other._dynaflags.items():
572 fullName = prefix+
"."+name
if prefix !=
"" else name
574 raise KeyError(
"Duplicated dynamic flags name: {}".
format( fullName ) )
575 _msg.debug(
"Joining dynamic flags with %s", fullName)
579 def dump(self, pattern=".*", evaluate=False, formatStr="{:40} : {}
", maxLength=None):
581 compiled = re.compile(pattern)
582 def truncate(s):
return s[:maxLength] + (
"..." if maxLength
and len(s)>maxLength
else "")
583 reverse_renames = {value: key
for key, value
in self.
_renames.
items()
if value !=
''}
586 if any([name.startswith(r)
for r
in reverse_renames.keys()]):
587 for oldprefix, newprefix
in reverse_renames.items():
588 if name.startswith(oldprefix):
589 renamed = name.replace(oldprefix, newprefix)
591 if compiled.match(renamed):
600 except Exception
as e:
606 print(
"Flag categories that can be loaded dynamically")
607 print(
"{:25} : {:>30} : {}".
format(
"Category",
"Generator name",
"Defined in" ) )
609 if compiled.match(name):
610 print(
"{:25} : {:>30} : {}".
format( name, gen_and_prefix[0].__name__,
'/'.
join(gen_and_prefix[0].__code__.co_filename.split(
'/')[-2:]) ) )
612 print(
"Flag categories that are redirected by the cloneAndReplace")
614 print(
"{:30} points to {:>30} ".
format( alias, src
if src
else "nothing") )
619 Mostly a self-test method
628 Scripts calling AthConfigFlags.fillFromArgs can extend this parser, and pass their version to fillFromArgs
631 from AthenaCommon.AthOptionsParser
import getArgumentParser
633 parser.add_argument(
"---",dest=
"terminator",action=
'store_true', help=argparse.SUPPRESS)
646 """Fill the flags from a string of type key=value"""
650 key, value = flag_string.split(
"=")
652 raise ValueError(f
"Cannot interpret argument {flag_string}, expected a key=value format")
655 value = value.strip()
666 raise KeyError(f
"{key} is not a known configuration flag")
669 if flag_type
is None:
672 ast.literal_eval(value)
676 elif isinstance(flag_type, EnumMeta):
678 ENUM = importlib.import_module(flag_type.__module__)
679 value=f
"ENUM.{value}"
682 exec(f
"self.{key}{oper}{value}")
686 def fillFromArgs(self, listOfArgs=None, parser=None, return_unknown=False):
688 Used to set flags from command-line parameters, like flags.fillFromArgs(sys.argv[1:])
690 if return_unknown=False, returns: args
691 otherwise returns: args, uknown_args
692 where unknown_args is the list of arguments that did not correspond to one of the flags
701 argList = listOfArgs
if listOfArgs
is not None else sys.argv[1:]
708 unrequiredActions = []
709 if "-h" in argList
or "--help" in argList:
711 if "-h" in argList: argList.remove(
"-h")
712 if "--help" in argList: argList.remove(
"--help")
714 for a
in parser._actions:
716 unrequiredActions.append(a)
718 (args,leftover)=parser.parse_known_args(argList)
719 for a
in unrequiredActions: a.required=
True
722 argList = [a
for a
in argList
if a
not in leftover]
726 """Check if dest is available in parser and has been set"""
727 return vars(args).
get(dest,
None)
is not None
730 self.Exec.DebugStage=args.debug
732 if arg_set(
'evtMax'):
733 self.Exec.MaxEvents=args.evtMax
735 if arg_set(
'interactive'):
736 self.Exec.Interactive=args.interactive
738 if arg_set(
'skipEvents'):
739 self.Exec.SkipEvents=args.skipEvents
741 if arg_set(
'filesInput'):
742 self.Input.Files = []
743 for f
in args.filesInput.split(
","):
746 self.Input.Files += found
if found
else [f]
748 if "-l" in argList
or "--loglevel" in argList:
749 from AthenaCommon
import Constants
750 self.Exec.OutputLevel = getattr(Constants, args.loglevel)
752 if arg_set(
'config_only')
and args.config_only
is not False:
753 from os
import environ
754 environ[
"PICKLECAFILE"] =
"" if args.config_only
is True else args.config_only
756 if arg_set(
'threads'):
757 self.Concurrency.NumThreads = args.threads
761 if args.concurrent_events
is None and self.Concurrency.NumConcurrentEvents==0:
762 self.Concurrency.NumConcurrentEvents = args.threads
764 if arg_set(
'concurrent_events'):
765 self.Concurrency.NumConcurrentEvents = args.concurrent_events
767 if arg_set(
'nprocs'):
768 self.Concurrency.NumProcs = args.nprocs
770 if arg_set(
'perfmon'):
771 from PerfMonComps.PerfMonConfigHelpers
import setPerfmonFlagsFromRunArgs
775 self.Exec.MTEventService = args.mtes
777 if arg_set(
'mtes_channel'):
778 self.Exec.MTEventServiceChannel = args.mtes_channel
780 if arg_set(
'profile_python'):
781 from AthenaCommon.Debugging
import dumpPythonProfile
782 import atexit, cProfile, functools
783 cProfile._athena_python_profiler = cProfile.Profile()
784 cProfile._athena_python_profiler.enable()
787 atexit.register(functools.partial(dumpPythonProfile, args.profile_python))
790 self.Exec.MPI = args.mpi
798 if do_help
and '=' not in arg:
799 argList += arg.split(
".")
803 except KeyError
as e:
805 unknown_args += [arg]
810 if parser.epilog
is None: parser.epilog=
""
811 parser.epilog +=
" Note: Specify additional flags in form <flagName>=<value>."
812 subparsers = {
"":[parser,parser.add_subparsers(help=argparse.SUPPRESS)]}
814 logging.root.setLevel(logging.ERROR)
815 def getParser(category):
816 if category
not in subparsers.keys():
817 cat1,cat2 = category.rsplit(
".",1)
if "." in category
else (
"",category)
818 p,subp = getParser(cat1)
819 if subp.help==argparse.SUPPRESS:
820 subp.help =
"Flag subcategories:"
821 newp = subp.add_parser(cat2,help=
"{} flags".
format(category),
822 formatter_class = argparse.ArgumentDefaultsHelpFormatter,usage=argparse.SUPPRESS)
823 newp._positionals.title =
"flags"
824 subparsers[category] = [newp,newp.add_subparsers(help=argparse.SUPPRESS)]
825 return subparsers[category]
828 category,flagName = name.rsplit(
".",1)
if "." in name
else (
"",name)
831 val =
repr(flag.get(self))
834 if flag._help != argparse.SUPPRESS:
836 if flag._help
is not None:
837 helptext = f
": {flag._help}"
838 if flag._type
is not None:
839 helptext += f
' [type: {flag._type.__name__}]'
840 if val
is not None and helptext ==
"":
842 getParser(category)[0].add_argument(name, nargs=
'?', default=val, help=helptext)
844 parser._positionals.title =
'flags and positional arguments'
845 parser.parse_known_args(argList + [
"--help"])
850 return args,unknown_args