ATLAS Offline Software
ConfigBlock.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
2 
3 import textwrap
4 import inspect
5 from functools import wraps
6 
7 from AnaAlgorithm.Logging import logging
8 logCPAlgCfgBlock = logging.getLogger('CPAlgCfgBlock')
9 
10 from AnalysisAlgorithmsConfig.ConfigAccumulator import DataType
11 import re
12 
13 def filter_dsids (filterList, config) :
14  """check whether the sample being run passes a"""
15  """possible DSID filter on the block"""
16  if len(filterList) == 0:
17  return True
18  for dsid_filter in filterList:
19  # Check if the pattern is enclosed in regex delimiters (e.g., starts with '^' or contains regex metacharacters)
20  if any(char in str(dsid_filter) for char in "^$*+?.()|[]{}\\"):
21  pattern = re.compile(dsid_filter)
22  if pattern.match(str(config.dsid())):
23  return True
24  else:
25  # Otherwise it's an exact DSID (but could be int or string)
26  if str(dsid_filter) == str(config.dsid()):
27  return True
28  return False
29 
31  """this wrapper ensures that the 'instanceName' of the various """
32  """config blocks is cleaned up of any non-alphanumeric characters """
33  """that may arise from using 'selectionName' in the naming."""
34  @wraps(func)
35  def wrapper(*args, **kwargs):
36  # Get the string returned by the 'instanceName()' method of a config block
37  orig_name = func(*args, **kwargs)
38 
39  if orig_name is None:
40  return ""
41 
42  # Allowed replacements - anything else is likely a mistake on the user-side
43  result = orig_name.replace("||", "OR").replace("&&", "AND").replace("(","LB").replace(")","RB").replace(" ","")
44 
45  return result
46  return wrapper
47 
49  """this meta class enforces the application of 'alphanumeric_block_names()' """
50  """to 'instanceName()' and will be used in the main ConfigBlock class in order """
51  """to propagate this rule also to all derived classes (the individual config blocks."""
52  def __new__(cls, name, bases, dct):
53  # Automatically apply alphanumeric-only decorator to 'instanceName()' method
54  if 'instanceName' in dct and callable(dct['instanceName']):
55  dct['instanceName'] = alphanumeric_block_name(dct['instanceName'])
56  return super().__new__(cls, name, bases, dct)
57 
59  """the information for a single option on a configuration block"""
60 
61  def __init__ (self, type=None, info='', noneAction='ignore', required=False,
62  default=None) :
63  self.type = type
64  self.info = info
65  self.required = required
66  self.noneAction = noneAction
67  self.default = default
68 
69 
70 
72  """Class encoding a blocks dependence on other blocks."""
73 
74  def __init__(self, blockName, required=True):
75  self.blockName = blockName
76  self.required = required
77 
78 
79  def __eq__(self, name):
80  return self.blockName == name
81 
82 
83  def __str__(self):
84  return self.blockName
85 
86 
87  def __repr__(self):
88  return f'ConfigBlockDependency(blockName="{self.blockName}", required={self.required})'
89 
90 
91 class ConfigBlock(metaclass=BlockNameProcessorMeta):
92  """the base class for classes implementing individual blocks of
93  configuration
94 
95  A configuration block is a sequence of one or more algorithms that
96  should always be scheduled together, e.g. the muon four momentum
97  corrections could be a single block, muon selection could then be
98  another block. The blocks themselves generally have their own
99  configuration options/properties specific to the block, and will
100  perform a dynamic configuration based on those options as well as
101  the overall job.
102 
103  The actual configuration of the algorithms in the block will
104  depend on what other blocks are scheduled before and afterwards,
105  most importantly some algorithms will introduce shallow copies
106  that subsequent algorithms will need to run on, and some
107  algorithms will add selection decorations that subquent algorithms
108  should use as preselections.
109 
110  The algorithms get created in a multi-step process (that may be
111  extended in the future): As a first step each block retrieves
112  references to the containers it uses (essentially marking its spot
113  in the processing chain) and also registering any shallow copies
114  that will be made. In the second/last step each block then
115  creates the fully configured algorithms.
116 
117  One goal is that when the algorithms get created they will have
118  their final configuration and there needs to be no
119  meta-configuration data attached to the algorithms, essentially an
120  inversion of the approach in AnaAlgSequence in which the
121  algorithms got created first with associated meta-configuration
122  and then get modified in susequent configuration steps.
123 
124  For now this is mostly an empty base class, but another goal of
125  this approach is to make it easier to build another configuration
126  layer on top of this one, and this class will likely be extended
127  and get data members at that point.
128 
129  The child class needs to implement the method `makeAlgs` which is
130  given a single `ConfigAccumulator` type argument. This is meant to
131  create the sequence of algorithms that this block configures. This
132  is currently (28 Jul 2025) called twice and should do the same thing
133  during both calls, but the plan is to change that to a single call.
134 
135  The child class should also implement the method `getInstanceName`
136  which should return a string that is used to distinguish between
137  multiple instances of the same block. This is used to append the
138  instance name to the names of all algorithms created by this block,
139  and may in the future also be used to distinguish between multiple
140  instances of the block.
141  """
142 
143  # Class-level dictionary to keep track of instance counts for each derived class
144  instance_counts = {}
145 
146  def __init__ (self) :
147  self._blockName = ''
148  self._factoryName = None
149  self._dependencies = []
150  self._options = {}
151  # used with block configuration to set arbitrary option
152  self.addOption('groupName', '', type=str,
153  info=('Used to specify this block when setting an'
154  ' option at an arbitrary location.'))
155  self.addOption('skipOnData', False, type=bool,
156  info=('User option to prevent the block from running'
157  ' on data. This only affects blocks that are'
158  ' intended to run on data.'))
159  self.addOption('skipOnMC', False, type=bool,
160  info=('User option to prevent the block from running'
161  ' on MC. This only affects blocks that are'
162  ' intended to run on MC.'))
163  self.addOption('onlyForDSIDs', [], type=list,
164  info=('Used to specify which MC DSIDs to allow this'
165  ' block to run on. Each element of the list'
166  ' can be a full DSID (e.g. 410470), or a regex'
167  ' (e.g. 410.* to select all 410xxx DSIDs, or'
168  ' ^(?!410) to veto them). An empty list means no'
169  ' DSID restriction.'))
170  self.addOption('propertyOverrides', {}, type=None,
171  info=('EXPERT USE ONLY: A dictionary of properties to'
172  ' override at the end of configuration. This should'
173  ' take the form'
174  ' {"algName.toolName.propertyName": value, ...},'
175  ' without any automatically applied postfixes for'
176  ' the algorithm name. THIS IS MEANT TO BE EXPERT'
177  ' USAGE ONLY. Properties that need to be set by'
178  ' the user should be declared as options on the'
179  ' block itself. EXPERT USE ONLY!'))
180  # Increment the instance count for the current class
181  cls = type(self) # Get the actual class of the instance (also derived!)
182  if cls not in ConfigBlock.instance_counts:
183  ConfigBlock.instance_counts[cls] = 0
184  # Note: we do need to check in the call stack that we are
185  # in a real makeConfig situation, and not e.g. printAlgs
186  stack = inspect.stack()
187  for frame_info in stack:
188  # Get the class name (if any) from the frame
189  parent_cls = frame_info.frame.f_locals.get('self', None)
190  if parent_cls is None or not isinstance(parent_cls, ConfigBlock):
191  # If the frame does not belong to an instance of ConfigBlock, it's an external caller
192  if frame_info.function == "makeConfig":
193  ConfigBlock.instance_counts[cls] += 1
194  break
195 
196 
197  def setBlockName(self, name):
198  """Set blockName"""
199  self._blockName = name
200 
201  def getBlockName(self):
202  """Get blockName"""
203  return self._blockName
204 
205  def factoryName(self):
206  """get the factory name for this block
207 
208  This is mostly to give a reliable means of identifying the type
209  of block we have in error messages. This is meant to be
210  automatically set by the factory based on the requested block
211  name, but there are a number of fallbacks. It is best not to
212  assume a specific format, this is mostly meant to be used as an
213  identifier in output messages.
214  """
215  if self._factoryName is not None and self._factoryName != '':
216  return self._factoryName
217  # If no factory name is set and the block has a name, use that
218  if self._blockName is not None and self._blockName != '':
219  return self._blockName
220  # Use the class name as a fallback
221  return self.__class__.__name__
222 
223  def setFactoryName(self, name):
224  """set the factory name for this block
225 
226  This is meant to be called automatically by the factory based on
227  the requested block name. If you are creating a block without a factory,
228  you can call this method to set the factory name manually.
229  """
230  self._factoryName = name
231 
232  def instanceName(self):
233  """Get the name of the instance
234 
235  The name of the instance is used to distinguish between multiple
236  instances of the same block. Most importantly, this will be
237  appended to the names of all algorithms created by this block.
238  This defaults to an empty string, but block implementations
239  should override it with an appropriate name based on identifying
240  options set on this instance. A typical example would be the
241  name of the (main) container, plus potentially the selection or
242  working point.
243 
244  Ideally all blocks should override this method, but for backward
245  compatibility (28 Jul 25) it defaults to an empty string.
246  """
247  return ''
248 
249  def isUsedForConfig(self, config):
250  """
251  whether this block should be used for the given configuration
252 
253  This is used by `ConfigSequence` to determine whether this block
254  should be included in the configuration.
255  """
256  if self.skipOnData and config.dataType() is DataType.Data:
257  return False
258  if self.skipOnMC and config.dataType() is not DataType.Data:
259  return False
260  if not filter_dsids(self.onlyForDSIDs, config):
261  return False
262  return True
263 
264  def applyConfigOverrides(self, config):
265  """
266  Apply any configuration overrides specified in the block's
267  `propertyOverrides` option. This is meant to be called at the
268  end of the configuration process, after all algorithms have been
269  created and configured.
270  """
271  for key, value in self.propertyOverrides.items():
272  # Split the key into algorithm name, tool name, and property name
273  parts = key.split('.')
274  if len(parts) < 2:
275  raise Exception(f"Invalid override key format: {key}")
276  alg = config.getAlgorithm(parts[0])
277  if alg is None:
278  raise Exception(f"Algorithm {parts[0]} not found in config for override: {key}")
279  for name in parts[1:-1]:
280  # Navigate through tools if necessary
281  if hasattr(alg, name):
282  alg = getattr(alg, name)
283  else:
284  raise Exception(f"Tool {name} not found for override: {key}")
285  # Set the property on the algorithm/tool. This is probably a
286  # horrible hack, but `setattr` didn't work for me.
287  alg.__setattr__(parts[-1], value)
288 
289  def addDependency(self, dependencyName, required=True):
290  """
291  Add a dependency for the block. Dependency is corresponds to the
292  blockName of another block. If required is True, will throw an
293  error if dependency is not present; otherwise will move this
294  block after the required block. If required is False, will do
295  nothing if required block is not present; otherwise, it will
296  move block after required block.
297  """
298  if not self.hasDependencies():
299  # add option to block ignore dependencies
300  self.addOption('ignoreDependencies', [], type=list,
301  info='List of dependencies defined in the ConfigBlock to ignore.')
302  self._dependencies.append(ConfigBlockDependency(dependencyName, required))
303 
304  def hasDependencies(self):
305  """Return True if there is a dependency."""
306  return bool(self._dependencies)
307 
308  def getDependencies(self):
309  """Return the list of dependencies. """
310  return self._dependencies
311 
312  def addOption (self, name, defaultValue, *,
313  type, info='', noneAction='ignore', required=False) :
314  """declare the given option on the configuration block
315 
316  This should only be called in the constructor of the
317  configuration block.
318 
319  NOTE: The backend to option handling is slated to be replaced
320  at some point. This particular function should essentially
321  stay the same, but some behavior may change.
322  """
323  if name in self._options :
324  raise KeyError (f'duplicate option: {name}')
325  if type not in [str, bool, int, float, list, None] :
326  raise TypeError (f'unknown option type: {type}')
327  noneActions = ['error', 'set', 'ignore']
328  if noneAction not in noneActions :
329  raise ValueError (f'invalid noneAction: {noneAction} [allowed values: {noneActions}]')
330  setattr (self, name, defaultValue)
331  self._options[name] = ConfigBlockOption(type=type, info=info,
332  noneAction=noneAction, required=required, default=defaultValue)
333 
334 
335  def setOptionValue (self, name, value) :
336  """set the given option on the configuration block
337 
338  NOTE: The backend to option handling is slated to be replaced
339  at some point. This particular function should essentially
340  stay the same, but some behavior may change.
341  """
342 
343  if name not in self._options :
344  raise KeyError (f'unknown option "{name}" in block "{self.__class__.__name__}"')
345  noneAction = self._options[name].noneAction
346  if value is not None or noneAction == 'set' :
347  # check type if specified
348  optType = self._options[name].type
349  # convert int to float to prevent crash
350  if optType is float and type(value) is int:
351  value = float(value)
352  if optType is not None and optType != type(value):
353  raise ValueError(f'{name} for block {self.__class__.__name__} should '
354  f'be of type {optType} not {type(value)}')
355  setattr (self, name, value)
356  elif noneAction == 'ignore' :
357  pass
358  elif noneAction == 'error' :
359  raise ValueError (f'passed None for setting option {name} with noneAction=error')
360 
361 
362  def getOptionValue(self, name):
363  """Returns config option value, if present; otherwise return None"""
364  if name in self._options:
365  return getattr(self, name)
366 
367 
368  def getOptions(self):
369  """Return a copy of the options associated with the block"""
370  return self._options.copy()
371 
372 
373  def printOptions(self, verbose=False, width=60, indent=" "):
374  """
375  Prints options and their values
376  """
377  def printWrap(text, width=60, indent=" "):
378  wrapper = textwrap.TextWrapper(width=width, initial_indent=indent,
379  subsequent_indent=indent)
380  for line in wrapper.wrap(text=text):
381  logCPAlgCfgBlock.info(line)
382 
383  for opt, vals in self.getOptions().items():
384  if verbose:
385  logCPAlgCfgBlock.info(indent + f"\033[4m{opt}\033[0m: {self.getOptionValue(opt)}")
386  logCPAlgCfgBlock.info(indent*2 + f"\033[4mtype\033[0m: {vals.type}")
387  logCPAlgCfgBlock.info(indent*2 + f"\033[4mdefault\033[0m: {vals.default}")
388  logCPAlgCfgBlock.info(indent*2 + f"\033[4mrequired\033[0m: {vals.required}")
389  logCPAlgCfgBlock.info(indent*2 + f"\033[4mnoneAction\033[0m: {vals.noneAction}")
390  printWrap(f"\033[4minfo\033[0m: {vals.info}", indent=indent*2)
391  else:
392  logCPAlgCfgBlock.info(indent + f"{ opt}: {self.getOptionValue(opt)}")
393 
394 
395  def hasOption (self, name) :
396  """whether the configuration block has the given option
397 
398  WARNING: The backend to option handling is slated to be
399  replaced at some point. This particular function may change
400  behavior, interface or be removed/replaced entirely.
401  """
402  return name in self._options
403 
404 
405  def __eq__(self, blockName):
406  """
407  Implementation of == operator. Used for seaching configSeque.
408  E.g. if blockName in configSeq:
409  """
410  return self._blockName == blockName
411 
412 
413  def __str__(self):
414  return self._blockName
415 
416 
417  @classmethod
419  # Access the current count for this class
420  return ConfigBlock.instance_counts.get(cls, 0)
python.ConfigBlock.ConfigBlock.hasOption
def hasOption(self, name)
Definition: ConfigBlock.py:395
replace
std::string replace(std::string s, const std::string &s2, const std::string &s3)
Definition: hcg.cxx:307
python.ConfigBlock.ConfigBlock.instanceName
def instanceName(self)
Definition: ConfigBlock.py:232
python.ConfigBlock.ConfigBlockDependency
Definition: ConfigBlock.py:71
python.ConfigBlock.ConfigBlockOption.noneAction
noneAction
Definition: ConfigBlock.py:65
python.ConfigBlock.ConfigBlock.hasDependencies
def hasDependencies(self)
Definition: ConfigBlock.py:304
python.ConfigBlock.ConfigBlock.addDependency
def addDependency(self, dependencyName, required=True)
Definition: ConfigBlock.py:289
python.ConfigBlock.ConfigBlock.applyConfigOverrides
def applyConfigOverrides(self, config)
Definition: ConfigBlock.py:264
python.ConfigBlock.ConfigBlock.__str__
def __str__(self)
Definition: ConfigBlock.py:413
python.ConfigBlock.BlockNameProcessorMeta
Definition: ConfigBlock.py:48
python.ConfigBlock.ConfigBlock.setOptionValue
def setOptionValue(self, name, value)
Definition: ConfigBlock.py:335
python.ConfigBlock.ConfigBlockOption.type
type
Definition: ConfigBlock.py:62
python.ConfigBlock.ConfigBlock._factoryName
_factoryName
Definition: ConfigBlock.py:148
python.ConfigBlock.ConfigBlock.getDependencies
def getDependencies(self)
Definition: ConfigBlock.py:308
python.ConfigBlock.ConfigBlockOption.info
info
Definition: ConfigBlock.py:63
python.ConfigBlock.alphanumeric_block_name
def alphanumeric_block_name(func)
Definition: ConfigBlock.py:30
python.ConfigBlock.ConfigBlock.getOptions
def getOptions(self)
Definition: ConfigBlock.py:368
dumpHVPathFromNtuple.append
bool append
Definition: dumpHVPathFromNtuple.py:91
python.ConfigBlock.ConfigBlockDependency.__str__
def __str__(self)
Definition: ConfigBlock.py:83
python.CaloAddPedShiftConfig.type
type
Definition: CaloAddPedShiftConfig.py:42
python.ConfigBlock.ConfigBlockOption.required
required
Definition: ConfigBlock.py:64
python.ConfigBlock.ConfigBlock.setFactoryName
def setFactoryName(self, name)
Definition: ConfigBlock.py:223
python.ConfigBlock.ConfigBlock.isUsedForConfig
def isUsedForConfig(self, config)
Definition: ConfigBlock.py:249
python.ConfigBlock.ConfigBlockOption.default
default
Definition: ConfigBlock.py:66
python.ConfigBlock.ConfigBlockDependency.__eq__
def __eq__(self, name)
Definition: ConfigBlock.py:79
python.ConfigBlock.ConfigBlock._dependencies
_dependencies
Definition: ConfigBlock.py:149
python.ConfigBlock.ConfigBlockDependency.blockName
blockName
Definition: ConfigBlock.py:75
python.ConfigBlock.ConfigBlock.get_instance_count
def get_instance_count(cls)
Definition: ConfigBlock.py:418
python.ConfigBlock.ConfigBlock.getBlockName
def getBlockName(self)
Definition: ConfigBlock.py:201
python.ConfigBlock.ConfigBlockDependency.required
required
Definition: ConfigBlock.py:76
python.ConfigBlock.ConfigBlock.__init__
def __init__(self)
Definition: ConfigBlock.py:146
python.ConfigBlock.BlockNameProcessorMeta.__new__
def __new__(cls, name, bases, dct)
Definition: ConfigBlock.py:52
python.ConfigBlock.ConfigBlock.printOptions
def printOptions(self, verbose=False, width=60, indent=" ")
Definition: ConfigBlock.py:373
python.ConfigBlock.ConfigBlock.addOption
def addOption(self, name, defaultValue, *type, info='', noneAction='ignore', required=False)
Definition: ConfigBlock.py:312
python.ConfigBlock.ConfigBlock.getOptionValue
def getOptionValue(self, name)
Definition: ConfigBlock.py:362
TrigJetMonitorAlgorithm.items
items
Definition: TrigJetMonitorAlgorithm.py:71
python.ConfigBlock.ConfigBlock.__eq__
def __eq__(self, blockName)
Definition: ConfigBlock.py:405
python.ConfigBlock.ConfigBlock._blockName
_blockName
Definition: ConfigBlock.py:147
python.ConfigBlock.filter_dsids
def filter_dsids(filterList, config)
Definition: ConfigBlock.py:13
python.ConfigBlock.ConfigBlock.factoryName
def factoryName(self)
Definition: ConfigBlock.py:205
python.ConfigBlock.ConfigBlockDependency.__init__
def __init__(self, blockName, required=True)
Definition: ConfigBlock.py:74
python.ConfigBlock.ConfigBlockDependency.__repr__
def __repr__(self)
Definition: ConfigBlock.py:87
str
Definition: BTagTrackIpAccessor.cxx:11
calibdata.copy
bool copy
Definition: calibdata.py:26
python.ConfigBlock.ConfigBlock._options
_options
Definition: ConfigBlock.py:150
python.ConfigBlock.ConfigBlockOption.__init__
def __init__(self, type=None, info='', noneAction='ignore', required=False, default=None)
Definition: ConfigBlock.py:61
xAOD::bool
setBGCode setTAP setLVL2ErrorBits bool
Definition: TrigDecision_v1.cxx:60
python.ConfigBlock.ConfigBlock
Definition: ConfigBlock.py:91
python.ConfigBlock.ConfigBlockOption
Definition: ConfigBlock.py:58
python.ConfigBlock.ConfigBlock.setBlockName
def setBlockName(self, name)
Definition: ConfigBlock.py:197
python.LArMinBiasAlgConfig.float
float
Definition: LArMinBiasAlgConfig.py:65