ATLAS Offline Software
ComponentAccumulator.py
Go to the documentation of this file.
1 
2 # Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
3 
4 import GaudiConfig2
5 from GaudiKernel.DataHandle import DataHandle
6 import GaudiKernel.GaudiHandles as GaudiHandles
7 
8 from AthenaCommon.Logging import logging
9 from AthenaCommon.Debugging import DbgStage
10 from AthenaCommon.CFElements import (isSequence, findSubSequence, findAlgorithm, iterSequences,
11  checkSequenceConsistency, findAllAlgorithmsByName)
12 
13 from AthenaConfiguration.AccumulatorCache import AccumulatorCachable
14 from AthenaConfiguration.ComponentFactory import CompFactory, isComponentAccumulatorCfg
15 from AthenaConfiguration.Deduplication import deduplicate, deduplicateOne, DeduplicationFailed
16 from AthenaConfiguration.DebuggingContext import (Context, raiseWithCurrentContext, shortCallStack,
17  createContextForDeduplication)
18 
19 import atexit
20 from collections.abc import Sequence
21 import sys,os
22 
23 class ConfigurationError(RuntimeError):
24  pass
25 
26 # Always create these services in this order:
27 _basicServicesToCreateOrder=("CoreDumpSvc/CoreDumpSvc",
28  "GeoModelSvc/GeoModelSvc",
29  "DetDescrCnvSvc/DetDescrCnvSvc")
30 
31 # Disable check of unmerged CA if we are exiting. This avoids an error on exceptions but
32 # also the need to mark CAs as merged in top-level scripts (e.g. unit tests).
33 def __exit():
34  ComponentAccumulator._checkUnmerged = False
35 atexit.register(__exit)
36 
37 
38 def printProperties(msg, c, nestLevel = 0, printDefaults=False):
39 
40  # Dictionary of (default and) explicitly set values with latter taking precedence
41  props = {**c.getDefaultProperties(), **c._properties} if printDefaults else c._properties
42 
43  for propname, propval in sorted(props.items()):
44 
45  # Recursively expand these types:
46  if isinstance(propval, GaudiConfig2.Configurable):
47  msg.info("%s * %s: %s", " "*nestLevel, propname, propval.getFullJobOptName())
48  printProperties(msg, propval, nestLevel+3, printDefaults)
49 
50  elif isinstance(propval, GaudiHandles.PrivateToolHandleArray):
51  msg.info( "%s * %s: PrivateToolHandleArray of size %s", " "*nestLevel, propname, len(propval))
52  for el in propval:
53  msg.info( "%s * %s/%s", " "*(nestLevel+3), el.__cpp_type__, el.getName())
54  printProperties(msg, el, nestLevel+6, printDefaults)
55 
56  # Only print handle keys:
57  elif isinstance(propval, DataHandle):
58  propval = propval.Path
59 
60  msg.info("%s * %s: %r", " "*nestLevel, propname, propval)
61 
62 
63 def filterComponents (comps, onlyComponents = []):
64  ret = []
65  for c in comps:
66  if not onlyComponents or c.getName() in onlyComponents:
67  ret.append((c, True))
68  elif c.getName()+'-' in onlyComponents:
69  ret.append((c, False))
70  return ret
71 
72 
73 class ComponentAccumulator(AccumulatorCachable):
74  # the debug mode is combination of the following strings:
75  # trackCA - to track CA creation,
76  # track[EventAlgo|CondAlgo|PublicTool|PrivateTool|Service|Sequence] - to track categories components addition
77  debugMode=""
78  _checkUnmerged = True
79 
80  def __init__(self,sequence='AthAlgSeq'):
81  # Ensure that we are not operating in the legacy Athena Configurable mode
82  # where only a single global instance exists
84  raise ConfigurationError(
85  """
86  ComponentAccumulator initialised with legacy (global) Configurable behavior!
87  CA Deduplication is impossible in this mode.
88  Create the CA using the AthenaCommon.Configurable.ConfigurableCABehavior context manager.
89  """
90  )
91  self._msg=logging.getLogger('ComponentAccumulator')
92  if isinstance(sequence, str):
93  kwargs={'IgnoreFilterPassed' : True,
94  'StopOverride' : True }
95  if sequence == 'AthAlgSeq' :
96  kwargs.setdefault('ProcessDynamicDataDependencies',True)
97  kwargs.setdefault('ExtraDataForDynamicConsumers',[])
98 
99  # (Nested) default sequence of event processing algorithms per sequence + their private tools
100  sequence = CompFactory.AthSequencer(sequence, **kwargs)
101 
102  self._sequence = sequence
103  self._allSequences = [self._sequence]
104  self._algorithms = {} #Dictionary of algorithm instances keyed by name
105  self._conditionsAlgs = [] #Unordered list of conditions algorithms + their private tools
106  self._services = [] #List of service, not yet sure if the order matters here in the MT age
108  self._auditors = [] #List of auditors
109  self._privateTools = None #A placeholder to carry a private tool(s) not yet attached to its parent
110  self._primaryComp = None #A placeholder to designate the primary service
111  self._currentDomain = None #Currently marked PerfMon domain
112  self._domainsRegistry = {} #PerfMon domains registry
113 
114  self._theAppProps = dict() #Properties of the ApplicationMgr
115 
116  #Backward compatibility hack: Allow also public tools:
117  self._publicTools = []
118 
119  #To check if this accumulator was merged:
120  self._wasMerged = False
121  self._isMergable = True
122  self._lastAddedComponent = "Unknown"
123  self._creationCallStack = Context.hint if "trackCA" not in ComponentAccumulator.debugMode else shortCallStack()
124  self._componentsContext = dict()
125  self._debugStage = DbgStage()
126  self.interactive = ""
127 
128  def setAsTopLevel(self):
129  self._isMergable = False
130 
131  def _inspect(self): #Create a string some basic info about this CA, useful for debugging
132  summary = "This CA contains {0} service(s), {1} conditions algorithm(s), {2} event algorithm(s) and {3} public tool(s):\n"\
133  .format(len(self._services),len(self._conditionsAlgs),len(self._algorithms),len(self._publicTools))
134 
135  if self._privateTools:
136  if isinstance(self._privateTools, list):
137  summary += " Private AlgTool: " + self._privateTools[-1].getFullJobOptName() + "\n"
138  else:
139  summary += " Private AlgTool: " + self._privateTools.getFullJobOptName() + "\n"
140 
141  if self._primaryComp:
142  summary += " Primary Component: " + self._primaryComp.getFullJobOptName() + "\n"
143 
144  summary += " Sequence(s): " + ", ".join([s.name+(" (main)" if s == self._sequence else "") for s in self._allSequences]) + "\n"
145  summary += " Last component added: " + self._lastAddedComponent+"\n"
146  summary += " Created by: " + self._creationCallStack
147  return summary
148 
149  def _cleanup(self):
150  # Delete internal data structures, to be called after all properties are transferred to the C++ application
151  del self._sequence
152  del self._allSequences
153  del self._algorithms
154  del self._conditionsAlgs
155  del self._services
156  del self._publicTools
157  del self._auditors
158 
159  # Clear all AccumulatorCaches
160  from AthenaConfiguration.AccumulatorCache import AccumulatorDecorator
161  AccumulatorDecorator.clearCache()
162 
163  # Run garbage collector
164  import gc
165  gc.collect()
166 
167  def empty(self):
168  return (len(self._sequence.Members)+len(self._conditionsAlgs)+len(self._services)+
169  len(self._publicTools)+len(self._theAppProps) == 0)
170 
171  def __del__(self):
172  if self._checkUnmerged and not getattr(self,'_wasMerged',True) and not self.empty():
173  log = logging.getLogger("ComponentAccumulator")
174  log.error("ComponentAccumulator was never merged. %s\n", self._inspect())
175  import traceback
176  traceback.print_stack()
177  if getattr(self,'_privateTools',None) is not None:
178  log = logging.getLogger("ComponentAccumulator")
179  log.error("Deleting a ComponentAccumulator with dangling private tool(s): %s",
180  " ".join([t.name for t in self._privateTools]) if isinstance(self._privateTools, Sequence) else self._privateTools.name)
181 
182  def _cacheEvict(self):
183  """Called by AccumulatorCache when deleting item from cache"""
184  self.popPrivateTools(quiet=True)
185  self.wasMerged()
186 
187  def __getstate__(self):
188  state = self.__dict__.copy()
189  # Remove the unpicklable entries.
190  del state['_msg']
191  return state
192 
193  def __setstate__(self,state):
194  self.__dict__.update(state)
195  #Re-enstate logger
196  self._msg=logging.getLogger('ComponentAccumulator')
197 
198 
199  def printCondAlgs(self, summariseProps=False, onlyComponents=[], printDefaults=False):
200  self._msg.info( "Condition Algorithms" )
201  for (c, flag) in filterComponents (self._conditionsAlgs, onlyComponents):
202  self._msg.info( " \\__ %s (cond alg)%s", c.name, self._componentsContext.get(c.name,""))
203  if summariseProps and flag:
204  printProperties(self._msg, c, 1, printDefaults)
205  return
206 
207 
208  # If onlyComponents is set, then only print components with names
209  # that appear in the onlyComponents list. If a name is present
210  # in the list with a trailing `-', then only the name of the component
211  # will be printed, not its properties.
212  def printConfig(self, withDetails=False, summariseProps=False,
213  onlyComponents = [], printDefaults=False, printSequenceTreeOnly=False, prefix=None):
214  msg = logging.getLogger(prefix) if prefix else self._msg
215 
216  msg.info( "Event Algorithm Sequences" )
217 
218  def printSeqAndAlgs(seq, nestLevel = 0,
219  onlyComponents = []):
220  def __prop(name):
221  if name in seq._properties:
222  return seq._properties[name]
223  return seq._descriptors[name].default
224  if withDetails:
225  msg.info( "%s\\__ %s (seq: %s %s)", " "*nestLevel, seq.name,
226  "SEQ" if __prop("Sequential") else "PAR",
227  "OR" if __prop("ModeOR") else "AND" + self._componentsContext.get(seq.name, "") )
228  else:
229  msg.info( "%s\\__ %s", " "*nestLevel, seq.name)
230 
231  nestLevel += 3
232  for (c, flag) in filterComponents(seq.Members, onlyComponents):
233  if isSequence(c):
234  printSeqAndAlgs(c, nestLevel, onlyComponents = onlyComponents )
235  else:
236  if withDetails:
237  msg.info( "%s\\__ %s (alg) %s", " "*nestLevel, c.getFullJobOptName(), self._componentsContext.get(c.name, ""))
238  else:
239  msg.info( "%s\\__ %s", " "*nestLevel, c.name )
240  if summariseProps and flag:
241  printProperties(msg, c, nestLevel, printDefaults)
242 
243 
244  for n,s in enumerate(self._allSequences):
245  msg.info( "Top sequence %d", n )
246  printSeqAndAlgs(s, onlyComponents = onlyComponents)
247 
248  if printSequenceTreeOnly:
249  return
250 
251  self.printCondAlgs (summariseProps = summariseProps,
252  onlyComponents = onlyComponents)
253  msg.info( "Services" )
254  msg.info( [ s[0].name + (" (created) " if s[0].name in self._servicesToCreate else "")
255  for s in filterComponents (self._services, onlyComponents) ] )
256  msg.info( "Public Tools" )
257  msg.info( "[" )
258  for (t, flag) in filterComponents (self._publicTools, onlyComponents):
259  msg.info( " %s,", t.getFullJobOptName() + self._componentsContext.get(t.name,""))
260  # Not nested, for now
261  if summariseProps and flag:
262  printProperties(msg, t, printDefaults)
263  msg.info( "]" )
264  msg.info( "Private Tools")
265  msg.info( "[" )
266  if self._privateTools:
267  for tool in self._privateTools if isinstance(self._privateTools, Sequence) else [self._privateTools]:
268  msg.info( " %s,", tool.getFullJobOptName() + self._componentsContext.get(tool.name,""))
269  if summariseProps:
270  printProperties(msg, tool, printDefaults)
271  msg.info( "]" )
272  if self._auditors:
273  msg.info( "Auditors" )
274  msg.info( [ a[0].name for a in filterComponents(self._auditors, onlyComponents) ] )
275 
276  msg.info( "theApp properties" )
277  for k, v in self._theAppProps.items():
278  msg.info(" %s : %s", k, v)
279 
280  def getIO(self):
281  """
282  Returns information about inputs needed and outputs produced by this CA
283 
284  It is a list of dictionaries containing the: type, key, R / W, the component and name of the property via which it is set
285  """
286  def __getHandles(comp):
287  io = []
288  for i in comp.ExtraInputs:
289  io.append({"type": i.split("#")[0],
290  "key": i.split("#")[1],
291  "comp": comp.getFullJobOptName(),
292  "mode": "R",
293  "prop": "ExtraInputs"})
294  for i in comp.ExtraOutputs:
295  io.append({"type": i.split("#")[0],
296  "key": i.split("#")[1],
297  "comp": comp.getFullJobOptName(),
298  "mode": "W",
299  "prop": "ExtraOutputs"})
300 
301  for prop, descr in comp._descriptors.items():
302  if isinstance(descr.default, DataHandle):
303  io.append( {"type": descr.default.type(),
304  "key": comp._properties[prop] if prop in comp._properties else descr.default.path(),
305  "comp": comp.getFullJobOptName(),
306  "mode": descr.default.mode(),
307  "prop": prop })
308  # TODO we should consider instantiating c++ defaults and fetching corresponsing props
309  if "PrivateToolHandle" == descr.cpp_type and prop in comp._properties:
310  io.extend( __getHandles(comp._properties[prop]) )
311  if "PrivateToolHandleArray" == descr.cpp_type and prop in comp._properties:
312  for tool in getattr(comp, prop):
313  io.extend( __getHandles(tool))
314  return io
315 
316  ret = []
317  for comp in self._allComponents():
318  ret.extend(__getHandles(comp))
319  return ret
320 
321 
322  def addSequence(self, newseq, primary=False, parentName = None ):
323  """ Adds new sequence. If second argument is present then it is added under another sequence """
324 
325  if not isSequence(newseq):
326  raise TypeError('{} is not a sequence'.format(newseq.name))
327 
328  if not isinstance(newseq, GaudiConfig2.Configurable):
329  raise ConfigurationError('{} is not the Conf2 Sequence, ComponentAccumulator handles only the former'.format(newseq.name))
330 
331  algorithmsInside = findAllAlgorithmsByName(newseq)
332  if len(algorithmsInside) != 0:
333  raise ConfigurationError('{} contains algorithms (or sub-sequences contain them). That is not supported. Construct ComponentAccumulator and merge it instead'.format(newseq.name))
334 
335 
336  if parentName is None:
337  parent=self._sequence
338  else:
339  parent = findSubSequence(self._sequence, parentName )
340  if parent is None:
341  raise ConfigurationError("Missing sequence {} to add new sequence to".format(parentName))
342 
343  parent.Members.append(newseq)
344  if "trackSequence" in ComponentAccumulator.debugMode:
345  self._componentsContext[newseq] = shortCallStack()
346 
347  if primary:
348  if self._primaryComp:
349  self._msg.warning("addEventAlgo: Overwriting primary component of this CA. Was %s/%s, now %s/%s",
350  self._primaryComp.__cpp_type__, self._primaryComp.name,
351  newseq.__cpp_type__, newseq.name)
352  #keep a ref of the sequence as primary component
353  self._primaryComp = newseq
354  return newseq
355 
356 
357  def getSequence(self,sequenceName=None):
358  if sequenceName is None:
359  return self._sequence
360  else:
361  return findSubSequence(self._sequence,sequenceName)
362 
363  def setPrivateTools(self,privTool):
364  """Use this method to carry private AlgTool(s) to the caller when returning this ComponentAccumulator.
365  The method accepts either a single private AlgTool or a list of private AlgTools (typically assigned to ToolHandleArray)
366  """
367  if self._privateTools is not None:
368  raise ConfigurationError("This ComponentAccumulator holds already a (list of) private tool(s). "
369  "Only one (list of) private tool(s) is allowed")
370 
371  if isinstance(privTool, Sequence):
372  for t in privTool:
373  if t.__component_type__ != 'AlgTool':
374  raise ConfigurationError("ComponentAccumulator.setPrivateTools accepts only ConfigurableAlgTools "
375  f"or lists of ConfigurableAlgTools. Encountered {type(t)} in a list")
376  else:
377  if privTool.__component_type__ != "AlgTool":
378  raise ConfigurationError("ComponentAccumulator.setPrivateTools accepts only ConfigurableAlgTools "
379  f"or lists of ConfigurableAlgTools. Encountered {type(privTool)}")
380 
381  self._privateTools=privTool
382  if "trackPrivateTool" in ComponentAccumulator.debugMode:
383  for tool in self._privateTools if isinstance(privTool, Sequence) else [self._privateTools]:
384  self._componentsContext[tool.name] = shortCallStack()
385 
386  return
387 
388  def popPrivateTools(self, quiet=False):
389  """Get the (list of) private AlgTools from this ComponentAccumulator.
390  The CA will not keep any reference to the AlgTool. Throw an exception if
391  no tools are available unless quiet=True.
392  """
393  tool = self._privateTools
394  if not quiet and tool is None:
395  raise ConfigurationError("Private tool(s) requested, but none are present")
396  self._privateTools=None
397  return tool
398 
399  def popToolsAndMerge(self, other):
400  """ Merging in the other accumulator and getting the (list of) private AlgTools
401  from this ComponentAccumulator.
402  """
403  if other is None:
404  raise RuntimeError("popToolsAndMerge called on object of type None: "
405  "did you forget to return a CA from a config function?")
406  tool = other.popPrivateTools()
407  self.merge(other)
408  return tool
409 
411  """ Get the current PerfMon domain. """
412  return self._currentDomain
413 
414  def flagPerfmonDomain(self, name):
415  """ Mark the beginning of a new PerfMon domain. """
416  self._msg.debug(f"Toggling the current algorithm domain to {name}")
417  self._currentDomain = name
418 
420  """ The actual registry keeps "alg":"domain".
421  This function inverts the registry to get "domain":["algs"].
422  """
423  result = {}
424  for i, v in self._domainsRegistry.items():
425  result[v] = [i] if v not in result.keys() else result[v] + [i]
426  return result
427 
428  def getAlgPerfmonDomain(self, name):
429  """ Return the PerfMon domain of the given algorithm """
430  if name in self._domainsRegistry:
431  return self._domainsRegistry[name]
432  else:
433  self._msg.info(f"Algorithm {name} is not in PerfMon domains registry")
434  return None
435 
436  def addAlgToPerfmonDomains(self, name, domain, overwrite=False):
437  """ Add the algorithm to the domains registry. """
438  if name not in self._domainsRegistry:
439  if domain:
440  self._domainsRegistry[name] = domain
441  self._msg.debug(f"Added algorithm {name} to the PerfMon domain {domain}")
442  else:
443  if overwrite and domain:
444  self._msg.info(f"Reassigned algorithm {name} "
445  f"from {self._domainsRegistry[name]} "
446  f"to {domain} PerfMon domain")
447  self._domainsRegistry[name] = domain
448  else:
449  self._msg.debug(f"Algorithm {name} is already in the PerfMon "
450  "domain, if you want to reassign do overwrite=True")
451 
453  """ Print the PerfMon domains. """
454  invertedDomains = self.getInvertedPerfmonDomains()
455  self._msg.info(":: This CA contains the following PerfMon domains ::")
456  self._msg.info(f":: There are a total of {len(self._domainsRegistry)} "
457  f"registered algorithms in {len(invertedDomains)} domains ::")
458  for domain, algs in invertedDomains.items():
459  self._msg.info(f"+ Domain : {domain}")
460  for alg in algs:
461  self._msg.info("\\_ %s", alg)
462  self._msg.info(":: End of PerfMon domains ::")
463 
464  def addEventAlgo(self, algorithms,sequenceName=None,primary=False,domain=None):
465  if not isinstance(algorithms, Sequence):
466  #Swallow both single algorithms as well as lists or tuples of algorithms
467  algorithms=[algorithms,]
468 
469  if sequenceName is None:
470  # If there is an AthAlgSeq add the event algorithm there by default
471  # See ATEAM-825 for a more detailed discussion for this choice
472  seq = findSubSequence(self._sequence, 'AthAlgSeq')
473  if seq is None:
474  seq = self._sequence
475  else:
476  seq = findSubSequence(self._sequence, sequenceName)
477  if seq is None:
478  self.printConfig()
479  raise ConfigurationError("Can not find sequence {}".format(sequenceName))
480 
481  for algo in algorithms:
482  if not isinstance(algo, GaudiConfig2.Configurable):
483  raise TypeError(f"Attempt to add wrong type: {type(algo).__name__} as event algorithm")
484 
485  if algo.__component_type__ != "Algorithm":
486  raise TypeError(f"Attempt to add an {algo.__component_type__} as event algorithm")
487 
488  if algo.name in self._algorithms:
489  context = createContextForDeduplication("Merging with existing Event Algorithm", algo.name, self._componentsContext) # noqa : F841
490  deduplicateOne(algo, self._algorithms[algo.name])
491  deduplicateOne(self._algorithms[algo.name], algo)
492  else:
493  self._algorithms[algo.name]=algo
494 
495  existingAlgInDest = findAlgorithm(seq, algo.name)
496  if not existingAlgInDest:
497  seq.Members.append(self._algorithms[algo.name])
498  # Assign the algorithm to a domain
499  self.addAlgToPerfmonDomains(algo.name, self._currentDomain if not domain else domain)
500 
501  if primary:
502  if len(algorithms)>1:
503  self._msg.warning("Called addEvenAlgo with a list of algorithms and primary==True. "
504  "Designating the first algorithm as primary component")
505  if self._primaryComp:
506  self._msg.warning("addEventAlgo: Overwriting primary component of this CA. Was %s/%s, now %s/%s",
507  self._primaryComp.__cpp_type__, self._primaryComp.name,
508  algorithms[0].__cpp_type__, algorithms[0].name)
509  #keep a ref of the algorithm as primary component
510  self._primaryComp = algorithms[0]
511  self._lastAddedComponent = algorithms[-1].name
512 
513  if "trackEventAlgo" in ComponentAccumulator.debugMode:
514  for algo in algorithms:
515  self._componentsContext[algo.name] = shortCallStack()
516 
517  return None
518 
519  def getEventAlgo(self, name=None):
520  """Get algorithm with `name`"""
521  if name not in self._algorithms:
522  raise ConfigurationError("Can not find an algorithm of name {} ".format(name))
523  return self._algorithms[name]
524 
525  def getEventAlgos(self, seqName=None):
526  """Get all algorithms within sequence"""
527  seq = self._sequence if seqName is None else findSubSequence(self._sequence, seqName )
528  return [s for s in iterSequences(seq) if not isSequence(s)]
529 
530  def addCondAlgo(self,algo,primary=False,domain=None):
531  """Add Conditions algorithm"""
532  if not isinstance(algo, GaudiConfig2.Configurable):
533  raise TypeError(f"Attempt to add wrong type: {type(algo).__name__} as conditions algorithm")
534 
535  if algo.__component_type__ != "Algorithm":
536  raise TypeError(f"Attempt to add wrong type: {algo.__component_type__} as conditions algorithm")
537 
538  context = createContextForDeduplication("Merging with existing Conditions Algorithm", algo.name, self._componentsContext) # noqa : F841
539 
540  deduplicate(algo, self._conditionsAlgs) #will raise on conflict
541  if primary:
542  if self._primaryComp:
543  self._msg.warning("addCondAlgo: Overwriting primary component of this CA. Was %s/%s, now %s/%s",
544  self._primaryComp.__cpp_type__, self._primaryComp.name,
545  algo.__cpp_type__, algo.name)
546  #keep a ref of the de-duplicated conditions algorithm as primary component
547  self._primaryComp = self.__getOne(self._conditionsAlgs, algo.name, "ConditionsAlgos")
548 
549  self._lastAddedComponent=algo.name
550  if "trackCondAlgo" in ComponentAccumulator.debugMode:
551  self._componentsContext[algo.name] = shortCallStack()
552 
553  # Assign the algorithm to a domain
554  self.addAlgToPerfmonDomains(algo.name, 'Conditions' if not domain else domain)
555 
556  return algo
557 
558  def getCondAlgos(self):
559  """Get all conditions algorithms"""
560  return self._conditionsAlgs
561 
562  def getCondAlgo(self, name):
563  """Get conditions algorithm by name"""
564  return self.__getOne( self._conditionsAlgs, name, "conditions algorithms")
565 
566  def addService(self, newSvc, primary=False, create=False):
567  """Add service and return the deduplicated instance"""
568  if not isinstance(newSvc, GaudiConfig2.Configurable):
569  raise TypeError(f"Attempt to add wrong type: {type(newSvc).__name__} as service")
570 
571  if newSvc.__component_type__ != "Service":
572  raise TypeError(f"Attempt to add wrong type: {newSvc.__component_type__} as service")
573 
574  context = createContextForDeduplication("Merging with existing Service", newSvc.name, self._componentsContext) # noqa : F841
575 
576  deduplicate(newSvc, self._services) #may raise on conflict
577  if primary:
578  if self._primaryComp:
579  self._msg.warning("addService: Overwriting primary component of this CA. Was %s/%s, now %s/%s",
580  self._primaryComp.__cpp_type__, self._primaryComp.name,
581  newSvc.__cpp_type__, newSvc.name)
582  #keep a ref of the de-duplicated service as primary component
583  self._primaryComp=self.__getOne( self._services, newSvc.name, "Services")
584  self._lastAddedComponent=newSvc.name
585 
586  if create:
587  sname = newSvc.getFullJobOptName()
588  if sname not in self._servicesToCreate:
589  self._servicesToCreate.append(sname)
590  if "trackService" in ComponentAccumulator.debugMode:
591  self._componentsContext[newSvc.name] = shortCallStack()
592  return self.__getOne( self._services, newSvc.name, "Services")
593 
594 
595  def addAuditor(self, auditor):
596  """Add Auditor to ComponentAccumulator and return the deduplicated instance.
597  This function will also create the required AuditorSvc."""
598  if not isinstance(auditor, GaudiConfig2.Configurable):
599  raise TypeError(f"Attempt to add wrong type: {type(auditor).__name__} as auditor")
600 
601  if auditor.__component_type__ != "Auditor":
602  raise TypeError(f"Attempt to add wrong type: {auditor.__component_type__} as auditor")
603 
604  context = createContextForDeduplication("Merging with existing auditors", auditor.name, self._componentsContext) # noqa : F841
605 
606  deduplicate(auditor, self._auditors) #may raise on conflict
607  newAuditor = self.addService(CompFactory.AuditorSvc(Auditors=[auditor.getFullJobOptName()]))
608  self._lastAddedComponent = auditor.name
609  return newAuditor
610 
611 
612  def addPublicTool(self, newTool, primary=False):
613  """Add public tool and return the deduplicated instance."""
614  if not isinstance(newTool, GaudiConfig2.Configurable):
615  raise TypeError(f"Attempt to add wrong type: {type(newTool).__name__} as public AlgTool")
616 
617  if newTool.__component_type__ != "AlgTool":
618  raise TypeError(f"Attempt to add wrong type: {newTool.__component_type__} as public AlgTool")
619 
620  context = createContextForDeduplication("Merging with existing Public Tool", newTool.name, self._componentsContext) # noqa : F841
621 
622  deduplicate(newTool,self._publicTools)
623  if primary:
624  if self._primaryComp:
625  self._msg.warning("addPublicTool: Overwriting primary component of this CA. Was %s/%s, now %s/%s",
626  self._primaryComp.__cpp_type__, self._primaryComp.name,
627  newTool.__cpp_type__, newTool.name)
628  #keep a ref of the de-duplicated tool as primary component
629  self._primaryComp=self.__getOne( self._publicTools, newTool.name, "Public Tool")
630  self._lastAddedComponent=newTool.name
631  if "trackPublicTool" in ComponentAccumulator.debugMode:
632  self._componentsContext[newTool.name] = shortCallStack()
633  # return the new public tool
634  return self.__getOne(self._publicTools, newTool.name, "Public Tool")
635 
636 
637  def getPrimary(self):
638  """Get designated primary component"""
639  if self._privateTools:
640  return self.popPrivateTools()
641  elif self._primaryComp:
642  return self._primaryComp
643  else:
644  raise ConfigurationError("Called getPrimary() but no primary component nor private AlgTool is known.\n{}".format(self._inspect()))
645 
646  def getPrimaryAndMerge(self, other):
647  """ Merging in the other accumulator and getting the primary component"""
648  if other is None:
649  raise RuntimeError("merge called on object of type None: did you forget to return a CA from a config function?")
650  comp = other.getPrimary()
651  self.merge(other)
652  return comp
653 
654  def __getOne(self, allcomps, name=None, typename="???"):
655  selcomps = allcomps if name is None else [ t for t in allcomps if t.name == name ]
656  if len( selcomps ) == 0:
657  raise ConfigurationError(f"Requested component of name {name} but is missing" )
658 
659  if len( selcomps ) == 1:
660  return selcomps[0]
661  nmstr = f'with name {name} ' if name else ''
662  raise ConfigurationError("Number of {} available {}{} which is != 1 expected by this API".format(typename, nmstr, len(selcomps)) )
663 
664  def getPublicTools(self):
665  return self._publicTools
666 
667  def getPublicTool(self, name=None):
668  """Returns single public tool, exception if either not found or to many found"""
669  return self.__getOne( self._publicTools, name, "PublicTools")
670 
671  def getServices(self):
672  return self._services
673 
674  def getService(self, name=None):
675  """Returns single service, exception if either not found or to many found"""
676  if name is None:
677  return self._primarySvc
678  else:
679  return self.__getOne( self._services, name, "Services")
680 
681  def getAuditor(self,name):
682  """Retuns a single auditor, exception if not found"""
683  return self.__getOne(self._auditors,name,"Auditors")
684 
685  def dropEventAlgo(self,name,sequence="AthAlgSeq"):
686  s=self.getSequence(sequence)
687  lenBefore=len(s.Members)
688  s.Members = [a for a in s.Members if not a.getName()==name]
689  lenAfter=len(s.Members)
690  if lenAfter == lenBefore:
691  self._msg.warning("Algorithm %s not found in sequence %s",name,sequence)
692  else:
693  self._msg.info("Removed algorithm %s from sequence %s",name,sequence)
694  try:
695  del self._algorithms[name]
696  except KeyError:
697  self._msg.warning("Algorithm %s not found in self._sequence ???",name)
698  return
699 
700  def popEventAlgo(self,name,sequence="AthAlgSeq"):
701  s=self.getSequence(sequence)
702  lenBefore=len(s.Members)
703  s.Members = [a for a in s.Members if not a.getName()==name]
704  lenAfter=len(s.Members)
705  if lenAfter == lenBefore:
706  self._msg.warning("Algorithm %s not found in sequence %s",name,sequence)
707  else:
708  self._msg.info("Removed algorithm %s from sequence %s",name,sequence)
709  try:
710  return self._algorithms.pop(name)
711  except KeyError:
712  self._msg.warning("Algorithm %s not found in self._sequence ??? Returning 'None'",name)
713  return None
714 
715  def dropCondAlgo(self,name):
716  lenBefore=len(self._conditionsAlgs)
717  self._conditionsAlgs = [a for a in self._conditionsAlgs if a.getName()!=name]
718  lenAfter=len(self._conditionsAlgs)
719  if lenAfter == lenBefore:
720  self._msg.warning("Condition Algorithm %s not found",name)
721  else:
722  self._msg.info("Removed conditions Algorithm %s",name)
723  return
724 
725  def dropService(self,name):
726  lenBefore=len(self._services)
727  self._services = [s for s in self._services if s.getName()!=name]
728  lenAfter=len(self._services)
729  if lenAfter == lenBefore:
730  self._msg.warning("Service %s not found",name)
731  else:
732  self._msg.info("Removed Service %s",name)
733  return
734 
735  def dropPublicTool(self,name):
736  lenBefore=len(self._publicTools)
737  self._publicTools = [p for p in self._publicTools if p.getName()!=name]
738  lenAfter=len(self._publicTools)
739  if lenAfter == lenBefore:
740  self._msg.warning("Public tool %s not found",name)
741  else:
742  self._msg.info("Removed public tool %s",name)
743  return
744 
745  def dropAuditor(self,name):
746  lenBefore=len(self._auditors)
747  self._auditors = [a for a in self._auditors if a.getName()!=name]
748  lenAfter=len(self._auditors)
749  if lenAfter == lenBefore:
750  self._msg.warning("Auditor %s not found",name)
751  else:
752  self._msg.info("Removed auditor %s",name)
753  return
754 
755  def getAppProps(self):
756  return self._theAppProps
757 
758  def setAppProperty(self,key,value,overwrite=False):
759  if (overwrite or key not in (self._theAppProps)):
760  self._theAppProps[key]=value
761  else:
762  if self._theAppProps[key] == value:
763  self._msg.debug("ApplicationMgr property '%s' already set to '%s'.", key, value)
764  elif isinstance(self._theAppProps[key], Sequence) and not isinstance(self._theAppProps[key],str):
765  value=self._theAppProps[key] + [el for el in value if el not in self._theAppProps[key]]
766  self._msg.info("ApplicationMgr property '%s' already set to '%s'. Overwriting with %s", key, self._theAppProps[key], value)
767  self._theAppProps[key]=value
768  else:
769  raise DeduplicationFailed("AppMgr property {} set twice: {} and {}".format(key, self._theAppProps[key], value))
770 
771 
772  def setDebugStage(self,stage):
773  if stage not in DbgStage.allowed_values:
774  raise RuntimeError("Allowed arguments for setDebugStage are [{}]".format(",".join(DbgStage.allowed_values)))
775  self._debugStage.value = stage
776 
777 
778  def merge(self,other, sequenceName=None):
779  """Merging in the other accumulator"""
780  if other is None:
781  raise RuntimeError("merge called on object of type None: "
782  "did you forget to return a CA from a config function?")
783 
784  if not isinstance(other,ComponentAccumulator):
785  raise TypeError(f"Attempt to merge wrong type {type(other).__name__}. "
786  "Only instances of ComponentAccumulator can be added")
787 
788  context = (Context.hint if not ComponentAccumulator.debugMode else # noqa: F841
789  Context("When merging the ComponentAccumulator:\n{} \nto:\n{}".format(other._inspect(), self._inspect())))
790 
791  if other._privateTools is not None:
792  if isinstance(other._privateTools, Sequence):
793  raiseWithCurrentContext(RuntimeError(
794  "merge called on ComponentAccumulator with a dangling (array of) private tools\n"))
795  else:
796  raiseWithCurrentContext(RuntimeError(
797  "merge called on ComponentAccumulator with a dangling private tool "
798  f"{other._privateTools.__cpp_type__}/{other._privateTools.name}"))
799 
800  if not other._isMergable:
802  "Attempted to merge a top level ComponentAccumulator. Revert the order of merging\n"))
803 
804  def mergeSequences( dest, src ):
805 
806  # Ensure sequences of same name have same properties. This is called many times
807  # and is highly optimized. Make sure you profile before modifying this code.
808  if dest.name == src.name:
809  # Compare all set properties ignoring 'Members'
810  props = (dest._properties.keys() | src._properties.keys()) - {'Members'}
811  for seqProp in props:
812  try:
813  if dest._properties[seqProp] != src._properties[seqProp]:
814  raise RuntimeError(
815  f"Merging two sequences with name '{dest.name}' but property '{seqProp}' "
816  f"has different values: {getattr(dest, seqProp)} vs {getattr(src, seqProp)}")
817  except KeyError as e:
818  raise RuntimeError(
819  f"Merging two sequences with name '{dest.name}' but property '{e}' is not "
820  f"set in one of them")
821 
822  for childIdx, c in enumerate(src.Members):
823  if isSequence( c ):
824  sub = findSubSequence( dest, c.name ) #depth=1 ???
825  if sub:
826  mergeSequences(sub, c )
827  else:
828  self._msg.debug(" Merging sequence %s to a destination sequence %s", c.name, dest.name )
829  algorithmsByName = findAllAlgorithmsByName(c) # dictionary: algName (alg, parentSeq, indexInParentSeq)
830  for name, existingAlgs in algorithmsByName.items():
831  # all algorithms from incoming CA are already deduplicated, so we can only handle the fist one
832  algInstance, _, _ = existingAlgs[0]
833  if name not in self._algorithms:
834  self._algorithms[name] = algInstance
835  else:
836  dedupContext1 = createContextForDeduplication("While merging sequences adding incoming algorithm", c.name, other._componentsContext) # noqa : F841
837  dedupContext2 = createContextForDeduplication("While merging sequences adding to existing algorithm", c.name, self._componentsContext) # noqa : F841
838  deduplicateOne(self._algorithms[name], algInstance)
839  deduplicateOne(algInstance, self._algorithms[name])
840  for _, parent, idx in existingAlgs: # put the deduplicated algo back into original sequences
841  parent.Members[idx] = self._algorithms[name]
842  # Add the algorithm to the PerfMon domains
843  self.addAlgToPerfmonDomains(name, other._domainsRegistry[name] if name in other._domainsRegistry else self._currentDomain)
844  dest.Members.append(c)
845 
846  else: # an algorithm
847  if c.name in self._algorithms:
848  dedupContext1 = createContextForDeduplication("While merging sequences adding incoming algorithm", c.name, other._componentsContext) # noqa : F841
849  dedupContext2 = createContextForDeduplication("While merging sequences adding to existing algorithm", c.name, self._componentsContext) # noqa : F841
850 
851  deduplicateOne(self._algorithms[c.name], c)
852  deduplicateOne(c, self._algorithms[c.name])
853  src.Members[childIdx] = self._algorithms[c.name]
854  else:
855  self._algorithms[c.name] = c
856 
857  existingAlgInDest = findAlgorithm( dest, c.name, depth=1 )
858  if not existingAlgInDest:
859  self._msg.debug(" Adding algorithm %s to a sequence %s", c.name, dest.name )
860  dest.Members.append(c)
861 
862  # Add the algorithm to the PerfMon domains
863  self.addAlgToPerfmonDomains(c.name, other._domainsRegistry[c.name] if c.name in other._domainsRegistry else self._currentDomain)
864 
865  # Merge sequences:
866  # mergeSequences(destSeq, other._sequence)
867  # if sequenceName is provided it means we should be ignoring the actual MAIN seq name there and use the sequenceName
868  # that means the first search in the destination sequence needs to be cheated
869  # the sequenceName argument is only relevant for the MAIN sequence,
870  # secondary top sequences are treated as if the sequenceName argument would not be provided
871 
872  for otherSeq in other._allSequences:
873  found=False
874  for ourSeq in self._allSequences:
875  destSeqName = otherSeq.name
876  if sequenceName and otherSeq == other._sequence: # if sequence moving is requested (sequenceName != None) it concerns only the main sequence
877  destSeqName = sequenceName
878  self._msg.verbose(" Will move sequence %s to %s", otherSeq.name, destSeqName )
879 
880  ourSeq = findSubSequence(ourSeq, destSeqName) # try to add sequence to the main structure first, to each seq in parent?
881  if ourSeq:
882  mergeSequences(ourSeq, otherSeq)
883  found=True
884  self._msg.verbose(" Succeeded to merge sequence %s to %s", otherSeq.name, ourSeq.name )
885  else:
886  self._msg.verbose(" Failed to merge sequence %s to any existing one, destination CA will have several top/dangling sequences", otherSeq.name )
887  if not found: # just copy the sequence as a dangling one
888  self._allSequences.append( otherSeq )
889  mergeSequences( self._allSequences[-1], otherSeq )
890 
891 
892 
893 
894 
895  # Additional checking and updating other accumulator's algorithms list
896  for name in other._algorithms:
897  if name not in self._algorithms:
898  raiseWithCurrentContext(ConfigurationError('Error in merging. Algorithm {} missing in destination accumulator\n'.format(name)))
899  other._algorithms[name] = self._algorithms[name]
900 
901  #self._conditionsAlgs+=other._conditionsAlgs
902  for condAlg in other._conditionsAlgs:
903  addContext = createContextForDeduplication("Merging incoming Conditions Algorithm", condAlg.name, other._componentsContext) # noqa : F841
904  self.addCondAlgo(condAlg) #Profit from deduplicaton here
905 
906  for svc in other._services:
907  addContext = createContextForDeduplication("Merging incoming Service", svc.name, other._componentsContext) # noqa : F841
908  self.addService(svc, create = svc.getFullJobOptName() in other._servicesToCreate) #Profit from deduplicaton here
909 
910  for pt in other._publicTools:
911  addContext = createContextForDeduplication("Merging incoming Public Tool", pt.name, other._componentsContext) # noqa : F841
912  self.addPublicTool(pt) #Profit from deduplicaton here
913 
914 
915  for aud in other._auditors:
916  addContext = createContextForDeduplication("Merging incoming Auditor", aud.name, other._componentsContext) # noqa : F841
917  self.addAuditor(aud) #Profit from deduplicaton here
918 
919  #Merge AppMgr properties:
920  for (k,v) in other._theAppProps.items():
921  self.setAppProperty(k,v) #Will warn about overrides
922  pass
923  other._wasMerged=True
924 
925  self._lastAddedComponent = other._lastAddedComponent #+ ' (Merged)'
926  self._componentsContext.update(other._componentsContext) # update the context so it contains an information about the new components (and refreshed old components)
927 
929  if len(self._allSequences) != 1:
930  raiseWithCurrentContext(ConfigurationError('It is not allowed for the storable CA to have more than one top sequence, now it has: {}'
931  .format(','.join([ s.name for s in self._allSequences]))))
932 
933 
934  def wasMerged(self):
935  """ Declares CA as merged
936 
937  This is temporarily needed by HLT and should not be used elsewhere
938  """
939  self._wasMerged=True
940 
941  def _allComponents(self):
942  """ returns iterable over all components """
943  import itertools
944  return itertools.chain(self._publicTools,
945  self._privateTools if self._privateTools else [],
946  self._algorithms.values(),
947  self._conditionsAlgs)
948 
949 
950  def store(self,outfile, withDefaultHandles=False):
951  """
952  Saves CA in pickle form
953 
954  when withDefaultHandles is True, also the handles that are not set are saved
955  """
956 
958 
959  self.wasMerged()
960  if withDefaultHandles:
961  from AthenaConfiguration.Utils import loadDefaultComps, exposeHandles
964  import pickle
965  pickle.dump(self,outfile)
966  return
967 
968 
969  def createApp(self):
970  # Create the Gaudi object early.
971  # Without this here, pyroot can sometimes get confused
972  # and report spurious type mismatch errors about this object.
973  import ROOT
974  ROOT.gROOT.SetBatch(True)
975  ROOT.Gaudi
976 
977  appPropsToSet, mspPropsToSet, bshPropsToSet = self.gatherProps()
978 
979  self._wasMerged = True
980  from Gaudi.Main import BootstrapHelper
981 
982  bsh = BootstrapHelper()
983  app = bsh.createApplicationMgr()
984 
985  for k, v in appPropsToSet.items():
986  self._msg.debug("Setting property %s : %s", k, v)
987  app.setProperty(k, v)
988 
989  app.configure()
990 
991  msp = app.getService("MessageSvc")
992  for k, v in mspPropsToSet.items():
993  self._msg.debug("Setting property %s : %s", k, v)
994  bsh.setProperty(msp, k.encode(), v.encode())
995 
996  # Feed the jobO service with the remaining options
997  for comp, name, value in bshPropsToSet:
998  self._msg.debug("Adding %s.%s = %s", comp, name, value)
999  app.setOption(f"{comp}.{name}", value)
1000 
1001  sys.stdout.flush()
1002  return app
1003 
1004  def gatherProps(self):
1005  appPropsToSet = {k: str(v) for k, v in self._theAppProps.items()}
1006  mspPropsToSet = {}
1007  bshPropsToSet = []
1008  svcToCreate = []
1009  extSvc = []
1010  for svc in self._services:
1011  extSvc += [
1012  svc.getFullJobOptName(),
1013  ]
1014  if svc.getFullJobOptName() in self._servicesToCreate:
1015  svcToCreate.append(svc.getFullJobOptName())
1016 
1017  # order basic services
1018  for bs in reversed(_basicServicesToCreateOrder):
1019  if bs in svcToCreate:
1020  svcToCreate.insert(0, svcToCreate.pop( svcToCreate.index(bs) ) )
1021 
1022  extSvc.append("PyAthena::PyComponentMgr/PyComponentMgr")
1023 
1024  appPropsToSet["ExtSvc"] = str(extSvc)
1025  appPropsToSet["CreateSvc"] = str(svcToCreate)
1026 
1027  def getCompsToBeAdded(comp, namePrefix=""):
1028  name = namePrefix + comp.getName()
1029  for k, v in comp._properties.items():
1030  # Handle special cases of properties:
1031  # 1.PrivateToolHandles
1032  if isinstance(v, GaudiConfig2.Configurable):
1033  # Add the name of the tool as property to the parent
1034  bshPropsToSet.append((name, k, v.getFullJobOptName()))
1035  # Recursively add properties of this tool to the JobOptionSvc
1036  getCompsToBeAdded(v, namePrefix=name + ".")
1037  # 2. PrivateToolHandleArray
1038  elif isinstance(v, GaudiHandles.PrivateToolHandleArray):
1039  # Add names of tools as properties to the parent
1040  bshPropsToSet.append(
1041  (name, k, str([v1.getFullJobOptName() for v1 in v]),)
1042  )
1043  # Recursively add properties of tools to JobOptionsSvc
1044  for v1 in v:
1045  getCompsToBeAdded(v1, namePrefix=name + ".")
1046  else:
1047  # For a list of DataHandle, we need to stringify
1048  # each element individually. Otherwise, we get the repr
1049  # version of the elements, which Gaudi JO will choke on.
1050  if isinstance(v, list) and v and isinstance(v[0], DataHandle):
1051  v = [str(x) for x in v]
1052  # For sequences, need to convert the list of algs to names
1053  elif isSequence(comp) and k == "Members":
1054  v = [alg.getFullJobOptName() for alg in comp.Members]
1055  vstr = "" if v is None else str(v)
1056  bshPropsToSet.append((name, k, vstr))
1057 
1058  try:
1059  from AthenaPython import PyAthenaComps
1060  PyAlg = PyAthenaComps.Alg
1061  PySvc = PyAthenaComps.Svc
1062  except ImportError:
1063  PyAlg = type(None)
1064  PySvc = type(None)
1065 
1066  # Services
1067  for svc in self._services:
1068  if svc.getName() != "MessageSvc": # MessageSvc will exist already! Needs special treatment
1069  getCompsToBeAdded(svc)
1070  if isinstance(svc, PySvc):
1071  svc.setup()
1072  else:
1073  mspPropsToSet.update((k,str(v)) for k,v in svc._properties.items())
1074 
1075  # Algorithms and Sequences
1076  for alg in iterSequences(self._sequence):
1077  getCompsToBeAdded(alg)
1078  if isinstance(alg, PyAlg):
1079  alg.setup()
1080 
1081  # Cond Algs
1082  condalgseq = []
1083  for alg in self._conditionsAlgs:
1084  getCompsToBeAdded(alg)
1085  condalgseq.append(alg.getFullJobOptName())
1086  if isinstance(alg, PyAlg):
1087  alg.setup()
1088  bshPropsToSet.append(("AthCondSeq", "Members", str(condalgseq)))
1089 
1090  # Public Tools
1091  for pt in self._publicTools:
1092  getCompsToBeAdded(pt, namePrefix="ToolSvc.")
1093 
1094  # Auditors
1095  for aud in self._auditors:
1096  getCompsToBeAdded(aud)
1097 
1098  return appPropsToSet, mspPropsToSet, bshPropsToSet
1099 
1100  def run(self,maxEvents=None):
1101  from os import environ
1102  outpklfile = environ.get("PICKLECAFILE", None)
1103  if outpklfile is not None:
1104  if outpklfile: # non-empty string
1105  self._msg.info("Storing configuration in pickle file %s",outpklfile)
1106  with open(outpklfile, "wb") as f:
1107  self.store(f)
1108  else: # empty string, just exit
1109  self.wasMerged()
1110  self._msg.info("Exiting after configuration stage")
1111  from Gaudi.Main import BootstrapHelper
1112  return BootstrapHelper.StatusCode(True)
1113 
1114  # Make sure python output is flushed before triggering output from Gaudi.
1115  # Otherwise, observed output ordering may differ between py2/py3.
1116  sys.stdout.flush()
1117 
1118 
1119  #Set TDAQ_ERS_NO_SIGNAL_HANDLERS to avoid interference with
1120  #TDAQ signal handling
1121  environ['TDAQ_ERS_NO_SIGNAL_HANDLERS']='1'
1122  from AthenaCommon.Debugging import allowPtrace, hookDebugger
1123  allowPtrace()
1124 
1126 
1127  app = self.createApp()
1129 
1130  #Determine maxEvents
1131  if maxEvents is None:
1132  if "EvtMax" in self._theAppProps:
1133  maxEvents=self._theAppProps["EvtMax"]
1134  else:
1135  maxEvents=-1
1136 
1137  if self.interactive == 'init':
1139  from sys import exit # noqa: F401
1140  startInteractive(locals())
1141 
1142  #At this point, we don't need the internal structures of this CA any more, clean them up
1143  self._cleanup()
1144 
1145  self._msg.info(f"Athena job with pid {os.getpid()}")
1146 
1147  if (self._debugStage.value == "init"):
1148  hookDebugger()
1149  sc = app.initialize()
1150  if not sc.isSuccess():
1151  self._msg.error("Failed to initialize AppMgr")
1152  return sc
1153 
1154  sc = app.start()
1155  if not sc.isSuccess():
1156  self._msg.error("Failed to start AppMgr")
1157  return sc
1158 
1159  if (self._debugStage.value=="exec"):
1160  hookDebugger()
1161 
1162 
1163  if self.interactive == 'run':
1165  from AthenaPython.PyAthena import py_svc
1166  sg=py_svc("StoreGateSvc/StoreGateSvc")
1167  startInteractive(locals())
1168  else:
1169  sc = app.run(maxEvents)
1170  if not sc.isSuccess():
1171  self._msg.error("Failure running application")
1172  return sc
1173 
1174  scStop=app.stop()
1175  if not scStop.isSuccess():
1176  self._msg.error("Failed to stop AppMgr")
1177  return scStop
1178 
1179  if (self._debugStage.value == "fini"):
1180  hookDebugger()
1181 
1182  scFin=app.finalize()
1183  if not scFin.isSuccess():
1184  self._msg.error("Failed to finalize AppMgr")
1185  return scFin
1186 
1187  sc1 = app.terminate()
1188  return sc1
1189 
1190  def foreach_component(self, path):
1191  """Utility to set properties of components using wildcards.
1192 
1193  Example:
1194  ca.foreach_component("*/HLTTop/*/*Hypo*").OutputLevel = VERBOSE
1195 
1196  The components name and location in the CF tree are translated into a UNIX-like path
1197  and are matched using the `fnmatch` library. If the property is set successfully
1198  an INFO message is printed else a WARNING.
1199 
1200  The convention for paths of nested components is as follows:
1201  Sequence : only the name is used in the path
1202  Algorithm : "type/name" is used
1203  Private Tool : ToolHandle property name plus "type/name" is used
1204  Public Tool : located under "ToolSvc/" and "type/name" is used
1205  Service : located under "SvcMgr/" and "type/name" is used
1206  """
1207  from AthenaConfiguration.PropSetterProxy import PropSetterProxy
1208  return PropSetterProxy(self, path)
1209 
1210 
1211 def startInteractive(localVarDic):
1212  """Setup and start a useful interactive session including auto-completion and history"""
1213  import code
1214 
1215  # collect all global and local variables
1216  vars = sys.modules['__main__'].__dict__
1217  vars.update(globals())
1218  vars.update(localVarDic)
1219 
1220  # configure the prompt
1221  from AthenaCommon.Interactive import configureInteractivePrompt
1223 
1224  # start the interpreter
1225  code.interact(local=vars)
1226 
1227 
1229  print("Interactive mode")
1230  print("\tThe ComponentAccumulator is known as 'self', you can inspect it but changes are not taken into account.")
1231  print("\tThe application is known as 'app' but not yet initialized.")
1232  print("\t^D will exit the interactive mode and athena will continue.")
1233  print("\texit() will terminate the program now.")
1234  return
1235 
1236 
1238  print("Interactive mode")
1239  print("\tThe application is known as 'app' and initialized.")
1240  print("\tYou can process N events with 'app.run(N)'.")
1241  print("\tStoreGate is accessible as 'sg'.")
1242  print("\t^D will exit the interactive mode and athena will finalize.")
1243  return
1244 
1245 
1246 # Make legacy support available in legacy jobs
1247 if not isComponentAccumulatorCfg():
1248  from AthenaConfiguration.LegacySupport import (conf2toConfigurable, # noqa: F401 (for client use)
1249  CAtoGlobalWrapper,
1250  appendCAtoAthena)
1251 # and the same names in CA (to support migration) but bomb on calling them
1252 else:
1253  def conf2toConfigurable(*args, **kwargs):
1254  raise RuntimeError("conf2toConfigurable cannot be called in a CA job")
1255  def CAtoGlobalWrapper(*args, **kwargs):
1256  raise RuntimeError("CAtoGlobalWrapper cannot be called in a CA job")
1257  def appendCAtoAthena(*args, **kwargs):
1258  raise RuntimeError("appendCAtoAthena cannot be called in a CA job")
python.ComponentAccumulator.ConfigurationError
Definition: ComponentAccumulator.py:23
python.ComponentAccumulator.ComponentAccumulator.setAsTopLevel
def setAsTopLevel(self)
Definition: ComponentAccumulator.py:128
python.ComponentAccumulator.ComponentAccumulator.getServices
def getServices(self)
Definition: ComponentAccumulator.py:671
DerivationFramework::TriggerMatchingUtils::sorted
std::vector< typename R::value_type > sorted(const R &r, PROJ proj={})
Helper function to create a sorted vector from an unsorted range.
python.ComponentAccumulator.ComponentAccumulator.__getstate__
def __getstate__(self)
Definition: ComponentAccumulator.py:187
python.ComponentAccumulator.ComponentAccumulator._cleanup
def _cleanup(self)
Definition: ComponentAccumulator.py:149
python.DebuggingContext.raiseWithCurrentContext
def raiseWithCurrentContext(exception)
Definition: DebuggingContext.py:44
python.ComponentAccumulator.ComponentAccumulator._theAppProps
_theAppProps
Definition: ComponentAccumulator.py:114
python.ComponentAccumulator.ComponentAccumulator._publicTools
_publicTools
Definition: ComponentAccumulator.py:117
python.ComponentAccumulator.ComponentAccumulator._allComponents
def _allComponents(self)
Definition: ComponentAccumulator.py:941
python.ComponentAccumulator.ComponentAccumulator.store
def store(self, outfile, withDefaultHandles=False)
Definition: ComponentAccumulator.py:950
python.ComponentAccumulator.ComponentAccumulator.dropCondAlgo
def dropCondAlgo(self, name)
Definition: ComponentAccumulator.py:715
python.ComponentAccumulator.ComponentAccumulator._creationCallStack
_creationCallStack
Definition: ComponentAccumulator.py:123
python.ComponentAccumulator.ComponentAccumulator.addEventAlgo
def addEventAlgo(self, algorithms, sequenceName=None, primary=False, domain=None)
Definition: ComponentAccumulator.py:464
python.ComponentAccumulator.ComponentAccumulator._checkUnmerged
_checkUnmerged
Definition: ComponentAccumulator.py:78
vtune_athena.format
format
Definition: vtune_athena.py:14
python.ComponentAccumulator.appendCAtoAthena
def appendCAtoAthena(*args, **kwargs)
Definition: ComponentAccumulator.py:1257
python.ComponentAccumulator.ComponentAccumulator.__getOne
def __getOne(self, allcomps, name=None, typename="???")
Definition: ComponentAccumulator.py:654
python.ComponentAccumulator.ComponentAccumulator._inspect
def _inspect(self)
Definition: ComponentAccumulator.py:131
python.ComponentAccumulator.ComponentAccumulator.empty
def empty(self)
Definition: ComponentAccumulator.py:167
python.ComponentAccumulator.ComponentAccumulator.getAppProps
def getAppProps(self)
Definition: ComponentAccumulator.py:755
python.ComponentAccumulator.CAtoGlobalWrapper
def CAtoGlobalWrapper(*args, **kwargs)
Definition: ComponentAccumulator.py:1255
python.ComponentAccumulator.ComponentAccumulator.createApp
def createApp(self)
Definition: ComponentAccumulator.py:969
python.ComponentAccumulator.ComponentAccumulator._msg
_msg
Definition: ComponentAccumulator.py:91
python.CFElements.findAllAlgorithmsByName
def findAllAlgorithmsByName(sequence, namesToLookFor=None)
Definition: CFElements.py:143
python.ComponentAccumulator.ComponentAccumulator._allSequences
_allSequences
Definition: ComponentAccumulator.py:103
python.ComponentAccumulator.ComponentAccumulator.addAuditor
def addAuditor(self, auditor)
Definition: ComponentAccumulator.py:595
python.ComponentAccumulator.printProperties
def printProperties(msg, c, nestLevel=0, printDefaults=False)
Definition: ComponentAccumulator.py:38
python.ComponentAccumulator.ComponentAccumulator._domainsRegistry
_domainsRegistry
Definition: ComponentAccumulator.py:112
python.PyAthenaComps.Alg
Definition: PyAthenaComps.py:62
python.ComponentAccumulator.ComponentAccumulator.setAppProperty
def setAppProperty(self, key, value, overwrite=False)
Definition: ComponentAccumulator.py:758
python.ComponentAccumulator.ComponentAccumulator.getService
def getService(self, name=None)
Definition: ComponentAccumulator.py:674
python.ComponentAccumulator.ComponentAccumulator._services
_services
Definition: ComponentAccumulator.py:106
python.ComponentAccumulator.ComponentAccumulator
Definition: ComponentAccumulator.py:73
python.ComponentAccumulator.ComponentAccumulator.getAlgPerfmonDomain
def getAlgPerfmonDomain(self, name)
Definition: ComponentAccumulator.py:428
python.ComponentAccumulator.ComponentAccumulator.addAlgToPerfmonDomains
def addAlgToPerfmonDomains(self, name, domain, overwrite=False)
Definition: ComponentAccumulator.py:436
python.ComponentAccumulator.conf2toConfigurable
def conf2toConfigurable(*args, **kwargs)
Definition: ComponentAccumulator.py:1253
python.ComponentAccumulator.ComponentAccumulator.__setstate__
def __setstate__(self, state)
Definition: ComponentAccumulator.py:193
python.ComponentAccumulator.ComponentAccumulator.run
def run(self, maxEvents=None)
Definition: ComponentAccumulator.py:1100
python.ComponentAccumulator.ComponentAccumulator.__del__
def __del__(self)
Definition: ComponentAccumulator.py:171
dumpHVPathFromNtuple.append
bool append
Definition: dumpHVPathFromNtuple.py:91
python.ComponentAccumulator.ComponentAccumulator.getPublicTools
def getPublicTools(self)
Definition: ComponentAccumulator.py:664
python.ComponentAccumulator.ComponentAccumulator._sequence
_sequence
Definition: ComponentAccumulator.py:102
python.CaloAddPedShiftConfig.type
type
Definition: CaloAddPedShiftConfig.py:42
python.ComponentAccumulator.ComponentAccumulator.merge
def merge(self, other, sequenceName=None)
Definition: ComponentAccumulator.py:778
python.DebuggingContext.createContextForDeduplication
def createContextForDeduplication(message, compName, destContext)
Definition: DebuggingContext.py:40
python.Bindings.values
values
Definition: Control/AthenaPython/python/Bindings.py:808
python.ComponentAccumulator.ComponentAccumulator.getAuditor
def getAuditor(self, name)
Definition: ComponentAccumulator.py:681
python.ComponentAccumulator.ComponentAccumulator.dropEventAlgo
def dropEventAlgo(self, name, sequence="AthAlgSeq")
Definition: ComponentAccumulator.py:685
python.ComponentAccumulator.ComponentAccumulator.foreach_component
def foreach_component(self, path)
Definition: ComponentAccumulator.py:1190
python.ComponentAccumulator.ComponentAccumulator._primaryComp
_primaryComp
Definition: ComponentAccumulator.py:110
python.ComponentAccumulator.ComponentAccumulator.getCondAlgo
def getCondAlgo(self, name)
Definition: ComponentAccumulator.py:562
python.ComponentAccumulator.__exit
def __exit()
Definition: ComponentAccumulator.py:33
python.ComponentAccumulator.ComponentAccumulator.wasMerged
def wasMerged(self)
Definition: ComponentAccumulator.py:934
python.Bindings.py_svc
def py_svc(svcName, createIf=True, iface=None)
Definition: Control/AthenaPython/python/Bindings.py:98
python.ComponentAccumulator.ComponentAccumulator.popPrivateTools
def popPrivateTools(self, quiet=False)
Definition: ComponentAccumulator.py:388
python.Debugging.allowPtrace
def allowPtrace()
Definition: Debugging.py:68
python.ComponentAccumulator.ComponentAccumulator.getCurrentPerfmonDomain
def getCurrentPerfmonDomain(self)
Definition: ComponentAccumulator.py:410
python.ComponentAccumulator.ComponentAccumulator.setPrivateTools
def setPrivateTools(self, privTool)
Definition: ComponentAccumulator.py:363
python.CFElements.checkSequenceConsistency
def checkSequenceConsistency(seq)
Definition: CFElements.py:52
python.ComponentAccumulator.ComponentAccumulator._currentDomain
_currentDomain
Definition: ComponentAccumulator.py:111
python.ComponentAccumulator.ComponentAccumulator.getPrimary
def getPrimary(self)
Definition: ComponentAccumulator.py:637
python.ComponentAccumulator.ComponentAccumulator.__init__
def __init__(self, sequence='AthAlgSeq')
Definition: ComponentAccumulator.py:80
python.ComponentAccumulator.ComponentAccumulator._wasMerged
_wasMerged
Definition: ComponentAccumulator.py:120
python.ComponentAccumulator.ComponentAccumulator.getPublicTool
def getPublicTool(self, name=None)
Definition: ComponentAccumulator.py:667
python.ComponentAccumulator.ComponentAccumulator.popToolsAndMerge
def popToolsAndMerge(self, other)
Definition: ComponentAccumulator.py:399
python.ComponentAccumulator.ComponentAccumulator.dropAuditor
def dropAuditor(self, name)
Definition: ComponentAccumulator.py:745
python.ComponentAccumulator.ComponentAccumulator.dropService
def dropService(self, name)
Definition: ComponentAccumulator.py:725
python.ComponentAccumulator.ComponentAccumulator._servicesToCreate
_servicesToCreate
Definition: ComponentAccumulator.py:107
python.ComponentAccumulator.ComponentAccumulator.printPerfmonDomains
def printPerfmonDomains(self)
Definition: ComponentAccumulator.py:452
python.ComponentAccumulator.ComponentAccumulator.addCondAlgo
def addCondAlgo(self, algo, primary=False, domain=None)
Definition: ComponentAccumulator.py:530
python.ComponentAccumulator.ComponentAccumulator.popEventAlgo
def popEventAlgo(self, name, sequence="AthAlgSeq")
Definition: ComponentAccumulator.py:700
python.Interactive.configureInteractivePrompt
def configureInteractivePrompt(completionDict=None)
Definition: Interactive.py:80
python.Utils.exposeHandles
def exposeHandles(allcomps)
Definition: Control/AthenaConfiguration/python/Utils.py:36
print
void print(char *figname, TCanvas *c1)
Definition: TRTCalib_StrawStatusPlots.cxx:25
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
python.ComponentAccumulator.ComponentAccumulator.__verifyFinalSequencesStructure
def __verifyFinalSequencesStructure(self)
Definition: ComponentAccumulator.py:928
debug
const bool debug
Definition: MakeUncertaintyPlots.cxx:53
python.ComponentAccumulator.ComponentAccumulator._debugStage
_debugStage
Definition: ComponentAccumulator.py:125
python.ComponentAccumulator.ComponentAccumulator.getInvertedPerfmonDomains
def getInvertedPerfmonDomains(self)
Definition: ComponentAccumulator.py:419
python.ComponentAccumulator.ComponentAccumulator._componentsContext
_componentsContext
Definition: ComponentAccumulator.py:124
TrigJetMonitorAlgorithm.items
items
Definition: TrigJetMonitorAlgorithm.py:71
python.JetAnalysisCommon.isComponentAccumulatorCfg
isComponentAccumulatorCfg
Definition: JetAnalysisCommon.py:263
python.ComponentAccumulator.ComponentAccumulator._auditors
_auditors
Definition: ComponentAccumulator.py:108
python.ComponentAccumulator.ComponentAccumulator.getEventAlgo
def getEventAlgo(self, name=None)
Definition: ComponentAccumulator.py:519
python.ComponentAccumulator.startInteractive
def startInteractive(localVarDic)
Definition: ComponentAccumulator.py:1211
python.ComponentAccumulator.ComponentAccumulator.printCondAlgs
def printCondAlgs(self, summariseProps=False, onlyComponents=[], printDefaults=False)
Definition: ComponentAccumulator.py:199
Trk::open
@ open
Definition: BinningType.h:40
python.ComponentAccumulator.ComponentAccumulator.addSequence
def addSequence(self, newseq, primary=False, parentName=None)
Definition: ComponentAccumulator.py:322
python.ComponentAccumulator.ComponentAccumulator.dropPublicTool
def dropPublicTool(self, name)
Definition: ComponentAccumulator.py:735
python.ComponentAccumulator.ComponentAccumulator.setDebugStage
def setDebugStage(self, stage)
Definition: ComponentAccumulator.py:772
python.Deduplication.deduplicateOne
def deduplicateOne(newComp, oldComp)
Definition: Deduplication.py:45
python.ComponentAccumulator.ComponentAccumulator._privateTools
_privateTools
Definition: ComponentAccumulator.py:109
python.TriggerHandler.verbose
verbose
Definition: TriggerHandler.py:296
python.ComponentAccumulator.ComponentAccumulator.getSequence
def getSequence(self, sequenceName=None)
Definition: ComponentAccumulator.py:357
python.ComponentAccumulator.ComponentAccumulator.addPublicTool
def addPublicTool(self, newTool, primary=False)
Definition: ComponentAccumulator.py:612
python.ComponentAccumulator.ComponentAccumulator.getCondAlgos
def getCondAlgos(self)
Definition: ComponentAccumulator.py:558
get
T * get(TKey *tobj)
get a TObject* from a TKey* (why can't a TObject be a TKey?)
Definition: hcg.cxx:127
python.ComponentAccumulator.ComponentAccumulator.addService
def addService(self, newSvc, primary=False, create=False)
Definition: ComponentAccumulator.py:566
python.ComponentAccumulator.ComponentAccumulator.flagPerfmonDomain
def flagPerfmonDomain(self, name)
Definition: ComponentAccumulator.py:414
python.ComponentAccumulator.ComponentAccumulator.gatherProps
def gatherProps(self)
Definition: ComponentAccumulator.py:1004
python.ComponentAccumulator.ComponentAccumulator._conditionsAlgs
_conditionsAlgs
Definition: ComponentAccumulator.py:105
python.ComponentAccumulator.ComponentAccumulator.printConfig
def printConfig(self, withDetails=False, summariseProps=False, onlyComponents=[], printDefaults=False, printSequenceTreeOnly=False, prefix=None)
Definition: ComponentAccumulator.py:212
python.ComponentAccumulator.ComponentAccumulator._algorithms
_algorithms
Definition: ComponentAccumulator.py:104
python.ComponentAccumulator.printInteractiveMsg_run
def printInteractiveMsg_run()
Definition: ComponentAccumulator.py:1237
python.ComponentAccumulator.ComponentAccumulator.interactive
interactive
Definition: ComponentAccumulator.py:126
python.ComponentAccumulator.ComponentAccumulator._isMergable
_isMergable
Definition: ComponentAccumulator.py:121
str
Definition: BTagTrackIpAccessor.cxx:11
python.CFElements.isSequence
def isSequence(obj)
Definition: CFElements.py:72
calibdata.copy
bool copy
Definition: calibdata.py:26
python.ComponentAccumulator.ComponentAccumulator._cacheEvict
def _cacheEvict(self)
Definition: ComponentAccumulator.py:182
python.CFElements.findSubSequence
def findSubSequence(start, nameToLookFor)
Definition: CFElements.py:76
python.CFElements.findAlgorithm
def findAlgorithm(startSequence, nameToLookFor, depth=1000000)
Definition: CFElements.py:121
python.ComponentAccumulator.printInteractiveMsg_init
def printInteractiveMsg_init()
Definition: ComponentAccumulator.py:1228
python.PyAthenaComps.Svc
Definition: PyAthenaComps.py:179
python.DebuggingContext.shortCallStack
def shortCallStack()
Definition: DebuggingContext.py:29
python.Utils.loadDefaultComps
def loadDefaultComps(allcomps)
Definition: Control/AthenaConfiguration/python/Utils.py:10
python.Deduplication.deduplicate
def deduplicate(newComp, compList)
Definition: Deduplication.py:15
python.ComponentAccumulator.ComponentAccumulator.getIO
def getIO(self)
Definition: ComponentAccumulator.py:280
error
Definition: IImpactPoint3dEstimator.h:70
python.ParticleTypeUtil.info
def info
Definition: ParticleTypeUtil.py:87
python.Debugging.hookDebugger
def hookDebugger()
Definition: Debugging.py:22
python.ComponentAccumulator.filterComponents
def filterComponents(comps, onlyComponents=[])
Definition: ComponentAccumulator.py:63
python.CFElements.iterSequences
def iterSequences(start)
Definition: CFElements.py:182
python.ComponentAccumulator.ComponentAccumulator._lastAddedComponent
_lastAddedComponent
Definition: ComponentAccumulator.py:122
python.ComponentAccumulator.ComponentAccumulator.getEventAlgos
def getEventAlgos(self, seqName=None)
Definition: ComponentAccumulator.py:525
python.ComponentAccumulator.ComponentAccumulator.getPrimaryAndMerge
def getPrimaryAndMerge(self, other)
Definition: ComponentAccumulator.py:646