ATLAS Offline Software
CFElements.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
2 from AthenaConfiguration.ComponentFactory import CompFactory
3 import collections
4 
5 AthSequencer = CompFactory.AthSequencer # cache lookup
6 
7 def parAND(name, subs=[]):
8  """parallel AND sequencer"""
9  return AthSequencer( name,
10  ModeOR = False,
11  Sequential = False,
12  StopOverride = True,
13  Members = subs.copy() )
14 
15 def parOR(name, subs=[]):
16  """parallel OR sequencer
17  This is the default sequencer and lets the DataFlow govern the execution entirely.
18  """
19  return AthSequencer( name,
20  ModeOR = True,
21  Sequential = False,
22  StopOverride = True,
23  Members = subs.copy() )
24 
25 def seqAND(name, subs=[]):
26  """sequential AND sequencer"""
27  return AthSequencer( name,
28  ModeOR = False,
29  Sequential = True,
30  StopOverride = False,
31  Members = subs.copy() )
32 
33 def seqOR(name, subs=[]):
34  """sequential OR sequencer
35  Used when a barrier needs to be set by all subs reached irrespective of the decision
36  """
37  return AthSequencer( name,
38  ModeOR = True,
39  Sequential = True,
40  StopOverride = True,
41  Members = subs.copy() )
42 
43 
45  """Return sequence children (empty if comp is not a sequence)"""
46  try:
47  return comp.Members
48  except AttributeError:
49  return []
50 
51 
53  """ Enforce rules for sequence graph - identical items can not be added to itself (even indirectly) """
54 
55  def __noSubSequenceOfName( s, n, seen = set() ):
56  seen = seen.copy()
57  seen.add (s)
58  for c in getSequenceChildren( s ):
59  if c in seen:
60  raise RuntimeError(f"Sequence {c.getName()} contains itself")
61  if isSequence( c ):
62  if c.getName() == n:
63  raise RuntimeError(f"Sequence {n} contains sub-sequence of the same name")
64  __noSubSequenceOfName( c, c.getName(), seen ) # check each sequence for repetition as well
65  __noSubSequenceOfName( c, n, seen )
66 
67  __noSubSequenceOfName( seq, seq.getName() )
68  for c in getSequenceChildren( seq ):
70 
71 
72 def isSequence( obj ):
73  return isinstance(obj, AthSequencer)
74 
75 
76 def findSubSequence( start, nameToLookFor ):
77  """ Traverse sequences tree to find a sequence of a given name. The first one is returned. """
78  if start.getName() == nameToLookFor:
79  return start
80  for c in getSequenceChildren(start):
81  if isSequence( c ):
82  if c.getName() == nameToLookFor:
83  return c
84  found = findSubSequence( c, nameToLookFor )
85  if found:
86  return found
87  return None
88 
89 
90 def findOwningSequence( start, nameToLookFor ):
91  """ find sequence that owns the sequence nameTooLookFor"""
92  for c in getSequenceChildren(start):
93  if c.getName() == nameToLookFor:
94  return start
95  if isSequence( c ):
96  found = findOwningSequence( c, nameToLookFor )
97  if found:
98  return found
99  return None
100 
101 
102 def findAlgorithmByPredicate( startSequence, predicate, depth = 1000000 ):
103  """ Traverse sequences tree to find the first algorithm satisfying given predicate. The first encountered is returned.
104 
105  Depth of the search can be controlled by the depth parameter.
106  Typical use is to limit search to the startSequence with depth parameter set to 1
107  """
108  for c in getSequenceChildren(startSequence):
109  if not isSequence(c):
110  if predicate(c):
111  return c
112  else:
113  if depth > 1:
114  found = findAlgorithmByPredicate( c, predicate, depth-1 )
115  if found:
116  return found
117 
118  return None
119 
120 
121 def findAlgorithm( startSequence, nameToLookFor, depth = 1000000 ):
122  """ Traverse sequences tree to find the algorithm of given name. The first encountered is returned.
123 
124  The name() method is used to obtain the algorithm name, that one has to match to the request.
125  """
126  return findAlgorithmByPredicate( startSequence, lambda alg: alg.getName() == nameToLookFor, depth )
127 
128 
129 def findAllAlgorithms(sequence, nameToLookFor=None):
130  """
131  Returns flat listof of all algorithm instances in this, and in sub-sequences
132  """
133  algorithms = []
134  for child in getSequenceChildren(sequence):
135  if isSequence(child):
136  algorithms += findAllAlgorithms(child, nameToLookFor)
137  else:
138  if nameToLookFor is None or child.getName() == nameToLookFor:
139  algorithms.append(child)
140  return algorithms
141 
142 
143 def findAllAlgorithmsByName(sequence, namesToLookFor=None):
144  """
145  Finds all algorithms in sequence and groups them by name
146 
147  Resulting dict has a following structure
148  {"Alg1Name":[(Alg1Instance, parentSequenceA, indexInSequenceA),(Alg1Instance, parentSequenceB, indexInSequenceB)],
149  "Alg2Name":(Alg2Instance, parentSequence, indexInThisSequence),
150  ....}
151  """
152  algorithms = collections.defaultdict(list)
153  for idx, child in enumerate(getSequenceChildren(sequence)):
154  if child.getName() == sequence.getName():
155  raise RuntimeError(f"Recursively-nested sequence: {child.getName()} contains itself")
156  if isSequence(child):
157  childAlgs = findAllAlgorithmsByName(child, namesToLookFor)
158  for algName in childAlgs:
159  algorithms[algName] += childAlgs[algName]
160  else:
161  if namesToLookFor is None or child.getName() in namesToLookFor:
162  algorithms[child.getName()].append( (child, sequence, idx) )
163  return algorithms
164 
165 
167  """ Converts tree like structure of sequences into dictionary
168  keyed by top/start sequence name containing lists of of algorithms & sequences."""
169 
170  def __inner( seq, collector ):
171  for c in getSequenceChildren(seq):
172  collector[seq.getName()].append( c )
173  if isSequence( c ):
174  __inner( c, collector )
175 
176  from collections import defaultdict,OrderedDict
177  c = defaultdict(list)
178  __inner(start, c)
179  return OrderedDict(c)
180 
181 
182 def iterSequences( start ):
183  """Iterator of sequences and their algorithms from (and including) the `start`
184  sequence object. Do start from a sequence name use findSubSequence."""
185  def __inner( seq ):
186  for c in getSequenceChildren(seq):
187  yield c
188  if isSequence(c):
189  yield from __inner(c)
190  yield start
191  yield from __inner(start)
192 
193 
194 
195 # self test
196 import unittest
197 class TestCF( unittest.TestCase ):
198  def setUp( self ):
199  import AthenaPython.PyAthena as PyAthena
200 
201  top = parOR("top")
202  top.Members += [parOR("nest1")]
203  nest2 = seqAND("nest2")
204  top.Members += [nest2]
205  top.Members += [PyAthena.Alg("SomeAlg0")]
206  nest2.Members += [parOR("deep_nest1")]
207  nest2.Members += [parOR("deep_nest2")]
208 
209  nest2.Members += [PyAthena.Alg("SomeAlg1")]
210  nest2.Members += [PyAthena.Alg("SomeAlg2")]
211  nest2.Members += [PyAthena.Alg("SomeAlg3")]
212  self.top = top
213 
214  def test_findTop( self ):
215  f = findSubSequence( self.top, "top")
216  self.assertIsNotNone( f, "Can not find sequence at start" )
217  self.assertEqual( f.getName(), "top", "Wrong sequence" )
218  # a one level deep search
219  nest2 = findSubSequence( self.top, "nest2" )
220  self.assertIsNotNone( nest2, "Can not find sub sequence" )
221  self.assertEqual( nest2.getName(), "nest2", "Sub sequence incorrect" )
222 
223  def test_findDeep( self ):
224  # deeper search
225  d = findSubSequence( self.top, "deep_nest2")
226  self.assertIsNotNone( d, "Deep searching for sub seqeunce fails" )
227  self.assertEqual( d.getName(), "deep_nest2", "Wrong sub sequence in deep search" )
228 
229  def test_findMissing( self ):
230  # algorithm is not a sequence
231  d = findSubSequence( self.top, "SomeAlg1")
232  self.assertIsNone( d, "Algorithm confused as a sequence" )
233 
234  # no on demand creation
235  inexistent = findSubSequence( self.top, "not_there" )
236  self.assertIsNone( inexistent, "ERROR, found sub sequence that does not relay exist" )
237 
238  # owner finding
239  inexistent = findOwningSequence(self.top, "not_there")
240  self.assertIsNone( inexistent, "ERROR, found owner of inexistent sequence " )
241 
243  owner = findOwningSequence( self.top, "deep_nest1")
244  self.assertEqual( owner.getName(), "nest2", "Wrong owner %s" % owner.getName() )
245 
246  owner = findOwningSequence( self.top, "deep_nest2")
247  self.assertEqual( owner.getName(), "nest2", "Wrong owner %s" % owner.getName() )
248 
249  owner = findOwningSequence( self.top, "SomeAlg1")
250  self.assertEqual( owner.getName(), "nest2", "Wrong owner %s" % owner.getName() )
251 
252  owner = findOwningSequence( self.top, "SomeAlg0")
253  self.assertEqual( owner.getName() , "top", "Wrong owner %s" % owner.getName() )
254 
255  def test_iterSequences( self ):
256  # Traverse from top
257  result = [seq.getName() for seq in iterSequences( self.top )]
258  self.assertEqual( result, ['top', 'nest1', 'nest2', 'deep_nest1', 'deep_nest2',
259  'SomeAlg1', 'SomeAlg2', 'SomeAlg3', 'SomeAlg0'] )
260 
261  # Traverse from nested sequence
262  nest2 = findSubSequence( self.top, "nest2" )
263  result = [seq.getName() for seq in iterSequences( nest2 )]
264  self.assertEqual( result, ['nest2', 'deep_nest1', 'deep_nest2',
265  'SomeAlg1', 'SomeAlg2', 'SomeAlg3'] )
266 
267  # Traverse empty sequence
268  deep_nest2 = findSubSequence( self.top, "deep_nest2" )
269  result = [seq.getName() for seq in iterSequences( deep_nest2 )]
270  self.assertEqual( result, ['deep_nest2'] )
271 
272  # Traverse from algorithm
273  alg1 = findAlgorithm( self.top, "SomeAlg1" )
274  result = [seq.getName() for seq in iterSequences( alg1 )]
275  self.assertEqual( result, ['SomeAlg1'] )
276 
277  def test_findAlgorithms( self ):
278  a1 = findAlgorithm( self.top, "SomeAlg0" )
279  self.assertIsNotNone( a1, "Can't find algorithm present in sequence" )
280 
281  a1 = findAlgorithm( self.top, "SomeAlg1" )
282  self.assertIsNotNone( a1, "Can't find nested algorithm " )
283 
284  nest2 = findSubSequence( self.top, "nest2" )
285 
286  a1 = findAlgorithm( nest2, "SomeAlg0" )
287  self.assertIsNone( a1, "Finding algorithm that is in the upper sequence" )
288 
289  a1 = findAlgorithm( nest2, "NonexistentAlg" )
290  self.assertIsNone( a1, "Finding algorithm that is does not exist" )
291 
292  a1 = findAlgorithm( self.top, "SomeAlg0", 1)
293  self.assertIsNotNone( a1, "Could not find algorithm within the required nesting depth == 1" )
294 
295  a1 = findAlgorithm( self.top, "SomeAlg1", 1)
296  self.assertIsNone( a1, "Could find algorithm even if it is deep in sequences structure" )
297 
298  a1 = findAlgorithm( self.top, "SomeAlg1", 2)
299  self.assertIsNotNone( a1, "Could not find algorithm within the required nesting depth == 2" )
300 
301  a1 = findAlgorithm( self.top, "SomeAlg3", 2)
302  self.assertIsNotNone( a1 is None, "Could find algorithm even if it is deep in sequences structure" )
303 
304 
305 class TestNest( unittest.TestCase ):
306  def test( self ):
307  global isComponentAccumulatorCfg
308  isComponentAccumulatorCfg = lambda : True # noqa: E731 (lambda for mockup)
309 
310  top = parOR("top")
311  nest1 = parOR("nest1")
312  nest2 = seqAND("nest2")
313  top.Members += [nest1, nest2]
314 
315  deep_nest1 = seqAND("deep_nest1")
316  nest1.Members += [deep_nest1]
317 
318  nest2.Members += [nest1] # that one is ok
320  deep_nest1.Members += [nest1] # introducing an issue
321  self.assertRaises( RuntimeError, checkSequenceConsistency, top )
322 
python.CFElements.TestCF
Definition: CFElements.py:197
python.CFElements.flatAlgorithmSequences
def flatAlgorithmSequences(start)
Definition: CFElements.py:166
python.CFElements.findAllAlgorithms
def findAllAlgorithms(sequence, nameToLookFor=None)
Definition: CFElements.py:129
python.CFElements.TestCF.test_findTop
def test_findTop(self)
Definition: CFElements.py:214
AthSequencer
ClassName: AthSequencer.
Definition: AthSequencer.h:40
python.CFElements.findOwningSequence
def findOwningSequence(start, nameToLookFor)
Definition: CFElements.py:90
python.CFElements.TestCF.test_iterSequences
def test_iterSequences(self)
Definition: CFElements.py:255
python.CFElements.findAlgorithmByPredicate
def findAlgorithmByPredicate(startSequence, predicate, depth=1000000)
Definition: CFElements.py:102
python.CFElements.findAllAlgorithmsByName
def findAllAlgorithmsByName(sequence, namesToLookFor=None)
Definition: CFElements.py:143
dumpHVPathFromNtuple.append
bool append
Definition: dumpHVPathFromNtuple.py:91
python.CFElements.TestNest.test
def test(self)
Definition: CFElements.py:306
python.CFElements.TestCF.test_findMissing
def test_findMissing(self)
Definition: CFElements.py:229
python.CFElements.seqAND
def seqAND(name, subs=[])
Definition: CFElements.py:25
python.CFElements.checkSequenceConsistency
def checkSequenceConsistency(seq)
Definition: CFElements.py:52
python.CFElements.seqOR
def seqOR(name, subs=[])
Definition: CFElements.py:33
python.CFElements.TestCF.test_findAlgorithms
def test_findAlgorithms(self)
Definition: CFElements.py:277
python.CFElements.TestCF.setUp
def setUp(self)
Definition: CFElements.py:198
python.CFElements.TestCF.top
top
Definition: CFElements.py:212
python.CFElements.TestNest
Definition: CFElements.py:305
python.CFElements.parOR
def parOR(name, subs=[])
Definition: CFElements.py:15
python.CFElements.isSequence
def isSequence(obj)
Definition: CFElements.py:72
PyAthena::Alg
Definition: PyAthenaAlg.h:33
python.CFElements.findSubSequence
def findSubSequence(start, nameToLookFor)
Definition: CFElements.py:76
python.CFElements.findAlgorithm
def findAlgorithm(startSequence, nameToLookFor, depth=1000000)
Definition: CFElements.py:121
python.CFElements.TestCF.test_findRespectingScope
def test_findRespectingScope(self)
Definition: CFElements.py:242
python.CFElements.getSequenceChildren
def getSequenceChildren(comp)
Definition: CFElements.py:44
python.CFElements.TestCF.test_findDeep
def test_findDeep(self)
Definition: CFElements.py:223
python.CFElements.iterSequences
def iterSequences(start)
Definition: CFElements.py:182
python.CFElements.parAND
def parAND(name, subs=[])
Definition: CFElements.py:7