ATLAS Offline Software
CFElements.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
2 from AthenaConfiguration.ComponentFactory import CompFactory
3 import collections
4 
5 AthSequencer = CompFactory.AthSequencer # cache lookup
6 
7 def parAND(name, subs=[], invert=False):
8  """parallel AND sequencer"""
9  return AthSequencer( name,
10  ModeOR = False,
11  Sequential = False,
12  StopOverride = True,
13  Invert = invert,
14  Members = subs.copy() )
15 
16 def parOR(name, subs=[], invert=False):
17  """parallel OR sequencer
18  This is the default sequencer and lets the DataFlow govern the execution entirely.
19  """
20  return AthSequencer( name,
21  ModeOR = True,
22  Sequential = False,
23  StopOverride = True,
24  Invert = invert,
25  Members = subs.copy() )
26 
27 def seqAND(name, subs=[], invert=False):
28  """sequential AND sequencer"""
29  return AthSequencer( name,
30  ModeOR = False,
31  Sequential = True,
32  StopOverride = False,
33  Invert = invert,
34  Members = subs.copy() )
35 
36 def seqOR(name, subs=[], invert=False):
37  """sequential OR sequencer
38  Used when a barrier needs to be set by all subs reached irrespective of the decision
39  """
40  return AthSequencer( name,
41  ModeOR = True,
42  Sequential = True,
43  StopOverride = True,
44  Invert = invert,
45  Members = subs.copy() )
46 
47 
49  """Return sequence children (empty if comp is not a sequence)"""
50  try:
51  return comp.Members
52  except AttributeError:
53  return []
54 
55 
57  """ Enforce rules for sequence graph - identical items can not be added to itself (even indirectly) """
58 
59  def __noSubSequenceOfName( s, n, seen = set() ):
60  seen = seen.copy()
61  seen.add (s)
62  for c in getSequenceChildren( s ):
63  if c in seen:
64  raise RuntimeError(f"Sequence {c.getName()} contains itself")
65  if isSequence( c ):
66  if c.getName() == n:
67  raise RuntimeError(f"Sequence {n} contains sub-sequence of the same name")
68  __noSubSequenceOfName( c, c.getName(), seen ) # check each sequence for repetition as well
69  __noSubSequenceOfName( c, n, seen )
70 
71  __noSubSequenceOfName( seq, seq.getName() )
72 
73 
74 def isSequence( obj ):
75  return isinstance(obj, AthSequencer)
76 
77 
78 def findSubSequence( start, nameToLookFor ):
79  """ Traverse sequences tree to find a sequence of a given name. The first one is returned. """
80  if start.getName() == nameToLookFor:
81  return start
82  for c in getSequenceChildren(start):
83  if isSequence( c ):
84  if c.getName() == nameToLookFor:
85  return c
86  found = findSubSequence( c, nameToLookFor )
87  if found:
88  return found
89  return None
90 
91 
92 def findOwningSequence( start, nameToLookFor ):
93  """ find sequence that owns the sequence nameTooLookFor"""
94  for c in getSequenceChildren(start):
95  if c.getName() == nameToLookFor:
96  return start
97  if isSequence( c ):
98  found = findOwningSequence( c, nameToLookFor )
99  if found:
100  return found
101  return None
102 
103 
104 def findAlgorithmByPredicate( startSequence, predicate, depth = 1000000 ):
105  """ Traverse sequences tree to find the first algorithm satisfying given predicate. The first encountered is returned.
106 
107  Depth of the search can be controlled by the depth parameter.
108  Typical use is to limit search to the startSequence with depth parameter set to 1
109  """
110  for c in getSequenceChildren(startSequence):
111  if not isSequence(c):
112  if predicate(c):
113  return c
114  else:
115  if depth > 1:
116  found = findAlgorithmByPredicate( c, predicate, depth-1 )
117  if found:
118  return found
119 
120  return None
121 
122 
123 def findAlgorithm( startSequence, nameToLookFor, depth = 1000000 ):
124  """ Traverse sequences tree to find the algorithm of given name. The first encountered is returned.
125 
126  The name() method is used to obtain the algorithm name, that one has to match to the request.
127  """
128  return findAlgorithmByPredicate( startSequence, lambda alg: alg.getName() == nameToLookFor, depth )
129 
130 
131 def findAllAlgorithms(sequence, nameToLookFor=None):
132  """
133  Returns flat listof of all algorithm instances in this, and in sub-sequences
134  """
135  algorithms = []
136  for child in getSequenceChildren(sequence):
137  if isSequence(child):
138  algorithms += findAllAlgorithms(child, nameToLookFor)
139  else:
140  if nameToLookFor is None or child.getName() == nameToLookFor:
141  algorithms.append(child)
142  return algorithms
143 
144 
145 def findAllAlgorithmsByName(sequence, namesToLookFor=None):
146  """
147  Finds all algorithms in sequence and groups them by name
148 
149  Resulting dict has a following structure
150  {"Alg1Name":[(Alg1Instance, parentSequenceA, indexInSequenceA),(Alg1Instance, parentSequenceB, indexInSequenceB)],
151  "Alg2Name":(Alg2Instance, parentSequence, indexInThisSequence),
152  ....}
153  """
154  algorithms = collections.defaultdict(list)
155  for idx, child in enumerate(getSequenceChildren(sequence)):
156  if child.getName() == sequence.getName():
157  raise RuntimeError(f"Recursively-nested sequence: {child.getName()} contains itself")
158  if isSequence(child):
159  childAlgs = findAllAlgorithmsByName(child, namesToLookFor)
160  for algName in childAlgs:
161  algorithms[algName] += childAlgs[algName]
162  else:
163  if namesToLookFor is None or child.getName() in namesToLookFor:
164  algorithms[child.getName()].append( (child, sequence, idx) )
165  return algorithms
166 
167 
169  """ Converts tree like structure of sequences into dictionary
170  keyed by top/start sequence name containing lists of of algorithms & sequences."""
171 
172  def __inner( seq, collector ):
173  for c in getSequenceChildren(seq):
174  collector[seq.getName()].append( c )
175  if isSequence( c ):
176  __inner( c, collector )
177 
178  from collections import defaultdict,OrderedDict
179  c = defaultdict(list)
180  __inner(start, c)
181  return OrderedDict(c)
182 
183 
184 def iterSequences( start ):
185  """Iterator of sequences and their algorithms from (and including) the `start`
186  sequence object. Do start from a sequence name use findSubSequence."""
187  def __inner( seq ):
188  for c in getSequenceChildren(seq):
189  yield c
190  if isSequence(c):
191  yield from __inner(c)
192  yield start
193  yield from __inner(start)
194 
195 
196 
197 # self test
198 import unittest
199 class TestCF( unittest.TestCase ):
200  def setUp( self ):
201  import AthenaPython.PyAthena as PyAthena
202 
203  top = parOR("top")
204  top.Members += [parOR("nest1")]
205  nest2 = seqAND("nest2")
206  top.Members += [nest2]
207  top.Members += [PyAthena.Alg("SomeAlg0")]
208  nest2.Members += [parOR("deep_nest1")]
209  nest2.Members += [parOR("deep_nest2")]
210 
211  nest2.Members += [PyAthena.Alg("SomeAlg1")]
212  nest2.Members += [PyAthena.Alg("SomeAlg2")]
213  nest2.Members += [PyAthena.Alg("SomeAlg3")]
214  self.top = top
215 
216  def test_findTop( self ):
217  f = findSubSequence( self.top, "top")
218  self.assertIsNotNone( f, "Can not find sequence at start" )
219  self.assertEqual( f.getName(), "top", "Wrong sequence" )
220  # a one level deep search
221  nest2 = findSubSequence( self.top, "nest2" )
222  self.assertIsNotNone( nest2, "Can not find sub sequence" )
223  self.assertEqual( nest2.getName(), "nest2", "Sub sequence incorrect" )
224 
225  def test_findDeep( self ):
226  # deeper search
227  d = findSubSequence( self.top, "deep_nest2")
228  self.assertIsNotNone( d, "Deep searching for sub seqeunce fails" )
229  self.assertEqual( d.getName(), "deep_nest2", "Wrong sub sequence in deep search" )
230 
231  def test_findMissing( self ):
232  # algorithm is not a sequence
233  d = findSubSequence( self.top, "SomeAlg1")
234  self.assertIsNone( d, "Algorithm confused as a sequence" )
235 
236  # no on demand creation
237  inexistent = findSubSequence( self.top, "not_there" )
238  self.assertIsNone( inexistent, "ERROR, found sub sequence that does not relay exist" )
239 
240  # owner finding
241  inexistent = findOwningSequence(self.top, "not_there")
242  self.assertIsNone( inexistent, "ERROR, found owner of inexistent sequence " )
243 
245  owner = findOwningSequence( self.top, "deep_nest1")
246  self.assertEqual( owner.getName(), "nest2", "Wrong owner %s" % owner.getName() )
247 
248  owner = findOwningSequence( self.top, "deep_nest2")
249  self.assertEqual( owner.getName(), "nest2", "Wrong owner %s" % owner.getName() )
250 
251  owner = findOwningSequence( self.top, "SomeAlg1")
252  self.assertEqual( owner.getName(), "nest2", "Wrong owner %s" % owner.getName() )
253 
254  owner = findOwningSequence( self.top, "SomeAlg0")
255  self.assertEqual( owner.getName() , "top", "Wrong owner %s" % owner.getName() )
256 
257  def test_iterSequences( self ):
258  # Traverse from top
259  result = [seq.getName() for seq in iterSequences( self.top )]
260  self.assertEqual( result, ['top', 'nest1', 'nest2', 'deep_nest1', 'deep_nest2',
261  'SomeAlg1', 'SomeAlg2', 'SomeAlg3', 'SomeAlg0'] )
262 
263  # Traverse from nested sequence
264  nest2 = findSubSequence( self.top, "nest2" )
265  result = [seq.getName() for seq in iterSequences( nest2 )]
266  self.assertEqual( result, ['nest2', 'deep_nest1', 'deep_nest2',
267  'SomeAlg1', 'SomeAlg2', 'SomeAlg3'] )
268 
269  # Traverse empty sequence
270  deep_nest2 = findSubSequence( self.top, "deep_nest2" )
271  result = [seq.getName() for seq in iterSequences( deep_nest2 )]
272  self.assertEqual( result, ['deep_nest2'] )
273 
274  # Traverse from algorithm
275  alg1 = findAlgorithm( self.top, "SomeAlg1" )
276  result = [seq.getName() for seq in iterSequences( alg1 )]
277  self.assertEqual( result, ['SomeAlg1'] )
278 
279  def test_findAlgorithms( self ):
280  a1 = findAlgorithm( self.top, "SomeAlg0" )
281  self.assertIsNotNone( a1, "Can't find algorithm present in sequence" )
282 
283  a1 = findAlgorithm( self.top, "SomeAlg1" )
284  self.assertIsNotNone( a1, "Can't find nested algorithm " )
285 
286  nest2 = findSubSequence( self.top, "nest2" )
287 
288  a1 = findAlgorithm( nest2, "SomeAlg0" )
289  self.assertIsNone( a1, "Finding algorithm that is in the upper sequence" )
290 
291  a1 = findAlgorithm( nest2, "NonexistentAlg" )
292  self.assertIsNone( a1, "Finding algorithm that is does not exist" )
293 
294  a1 = findAlgorithm( self.top, "SomeAlg0", 1)
295  self.assertIsNotNone( a1, "Could not find algorithm within the required nesting depth == 1" )
296 
297  a1 = findAlgorithm( self.top, "SomeAlg1", 1)
298  self.assertIsNone( a1, "Could find algorithm even if it is deep in sequences structure" )
299 
300  a1 = findAlgorithm( self.top, "SomeAlg1", 2)
301  self.assertIsNotNone( a1, "Could not find algorithm within the required nesting depth == 2" )
302 
303  a1 = findAlgorithm( self.top, "SomeAlg3", 2)
304  self.assertIsNotNone( a1 is None, "Could find algorithm even if it is deep in sequences structure" )
305 
306 
307 class TestNest( unittest.TestCase ):
308  def test( self ):
309  global isComponentAccumulatorCfg
310  isComponentAccumulatorCfg = lambda : True # noqa: E731 (lambda for mockup)
311 
312  top = parOR("top")
313  nest1 = parOR("nest1")
314  nest2 = seqAND("nest2")
315  top.Members += [nest1, nest2]
316 
317  deep_nest1 = seqAND("deep_nest1")
318  nest1.Members += [deep_nest1]
319 
320  nest2.Members += [nest1] # that one is ok
322  deep_nest1.Members += [nest1] # introducing an issue
323  self.assertRaises( RuntimeError, checkSequenceConsistency, top )
324 
python.CFElements.TestCF
Definition: CFElements.py:199
python.CFElements.flatAlgorithmSequences
def flatAlgorithmSequences(start)
Definition: CFElements.py:168
python.CFElements.findAllAlgorithms
def findAllAlgorithms(sequence, nameToLookFor=None)
Definition: CFElements.py:131
python.CFElements.TestCF.test_findTop
def test_findTop(self)
Definition: CFElements.py:216
AthSequencer
ClassName: AthSequencer.
Definition: AthSequencer.h:40
python.CFElements.findOwningSequence
def findOwningSequence(start, nameToLookFor)
Definition: CFElements.py:92
python.CFElements.TestCF.test_iterSequences
def test_iterSequences(self)
Definition: CFElements.py:257
python.CFElements.findAlgorithmByPredicate
def findAlgorithmByPredicate(startSequence, predicate, depth=1000000)
Definition: CFElements.py:104
python.CFElements.findAllAlgorithmsByName
def findAllAlgorithmsByName(sequence, namesToLookFor=None)
Definition: CFElements.py:145
python.CFElements.parOR
def parOR(name, subs=[], invert=False)
Definition: CFElements.py:16
dumpHVPathFromNtuple.append
bool append
Definition: dumpHVPathFromNtuple.py:91
python.CFElements.TestNest.test
def test(self)
Definition: CFElements.py:308
python.CFElements.parAND
def parAND(name, subs=[], invert=False)
Definition: CFElements.py:7
python.CFElements.TestCF.test_findMissing
def test_findMissing(self)
Definition: CFElements.py:231
python.CFElements.checkSequenceConsistency
def checkSequenceConsistency(seq)
Definition: CFElements.py:56
python.CFElements.TestCF.test_findAlgorithms
def test_findAlgorithms(self)
Definition: CFElements.py:279
python.CFElements.TestCF.setUp
def setUp(self)
Definition: CFElements.py:200
python.CFElements.TestCF.top
top
Definition: CFElements.py:214
python.CFElements.TestNest
Definition: CFElements.py:307
python.CFElements.seqAND
def seqAND(name, subs=[], invert=False)
Definition: CFElements.py:27
python.CFElements.isSequence
def isSequence(obj)
Definition: CFElements.py:74
PyAthena::Alg
Definition: PyAthenaAlg.h:33
python.CFElements.findSubSequence
def findSubSequence(start, nameToLookFor)
Definition: CFElements.py:78
python.CFElements.findAlgorithm
def findAlgorithm(startSequence, nameToLookFor, depth=1000000)
Definition: CFElements.py:123
python.CFElements.TestCF.test_findRespectingScope
def test_findRespectingScope(self)
Definition: CFElements.py:244
python.CFElements.getSequenceChildren
def getSequenceChildren(comp)
Definition: CFElements.py:48
python.CFElements.TestCF.test_findDeep
def test_findDeep(self)
Definition: CFElements.py:225
python.CFElements.iterSequences
def iterSequences(start)
Definition: CFElements.py:184
python.CFElements.seqOR
def seqOR(name, subs=[], invert=False)
Definition: CFElements.py:36