ATLAS Offline Software
HLTCFConfig.py
Go to the documentation of this file.
1 # Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
2 
3 """
4  ------ Documentation on HLT Tree creation -----
5 
6 ++ Filter creation strategy
7 
8 ++ Connections between InputMaker/HypoAlg/Filter
9 
10 ++ Seeds
11 
12 ++ Combined chain strategy
13 
14 - The combined chains use duplicates of the single-object-HypoAlg, called HypoAlgName_for_stepName.
15  These duplicates are connected to a dedicated ComboHypoAlg (added by the framework), able to count object multiplicity
16  -- This is needed for two reasons:
17  -- the HypoAlg is designed to have only one input TC (that is already for the single object)
18  -- otherwise the HypoAlg would be equipped with differnt HypoTools with the same name (see for example e3_e8)
19  -- If the combined chain is symmetric (with multiplicity >1), the Hypo is duplicated only once,
20  equipped with a HypoTool configured as single object and followed by one ComboHypoAlg
21 
22 
23 """
24 
25 from TriggerMenuMT.HLT.Config.ControlFlow.HLTCFDot import stepCF_DataFlow_to_dot, stepCF_ControlFlow_to_dot, all_DataFlow_to_dot
26 from TriggerMenuMT.HLT.Config.ControlFlow.HLTCFComponents import RoRSequenceFilterNode, PassFilterNode, CFGroup
27 from TriggerMenuMT.HLT.Config.ControlFlow.MenuComponentsNaming import CFNaming
28 from AthenaCommon.CFElements import parOR, isSequence
29 from AthenaCommon.AlgSequence import dumpSequence
30 from AthenaCommon.Logging import logging
31 
32 from AthenaConfiguration.ComponentFactory import CompFactory
33 
34 from DecisionHandling.DecisionHandlingConfig import TriggerSummaryAlg
35 from AthenaConfiguration.ComponentAccumulator import ComponentAccumulator
36 
37 from builtins import map, range, str, zip
38 from collections import defaultdict
39 import re
40 
41 log = logging.getLogger( __name__ )
42 
43 
44 def createStepFilterNode(name, seq_list, dump=False):
45  """ Elementary HLT filter step: OR node containing all Filters of the sequences. The node gates execution of next reco step """
46 
47  log.debug("Create filter step %s with %d filters", name, len(seq_list))
48  stepCF = parOR(name + CFNaming.FILTER_POSTFIX)
49  filter_list=[]
50  for seq in seq_list:
51  filterAlg = seq.filter.Alg
52  log.debug("createStepFilterNode: Add %s to filter node %s", filterAlg.getName(), name)
53  if filterAlg not in filter_list:
54  filter_list.append(filterAlg)
55 
56  stepCF = parOR(name + CFNaming.FILTER_POSTFIX, subs=filter_list)
57  if dump:
58  dumpSequence (stepCF, indent=0)
59  return stepCF
60 
61 
62 
63 
66 
67 def matrixDisplay( allCFSeq ):
68 
69  def __getHyposOfStep( step ):
70  if len(step.sequences):
71  step.getChainNames()
72  return []
73 
74  # fill dictionary to cumulate chains on same sequences, in steps (dict with composite keys)
75  mx = defaultdict(list)
76 
77  for stepNumber,cfseq_list in enumerate(allCFSeq, 1):
78  for cfseq in cfseq_list:
79  chains = __getHyposOfStep(cfseq.step)
80  for seq in cfseq.step.sequences:
81  if seq.name == "Empty":
82  mx[stepNumber, "Empty"].extend(chains)
83  else:
84  mx[stepNumber, seq.sequence.Alg.getName()].extend(chains)
85 
86  # sort dictionary by fist key=step
87  sorted_mx = dict(sorted( list(mx.items()), key= lambda k: k[0]))
88 
89  log.debug( "" )
90  log.debug( "="*90 )
91  log.debug( "Cumulative Summary of steps")
92  log.debug( "="*90 )
93  for (step, seq), chains in list(sorted_mx.items()):
94  log.debug( "(step, sequence) ==> (%d, %s) is in chains: ", step, seq)
95  for chain in chains:
96  log.debug( " %s",chain)
97 
98  log.debug( "="*90 )
99 
100 
101 def sequenceScanner( HLTNode ):
102  """ Checks the alignement of sequences and steps in the tree"""
103  # +-- AthSequencer/HLTAllSteps
104  # +-- AthSequencer/Step1_filter
105  # +-- AthSequencer/Step1_reco
106 
107  _seqMapInStep = defaultdict(set)
108  _status = True
109  def _mapSequencesInSteps(seq, stepIndex, childInView):
110  """ Recursively finds the steps in which sequences are used"""
111  if not isSequence(seq):
112  return stepIndex
113  match=re.search('^Step([0-9]+)_filter',seq.name)
114  if match:
115  stepIndex = match.group(1)
116  log.debug("sequenceScanner: This is another step: %s %s", seq.name, stepIndex)
117  inViewSequence = ""
118  inView = False
119  for c in seq.Members:
120  if isSequence(c):
121  # Detect whether this is the view sequence pointed to
122  # by the EV creator alg, or if it is in such a sequence
123  inView = c.getName()==inViewSequence or childInView
124  stepIndex = _mapSequencesInSteps(c, stepIndex, childInView=inView)
125  _seqMapInStep[c.name].add((stepIndex,inView))
126  log.verbose("sequenceScanner: Child %s of sequence %s is in view? %s --> '%s'", c.name, seq.name, inView, inViewSequence)
127  else:
128  if isinstance(c, CompFactory.EventViewCreatorAlgorithm):
129  inViewSequence = c.ViewNodeName
130  log.verbose("sequenceScanner: EventViewCreatorAlg %s is child of sequence %s with ViewNodeName %s", c.name, seq.name, c.ViewNodeName)
131  log.debug("sequenceScanner: Sequence %s is in view? %s --> '%s'", seq.name, inView, inViewSequence)
132  return stepIndex
133 
134  # do the job:
135  final_step=_mapSequencesInSteps(HLTNode, 0, childInView=False)
136 
137  for alg, steps in _seqMapInStep.items():
138  if 'PassSequence' in alg or 'HLTCaloClusterMakerFSRecoSequence' in alg \
139  or 'HLTCaloCellMakerFSRecoSequence' in alg: # do not count PassSequences, which is used many times. Harcoding HLTCaloClusterMakerFSRecoSequence and HLTCaloCellMakerFSRecoSequence when using FullScanTopoCluster building for photon triggers with RoI='' (also for Jets and MET) following discussion in ATR-24722. To be fixed
140  continue
141  # Sequences in views can be in multiple steps
142  nonViewSteps = sum([0 if isInViews else 1 for (stepIndex,isInViews) in steps])
143  if nonViewSteps > 1:
144  steplist = [stepIndex for stepIndex,inViewSequence in steps]
145  log.error("sequenceScanner: Sequence %s is expected outside of a view in more than one step: %s", alg, steplist)
146  match=re.search('Step([0-9]+)',alg)
147  if match:
148  candidateStep=match.group(1)
149  log.error("sequenceScanner: ---> candidate good step is %s", candidateStep)
150  _status=False
151  raise RuntimeError(f"Duplicated event-scope sequence {alg} in steps {steplist}")
152 
153  log.debug("sequenceScanner: scanned %s steps with status %d", final_step, _status)
154  return _status
155 
156 
157 def decisionTreeFromChains(flags, HLTNode, chains, allDicts):
158  """ Creates the decision tree, given the starting node and the chains containing the sequences """
159  log.info("[decisionTreeFromChains] Run decisionTreeFromChains on %s", HLTNode.getName())
160  HLTNodeName = HLTNode.getName()
161  acc = ComponentAccumulator()
162 
163  if len(chains) == 0:
164  log.info("[decisionTreeFromChains] Configuring empty decisionTree")
165  acc.addSequence(HLTNode)
166  return ([], [], acc)
167 
168  ( finalDecisions, CFseq_list) = createDataFlow(flags, chains)
169  addChainsToDataFlow(flags, CFseq_list, allDicts)
170  # now connect all algorithms and creates the CAs
171  cfAcc = createControlFlow(flags, HLTNode, CFseq_list)
172  acc.merge(cfAcc)
173 
174  # create dot graphs
175  log.debug("finalDecisions: %s", finalDecisions)
176  if flags.Trigger.generateMenuDiagnostics:
177  all_DataFlow_to_dot(HLTNodeName, CFseq_list)
178 
179  # matrix display
180  # uncomment for serious debugging
181  # matrixDisplay( CFseq_list )
182 
183  return (finalDecisions,CFseq_list, acc)
184 
185 def createDataFlow(flags, chains):
186  """ Creates the filters and connect them to the menu sequences"""
187 
188  # find tot nsteps
189  chainWithMaxSteps = max(chains, key = lambda chain: len(chain.steps))
190  NSTEPS = len(chainWithMaxSteps.steps)
191  log.info("[createDataFlow] creating DF for %d chains and total %d steps", len(chains), NSTEPS)
192 
193  # initialize arrays for monitor
194  finalDecisions = [ [] for n in range(NSTEPS) ]
195  CFseqList = [ [] for n in range(NSTEPS) ]
196  CFSeqByFilterName = [ {} for n in range(NSTEPS) ] # CFSeqeunces keyed by filter name (speedup)
197 
198  # loop over chains
199  for chain in chains:
200  log.debug("\n Configuring chain %s with %d steps: \n - %s ", chain.name,len(chain.steps),'\n - '.join(map(str, [{step.name:step.nLegs} for step in chain.steps])))
201 
202  lastCFgroup = None
203  lastDecisions = []
204  for nstep, chainStep in enumerate( chain.steps ):
205  if not flags.Trigger.fastMenuGeneration:
206  #create all sequences CA in all steps to allow data flow connections
207  chainStep.createSequences()
208  log.debug("\n************* Start connecting step %d %s for chain %s", nstep+1, chainStep.name, chain.name)
209  if nstep == 0:
210  filterInput = chain.L1decisions
211  else:
212  filterInput = lastDecisions
213  if len(filterInput) == 0 :
214  log.error("[createDataFlow] Filter for step %s has %d inputs! At least one is expected", chainStep.name, len(filterInput))
215  raise Exception("[createDataFlow] Cannot proceed, exiting.")
216 
217  log.debug("Set Filter input: %s while setting the chain: %s", filterInput, chain.name)
218 
219  # make one filter per step:
220  sequenceFilter = None
221  filterName = CFNaming.filterName(chainStep.name)
222  if chainStep.isEmpty:
223  filterOutput = filterInput
224  else:
225  filterOutput = [CFNaming.filterOutName(filterName, inputName) for inputName in filterInput ]
226 
227  # TODO: Check sequence consistency if skipping, to avoid issues like https://its.cern.ch/jira/browse/ATR-28617
228  foundCFgroup = CFSeqByFilterName[nstep].get(filterName, None)
229  log.debug("%s CF sequences with filter name %s", "Not found" if foundCFgroup is None else "Found", filterName)
230  if foundCFgroup is None:
231  sequenceFilter = buildFilter(filterName, filterInput, chainStep.isEmpty)
232  if flags.Trigger.fastMenuGeneration:
233  #create the sequences CA of this step in fast mode
234  chainStep.createSequences()
235  # add the step to a new group
236  CFgroup = CFGroup( ChainStep = chainStep, FilterAlg = sequenceFilter)
237  CFgroup.connect(filterOutput)
238  CFSeqByFilterName[nstep][sequenceFilter.Alg.getName()] = CFgroup
239  CFseqList[nstep].append(CFgroup)
240  lastCFgroup = CFgroup
241  else:
242  lastCFgroup = foundCFgroup
243  sequenceFilter = lastCFgroup.sequenceCA.filterNode
244  if len(list(set(sequenceFilter.getInputList()).intersection(filterInput))) != len(list(set(filterInput))):
245  [ sequenceFilter.addInput(inputName) for inputName in filterInput ]
246  [ sequenceFilter.addOutput(outputName) for outputName in filterOutput ]
247  lastCFgroup.connect(filterOutput)
248 
249  lastDecisions = lastCFgroup.sequenceCA.decisions
250 
251  # add chains to the filter:
252  chainLegs = chainStep.getChainLegs()
253  if len(chainLegs) != len(filterInput):
254  log.error("[createDataFlow] lengths of chainlegs = %s differ from inputs = %s", str(chainLegs), str(filterInput))
255  raise Exception("[createDataFlow] Cannot proceed, exiting.")
256  for finput, leg in zip(filterInput, chainLegs):
257  log.debug("Adding chain %s to input %s of %s", leg, finput, sequenceFilter.Alg.name)
258  sequenceFilter.addChain(leg, finput)
259 
260  log.debug("Now Filter has chains: %s", sequenceFilter.getChains())
261  log.debug("Now Filter has chains/input: %s", sequenceFilter.getChainsPerInput())
262  # store legs and mult in the CFGroup
263  lastCFgroup.addStepLeg(chainStep, chain.name)
264 
265  if len(chain.steps) == nstep+1:
266  log.debug("Adding finalDecisions for chain %s at step %d:", chain.name, nstep+1)
267  for dec in lastDecisions:
268  finalDecisions[nstep].append(dec)
269  log.debug(dec)
270 
271  #end of loop over steps
272  log.debug("\n Built CD for chain %s with %d steps: \n - %s ", chain.name,len(chain.steps),'\n - '.join(map(str, [{step.name:step.nLegs} for step in chain.steps])))
273  #end of loop over chains
274 
275  log.debug("End of createDataFlow for %d chains and total %d steps", len(chains), NSTEPS)
276  return (finalDecisions, CFseqList)
277 
278 
279 def createControlFlow(flags, HLTNode, CFseqList):
280  """ Creates Control Flow Tree starting from the CFSequences"""
281  HLTNodeName = HLTNode.getName()
282  log.debug("[createControlFlow] on node %s with %d CFsequences",HLTNodeName, len(CFseqList))
283 
284  acc = ComponentAccumulator()
285  acc.addSequence(HLTNode)
286 
287  for nstep, sequences in enumerate(CFseqList):
288  stepSequenceName = CFNaming.stepName(nstep)
289  log.debug("\n******** Create CF Tree %s with %d AthSequencers", stepSequenceName, len(sequences))
290 
291  # create filter node
292  log.debug("[createControlFlow] Create filter step %s with %d filters", stepSequenceName, len(CFseqList[nstep]))
293  stepCFFilter = parOR(stepSequenceName + CFNaming.FILTER_POSTFIX)
294  acc.addSequence(stepCFFilter, parentName=HLTNodeName)
295 
296  filter_list = []
297  # add the filter to the node
298  for cgroup in sequences:
299  filterAlg = cgroup.sequenceCA.filterNode.Alg
300  if filterAlg.getName() not in filter_list:
301  log.debug("[createControlFlow] Add %s to filter node %s", filterAlg.getName(), stepSequenceName)
302  filter_list.append(filterAlg.getName())
303  stepCFFilter.Members += [filterAlg]
304 
305  # create reco step node
306  log.debug("[createControlFlow] Create reco step %s with %d sequences", stepSequenceName, len(CFseqList))
307  stepCFReco = parOR(stepSequenceName + CFNaming.RECO_POSTFIX)
308  acc.addSequence(stepCFReco, parentName = HLTNodeName)
309 
310  # add the sequences to the reco node
311  addedEmtpy = False
312  for cgroup in sequences:
313  cseq=cgroup.sequenceCA
314  if cseq.step.isEmpty and addedEmtpy:
315  cseq.ca.wasMerged()
316  continue
317  if cseq.step.isEmpty: # adding Empty only once to avoid merging multiple times the PassSequence
318  addedEmtpy= True
319  log.debug(" *** Create CF Tree for CFSequence %s", cseq.step.name)
320  acc.merge(cseq.ca, sequenceName=stepCFReco.getName())
321 
322  # add the monitor summary
323  stepDecisions = []
324  for CFseq in CFseqList[nstep]:
325  stepDecisions.extend(CFseq.sequenceCA.decisions)
326 
327  summaryAlg = TriggerSummaryAlg( flags, CFNaming.stepSummaryName(stepSequenceName),
328  InputDecision = "HLTSeedingSummary",
329  FinalDecisions = list(dict.fromkeys(stepDecisions)) )
330 
331  acc.addEventAlgo(summaryAlg, sequenceName = HLTNode.getName())
332 
333  if flags.Trigger.generateMenuDiagnostics:
334  log.debug("Now Draw Menu Diagnostic dot graphs...")
335  stepCF_DataFlow_to_dot( stepCFReco.getName(), CFseqList[nstep] )
336  stepCF_ControlFlow_to_dot( stepCFReco )
337 
338  log.debug("************* End of step %d, %s", nstep+1, stepSequenceName)
339 
340  return acc
341 
342 def addChainsToDataFlow(flags, CFseq_list, allDicts):
343  for groupsInStep in CFseq_list:
344  for cfgroup in groupsInStep:
345  chains = cfgroup.chains
346  CFS = cfgroup.sequenceCA
347  # add chains to the ComboHypo:
348  if CFS.step.combo is not None:
349  for chain in chains:
350  CFS.step.combo.addChain( [d for d in allDicts if d['chainName'] == chain ][0])
351  log.debug("Added chains to ComboHypo: %s",CFS.step.combo.getChains())
352 
353  # add HypoTools to this step (cumulating all same steps)
354  cfgroup.createHypoTools(flags)
355 
356 
357 
358 def buildFilter(filter_name, filter_input, empty):
359  """
360  Build the FILTER
361  one filter per previous sequence at the start of the sequence: always create a new one
362  if the previous hypo has more than one output, try to get all of them
363  one filter per previous sequence: 1 input/previous seq, 1 output/next seq
364  """
365  if empty:
366  log.debug("Calling PassFilterNode %s", filter_name)
367  sfilter = PassFilterNode(name = filter_name)
368  for i in filter_input:
369  sfilter.addInput(i)
370  sfilter.addOutput(i)
371  else:
372  sfilter = RoRSequenceFilterNode(name = filter_name)
373  for i in filter_input:
374  sfilter.addInput(i)
375  sfilter.addOutput(CFNaming.filterOutName(filter_name, i))
376 
377  log.debug("Added inputs to filter: %s", sfilter.getInputList())
378  log.debug("Added outputs to filter: %s", sfilter.getOutputList())
379  log.debug("Filter Done: %s", sfilter.Alg.name)
380 
381 
382  return (sfilter)
383 
384 
385 
HLTCFConfig.matrixDisplay
def matrixDisplay(allCFSeq)
CORE of Decision Handling.
Definition: HLTCFConfig.py:67
DerivationFramework::TriggerMatchingUtils::sorted
std::vector< typename R::value_type > sorted(const R &r, PROJ proj={})
Helper function to create a sorted vector from an unsorted range.
HLTCFConfig.createDataFlow
def createDataFlow(flags, chains)
Definition: HLTCFConfig.py:185
python.JetAnalysisCommon.ComponentAccumulator
ComponentAccumulator
Definition: JetAnalysisCommon.py:302
HLTCFConfig.createControlFlow
def createControlFlow(flags, HLTNode, CFseqList)
Definition: HLTCFConfig.py:279
max
constexpr double max()
Definition: ap_fixedTest.cxx:33
HLTCFConfig.createStepFilterNode
def createStepFilterNode(name, seq_list, dump=False)
Definition: HLTCFConfig.py:44
HLTCFConfig.decisionTreeFromChains
def decisionTreeFromChains(flags, HLTNode, chains, allDicts)
Definition: HLTCFConfig.py:157
dumpHVPathFromNtuple.append
bool append
Definition: dumpHVPathFromNtuple.py:91
intersection
std::vector< std::string > intersection(std::vector< std::string > &v1, std::vector< std::string > &v2)
Definition: compareFlatTrees.cxx:25
HLTCFConfig.addChainsToDataFlow
def addChainsToDataFlow(flags, CFseq_list, allDicts)
Definition: HLTCFConfig.py:342
convertTimingResiduals.sum
sum
Definition: convertTimingResiduals.py:55
python.DecisionHandlingConfig.TriggerSummaryAlg
def TriggerSummaryAlg(flags, name, **kwargs)
Definition: DecisionHandlingConfig.py:25
HLTCFConfig.sequenceScanner
def sequenceScanner(HLTNode)
Definition: HLTCFConfig.py:101
python.JetAnalysisCommon.parOR
parOR
Definition: JetAnalysisCommon.py:271
plotBeamSpotVxVal.range
range
Definition: plotBeamSpotVxVal.py:194
add
bool add(const std::string &hname, TKey *tobj)
Definition: fastadd.cxx:55
histSizes.list
def list(name, path='/')
Definition: histSizes.py:38
CxxUtils::set
constexpr std::enable_if_t< is_bitmask_v< E >, E & > set(E &lhs, E rhs)
Convenience function to set bits in a class enum bitmask.
Definition: bitmask.h:232
TCS::join
std::string join(const std::vector< std::string > &v, const char c=',')
Definition: Trigger/TrigT1/L1Topo/L1TopoCommon/Root/StringUtils.cxx:10
HLTCFDot.all_DataFlow_to_dot
def all_DataFlow_to_dot(name, step_list)
Definition: HLTCFDot.py:87
ActsTrk::detail::MakeDerivedVariant::extend
constexpr std::variant< Args..., T > extend(const std::variant< Args... > &, const T &)
Definition: MakeDerivedVariant.h:17
HLTCFDot.stepCF_ControlFlow_to_dot
def stepCF_ControlFlow_to_dot(stepCF)
Definition: HLTCFDot.py:23
HLTCFDot.stepCF_DataFlow_to_dot
def stepCF_DataFlow_to_dot(name, cfseq_list)
Definition: HLTCFDot.py:146
get
T * get(TKey *tobj)
get a TObject* from a TKey* (why can't a TObject be a TKey?)
Definition: hcg.cxx:127
str
Definition: BTagTrackIpAccessor.cxx:11
python.CFElements.isSequence
def isSequence(obj)
Definition: CFElements.py:72
HLTCFConfig.buildFilter
def buildFilter(filter_name, filter_input, empty)
Definition: HLTCFConfig.py:358