151 def getChainDicts(self, flags):
153 def validSignature(currentSig, chainSig):
154 """Check if chain is assigned to the correct signature"""
155 reqd = GenerateMenuMT.getRequiredSignatures(currentSig)
156 isValid = chainSig.issubset( reqd )
157 log.debug("Chain signatures: %s, required signatures: %s",chainSig,reqd)
159 log.error("Chain signatures %s not a subset of required signatures %s",set(chainSig),reqd)
162 from TriggerMenuMT.HLT.Config.Utility.DictFromChainName import dictFromChainName
166 for sig, chains in self.chainsInMenu.items():
168 log.debug("Now processing chain: %s from signature %s", chain, sig)
170 chainDict = dictFromChainName(flags, chain)
171 chainDict['chainCounter'] = chainCounter
172 chainDict['prescale'] = 1 # set default chain prescale
174 # Pick out the folder and subsignature directories to import
175 for sigfo, subsig in chainDict['sigDicts'].items():
176 if sigfo not in self.sigDicts:
177 self.sigDicts[sigfo] = subsig
180 if ss not in self.sigDicts[sigfo]:
181 self.sigDicts[sigfo].append(ss)
183 self.chainDicts.append(chainDict)
185 if not validSignature(sig, set(chainDict['signatures'])):
187 log.error('Chain %s assigned to signature %s but creates %s',
188 chainDict['chainName'], sig, set(chainDict['signatures']))
190 raise RuntimeError('Incorrect assignment of chains to slices -- see preceding messages.')
213 def generateChains(self, flags):
215 combinations_in_menu = []
216 alignmentGroups_to_align = set()
217 length_of_configs = {}
219 nchainDicts = len(self.chainDicts)
220 notify_increment = max(int(nchainDicts / 10),1)
221 for ichainDict, chainDict in enumerate(self.chainDicts):
222 log.debug("Next: getting chain configuration for chain %s ", chainDict['chainName'])
223 if ichainDict % notify_increment==0:
224 log.info("Generating HLT chain %d / %d", ichainDict+1, nchainDicts)
225 chainConfig,lengthOfChainConfigs = self.__generateChainConfig(flags, chainDict)
226 all_chains += [(chainDict,chainConfig,lengthOfChainConfigs)]
228 #update the alignment group length dictionary if we have a longer number of steps
229 #or the signature isn't registered in the dictionary yet
230 for config_length, config_grp in lengthOfChainConfigs:
231 if config_grp in length_of_configs:
232 if config_length > length_of_configs[config_grp]:
233 length_of_configs[config_grp] = config_length
235 length_of_configs[config_grp] = config_length
237 # find the chains that contain more than one alignment group, to keep track
238 # of what combinations do we need to deal with.
239 # we're using sets here so we don't end up with duplicates
240 if len(set(chainDict['alignmentGroups'])) > 1:
241 combinations_in_menu += [list(set(chainDict['alignmentGroups']))]
242 for align_group in list(set(chainDict['alignmentGroups'])):
243 alignmentGroups_to_align.update([align_group])
245 self.allChainsForAlignment = all_chains
246 self.combinationsInMenu = combinations_in_menu
247 self.alignmentGroupsToAlign = alignmentGroups_to_align
248 self.configLengthDict = length_of_configs
253 def generateAllChainConfigs(self, flags):
255 == Obtains chain configs for all chains in menu
258 from TriggerMenuMT.HLT.Config.Utility.MenuAlignmentTools import MenuAlignment
259 from TriggerMenuMT.HLT.CommonSequences import EventBuildingSequences, TLABuildingSequences
261 # get all chain names from menu
262 log.info("Will now get chains from the menu")
263 self.getChainsFromMenu(flags)
265 # decoding of the chain name
266 log.info("Will now get chain dictionaries for each chain")
267 self.getChainDicts(flags)
269 if flags.Trigger.disableCPS:
270 log.warning('Removing all CPS group because the flag Trigger.disableCPS is set')
271 for chainDict in self.chainDicts:
272 chainDict['groups'] = [g for g in chainDict['groups'] if not g.startswith('RATE:CPS_')]
274 #import the necessary signatures
275 log.debug("Importing the necessary signatures")
276 self.importSignaturesToGenerate()
278 log.info("Will now generate the chain configuration for each chain")
279 self.generateChains(flags)
281 log.info("Will now calculate the alignment parameters")
282 #dict of signature: set it belongs to
283 #e.g. {'Electron': ['Electron','Muon','Photon']}
284 menuAlignment = MenuAlignment(self.combinationsInMenu,
285 self.alignmentGroupsToAlign,
286 self.configLengthDict)
287 menuAlignment.analyse_combinations()
289 # alignmentGroups_to_align = menuAlignment.groupsToAlign
290 # lengthOfChainConfigs = self.configLengthDict
291 # combinationsInMenu = menuAlignment.combinationsInMenu
292 # alignmentGroup_sets_to_align = menuAlignment.setsToAlign
294 log.info('Aligning the following signatures: %s',sorted(menuAlignment.sets_to_align))
295 log.debug('Length of each of the alignment groups: %s',self.configLengthDict)
299 for chainDict,chainConfig,lengthOfChainConfigs in self.allChainsForAlignment:
301 # start by ordering electron, photon, muon by having e+mu, g+mu, e+g chains
302 # desired ordering: electron, photon, muon, tau, jet, met, b-jet
304 # lengthOfChainConfigs is something like this: [(4, 'Photon'), (5, 'Muon')]
305 # needs to match up with the maximum number of steps in a signature in the menu (length_of_configs)
306 # start with electron! Only need to add post-steps for combined electron chains if the max length in a combined chain
307 # is greater than the number of electron steps combined chain. Assume that the max length of an electron chain occurs
308 # in a combined chain.
310 log.debug("[generateAllChainConfigs] chain %s has config lengths %s and alignment groups %s", chainDict['chainName'], lengthOfChainConfigs, chainDict['alignmentGroups'])
312 alignmentGroups = chainDict['alignmentGroups']
314 #parallel-merged single-signature chains or single signature chains. Anything that needs no splitting!
315 if len(set(alignmentGroups)) == 1:
316 alignedChainConfig = menuAlignment.single_align(chainDict, chainConfig)
317 HLTMenuConfig.registerChain( chainDict )
318 chainConfigs.append( alignedChainConfig )
320 elif len(alignmentGroups) >= 2:
321 alignedChainConfig = menuAlignment.multi_align(chainDict, chainConfig, lengthOfChainConfigs)
322 HLTMenuConfig.registerChain( chainDict )
323 chainConfigs.append( alignedChainConfig )
326 log.error("Menu can't deal with combined chains with more than two alignmentGroups at the moment. oops...")
327 raise NotImplementedError("more than three alignment groups still needs implementing in ChainMerging.py, ATR-22206")
329 if not HLTMenuConfig.isChainRegistered(chainDict['chainName']):
330 log.error("Chain %s has not been registered in the menu!", chainDict['chainName'])
332 pp = pprint.PrettyPrinter(indent=4, depth=8)
333 log.error('The chain dictionary is: %s', pp.pformat(chainDict))
334 raise Exception("Please fix the menu or the chain.")
336 # align event building sequences
337 log.info("[generateAllChainConfigs] general alignment complete, will now align TLA chains")
338 TLABuildingSequences.alignTLASteps(chainConfigs, HLTMenuConfig.dicts())
339 log.info("[generateAllChainConfigs] general and TLA alignment complete, will now align PEB chains")
340 EventBuildingSequences.alignEventBuildingSteps(chainConfigs, HLTMenuConfig.dicts())
342 log.info("[generateAllChainConfigs] all chain configurations have been generated.")
346 def getChainsFromMenu(self, flags):
348 == Returns the list of chain names that are in the menu
351 log.info(f'Menu name: {flags.Trigger.triggerMenuSetup}')
353 # Get the base menu name (without prescale set suffix)
354 from TriggerMenuMT.HLT.Config.Utility.MenuPrescaleSet import AutoPrescaleSetGen
355 self.base_menu_name, self.prescale_set_name = AutoPrescaleSetGen.split_menu_name(flags.Trigger.triggerMenuSetup)
356 log.debug('Base menu name: %s', self.base_menu_name)
358 # Generate the list of chains from the basic menu (terminated in a version number)
360 menu_module = importlib.import_module(f'TriggerMenuMT.HLT.Menu.{self.base_menu_name}')
361 except Exception as e:
362 log.fatal(f'Failed to import menu module "{self.base_menu_name}" inferred from menu "{flags.Trigger.triggerMenuSetup}"')
366 self.chainsInMenu = menu_module.setupMenu()
368 # Temporary HLT->EF migration helper block.
369 # Replaces EF_ with HLT_ just after the menu is read, before it is processed any further.
370 if flags.Trigger.useHLTPrefixForRun4:
372 for grp in self.chainsInMenu:
373 for i, chn in enumerate(self.chainsInMenu[grp]):
374 if chn.name.startswith("EF_"):
375 self.chainsInMenu[grp][i] = dataclasses.replace(chn, name="HLT_" + chn.name[3:])
377 # Filter chains if requested
378 if self.chainFilter is not None:
379 self.signaturesOverwritten = True
381 # Verify that if the chain filter has lists of chains
382 # they are all in the menu
384 if hasattr(self.chainFilter,'selectChains'):
385 chainsToCheck += self.chainFilter.selectChains
386 if hasattr(self.chainFilter,'disableChains'):
387 chainsToCheck += self.chainFilter.disableChains
388 for chain in chainsToCheck:
390 for signame in self.chainsInMenu:
391 if chain in [c.name for c in self.chainsInMenu[signame]]:
395 raise RuntimeError(f'Request to enable/disable chain {chain} that is not in menu')
397 for signame in self.chainsInMenu:
398 self.chainsInMenu[signame] = [c for c in self.chainsInMenu[signame]
399 if self.chainFilter(signame, c)]
401 if not self.chainsInMenu:
402 log.warning("There seem to be no chains in the menu - please check")
403 elif log.isEnabledFor(logging.DEBUG):
405 log.debug("The following chains were found in the menu:")
406 pprint.pprint(self.chainsInMenu)
409 def __generateChainConfig(self, flags, mainChainDict):
411 # Assembles the chain configuration and returns a chain object with (name, L1see and list of ChainSteps)
414 from TriggerMenuMT.HLT.Config.Utility.ChainDictTools import splitInterSignatureChainDict
415 from TriggerMenuMT.HLT.Config.Utility.ComboHypoHandling import addTopoInfo, comboConfigurator, topoLegIndices, anomdetWPIndices, topo3VarLegIndices
416 from TriggerMenuMT.HLT.Config.Utility.ChainMerging import mergeChainDefs
417 from TriggerMenuMT.HLT.CommonSequences import EventBuildingSequences, TLABuildingSequences
419 # split the the chainDictionaries for each chain and print them in a pretty way
420 chainDicts = splitInterSignatureChainDict(mainChainDict)
422 if log.isEnabledFor(logging.DEBUG):
424 pp = pprint.PrettyPrinter(indent=4, depth=8)
425 log.debug('dictionary is: %s', pp.pformat(chainDicts))
427 # Loop over all chainDicts and send them off to their respective assembly code
428 listOfChainConfigs = []
429 perSig_lengthOfChainConfigs = []
431 for chainPartDict in chainDicts:
432 chainPartConfig = None
433 currentSig = chainPartDict['signature']
434 currentAlignGroup = None
435 if len(chainPartDict['chainParts']) == 1:
436 currentAlignGroup = chainPartDict['chainParts'][0]['alignmentGroup']
438 chainName = chainPartDict['chainName']
439 log.debug('Checking chainDict for chain %s in signature %s, alignment group %s' , chainName, currentSig, currentAlignGroup)
441 if currentSig in self.availableSignatures:
443 log.debug("[__generateChainConfigs] Trying to get chain config for %s", currentSig)
444 if currentSig in self.defaultFlagsForSignature:
445 sigFlags = self.defaultFlagsForSignature[currentSig]
448 sigFlags = self.chainDefModule[currentSig].prepareDefaultSignatureFlags(flags)
449 except AttributeError:
450 log.debug("prepareDefaultSignatureFlags not present")
452 except Exception as e:
453 log.error(f"Unexpected error invoking prepareDefaultSignatureFlags {e}")
456 self.defaultFlagsForSignature[currentSig] = sigFlags
458 if currentSig in ['Electron', 'Photon', 'Muon', 'Tau', 'Bphysics'] :
459 chainPartConfig, perSig_lengthOfChainConfigs = self.chainDefModule[currentSig].generateChainConfigs(sigFlags, chainPartDict, perSig_lengthOfChainConfigs)
461 chainPartConfig = self.chainDefModule[currentSig].generateChainConfigs(sigFlags, chainPartDict)
462 if currentSig == 'Test' and isinstance(chainPartConfig, tuple):
463 chainPartConfig = chainPartConfig[0]
465 log.error('[__generateChainConfigs] Problems creating ChainDef for chain %s ', chainName)
466 log.error('[__generateChainConfigs] I am in chain part\n %s ', chainPartDict)
467 log.exception('[__generateChainConfigs] Full chain dictionary is\n %s ', mainChainDict)
468 raise Exception('[__generateChainConfigs] Stopping menu generation. Please investigate the exception shown above.')
470 log.error('Chain %s cannot be generated - Signature "%s" not available', chainPartDict['chainName'], currentSig)
471 log.error('Available signature(s): %s', self.availableSignatures)
472 raise Exception('Stopping the execution. Please correct the configuration.')
474 log.debug("Chain %s \n chain config: %s",chainPartDict['chainName'],chainPartConfig)
476 listOfChainConfigs.append(chainPartConfig)
477 log.debug("[__generateChainConfigs] adding to the perSig_lengthOfChainConfigs list (%s, %s)",chainPartConfig.nSteps,chainPartConfig.alignmentGroups)
478 perSig_lengthOfChainConfigs.append((chainPartConfig.nSteps,chainPartConfig.alignmentGroups))
480 # this will be a list of lists for inter-sig combined chains and a list with one
481 # multi-element list for intra-sig combined chains
482 # here, we flatten it accordingly (works for both cases!)
483 lengthOfChainConfigs = []
484 for nSteps, aGrps in perSig_lengthOfChainConfigs:
485 if len(nSteps) != len(aGrps):
486 log.error("Chain part has %s steps and %s alignment groups - these don't match!",nSteps,aGrps)
488 for a,b in zip(nSteps,aGrps):
489 lengthOfChainConfigs.append((a,b))
492 # This part is to deal with combined chains between different signatures
494 if len(listOfChainConfigs) == 0:
495 raise Exception('[__generateChainConfigs] No Chain Configuration found for {0}'.format(mainChainDict['chainName']))
497 if len(listOfChainConfigs)>1:
498 log.debug("Merging strategy from dictionary: %s", mainChainDict["mergingStrategy"])
499 theChainConfig, perSig_lengthOfChainConfigs = mergeChainDefs(listOfChainConfigs, mainChainDict, perSig_lengthOfChainConfigs)
500 lengthOfChainConfigs = []
501 for nSteps, aGrps in perSig_lengthOfChainConfigs:
502 if len(nSteps) != len(aGrps):
503 log.error("Post-merged chain part has %s steps and %s alignment groups - these don't match!",nSteps,aGrps)
505 for a,b in zip(nSteps,aGrps):
506 lengthOfChainConfigs.append((a,b))
508 theChainConfig = listOfChainConfigs[0]
510 for topoID in range(len(mainChainDict['extraComboHypos'])):
511 thetopo = mainChainDict['extraComboHypos'][topoID].strip(string.digits).rstrip(topoLegIndices)
514 if "anomdet" in thetopo:
515 thetopo = thetopo.rstrip(anomdetWPIndices)
517 if "masswiso" in thetopo:
518 thetopo = thetopo.rstrip(topo3VarLegIndices)
520 theChainConfig.addTopo((comboConfigurator[thetopo],thetopo))
522 # Now we know where the topos should go, we can insert them in the right steps
523 if len(theChainConfig.topoMap) > 0:
524 log.debug("Trying to add extra ComboHypoTool for %s",mainChainDict['extraComboHypos'])
525 addTopoInfo(theChainConfig,mainChainDict,listOfChainConfigs,lengthOfChainConfigs)
527 log.error('[__generateChainConfigs] Problems creating ChainDef for chain %s ', chainName)
528 log.error('[__generateChainConfigs] I am in the extraComboHypos section, for %s ', mainChainDict['extraComboHypos'])
529 log.exception('[__generateChainConfigs] Full chain dictionary is\n %s ', mainChainDict)
530 raise Exception('[__generateChainConfigs] Stopping menu generation. Please investigate the exception shown above.')
531 except AttributeError:
532 raise Exception('[__generateChainConfigs] Stopping menu generation. Please investigate the exception shown above.')
534 # Configure event building strategy
535 eventBuildType = mainChainDict['eventBuildType']
536 TLAEventBuildTypes = ('PhysicsTLA', 'FTagPEBTLA', 'EgammaPEBTLA', 'DarkJetPEBTLA')
539 if any(ebtype in eventBuildType for ebtype in TLAEventBuildTypes):
540 log.debug("Adding TLA Step for chain %s", mainChainDict['chainName'])
541 TLABuildingSequences.addTLAStep(flags, theChainConfig, mainChainDict)
542 log.debug('Configuring event building sequence %s for chain %s', eventBuildType, mainChainDict['chainName'])
543 EventBuildingSequences.addEventBuildingSequence(flags, theChainConfig, eventBuildType, mainChainDict)
544 except TypeError as ex:
546 raise Exception('[__generateChainConfigs] Stopping menu generation for EventBuilding/TLA sequences. Please investigate the exception shown above.')
548 log.debug('[__generateChainConfigs] lengthOfChainConfigs %s, ChainConfigs %s ', lengthOfChainConfigs, theChainConfig)
549 return theChainConfig,lengthOfChainConfigs
552 def resolveEmptySteps(self,chainConfigs):
553 max_steps = max([len(cc.steps) for cc in chainConfigs], default=0)
554 steps_are_empty = [True for i in range(0,max_steps)]
556 for cc in chainConfigs:
557 for istep, the_step in enumerate(cc.steps):
558 if not the_step.isEmpty:
559 steps_are_empty[istep] = False
561 emptySteps.append(the_step)
563 log.debug("Are there any fully empty steps? %s", steps_are_empty)
564 log.debug("The empty step(s) and associated chain(s) are: %s", emptySteps)
565 empty_step_indices = [i for i,is_empty in enumerate(steps_are_empty) if is_empty]
567 if len(empty_step_indices) == 0:
570 special_test_menu = self.chainFilter and ( getattr(self.chainFilter, "selectChains", False) or \
571 getattr(self.chainFilter, "disableChains", False) or \
572 getattr(self.chainFilter, "disabledSignatures", False) or \
573 getattr(self.chainFilter, "enabledSignatures", False) )
576 if len(self.availableSignatures) != 1 and not special_test_menu:
577 raise Exception("[resolveEmptySteps] Please find the reason for this empty step and resolve it / remove it from the menu: %s", emptySteps)
579 log.info("Will now delete steps %s (indexed from zero)",empty_step_indices)
581 for cc in chainConfigs:
583 #only add non-empty steps to the new steps list!
584 for istep,step in enumerate(cc.steps):
585 if istep not in empty_step_indices: