10 from TrigEDMConfig.TriggerEDMRun1
import TriggerL2List,TriggerEFList,TriggerResultsRun1List
11 from TrigEDMConfig.TriggerEDMRun2
import TriggerResultsList,TriggerLvl1List,TriggerIDTruth,TriggerHLTList,EDMDetails,EDMLibraries,TriggerL2EvolutionList,TriggerEFEvolutionList
12 from TrigEDMConfig.TriggerEDMRun3
import TriggerHLTListRun3,varToRemoveFromAODSLIM,EDMDetailsRun3,getSafeEDMInsertPosition
13 from TrigEDMConfig.TriggerEDMRun4
import TriggerHLTListRun4
14 from TrigEDMConfig.TriggerEDMDefs
import allowTruncation
15 from CLIDComps.clidGenerator
import clidGenerator
16 from collections
import defaultdict
20 from AthenaCommon.Logging
import logging
21 log = logging.getLogger(
'TriggerEDM')
32 AllowedOutputFormats = [
'BS',
'ESD',
'AODFULL',
'AODSLIM',
'AODBLSSLIM' ]
33 from TrigEDMConfig
import DataScoutingInfo
34 AllowedOutputFormats.extend(DataScoutingInfo.getAllDataScoutingIdentifiers())
36 _allowedEDMPrefixes = [
'HLT_',
'L1_',
'LVL1']
39 Verify that the name is in the list of recorded objects and conform to the name convention
41 In Run 2 it was a delicate process to configure correctly what got recorded
42 as it had to be set in the algorithm that produced it as well in the TriggerEDM.py in a consistent manner.
44 For Run 3 every alg input/output key can be crosschecked against the list of objects to record which is defined here.
45 I.e. in the configuration alg developer would do this:
46 from TriggerEDM.TriggerEDMRun3 import recordable
48 alg.outputKey = recordable("SomeKey")
49 If the names are correct the outputKey is assigned with SomeKey, if there is a missmatch an exception is thrown.
57 log.error(
"Don't call recordable({0}), or add any \"HLTNav_\" collection manually to the EDM. See:collectDecisionObjects.".
format( name ) )
60 if not any([name.startswith(p)
for p
in _allowedEDMPrefixes]):
61 raise RuntimeError( f
"The collection name {name} does not start with any of the allowed prefixes: {_allowedEDMPrefixes}" )
62 if "Aux" in name
and not name[-1] !=
".":
63 raise RuntimeError( f
"The collection name {name} is Aux but the name does not end with the '.'" )
66 for entry
in TriggerHLTListRun3:
67 if entry[0].
split(
"#" )[1] == name:
69 msg =
"The collection name {0} is not declared to be stored by HLT. Add it to TriggerEDMRun3.py".
format( name )
70 log.error(
"ERROR in recordable() - see following stack trace.")
71 raise RuntimeError( msg )
75 Extend edmList with extraList, keeping track whether a completely new
76 collection is being added, or a dynamic variable is added to an existing collection, or new targets are added to an existing collection.
77 The format of extraList is the same as those of TriggerHLTListRun3.
79 existing_collections = [(c[0].
split(
"#")[1]).
split(
".")[0]
for c
in edmList]
81 for item
in extraList:
82 colname = (item[0].
split(
"#")[1]).
split(
".")[0]
83 if colname
not in existing_collections:
86 edmList.insert(insert_idx+1,item)
88 edmList.insert(insert_idx,item)
89 log.info(
"added new item to Trigger EDM: {}".
format(item))
92 isAux =
"Aux." in item[0]
94 existing_item_nr = [i
for i,s
in enumerate(edmList)
if colname == (s[0].
split(
"#")[1]).
split(
".")[0]]
95 if len(existing_item_nr) != 1:
96 log.error(
"Found {} existing edm items corresponding to new item {}, but it must be exactly one!".
format(len(existing_item_nr), item))
97 existingItem = edmList[existing_item_nr[0]]
99 dynVars = (item[0].
split(
"#")[1]).
split(
".")[1:]
100 existing_dynVars = (existingItem[0].
split(
"#")[1]).
split(
".")[1:]
101 existing_dynVars.extend(dynVars)
102 dynVars =
list(dict.fromkeys(existing_dynVars))
105 newVars =
'.'.
join(dynVars)
106 edmTargets = item[1].
split(
" ")
if len(item) > 1
else []
107 existing_edmTargets = existingItem[1].
split(
" ")
108 edmTargets.extend(existing_edmTargets)
109 edmTargets =
list(dict.fromkeys(edmTargets))
110 newTargets =
" ".
join(edmTargets)
111 typename = item[0].
split(
"#")[0]
112 log.info(
"old item in Trigger EDM : {}".
format(existingItem))
113 signature = existingItem[2]
114 tags = existingItem[3]
if len(existingItem) > 3
else None
115 edmList.pop(existing_item_nr[0])
116 combName = typename +
"#" + colname
118 combName +=
"." + newVars
120 edmList.insert(existing_item_nr[0], (combName, newTargets, signature, tags))
122 edmList.insert(existing_item_nr[0] , (combName, newTargets, signature))
123 log.info(
"updated item in Trigger EDM: {}".
format(edmList[existing_item_nr[0]]))
125 if testEDMList(edmList, error_on_edmdetails=
False):
126 log.error(
"edmList contains inconsistencies!")
130 The static EDM list does still need some light manipulation before it can be used commonly.
131 Never import TriggerHLTListRun3 or TriggerHLTListRun4 directly, always fetch them via this function
134 runVersion = flags.Trigger.EDMVersion
137 edmListCopy = TriggerHLTListRun3.copy()
138 elif runVersion == 4:
139 edm4ListCopy = TriggerHLTListRun4.copy()
140 edm3ListCopy = TriggerHLTListRun3.copy()
141 log.debug(
"Removing duplicated item between TriggerHLTListRun3 and TriggerHLTListRun4.")
142 for i
in range(len(edm3ListCopy)-1, -1, -1):
143 for r4item
in edm4ListCopy:
144 if edm3ListCopy[i][0] == r4item[0]:
146 log.debug(f
"- Dupe: {r4item} is removed from TriggerHLTListRun3 to be updated by TriggerHLTListRun4")
148 lenPreMerge = len(edm3ListCopy)
149 edm4ListCopy.extend(edm3ListCopy)
150 lenPostMerge = len(edm4ListCopy)
151 edmListCopy = edm4ListCopy
152 log.info(f
"Added TriggerHLTListRun4 to TriggerHLTListRun3. EDM entries {lenPreMerge} -> {lenPostMerge}")
153 if testEDMList(edmListCopy, error_on_edmdetails=
False):
154 log.error(
"edmList contains inconsistencies!")
156 errMsg=
"ERROR the getRawTriggerEDMList function supports runs 3 and 4."
158 raise RuntimeError(errMsg)
160 if flags
and flags.Trigger.ExtraEDMList:
161 log.info(
"Adding extra collections to EDM %i: %s", runVersion,
str(flags.Trigger.ExtraEDMList))
168 List (Literally Python dict) of trigger objects to be placed with flags:
169 flags is the CA flag container
170 key can be" 'ESD', 'AODSLIM', 'AODFULL'
171 runVersion can be: '-1 (Auto-configure)', '1 (Run1)', '2 (Run2)', '3' (Run 3), '4' (Run 4)
175 runVersion = flags.Trigger.EDMVersion
180 elif runVersion == 2:
184 elif runVersion >= 3:
187 if key
not in AllowedOutputFormats:
188 log.warning(
'Output format: %s is not in list of allowed formats, please check!', key)
198 elif "AODSLIM" in key:
202 if len(varToRemoveFromAODSLIM) == 0:
204 log.info(
"No decorations are listed to be removed from AODSLIM")
207 log.info(
"The following decorations are going to be removed from the listed collections in AODSLIM {}".
format(varToRemoveFromAODSLIM))
210 for cont, values
in Run3TrigEDMSLIM.items():
211 if (isinstance(values, list)):
215 coll = value.split(
'.')[0]
217 varRemovedFlag =
False
218 for myTuple
in varToRemoveFromAODSLIM:
221 if var
in value
and coll
in myTuple:
222 varRemovedFlag =
True
223 removeVar =
'.'+var+
'.'
224 newValue = newValue.replace(removeVar,
'.')
226 if newValue[-1:] ==
'.':
227 newValue = newValue[:-1]
229 if varRemovedFlag
is False:
230 newValues.append(value)
231 elif varRemovedFlag
is True:
232 newValues.append(newValue)
234 raise RuntimeError(
"Decoration removed but no new Value was available, not sure what to do...")
237 Run3TrigEDM[cont] = newValues
239 raise RuntimeError(
"Value in Run3TrigEDM dictionary is not a list")
244 log.debug(
'TriggerEDM for EDM set {} contains the following collections: {}'.
format(key, Run3TrigEDM) )
249 raise RuntimeError(
"Invalid runVersion=%s supplied to getTriggerEDMList" % runVersion)
256 Finds a given key from within the trigEDMList.
257 Returns true if this collection is produced inside EventViews
258 (Hence, has the special viewIndex Aux decoration applied by steering)
260 from TrigEDMConfig.TriggerEDMRun3
import InViews
262 return any(coll
for coll
in itertools.chain(*trigEDMList)
if
263 len(coll)>3
and theKey==coll[0].
split(
'#')[1]
and
264 any(isinstance(v, InViews)
for v
in coll[3]))
270 keyNoAux = el.split(
'.')[0].
replace(
'Aux',
'')
273 if el.split(
'.')[1] ==
'':
277 return el.split(
'.')[0]+
'.viewIndex'
280 return el.split(
'.')[0]+
'.-'
285 return el+
'.viewIndex'
295 The keys should contain BS and all the identifiers used for scouting.
296 Returns list of tuples (typename#key, [keys], [properties]).
299 from TrigEDMConfig.TriggerEDMRun3
import persistent
303 for definition
in _HLTList:
305 typename,collkey = definition[0].
split(
"#")
309 destination = keys &
set(definition[1].
split())
310 if len(destination) > 0:
311 collections.append( (typename+
"#"+collkey,
list(destination),
312 definition[3]
if len(definition)>3
else []) )
320 Gives back the Python dictionary with the content of ESD/AOD (dst) which can be inserted in OKS.
322 dset =
set(destination.split())
323 toadd = defaultdict(list)
325 for item
in itertools.chain(*trigEDMList):
334 if k
not in toadd[colltype]:
335 toadd[colltype] += [k]
343 Modified EDM list to remove all dynamic variables
344 Requires changing the list to have 'Aux.-'
349 for k,v
in _edmList.items():
361 Modified EDM list to remove all dynamic variables
362 Requires changing the list to have 'Aux.-'
365 for k,v
in edmList.items():
369 newnames+=[el.split(
'.')[0]+
'.-']
376 """ From name of object in AOD/ESD found by checkFileTrigSize.py, return category """
378 """ Clean up object name """
387 if s.count(
'.') : s = s[:s.index(
'.')]
388 if s.count(
'::'): s = s[s.index(
':')+2:]
389 if s.count(
'<'): s = s[s.index(
'<')+1:]
390 if s.count(
'>'): s = s[:s.index(
'>')]
391 if s.count(
'.') : s = s[:s.index(
'.')]
392 if s.count(
'Dyn') : s = s[:s.index(
'Dyn')]
398 if s12.startswith(
'HLT_xAOD__')
or s12.startswith(
'HLT_Rec__')
or s12.startswith(
'HLT_Analysis__') :
399 s12 = s12[s12.index(
'__')+2:]
400 s12 = s12[s12.index(
'_')+1:]
403 elif s12.startswith(
'HLT_'):
405 if s12.count(
'_'): s12 = s12[s12.index(
'_')+1:]
406 if s12.count(
'_'): s12 = s12[s12.index(
'_')+1:]
409 TriggerListRun1 = TriggerL2List + TriggerEFList + TriggerResultsRun1List
410 TriggerListRun2 = TriggerResultsList + TriggerLvl1List + TriggerIDTruth + TriggerHLTList
416 """ Loop over all objects already defined in lists (and hopefully categorized!!) """
417 for item
in TriggerListRun1+TriggerListRun2:
420 """ Clean up type name """
421 if t.count(
'::'): t = t[t.index(
':')+2:]
422 if t.count(
'<'): t = t[t.index(
'<')+1:]
423 if t.count(
'>'): t = t[:t.index(
'>')]
424 if (s12.startswith(t)
and s12.endswith(k))
and (len(t) > len(bestMatch)):
428 if k.count(
'.'): k = k[:k.index(
'.')]
433 for item
in TriggerListRun3:
436 """ Clean up type name """
437 if t.count(
'::'): t = t[t.index(
':')+2:]
438 if t.count(
'<'): t = t[t.index(
'<')+1:]
439 if t.count(
'>'): t = t[:t.index(
'>')]
441 if (s.startswith(t)
and s.endswith(k))
and (len(t) > len(bestMatch)):
445 if k.count(
'.'): k = k[:k.index(
'.')]
450 if category ==
'' and 'HLTNav' in s:
453 if category ==
'':
return 'NOTFOUND'
459 """ From the strings containing type and key of trigger EDM extract type and key
461 return s[:s.index(
'#')], s[s.index(
'#')+1:]
464 """ The key is usually HLT_*, this function returns second part of it or empty string
469 return key[key.index(
'_'):].lstrip(
'_')
473 Gives back the Python dictionary with the content of ESD/AOD (dst) which can be inserted in OKS.
475 dset =
set(destination.split())
479 for item
in itertools.chain(*lst):
486 if 'collection' in EDMDetails[t]:
487 colltype = EDMDetails[t][
'collection']
488 if colltype
in toadd:
489 if k
not in toadd[colltype]:
490 toadd[colltype] += [k]
492 toadd[colltype] = [k]
498 Gives back the Python dictionary with the truth trigger content of ESD/AOD (dst) which can be inserted in OKS.
504 Gives back the Python dictionary with the lvl1 trigger result content of ESD which can be inserted in OKS.
510 Gives back the Python dictionary with the lvl1 trigger result content of AOD which can be inserted in OKS.
518 List (Literally Python list) of trigger objects to be preregistered i.e. this objects we want in every event for L2
521 for item
in TriggerL2List:
522 if len (item[1]) == 0:
continue
531 List (Literally Python list) of trigger objects to be preregistered i.e. this objects we want in every event for EF
534 for item
in TriggerEFList:
535 if len (item[1]) == 0:
continue
544 List (Literally Python list) of trigger objects to be preregistered i.e. this objects we want in every event for merged L2/EF in addition to default L2 and EF
547 for item
in TriggerHLTList:
548 if len (item[1]) == 0:
continue
558 List (Literally Python list) of trigger objects to be preregistered i.e. this objects we want for all levels
559 version can be: '1 (Run1)', '2 (Run2)'
570 l=
list(dict.fromkeys(objs))
572 raise RuntimeError(
"Invalid version=%s supplied to getPreregistrationList" % version)
577 """ List of L2 types to be read from BS, used by the TP
580 for item
in TriggerL2List:
583 if 'collection' in EDMDetails[t]:
584 ctype = EDMDetails[t][
'collection']
589 """ List of EF types to be read from BS, used by the TP
592 for item
in TriggerEFList:
595 if 'collection' in EDMDetails[t]:
596 ctype = EDMDetails[t][
'collection']
601 """ List of HLT types to be read from BS, used by the TP
604 for item
in TriggerHLTList:
607 if 'collection' in EDMDetails[t]:
608 ctype = EDMDetails[t][
'collection']
614 Mapping of Transient objects to Peristent during serialization (BS creation)
615 version can be: '1 (Run1)', '2 (Run2)'
623 raise RuntimeError(
"Invalid version=%s supplied to getTPList" % version)
625 for t,d
in EDMDetails.items():
627 if 'collection' in d:
628 colltype = EDMDetails[t][
'collection']
629 if colltype
in bslist:
630 l[colltype] = d[
'persistent']
640 for k,v
in typedict.items():
643 if el.startswith(
'HLT_')
or el ==
'HLT':
644 prefixAndLabel = el.split(
'_',1)
645 containername = k
if 'Aux' not in k
else EDMDetails[k][
'parent']
647 containername = re.sub(
'::',
'__',re.sub(
'_v[0-9]+$',
'',containername))
648 newnames+=[
'_'.
join([prefixAndLabel[0],containername]+([prefixAndLabel[1]]
if len(prefixAndLabel) > 1
else []))]
656 List of EF trigger objects that were written to ByteStream in Run 1
659 for item
in TriggerEFEvolutionList:
660 if len (item[1]) == 0:
continue
667 List of Run-2 containers equivalent to Run-1 EF containers
670 for item
in TriggerEFEvolutionList:
671 if len (item[1]) == 0:
continue
678 List of L2 trigger objects that were written to ByteStream in Run 1
681 for item
in TriggerL2EvolutionList:
682 if len (item[1]) == 0:
continue
689 List of Run-2 containers equivalent to Run-1 L2 containers
692 for item
in TriggerL2EvolutionList:
693 if len (item[1]) == 0:
continue
700 Checks container type name is hashable
702 c = cgen.genClidFromName(typename)
703 return (cgen.getNameFromClid(c)
is not None)
707 Checks EDM list entries for serialization and configuration compliance.
711 _noAuxList = [
"xAOD::" + contType
for contType
in [
"TrigConfKeys",
"BunchConfKey"]]
713 cgen = clidGenerator(
"",
False)
715 found_allow_truncation =
False
716 serializable_names = []
717 serializable_names_no_label = []
718 serializable_names_no_properties = []
720 for i, edm
in enumerate(edm_list):
724 log.error(
"EDM entry too short for " + edm[0])
728 serializable_name = edm[0]
729 serializable_name_no_label = re.sub(
r"\#.*",
"", serializable_name)
730 serializable_name_no_properties = serializable_name.split(
'.')[0]
732 serializable_names.append(serializable_name)
733 serializable_names_no_label.append(serializable_name_no_label)
734 serializable_names_no_properties.append(serializable_name_no_properties)
738 log.error(
"no CLID for " + serializable_name)
742 if "Aux" not in serializable_name
and "." in serializable_name:
743 log.error(
"A '.' found in non-Aux container name " + serializable_name)
747 if "Aux" in serializable_name
and "Aux." not in serializable_name:
748 log.error(
"no final Aux. in label for " + serializable_name)
752 if serializable_name.count(
"#") != 1:
753 log.error(
"Invalid naming structure for " + serializable_name)
757 if serializable_name.startswith(
"xAOD")
and "Aux" not in serializable_name:
758 cont_type,cont_name = serializable_name.split(
"#")
759 if cont_type
not in _noAuxList:
761 if len(edm_list) == i+1:
763 cont_to_test =
"nothing"
765 cont_to_test = edm_list[i+1][0].
split(
".")[0]+
"."
766 cont_type_short = cont_type.replace(
"Container",
"")
767 pattern = re.compile(cont_type_short+
r".*Aux.*\#"+cont_name+
"Aux.")
768 if not pattern.match(cont_to_test):
770 shallow_pattern = re.compile(
"xAOD::ShallowAuxContainer#"+cont_name+
"Aux.")
771 if not shallow_pattern.match(cont_to_test):
774 log.error(
"Expected relevant Aux container following interface container %s, but found %s instead.",serializable_name,cont_to_test)
779 if len(edm_list[i+1]) > 1
and cont_to_test.count(
"#") == 1:
780 contname_to_test = cont_to_test.split(
"#")[1]
781 targets_to_test =
set(edm_list[i+1][1].
split())
782 if len(targets ^ targets_to_test) > 0:
783 log.error(
"Targets of %s (%s) and %s (%s) do not match",cont_name,targets,contname_to_test,targets_to_test)
787 if i>0
and "Aux" in serializable_name
and "Aux" in edm_list[i-1][0]:
788 log.error(f
"Aux container {serializable_name} needs to folow the "
789 "associated interface container in the EDM list")
794 file_types = edm[1].
split()
795 for file_type
in file_types:
796 if file_type
not in AllowedOutputFormats:
797 log.error(
"unknown file type " + file_type +
" for " + serializable_name)
799 for higher_level,required_lower_level
in [(
'AOD',
'ESD'), (
'SLIM',
'AODFULL')]:
800 if higher_level
in file_type
and required_lower_level
not in file_types:
801 log.error(
"Target list for %s containing '%s' must also contain lower level target '%s'.",serializable_name,file_type,required_lower_level)
805 tags = edm[3]
if len(edm) > 3
else None
806 allow_truncation_flag =
False
808 allow_truncation_flag = allowTruncation
in tags
809 if allow_truncation_flag:
810 found_allow_truncation =
True
812 if found_allow_truncation
and not allow_truncation_flag:
813 log.error(
"All instances of 'allowTruncation' need to be at the END of the EDM serialisation list")
820 if not len(
set(serializable_names_no_properties)) == len(serializable_names_no_properties):
821 log.error(
"Duplicates in EDM list! Duplicates found:")
822 import collections.abc
823 for item, count
in collections.Counter(serializable_names_no_properties).
items():
825 log.error(
str(count) +
"x: " +
str(item))
829 for EDMDetail
in EDMDetailsRun3.keys():
830 if EDMDetail
not in serializable_names_no_label:
831 msg =
"EDMDetail for " + EDMDetail +
" does not correspond to any name in TriggerList"
832 if error_on_edmdetails: