3 __author__ =
'Javier Montejo'
4 __version__=
"$Revision: 2.0 $"
5 __doc__=
"Access to Trigger DB and TriggerMenu to read past and future prescales"
9 from TriggerMenuMT.TriggerAPI.TriggerEnums
import TriggerPeriod, LBexceptions, TriggerRenaming
10 from TriggerMenuMT.TriggerAPI.TriggerPeriodData
import TriggerPeriodData
11 from AthenaCommon.Logging
import logging
12 log = logging.getLogger(__name__)
18 lb = runlb & ((1<<32)-1)
19 return (
int(run),
int(lb) )
24 returns all runs in the given period which have the ReadyForPhysics flag set in at least 1 LB
27 log.info(
"Loading COOL libs...")
28 from CoolLumiUtilities.CoolDataReader
import CoolDataReader
29 log.info(
"Done loading libs, starting now ...")
31 myReader = CoolDataReader(
'COOLONL_TDAQ/CONDBR2',
'/TDAQ/RunCtrl/DataTakingMode')
34 firstRun =
min([x
for x
in period.keys()])
35 lastRun =
max([x
for x
in period.keys()])+1
36 since = (firstRun << 32)
37 until = (lastRun << 32)
39 myReader.setIOVRange( since, until )
42 for obj
in myReader.data:
43 isReady = (obj.payload()[
'ReadyForPhysics'] == 1)
48 if sincerun != untilrun:
49 log.info(
"WARNING: ready block crosses run boundaries: %d %d", sincerun, untilrun)
50 if sincerun
not in period:
continue
51 if sincerun
in runsWithReady:
52 runsWithReady[sincerun] += [ (sincelb, untillb) ]
54 runsWithReady[sincerun] = [ (sincelb, untillb) ]
60 def getKeys( listOfRuns, doPrint = False ):
62 from CoolLumiUtilities.CoolDataReader
import CoolDataReader
66 mySmkReader = CoolDataReader(
'COOLONL_TRIGGER/CONDBR2',
'/TRIGGER/HLT/HltConfigKeys')
67 myL1pskReader = CoolDataReader(
'COOLONL_TRIGGER/CONDBR2',
'/TRIGGER/LVL1/Lvl1ConfigKey')
68 myHltpskReader = CoolDataReader(
'COOLONL_TRIGGER/CONDBR2',
'/TRIGGER/HLT/PrescaleKey')
72 pbar = tqdm.tqdm(
sorted(listOfRuns),unit=
" runs",bar_format=
'{l_bar}{bar:10}{r_bar}{bar:-10b}')
75 listOfReadyBlocks = listOfRuns[run]
77 pbar.set_description(f
"Getting keys for run {run}")
80 until = ((run+1) << 32)
83 mySmkReader.setIOVRange( since, until - 1 )
84 mySmkReader.readData()
85 for obj
in mySmkReader.data:
86 smk = obj.payload()[
'MasterConfigurationKey']
89 keysByRun.setdefault(run,{})[
'smk'] = smk
91 for sincelb, untillb
in listOfReadyBlocks:
93 since = (run << 32) + sincelb
94 until = (run << 32) + untillb
97 myL1pskReader.setIOVRange( since, until )
98 myL1pskReader.readData()
99 for obj
in myL1pskReader.data:
100 l1psk = obj.payload()[
'Lvl1PrescaleConfigurationKey']
103 if sincelb2 == untillb:
break
104 keysByRun.setdefault(run,{}).setdefault(
'l1psk',[]).
append((l1psk,sincerun2, sincelb2,untilrun2, untillb2-1))
107 myHltpskReader.setIOVRange( since, until )
108 myHltpskReader.readData()
109 for obj
in myHltpskReader.data:
110 hltpsk = obj.payload()[
'HltPrescaleKey']
113 if sincelb2 == untillb:
break
114 keysByRun.setdefault(run,{}).setdefault(
'hltpsk',[]).
append((hltpsk,sincerun2, sincelb2,untilrun2, untillb2-1))
131 """returns set name, prescale and passthrough
132 values for a given HLT prescale key
133 @connection - connection string, e.g. TRIGGERDB
134 @psk - HLT prescale key
135 @return (ps name, [('L2/EF',chainId,prescale,pass-through),...])
139 prescales = {r[0]:r[3]
for r
in res
if r
and r[1]==
'Prescale'}
140 rerun = {r[0]:r[3]
for r
in res
if r
and r[1]==
'ReRun'}
142 if x
not in rerun: rerun[x] =
False
143 return {x: (prescales[x],rerun[x])
for x
in prescales}
147 from TrigConfigSvc.TrigConfigSvcUtils
import getTriggerDBCursor, executeQuery
149 global cursor, schemaname
152 output = [
"PS.HPR_CHAIN_COUNTER",
"PS.HPR_TYPE",
"PS.HPR_CONDITION" ,
"PS.HPR_VALUE"]
155 tables[
'PS'] =
'HLT_PRESCALE'
157 condition = [
"PS.HPR_PRESCALE_SET_ID = :psk" ]
159 bindvars = {
"psk": psk }
161 res =
executeQuery(cursor, output, condition, schemaname, tables, bindvars)
166 def fillHLTmap( info, hltMap_prev , lbCount, run, grlblocks):
167 from TrigConfigSvc.TrigConfigSvcUtils
import getL1Items, getL1Prescales
169 from TrigConfIO.L1TriggerConfigAccess
import L1MenuAccess,L1PrescalesSetAccess
170 from TrigConfIO.HLTTriggerConfigAccess
import HLTMenuAccess,HLTPrescalesSetAccess
172 from collections
import defaultdict
175 tcsLogger = logging.getLogger(
"TrigConfigSvcUtils.py")
176 tcsLogLevel = tcsLogger.level
177 tcsLogger.setLevel(logging.ERROR)
179 lvl =
int(logging.root.level)
180 logging.root.setLevel(logging.WARNING)
185 items = {}; chainsHLT = {}
186 for name,value
in L1MenuAccess(dbalias =
'TRIGGERDB_RUN3', smkey = info[
'smk']).
items().
items():
187 items[name] = value[
"ctpid"]
188 for name,value
in HLTMenuAccess(dbalias =
'TRIGGERDB_RUN3', smkey = info[
'smk']).
chains().
items():
189 if "L1" not in value[
"l1item"]:
continue
190 chainsHLT[value[
"nameHash"]] = (name,value[
"l1item"])
192 items =
getL1Items(
'oracle://ATLAS_CONFIG/ATLAS_CONF_TRIGGER_RUN2', info[
'smk'])
193 chainsHLT =
getChainsWithL1seed(
'oracle://ATLAS_CONFIG/ATLAS_CONF_TRIGGER_RUN2', info[
'smk'])
194 chainsHLT = {k:v
for (k,v)
in chainsHLT.items()
if "L1" in v[1]}
199 for lbrange
in info[
'hltpsk']:
200 lbstart, lbend = lbrange[2], lbrange[4]
201 if lbend ==-1: lbend = 2000
204 for name,value
in HLTPrescalesSetAccess(dbalias=
'TRIGGERDB_RUN3',hltpskey=lbrange[0]).
prescales().
items():
208 hltprescales[value[
"hash"]] = (
float(value[
"prescale"]),rerun)
210 hltprescales =
getHLTPrescalesRun2(
'oracle://ATLAS_CONFIG/ATLAS_CONF_TRIGGER_RUN2', lbrange[0])
211 tmphltList.append(( lbstart, lbend,hltprescales) )
214 for lbrange
in info[
'l1psk']:
215 lbstart, lbend = lbrange[2], lbrange[4]
216 if lbend ==-1: lbend = 2000
218 l1ps = L1PrescalesSetAccess(dbalias=
'TRIGGERDB_RUN3',l1pskey=lbrange[0])
219 l1prescales = {name: l1ps.prescale(name)
for name
in l1ps.itemNames()}
221 l1psname, l1prescales =
getL1Prescales(
'oracle://ATLAS_CONFIG/ATLAS_CONF_TRIGGER_RUN2', lbrange[0])
222 l1prescales =
list(l1prescales)
223 l1prescales = {l1name: l1prescales[
int(l1id)]
for (l1name, l1id)
in items.items()}
224 tmpl1List.append(( lbstart, lbend,l1prescales) )
226 logging.root.setLevel(lvl)
227 tcsLogger.setLevel(tcsLogLevel)
230 hltindex, l1index = 0,0
232 while hltindex < len(tmphltList)
and l1index < len(tmpl1List) :
233 if tmphltList[hltindex][1] == tmpl1List[l1index][1]:
234 lbstart, lbend =
max(tmphltList[hltindex][0],tmpl1List[l1index][0]), tmphltList[hltindex][1]
235 mergedList.append((lbstart, lbend,tmphltList[hltindex][2],tmpl1List[l1index][2]))
238 elif tmphltList[hltindex][1] > tmpl1List[l1index][1]:
239 lbstart, lbend =
max(tmphltList[hltindex][0],tmpl1List[l1index][0]), tmpl1List[l1index][1]
240 mergedList.append((lbstart, lbend,tmphltList[hltindex][2],tmpl1List[l1index][2]))
243 lbstart, lbend =
max(tmphltList[hltindex][0],tmpl1List[l1index][0]), tmphltList[hltindex][1]
244 mergedList.append((lbstart, lbend,tmphltList[hltindex][2],tmpl1List[l1index][2]))
248 f =
open(
"liveFractions.txt",
"a")
if os.path.exists(
"liveFractions.txt")
else None
251 for lbstart, lbend, hltprescales, l1prescales
in mergedList:
252 if run
in LBexceptions.exceptions:
253 if any([lbstart>=exc_start
and lbstart<=exc_end
for exc_start, exc_end
in LBexceptions.exceptions[run]]):
continue
254 if any([lbend>=exc_start
and lbend<=exc_end
for exc_start, exc_end
in LBexceptions.exceptions[run]]):
continue
256 for grllbstart,grllbend
in grlblocks:
257 lboverlap = (
min(lbend,grllbend) -
max(lbstart,grllbstart))+1
258 if lboverlap <= 0:
continue
261 for hltid, (hltps, hltrerun)
in hltprescales.items():
262 if hltid
not in chainsHLT:
continue
263 if hltps < 1: hltps = 1e99
264 l1seeds = chainsHLT[hltid][1]
266 for l1seed
in l1seeds.split(
","):
267 if l1seed
not in l1prescales
and len(l1seeds) > 10:
continue
268 tmpl1ps = l1prescales[l1seed]
269 if tmpl1ps < 1: tmpl1ps = 1e99
270 l1ps =
min(l1ps, tmpl1ps)
273 if hltps*l1ps < 1e99: efflb = lboverlap/(hltps*l1ps)
275 if not chainsHLT[hltid][0]
in hltMap: hltMap[chainsHLT[hltid][0]] = [l1seeds, 0, hltrerun>0, defaultdict(int)]
276 hltMap[chainsHLT[hltid][0]][1] += efflb
277 hltMap[chainsHLT[hltid][0]][3][run] += efflb
278 if f: f.write(f
"{chainsHLT[hltid][0]},{run},{lbstart},{lbend},{grllbstart},{grllbend},{lboverlap},{l1ps},{hltps}\n")
282 for hlt,(l1,efflb,rerun,efflbByRun)
in hltMap_prev.items():
284 hltMap[hlt][1] += efflb
285 hltMap[hlt][2] |= rerun
286 for run,runefflb
in efflbByRun.items():
287 hltMap[hlt][3][run] += runefflb
288 else: hltMap[hlt] = [l1, efflb,rerun, efflbByRun]
289 return hltMap, lbCount
293 '''copy of getChains but retrieving also the L1 seed and assuming always run2
294 https://gitlab.cern.ch/atlas/athena/blob/master/Trigger/TrigConfiguration/TrigConfigSvc/python/TrigConfigSvcUtils.py#L611
296 from TrigConfigSvc.TrigConfigSvcUtils
import getTriggerDBCursor, executeQuery
299 output = [
'TC.HTC_ID',
'TC.HTC_CHAIN_COUNTER',
'TC.HTC_NAME',
'TC.HTC_LOWER_CHAIN_NAME']
301 tables[
'SM'] =
'SUPER_MASTER_TABLE'
302 tables[
'M2C'] =
'HLT_TM_TO_TC'
303 tables[
'TC'] =
'HLT_TRIGGER_CHAIN'
304 tables[
'MT'] =
'HLT_MASTER_TABLE'
306 condition = [
"SM.SMT_ID = :smk",
307 'SM.SMT_HLT_MASTER_TABLE_ID = MT.HMT_ID',
308 'MT.HMT_TRIGGER_MENU_ID = M2C.HTM2TC_TRIGGER_MENU_ID',
309 'M2C.HTM2TC_TRIGGER_CHAIN_ID = TC.HTC_ID' ]
311 bindvars = {
"smk": smk }
313 res =
executeQuery(cursor, output, condition, schemaname, tables, bindvars)
317 if len(x)!=4:
continue
318 chainsef[x[1]] = (x[2],x[3])
325 ''' Return a map of HLT chain: (L1 seed, active LBs, is-rerun) for a given period
328 triggerPeriod = TriggerPeriodData( period, customGRL ).grl
329 if not triggerPeriod:
return {},0
331 keys =
getKeys( runsWithReadyForPhysics)
337 pbar = tqdm.tqdm(keys,unit=
" runs",bar_format=
'{l_bar}{bar:10}{r_bar}{bar:-10b}')
339 pbar.set_description(f
"Getting prescales for run {run}")
340 prev_lbCount =
int(lbCount)
341 hltMap, lbCount =
fillHLTmap( keys[run], hltMap, lbCount , run, triggerPeriod[run])
342 lbByRun[run] = (lbCount - prev_lbCount)
344 return hltMap, lbCount, lbByRun
347 ''' Return a map of HLT chain: (L1 seed, active LBs, is-rerun) for a given period
348 Only "Future" periods make sense here
349 The format is the same as for TriggerDBAccess for compatibility but rerun is always false
352 if not period & TriggerPeriod.future:
return {}, 0
356 from TrigConfigSvc.TrigConfigSvcCfg
import getTrigConfigFromFlag
357 if (flags.Input.Files
and os.path.exists(flags.Input.Files[0])
and getTrigConfigFromFlag(flags)[
"SOURCE"]==
'INFILE' and
358 (flags.Input.isMC
or flags.Input.DataYear>=2022) ):
360 from TrigConfigSvc.TriggerConfigAccess
import getHLTMenuAccess
363 for hltname,chain
in menu.chains().
items():
364 l1seed = chain[
"l1item"]
365 if l1seed ==
"": l1seed =
"All"
366 primary = any(
'Primary' in g
or 'TagAndProbe' in g
for g
in chain[
"groups"])
367 ps = 1
if primary
else 0
368 hltMap[hltname] = (l1seed, dummyfutureLBs*ps,
False, {})
370 log.info(
"Failed to read infile menu, which can happen with old MC, reverting to release menu")
374 from TriggerMenuMT.HLT.Config.GenerateMenuMT
import GenerateMenuMT
376 menu.getChainsFromMenu(flags)
377 for chain
in itertools.chain.from_iterable(menu.chainsInMenu.values()):
379 l1seed = chain.name[chain.name.rfind(
"_L1")+3:]
380 primary = any(
'Primary' in g
or 'TagAndProbe' in g
for g
in chain.groups)
381 ps = 1
if primary
else 0
382 hltMap[hltname] = (l1seed, dummyfutureLBs*ps,
False, {})
384 return hltMap, dummyfutureLBs, {}
388 return "/cvmfs/atlas.cern.ch/repo/sw/software/21.1/AthenaP1/%s/InstallArea/x86_64-slc6-gcc62-opt/python/TriggerMenu/menu"%release
390 return "/cvmfs/atlas.cern.ch/repo/sw/software/21.1/AthenaP1/21.1.50/InstallArea/x86_64-slc6-gcc62-opt/python/TriggerMenu/menu"
394 ''' For a given period it returns: [HLT chain, L1 seed, average livefraction, active LB, is-rerun], total LB
395 The average livefraction is an approximation weighting the PS by number of lumiblocks.
396 *** Don't use this number in analysis!!! ***
397 For "future" periods, the average livefraction is 1 for items flagged as primary in TM and 0 for non-primaries
399 if not period & TriggerPeriod.future
or TriggerPeriod.isRunNumber(period):
406 raise RuntimeError(
'ConfigFlags need to be provided via TriggerAPI.setConfigFlags for "future" periods.')
410 return (hltlist, totalLB, totalLBByRun)
414 from copy
import deepcopy
415 for name, (l1seed, activeLB, hasRerun,activLBByRun)
in deepcopy(hltmap).
items():
416 for pair
in TriggerRenaming.pairs:
417 if name==pair[0]
and pair[1]
in hltmap:
418 hltmap[pair[1]][1] += activeLB
419 for run,efflb
in activLBByRun.items(): hltmap[pair[1]][3][run] += efflb
421 if name==pair[1]
and pair[0]
in hltmap:
422 hltmap[pair[0]][1] += activeLB
423 for run,efflb
in activLBByRun.items(): hltmap[pair[0]][3][run] += efflb
426 vetoes = [
'calib',
'noise',
'noalg',
'satmon',
'peb']
427 hltlist = [(name, l1seed, activeLB/totalLB, activeLB, hasRerun,activLBByRun)
for name, (l1seed, activeLB, hasRerun,activLBByRun)
in hltmap.items()
if not any(v
in name
for v
in vetoes)]
431 log.info(
getHLTlist(TriggerPeriod.future,
None,
None))
433 if __name__ ==
"__main__":