3 __author__ =
'Javier Montejo'
4 __version__=
"$Revision: 2.0 $"
5 __doc__=
"Access to Trigger DB and TriggerMenu to read past and future prescales"
9 from TriggerMenuMT.TriggerAPI.TriggerEnums
import TriggerPeriod, LBexceptions, TriggerRenaming
10 from TriggerMenuMT.TriggerAPI.TriggerPeriodData
import TriggerPeriodData
11 from AthenaCommon.Logging
import logging
12 log = logging.getLogger(__name__)
19 lb = runlb & ((1<<32)-1)
20 return (
int(run),
int(lb) )
25 returns all runs in the given period which have the ReadyForPhysics flag set in at least 1 LB
28 log.info(
"Loading COOL libs...")
29 from CoolLumiUtilities.CoolDataReader
import CoolDataReader
30 log.info(
"Done loading libs, starting now ...")
32 myReader = CoolDataReader(
'COOLONL_TDAQ/CONDBR2',
'/TDAQ/RunCtrl/DataTakingMode')
35 firstRun =
min([x
for x
in period.keys()])
36 lastRun =
max([x
for x
in period.keys()])+1
37 since = (firstRun << 32)
38 until = (lastRun << 32)
40 myReader.setIOVRange( since, until )
43 for obj
in myReader.data:
44 isReady = (obj.payload()[
'ReadyForPhysics'] == 1)
49 if sincerun != untilrun:
50 log.info(
"WARNING: ready block crosses run boundaries:", sincerun, untilrun)
51 if sincerun
not in period:
continue
52 if sincerun
in runsWithReady:
53 runsWithReady[sincerun] += [ (sincelb, untillb) ]
55 runsWithReady[sincerun] = [ (sincelb, untillb) ]
61 def getKeys( listOfRuns, doPrint = False ):
63 from CoolLumiUtilities.CoolDataReader
import CoolDataReader
67 mySmkReader = CoolDataReader(
'COOLONL_TRIGGER/CONDBR2',
'/TRIGGER/HLT/HltConfigKeys')
68 myL1pskReader = CoolDataReader(
'COOLONL_TRIGGER/CONDBR2',
'/TRIGGER/LVL1/Lvl1ConfigKey')
69 myHltpskReader = CoolDataReader(
'COOLONL_TRIGGER/CONDBR2',
'/TRIGGER/HLT/PrescaleKey')
73 pbar = tqdm.tqdm(
sorted(listOfRuns),unit=
" runs",bar_format=
'{l_bar}{bar:10}{r_bar}{bar:-10b}')
76 listOfReadyBlocks = listOfRuns[run]
78 pbar.set_description(f
"Getting keys for run {run}")
81 until = ((run+1) << 32)
84 mySmkReader.setIOVRange( since, until - 1 )
85 mySmkReader.readData()
86 for obj
in mySmkReader.data:
87 smk = obj.payload()[
'MasterConfigurationKey']
90 keysByRun.setdefault(run,{})[
'smk'] = smk
92 for sincelb, untillb
in listOfReadyBlocks:
94 since = (run << 32) + sincelb
95 until = (run << 32) + untillb
98 myL1pskReader.setIOVRange( since, until )
99 myL1pskReader.readData()
100 for obj
in myL1pskReader.data:
101 l1psk = obj.payload()[
'Lvl1PrescaleConfigurationKey']
104 if sincelb2 == untillb:
break
105 keysByRun.setdefault(run,{}).setdefault(
'l1psk',[]).
append((l1psk,sincerun2, sincelb2,untilrun2, untillb2-1))
108 myHltpskReader.setIOVRange( since, until )
109 myHltpskReader.readData()
110 for obj
in myHltpskReader.data:
111 hltpsk = obj.payload()[
'HltPrescaleKey']
114 if sincelb2 == untillb:
break
115 keysByRun.setdefault(run,{}).setdefault(
'hltpsk',[]).
append((hltpsk,sincerun2, sincelb2,untilrun2, untillb2-1))
132 """returns set name, prescale and passthrough
133 values for a given HLT prescale key
134 @connection - connection string, e.g. TRIGGERDB
135 @psk - HLT prescale key
136 @return (ps name, [('L2/EF',chainId,prescale,pass-through),...])
140 prescales = {r[0]:r[3]
for r
in res
if r
and r[1]==
'Prescale'}
141 rerun = {r[0]:r[3]
for r
in res
if r
and r[1]==
'ReRun'}
143 if x
not in rerun: rerun[x] =
False
144 return {x: (prescales[x],rerun[x])
for x
in prescales}
148 from TrigConfigSvc.TrigConfigSvcUtils
import getTriggerDBCursor, executeQuery
150 global cursor, schemaname
153 output = [
"PS.HPR_CHAIN_COUNTER",
"PS.HPR_TYPE",
"PS.HPR_CONDITION" ,
"PS.HPR_VALUE"]
156 tables[
'PS'] =
'HLT_PRESCALE'
158 condition = [
"PS.HPR_PRESCALE_SET_ID = :psk" ]
160 bindvars = {
"psk": psk }
162 res =
executeQuery(cursor, output, condition, schemaname, tables, bindvars)
167 def fillHLTmap( info, hltMap_prev , lbCount, run, grlblocks):
168 from TrigConfigSvc.TrigConfigSvcUtils
import getL1Items, getL1Prescales
170 from TrigConfIO.L1TriggerConfigAccess
import L1MenuAccess,L1PrescalesSetAccess
171 from TrigConfIO.HLTTriggerConfigAccess
import HLTMenuAccess,HLTPrescalesSetAccess
173 from collections
import defaultdict
175 lvl =
int(logging.root.level)
176 logging.root.setLevel(logging.WARNING)
181 items = {}; chainsHLT = {}
182 for name,value
in L1MenuAccess(dbalias =
'TRIGGERDB_RUN3', smkey = info[
'smk']).
items().
items():
183 items[name] = value[
"ctpid"]
184 for name,value
in HLTMenuAccess(dbalias =
'TRIGGERDB_RUN3', smkey = info[
'smk']).
chains().
items():
185 if "L1" not in value[
"l1item"]:
continue
186 chainsHLT[value[
"nameHash"]] = (name,value[
"l1item"])
190 chainsHLT = {k:v
for (k,v)
in six.iteritems (chainsHLT)
if "L1" in v[1]}
195 for lbrange
in info[
'hltpsk']:
196 lbstart, lbend = lbrange[2], lbrange[4]
197 if lbend ==-1: lbend = 2000
200 for name,value
in HLTPrescalesSetAccess(dbalias=
'TRIGGERDB_RUN3',hltpskey=lbrange[0]).
prescales().
items():
203 hltprescales[value[
"hash"]] = (value[
"prescale"],rerun)
206 tmphltList.append(( lbstart, lbend,hltprescales) )
209 for lbrange
in info[
'l1psk']:
210 lbstart, lbend = lbrange[2], lbrange[4]
211 if lbend ==-1: lbend = 2000
213 l1ps = L1PrescalesSetAccess(dbalias=
'TRIGGERDB_RUN3',l1pskey=lbrange[0])
214 l1prescales = {name: l1ps.prescale(name)
for name
in l1ps.itemNames()}
217 l1prescales = {l1name: l1prescales[
int(l1id)]
for (l1name, l1id)
in six.iteritems (items)}
218 tmpl1List.append(( lbstart, lbend,l1prescales) )
220 logging.root.setLevel(lvl)
223 hltindex, l1index = 0,0
225 while hltindex < len(tmphltList)
and l1index < len(tmpl1List) :
226 if tmphltList[hltindex][1] == tmpl1List[l1index][1]:
227 lbstart, lbend =
max(tmphltList[hltindex][0],tmpl1List[l1index][0]), tmphltList[hltindex][1]
228 mergedList.append((lbstart, lbend,tmphltList[hltindex][2],tmpl1List[l1index][2]))
231 elif tmphltList[hltindex][1] > tmpl1List[l1index][1]:
232 lbstart, lbend =
max(tmphltList[hltindex][0],tmpl1List[l1index][0]), tmpl1List[l1index][1]
233 mergedList.append((lbstart, lbend,tmphltList[hltindex][2],tmpl1List[l1index][2]))
236 lbstart, lbend =
max(tmphltList[hltindex][0],tmpl1List[l1index][0]), tmphltList[hltindex][1]
237 mergedList.append((lbstart, lbend,tmphltList[hltindex][2],tmpl1List[l1index][2]))
241 f =
open(
"liveFractions.txt",
"a")
if os.path.exists(
"liveFractions.txt")
else None
244 for lbstart, lbend, hltprescales, l1prescales
in mergedList:
245 if run
in LBexceptions.exceptions:
246 if any([lbstart>=exc_start
and lbstart<=exc_end
for exc_start, exc_end
in LBexceptions.exceptions[run]]):
continue
247 if any([lbend>=exc_start
and lbend<=exc_end
for exc_start, exc_end
in LBexceptions.exceptions[run]]):
continue
249 for grllbstart,grllbend
in grlblocks:
250 lboverlap = (
min(lbend,grllbend) -
max(lbstart,grllbstart))+1
251 if lboverlap <= 0:
continue
254 for hltid, (hltps, hltrerun)
in six.iteritems (hltprescales):
255 if hltid
not in chainsHLT:
continue
256 if hltps < 1: hltps = 1e99
257 l1seeds = chainsHLT[hltid][1]
259 for l1seed
in l1seeds.split(
","):
260 if l1seed
not in l1prescales
and len(l1seeds) > 10:
continue
261 tmpl1ps = l1prescales[l1seed]
262 if tmpl1ps < 1: tmpl1ps = 1e99
263 l1ps =
min(l1ps, tmpl1ps)
266 if hltps*l1ps < 1e99: efflb = lboverlap/(hltps*l1ps)
268 if not chainsHLT[hltid][0]
in hltMap: hltMap[chainsHLT[hltid][0]] = [l1seeds, 0, hltrerun>0, defaultdict(int)]
269 hltMap[chainsHLT[hltid][0]][1] += efflb
270 hltMap[chainsHLT[hltid][0]][3][run] += efflb
271 if f: f.write(f
"{chainsHLT[hltid][0]},{run},{lbstart},{lbend},{grllbstart},{grllbend},{lboverlap},{l1ps},{hltps}\n")
275 for hlt,(l1,efflb,rerun,efflbByRun)
in six.iteritems (hltMap_prev):
277 hltMap[hlt][1] += efflb
278 hltMap[hlt][2] |= rerun
279 for run,runefflb
in efflbByRun.items():
280 hltMap[hlt][3][run] += runefflb
281 else: hltMap[hlt] = [l1, efflb,rerun, efflbByRun]
282 return hltMap, lbCount
286 '''copy of getChains but retrieving also the L1 seed and assuming always run2
287 https://gitlab.cern.ch/atlas/athena/blob/master/Trigger/TrigConfiguration/TrigConfigSvc/python/TrigConfigSvcUtils.py#L611
289 from TrigConfigSvc.TrigConfigSvcUtils
import getTriggerDBCursor, executeQuery
292 output = [
'TC.HTC_ID',
'TC.HTC_CHAIN_COUNTER',
'TC.HTC_NAME',
'TC.HTC_LOWER_CHAIN_NAME']
294 tables[
'SM'] =
'SUPER_MASTER_TABLE'
295 tables[
'M2C'] =
'HLT_TM_TO_TC'
296 tables[
'TC'] =
'HLT_TRIGGER_CHAIN'
297 tables[
'MT'] =
'HLT_MASTER_TABLE'
299 condition = [
"SM.SMT_ID = :smk",
300 'SM.SMT_HLT_MASTER_TABLE_ID = MT.HMT_ID',
301 'MT.HMT_TRIGGER_MENU_ID = M2C.HTM2TC_TRIGGER_MENU_ID',
302 'M2C.HTM2TC_TRIGGER_CHAIN_ID = TC.HTC_ID' ]
304 bindvars = {
"smk": smk }
306 res =
executeQuery(cursor, output, condition, schemaname, tables, bindvars)
310 if len(x)!=4:
continue
311 chainsef[x[1]] = (x[2],x[3])
318 ''' Return a map of HLT chain: (L1 seed, active LBs, is-rerun) for a given period
321 triggerPeriod = TriggerPeriodData( period, customGRL ).grl
322 if not triggerPeriod:
return {},0
324 keys =
getKeys( runsWithReadyForPhysics)
330 pbar = tqdm.tqdm(keys,unit=
" runs",bar_format=
'{l_bar}{bar:10}{r_bar}{bar:-10b}')
332 pbar.set_description(f
"Getting prescales for run {run}")
333 prev_lbCount =
int(lbCount)
334 hltMap, lbCount =
fillHLTmap( keys[run], hltMap, lbCount , run, triggerPeriod[run])
335 lbByRun[run] = (lbCount - prev_lbCount)
337 return hltMap, lbCount, lbByRun
340 ''' Return a map of HLT chain: (L1 seed, active LBs, is-rerun) for a given period
341 Only "Future" periods make sense here
342 The format is the same as for TriggerDBAccess for compatibility but rerun is always false
345 from TriggerMenuMT.HLT.Config.GenerateMenuMT
import GenerateMenuMT
347 menu.getChainsFromMenu(flags)
349 if not period & TriggerPeriod.future:
return {}, 0
353 for chain
in itertools.chain.from_iterable(menu.chainsInMenu.values()):
355 l1seed = chain.name[chain.name.rfind(
"_L1")+3:]
356 primary = any(
'Primary' in g
or 'TagAndProbe' in g
for g
in chain.groups)
357 ps = 1
if primary
else 0
358 hltMap[hltname] = (l1seed, dummyfutureLBs*ps,
False, {})
359 return hltMap, dummyfutureLBs, {}
363 return "/cvmfs/atlas.cern.ch/repo/sw/software/21.1/AthenaP1/%s/InstallArea/x86_64-slc6-gcc62-opt/python/TriggerMenu/menu"%release
365 return "/cvmfs/atlas.cern.ch/repo/sw/software/21.1/AthenaP1/21.1.50/InstallArea/x86_64-slc6-gcc62-opt/python/TriggerMenu/menu"
369 ''' For a given period it returns: [HLT chain, L1 seed, average livefraction, active LB, is-rerun], total LB
370 The average livefraction is an approximation weighting the PS by number of lumiblocks.
371 *** Don't use this number in analysis!!! ***
372 For "future" periods, the average livefraction is 1 for items flagged as primary in TM and 0 for non-primaries
374 if not period & TriggerPeriod.future
or TriggerPeriod.isRunNumber(period):
381 raise RuntimeError(
'ConfigFlags need to be provided via TriggerAPI.setConfigFlags for "future" periods.')
385 return (hltlist, totalLB, totalLBByRun)
389 from copy
import deepcopy
390 for name, (l1seed, activeLB, hasRerun,activLBByRun)
in six.iteritems (deepcopy(hltmap)):
391 for pair
in TriggerRenaming.pairs:
392 if name==pair[0]
and pair[1]
in hltmap:
393 hltmap[pair[1]][1] += activeLB
394 for run,efflb
in activLBByRun.items(): hltmap[pair[1]][3][run] += efflb
396 if name==pair[1]
and pair[0]
in hltmap:
397 hltmap[pair[0]][1] += activeLB
398 for run,efflb
in activLBByRun.items(): hltmap[pair[0]][3][run] += efflb
401 vetoes = [
'calib',
'noise',
'noalg',
'satmon',
'peb']
402 hltlist = [(name, l1seed, activeLB/totalLB, activeLB, hasRerun,activLBByRun)
for name, (l1seed, activeLB, hasRerun,activLBByRun)
in six.iteritems (hltmap)
if not any(v
in name
for v
in vetoes)]
406 log.info(
getHLTlist(TriggerPeriod.future,
None,
None))
408 if __name__ ==
"__main__":