2 from functools
import cache
3 from typing
import Any, cast
4 from collections.abc
import Iterable
5 from pycrest.api.crest_api
import CrestApi, HTTPResponse, IovSetDto, TagMetaSetDto
8 from pprint
import pprint
9 from datetime
import datetime
as dt
11 from AthenaCommon.Logging
import logging
12 log = logging.getLogger(
'TriggerCrestUtil.py')
18 dbname_crestconn_mapping: dict[str, str] = {
20 "ATLAS_CONF_TRIGGER_RUN3":
"CONF_DATA_RUN3",
21 "ATLAS_CONF_TRIGGER_MC_RUN3":
"CONF_MC_RUN3",
22 "ATLAS_CONF_TRIGGER_REPR_RUN3":
"CONF_REPR_RUN3",
23 "ATLAS_CONF_TRIGGER_RUN4":
"CONF_DATA_RUN4",
24 "ATLAS_CONF_TRIGGER_MC_RUN4":
"CONF_MC_RUN4",
25 "ATLAS_CONF_TRIGGER_REPR_RUN4":
"CONF_REPR_RUN4",
26 "ATLAS_CONF_TRIGGER_LS3_DEV":
"CONF_DEV_LS3",
27 "ATLAS_CONF_TRIGGER_LS3_L0":
"CONF_DEV_L0",
28 "ATLAS_CONF_TRIGGER_RUN2_NF":
"CONF_DATA_RUN2",
30 "TRIGGERDB_RUN3":
"CONF_DATA_RUN3",
31 "TRIGGERDBREPR_RUN3":
"CONF_REPR_RUN3",
32 "TRIGGERDBMC_RUN3":
"CONF_MC_RUN3",
33 "TRIGGERDB_RUN4":
"CONF_DATA_RUN4",
34 "TRIGGERDBREPR_RUN4":
"CONF_REPR_RUN4",
35 "TRIGGERDBMC_RUN4":
"CONF_MC_RUN4",
36 "TRIGGERDBLS3_DEV":
"CONF_DEV_LS3",
37 "TRIGGERDBLS3_L0":
"CONF_DEV_L0",
38 "TRIGGERDB_RUN2_NF":
"CONF_DATA_RUN2_NF"
41 crestconn_dbname_mapping: dict[str, str] = {
43 "CONF_DATA_RUN3":
"ATLAS_CONF_TRIGGER_RUN3",
44 "CONF_MC_RUN3":
"ATLAS_CONF_TRIGGER_MC_RUN3",
45 "CONF_REPR_RUN3":
"ATLAS_CONF_TRIGGER_REPR_RUN3",
46 "CONF_DATA_RUN4":
"ATLAS_CONF_TRIGGER_RUN4",
47 "CONF_MC_RUN4" :
"ATLAS_CONF_TRIGGER_MC_RUN4",
48 "CONF_REPR_RUN4" :
"ATLAS_CONF_TRIGGER_REPR_RUN4",
49 "CONF_DEV_LS3" :
"ATLAS_CONF_TRIGGER_LS3_DEV",
50 "CONF_DEV_L0" :
"ATLAS_CONF_TRIGGER_LS3_L0",
51 "CONF_DATA_RUN2" :
"ATLAS_CONF_TRIGGER_RUN2_NF"
56 """list of all known crest connections
59 list[str]: list of crest connection names
61 return list(TriggerCrestUtil.dbname_crestconn_mapping.values())
65 """maps from triggerdb schema or triggerdb-alias to crest connection
66 See https://its.cern.ch/jira/browse/ATR-32030
68 If the input dbname is already a crest connection name, it is returned as is.
71 dbname (str): triggerdb name or alias or crest connection name.
73 str | None: crest connection name or None if not found.
75 if dbname
in TriggerCrestUtil.dbname_crestconn_mapping.values():
77 return TriggerCrestUtil.dbname_crestconn_mapping.get(dbname,
None)
82 return TriggerCrestUtil.dbname_crestconn_mapping
89 server (str): crest server name.
92 CrestApi: Crest API instance.
94 return CrestApi(host=server)
98 get_time_type: bool =
False) -> dict[str, dict[str, Any]]:
99 """ Conditions data for tag at a specific timestamp
101 Conditions data is returned as a dict with 'since' and 'payload' keys.
102 The 'payload' itself is a dict of channel:'data dicts'. The 'data dict' has attribute names as
103 keys and the corresponding values. If get_time_type is True, also 'time_type' key is added to
104 each IOV dict and the since is further expanded into 'since_run' and 'since_lb' (for run/lb-based
105 timestamps) or 'since_formatted' (for time-based timestamps).
108 tag (str): The tag name
109 timestamp (int): The timestamp to query (either run<32+lb or time in nanoseconds since 1.1.1970)
110 server (str, optional): Crest server name. Only needed if api is not provided. Defaults to "".
111 api (_type_, optional): Crest API instance. Defaults to None.
112 get_time_type (bool, optional): Whether to retrieve the time type. Defaults to False.
115 RuntimeError: If both server and api are missing
118 dict[str, dict[str, any]]: Conditions data
120 if server==
"" and api
is None:
121 log.error(
"Need either crest server name or crest api specified for accessing Crest")
122 raise RuntimeError(
"Crest access information missing")
124 api = CrestApi(host=server)
126 attr_list, _ = TriggerCrestUtil._get_payload_spec(tag, api)
128 iov = TriggerCrestUtil._get_iov_for_timestamp(tag, timestamp=timestamp, api=api)
131 time_type = TriggerCrestUtil.getTagTimeType(tag, api=api)
134 payload_hash: str = cast(list, iov[
'resources'])[0][
'payload_hash']
135 since: int = cast(list, iov[
'resources'])[0][
'since']
136 payload = TriggerCrestUtil.getPayloadFromHash(payload_hash, api=api)
137 for channel, data
in payload.items():
138 payload_for_iov[channel] = dict(zip(attr_list, data))
140 result: dict[str, Any] = {
142 'payload': payload_for_iov
145 TriggerCrestUtil._update_with_time_type(result, time_type, since)
151 get_time_type: bool =
False) -> list[dict[str, dict[str, Any]]]:
152 """ Conditions data for tag in given range
154 List goes over the IOVs found. For each IOV, there is a dict with 'since' and 'payload' keys.
155 The 'payload' itself is a dict of channel:'data dicts'. The 'data dict' has attribute names as
156 keys and the corresponding values. If get_time_type is True, also 'time_type' key is added to
157 each IOV dict and the since is further expanded into 'since_run' and 'since_lb' (for run/lb-based
158 timestamps) or 'since_formatted' (for time-based timestamps).
162 since (int): start of the range (either run<32+lb or time in nanoseconds since 1.1.1970, inclusive)
163 until (int): end of the range (either run<32+lb or time in nanoseconds since 1.1.1970, exclusive)
164 server (str, optional): crest server name. Only needed if api is not provided. Defaults to "".
165 api (CrestApi, optional): Crest API instance. Defaults to None.
166 get_time_type (bool, optional): whether to retrieve the time type. Defaults to False.
169 RuntimeError: if both server and api are missing
172 list[dict[str, dict[str, any]]]: conditions data
174 if server==
"" and api
is None:
175 log.error(
"Need either crest server name or crest api specified for accessing Crest")
176 raise RuntimeError(
"Crest access information missing")
178 api = CrestApi(host=server)
180 attr_list, type_dict = TriggerCrestUtil._get_payload_spec(tag, api)
182 all_iovs = TriggerCrestUtil._get_iovs_range(since=since, until=until, api=api, tag=tag)
185 time_type = TriggerCrestUtil.getTagTimeType(tag, api=api)
187 result: list[dict[str, dict[str, Any]]] = []
188 for iov
in cast(Iterable, all_iovs[
'resources']):
190 payload_hash = iov[
'payload_hash']
192 payload = TriggerCrestUtil.getPayloadFromHash(payload_hash, api=api)
193 for channel, data
in payload.items():
194 payload_for_iov[channel] = dict(zip(attr_list, data))
195 entry = {
'since': since,
'payload': payload_for_iov}
197 TriggerCrestUtil._update_with_time_type(entry, time_type, since)
202 def getTagTimeType(tag: str, *, server: str =
"", api: CrestApi |
None =
None) -> str:
203 """ time type of the tag, either 'run-lumi' or 'time'
207 server (str, optional): crest server name. Only needed if api is not provided. Defaults to "".
208 api (CrestApi, optional): Crest API instance. Defaults to None.
211 RuntimeError: if both server and api are missing
214 str: time type of the tag ['run-lumi' or 'time']
216 if server==
"" and api
is None:
217 log.error(
"Need either crest_server name or crest api for retrieving the payload")
218 raise RuntimeError(
"Crest access information missing")
220 api = CrestApi(host=server)
221 tag_info = api.find_tag(tag)
222 time_type: str = tag_info[
'time_type']
226 def getAttribs(tag: str, *, server: str =
"", api: CrestApi |
None =
None) -> list[str]:
227 """ list of attributes for this tag
231 server (str, optional): crest server name. Only needed if api is not provided. Defaults to "".
232 api (CrestApi, optional): Crest API instance. Defaults to None.
235 RuntimeError: if both server and api are missing
238 dict: list of attributes
240 if server==
"" and api
is None:
241 log.error(
"Need either crest_server name or crest api for retrieving the payload")
242 raise RuntimeError(
"Crest access information missing")
244 api = CrestApi(host=server)
245 attr_list, _ = TriggerCrestUtil._get_payload_spec(tag, api)
250 """ Get payload from hash
253 payload_hash (str): hash of the payload
254 server (str, optional): crest server name. Defaults to "".
255 api (CrestApi, optional): Crest API instance. Defaults to None.
258 RuntimeError: if both server and api are missing
261 dict: payload retrieved from the hash
263 if server ==
"" and api
is None:
264 log.error(
"Need either crest_server name or crest api for retrieving the payload")
265 raise RuntimeError(
"Crest access information missing")
270 return TriggerCrestUtil._get_payload_workaround(payload_hash, server)
273 api = CrestApi(host=server)
274 return TriggerCrestUtil._get_payload(payload_hash, api)
278 def getEORParams(run: int, *, server: str =
"", api: CrestApi |
None =
None) -> dict[str, Any] |
None:
280 api = CrestApi(host=server)
281 start_of_run = (run << 32)
282 cond_data = TriggerCrestUtil.getConditionsForTimestamp(
"TDAQRunCtrlEOR-HEAD", timestamp=start_of_run, api=api, get_time_type=
False)
284 log.error(
"No EOR params found for run %s", run)
286 return cond_data[
'payload'][
'0']
289 def getHLTPrescaleKeys(run: int, *, server: str =
"", api: CrestApi |
None =
None) -> list[dict[str, dict[str, Any]]]:
291 api = CrestApi(host=server)
292 run_start = (run << 32)
293 run_end = ((run + 1) << 32) - 1
294 cond = TriggerCrestUtil.getConditionsInRange(
"TRIGGERHLTPrescaleKey-HEAD", since=run_start, until=run_end, api=api, get_time_type=
True)
296 entry.update(entry.pop(
'payload')[
'0'])
297 entry[
'key'] = entry[
'HltPrescaleKey']
300 cond: list[dict[str, dict[str, Any]]] = [entry
for entry
in cond
if entry[
'since_run']==run]
304 def getHLTPrescaleKey(run: int, lb: int, *, server: str =
"", api: CrestApi |
None =
None) -> int |
None:
306 api = CrestApi(host=server)
307 run_lb = (run << 32) + lb
308 cond_entry: dict[str, Any] = TriggerCrestUtil.getConditionsForTimestamp(
"TRIGGERHLTPrescaleKey-HEAD", timestamp=run_lb, api=api, get_time_type=
True)
309 if cond_entry[
'since_run'] < run:
311 return cond_entry[
'payload'][
'0'][
'HltPrescaleKey']
314 def getL1PrescaleKeys(run: int, *, server: str =
"", api: CrestApi |
None =
None) -> list[dict[str, dict[str, Any]]]:
316 api = CrestApi(host=server)
317 run_start = (run << 32)
318 run_end = ((run + 1) << 32) - 1
319 cond = TriggerCrestUtil.getConditionsInRange(
"TRIGGERLVL1Lvl1ConfigKey-HEAD", since=run_start, until=run_end, api=api, get_time_type=
True)
321 entry.update(entry.pop(
'payload')[
'0'])
322 entry[
'key'] = entry[
'Lvl1PrescaleConfigurationKey']
325 cond: list[dict[str, dict[str, Any]]] = [entry
for entry
in cond
if entry[
'since_run']==run]
329 def getL1PrescaleKey(run: int, lb: int, *, server: str =
"", api: CrestApi |
None =
None) -> int |
None:
331 api = CrestApi(host=server)
332 run_lb = (run << 32) + lb
333 cond_entry: dict[str, Any] = TriggerCrestUtil.getConditionsForTimestamp(
"TRIGGERLVL1Lvl1ConfigKey-HEAD", timestamp=run_lb, api=api, get_time_type=
True)
334 if cond_entry[
'since_run'] < run:
336 return cond_entry[
'payload'][
'0'][
'Lvl1PrescaleConfigurationKey']
339 def getBunchGroupKeys(run: int, *, server: str =
"", api: CrestApi |
None =
None) -> list[dict[str, dict[str, Any]]]:
341 api = CrestApi(host=server)
342 run_start = (run << 32)
343 run_end = ((run+1) << 32)-1
344 cond = TriggerCrestUtil.getConditionsInRange(
"TRIGGERLVL1BunchGroupKey-HEAD", since=run_start, until=run_end, api=api, get_time_type=
True)
346 entry.update(entry.pop(
'payload')[
'0'])
347 entry[
'key'] = entry[
'Lvl1BunchGroupConfigurationKey']
350 cond: list[dict[str, dict[str, Any]]] = [entry
for entry
in cond
if entry[
'since_run']==run]
354 def getBunchGroupKey(run: int, lb: int, *, server: str =
"", api: CrestApi |
None =
None) -> int |
None:
356 api = CrestApi(host=server)
357 run_lb = (run << 32) + lb
358 cond_entry: dict[str, Any] = TriggerCrestUtil.getConditionsForTimestamp(
"TRIGGERLVL1BunchGroupKey-HEAD", timestamp=run_lb, api=api, get_time_type=
True)
359 if cond_entry[
'since_run'] < run:
361 return cond_entry[
'payload'][
'0'][
'Lvl1BunchGroupConfigurationKey']
364 def getMenuConfigKey(run: int, *, server: str =
"", api: CrestApi |
None =
None) -> dict[str, Any] |
None:
366 def _parse_info(run, cond):
373 confsrc = cond[
'ConfigSource'].
split(
';')
376 release = confsrc[2].
split(
',')[0] + f
",{confsrc[1]}"
378 confsrc = cond[
'ConfigSource'].
split(
',', maxsplit=1)
381 release = f
"{confsrc[2]},{confsrc[1]}"
383 "SMK": cond[
'MasterConfigurationKey'],
384 "HLTPSK": cond[
'HltPrescaleConfigurationKey'],
390 api = CrestApi(host=server)
392 run_start = (run << 32) + 1
393 cond: dict[str, Any] = TriggerCrestUtil.getConditionsForTimestamp(
"TRIGGERHLTHltConfigKeys-HEAD", timestamp=run_start, api=api, get_time_type=
True)
394 if cond[
'since_run'] < run:
396 cond.update(cond.pop(
'payload')[
'0'])
397 return _parse_info(run, cond)
400 def getTrigConfKeys(runNumber: int, lumiBlock: int, server: str =
"", api: CrestApi |
None =
None) -> dict[str, Any]:
402 api = CrestApi(host=server)
403 bgkey: int |
None = TriggerCrestUtil.getBunchGroupKey(runNumber, lumiBlock, api=api)
404 l1pskey: int |
None = TriggerCrestUtil.getL1PrescaleKey(runNumber, lumiBlock, api=api)
405 hltpskey: int |
None = TriggerCrestUtil.getHLTPrescaleKey(runNumber, lumiBlock, api=api)
406 menucfg: dict[str, Any] |
None = TriggerCrestUtil.getMenuConfigKey(runNumber, api=api)
408 "SMK": menucfg[
'SMK']
if menucfg
else None,
409 "DB": menucfg[
'DB']
if menucfg
else None,
418 result[
'time_type'] = time_type
419 if time_type ==
'run-lumi':
421 'since_run': since >> 32,
422 'since_lb': since & 0xFFFFFFFF
424 elif time_type ==
'time':
426 'since_formatted': dt.fromtimestamp(since / 1e9)
432 return api.get_payload(hash=payload_hash).
decode(
'utf-8')
438 url = f
"{server}/payloads/data"
443 preq = requests.Request(method=
'GET', url=url, params=params).prepare()
444 with requests.Session()
as session:
446 resp = session.send(preq)
447 except requests.ConnectionError
as exc:
448 raise RuntimeError(f
"Could not connect to CREST server {server}")
from exc
450 if resp.status_code != 200:
451 raise RuntimeError(f
"Query {payload_hash} to crest failed with status code {resp.status_code}")
453 return json.loads(resp.content)
457 """Helper to retrieve all IOVs for a run"""
458 run_start = (run << 32) + 1
459 run_end = ((run + 1) << 32) - 1
460 return TriggerCrestUtil._get_iovs_range(since=run_start, until=run_end, api=api, tag=tag)
464 """Helper to retrieve the IOV for a given timestamp
468 timestamp (int): time-stamp in format run<32+lb or time in nanoseconds since 1.1.1970
469 api (CrestApi): Crest API instance.
472 IovSetDto: set of IOVs (of size 1)
475 return api.select_iovs(tag,
"0",
str(timestamp+1), sort=
'id.since:DESC', size=1, snapshot=0)
478 def _get_iovs_range(*, since: int, until: int, api: CrestApi, tag: str) -> IovSetDto | HTTPResponse:
479 """Helper to retrieve the IOV in a given range
483 since (int): start of the range (either run<32+lb or time in nanoseconds since 1.1.1970, inclusive)
484 until (int): end of the range (either run<32+lb or time in nanoseconds since 1.1.1970, exclusive)
485 api (CrestApi): Crest API instance.
488 IovSetDto: set of IOVs
491 iovs = api.select_iovs(tag,
"0",
str(since+1), sort=
'id.since:DESC', size=1, snapshot=0)
492 if cast(int, iovs[
'size']) < 1:
493 raise RuntimeError(f
"Did not get an iov which includes the start of run {since}")
494 firstiov = cast(list, iovs[
'resources'])[0]
495 all_iovs = api.select_iovs(tag,
str(firstiov[
'since']),
str(until), sort=
'id.since:ASC', snapshot=0)
500 """Helper to retrieve the payload spec for a given tag"""
501 meta: TagMetaSetDto | HTTPResponse = api.find_tag_meta(tag)
502 tag_info = cast(list, meta[
'resources'])[0][
'tag_info']
503 tag_info_dict = json.loads(tag_info)
506 for d
in tag_info_dict[
'payload_spec']:
509 return attr_list, type_dict
512 if __name__ ==
"__main__":
516 crest_server =
"https://crest.cern.ch/api-v5.0"
517 api = CrestApi(crest_server)
519 print(f
"run {testrun}:")
520 print(
"\nSuper master key, etc:")
521 pprint(TriggerCrestUtil.getMenuConfigKey(testrun, api=api))
522 print(
"\nL1 prescale keys:")
523 pprint(TriggerCrestUtil.getL1PrescaleKeys(testrun, api=api))
524 pprint(TriggerCrestUtil.getL1PrescaleKey(testrun, 1, api=api))
525 print(
"\nL1 bunchgroup keys:")
526 pprint(TriggerCrestUtil.getBunchGroupKeys(testrun, api=api))
527 pprint(TriggerCrestUtil.getBunchGroupKey(testrun, 1, api=api))
528 print(
"\nHLT prescale keys:")
529 pprint(TriggerCrestUtil.getHLTPrescaleKeys(testrun, api=api))
530 pprint(TriggerCrestUtil.getHLTPrescaleKey(testrun, 506, api=api))
531 pprint(TriggerCrestUtil.getHLTPrescaleKey(testrun, 507, api=api))
532 print(
"\nEOR params:")
533 pprint(TriggerCrestUtil.getEORParams(testrun, api=api),sort_dicts=
False)
535 print(
"\nNon existing lb:")
536 pprint(TriggerCrestUtil.getHLTPrescaleKey(testrun, 1_000_000, api=api))
538 print(
"\nNon existing run:")
539 pprint(TriggerCrestUtil.getHLTPrescaleKey(1_000_000, 1, api=api))
540 pprint(TriggerCrestUtil.getHLTPrescaleKeys(1_000_000, api=api))