2from functools
import cache
3from typing
import Any, cast
5from pycrest.api.crest_api
import CrestApi, HTTPResponse, IovSetDto, TagMetaDto, TagDto
8from pprint
import pprint
9from datetime
import datetime
as dt
11from AthenaCommon.Logging
import logging
12log = logging.getLogger(
'TriggerCrestUtil.py')
19 dbname_crestconn_mapping: dict[str, str] = {
21 "ATLAS_CONF_TRIGGER_RUN3":
"CONF_DATA_RUN3",
22 "ATLAS_CONF_TRIGGER_MC_RUN3":
"CONF_MC_RUN3",
23 "ATLAS_CONF_TRIGGER_REPR_RUN3":
"CONF_REPR_RUN3",
24 "ATLAS_CONF_TRIGGER_RUN4":
"CONF_DATA_RUN4",
25 "ATLAS_CONF_TRIGGER_MC_RUN4":
"CONF_MC_RUN4",
26 "ATLAS_CONF_TRIGGER_REPR_RUN4":
"CONF_REPR_RUN4",
27 "ATLAS_CONF_TRIGGER_LS3_DEV":
"CONF_DEV_LS3",
28 "ATLAS_CONF_TRIGGER_LS3_L0":
"CONF_DEV_L0",
29 "ATLAS_CONF_TRIGGER_RUN2_NF":
"CONF_DATA_RUN2",
31 "TRIGGERDB_RUN3":
"CONF_DATA_RUN3",
32 "TRIGGERDBREPR_RUN3":
"CONF_REPR_RUN3",
33 "TRIGGERDBMC_RUN3":
"CONF_MC_RUN3",
34 "TRIGGERDB_RUN4":
"CONF_DATA_RUN4",
35 "TRIGGERDBREPR_RUN4":
"CONF_REPR_RUN4",
36 "TRIGGERDBMC_RUN4":
"CONF_MC_RUN4",
37 "TRIGGERDBLS3_DEV":
"CONF_DEV_LS3",
38 "TRIGGERDBLS3_L0":
"CONF_DEV_L0",
39 "TRIGGERDB_RUN2_NF":
"CONF_DATA_RUN2_NF"
42 crestconn_dbname_mapping: dict[str, str] = {
44 "CONF_DATA_RUN3":
"ATLAS_CONF_TRIGGER_RUN3",
45 "CONF_MC_RUN3":
"ATLAS_CONF_TRIGGER_MC_RUN3",
46 "CONF_REPR_RUN3":
"ATLAS_CONF_TRIGGER_REPR_RUN3",
47 "CONF_DATA_RUN4":
"ATLAS_CONF_TRIGGER_RUN4",
48 "CONF_MC_RUN4" :
"ATLAS_CONF_TRIGGER_MC_RUN4",
49 "CONF_REPR_RUN4" :
"ATLAS_CONF_TRIGGER_REPR_RUN4",
50 "CONF_DEV_LS3" :
"ATLAS_CONF_TRIGGER_LS3_DEV",
51 "CONF_DEV_L0" :
"ATLAS_CONF_TRIGGER_LS3_L0",
52 "CONF_DATA_RUN2" :
"ATLAS_CONF_TRIGGER_RUN2_NF"
57 """list of all known crest connections
60 list[str]: list of crest connection names
62 return list(TriggerCrestUtil.dbname_crestconn_mapping.values())
66 """maps from triggerdb schema or triggerdb-alias to crest connection
67 See https://its.cern.ch/jira/browse/ATR-32030
69 If the input dbname is already a crest connection name, it is returned as is.
72 dbname (str): triggerdb name or alias or crest connection name.
74 str | None: crest connection name or None if not found.
76 if dbname
in TriggerCrestUtil.dbname_crestconn_mapping.values():
78 return TriggerCrestUtil.dbname_crestconn_mapping.get(dbname,
None)
83 return TriggerCrestUtil.dbname_crestconn_mapping
90 server (str): crest server name.
93 CrestApi: Crest API instance.
95 return CrestApi(host=server)
99 get_time_type: bool =
False) -> dict[str, dict[str, Any]]:
100 """ Conditions data for tag at a specific timestamp
102 Conditions data is returned as a dict with 'since' and 'payload' keys.
103 The 'payload' itself is a dict of channel:'data dicts'. The 'data dict' has attribute names as
104 keys and the corresponding values. If get_time_type is True, also 'time_type' key is added to
105 each IOV dict and the since is further expanded into 'since_run' and 'since_lb' (for run/lb-based
106 timestamps) or 'since_formatted' (for time-based timestamps).
109 tag (str): The tag name
110 timestamp (int): The timestamp to query (either run<32+lb or time in nanoseconds since 1.1.1970)
111 server (str, optional): Crest server name. Only needed if api is not provided. Defaults to "".
112 api (_type_, optional): Crest API instance. Defaults to None.
113 get_time_type (bool, optional): Whether to retrieve the time type. Defaults to False.
116 RuntimeError: If both server and api are missing
119 dict[str, dict[str, any]]: Conditions data
121 if server==
"" and api
is None:
122 log.error(
"Need either crest server name or crest api specified for accessing Crest")
123 raise RuntimeError(
"Crest access information missing")
125 api = CrestApi(host=server)
127 attr_list, _ = TriggerCrestUtil._get_payload_spec(tag, api)
129 iov: IovSetDto | HTTPResponse = TriggerCrestUtil._get_iov_for_timestamp(tag, timestamp=timestamp, api=api)
132 time_type = TriggerCrestUtil.getTagTimeType(tag, api=api)
135 payload_hash: str = iov.resources[0].payload_hash
136 since: int = iov.resources[0].since
137 payload = TriggerCrestUtil.getPayloadFromHash(payload_hash, api=api)
138 for channel, data
in payload.items():
139 payload_for_iov[channel] = dict(zip(attr_list, data))
141 result: dict[str, Any] = {
143 'payload': payload_for_iov
146 TriggerCrestUtil._update_with_time_type(result, time_type, since)
152 get_time_type: bool =
False) -> list[dict[str, dict[str, Any]]]:
153 """ Conditions data for tag in given range
155 List goes over the IOVs found. For each IOV, there is a dict with 'since' and 'payload' keys.
156 The 'payload' itself is a dict of channel:'data dicts'. The 'data dict' has attribute names as
157 keys and the corresponding values. If get_time_type is True, also 'time_type' key is added to
158 each IOV dict and the since is further expanded into 'since_run' and 'since_lb' (for run/lb-based
159 timestamps) or 'since_formatted' (for time-based timestamps).
163 since (int): start of the range (either run<32+lb or time in nanoseconds since 1.1.1970, inclusive)
164 until (int): end of the range (either run<32+lb or time in nanoseconds since 1.1.1970, exclusive)
165 server (str, optional): crest server name. Only needed if api is not provided. Defaults to "".
166 api (CrestApi, optional): Crest API instance. Defaults to None.
167 get_time_type (bool, optional): whether to retrieve the time type. Defaults to False.
170 RuntimeError: if both server and api are missing
173 list[dict[str, dict[str, any]]]: conditions data
175 if server==
"" and api
is None:
176 log.error(
"Need either crest server name or crest api specified for accessing Crest")
177 raise RuntimeError(
"Crest access information missing")
179 api = CrestApi(host=server)
181 attr_list, type_dict = TriggerCrestUtil._get_payload_spec(tag, api)
183 all_iovs = TriggerCrestUtil._get_iovs_range(since=since, until=until, api=api, tag=tag)
186 time_type = TriggerCrestUtil.getTagTimeType(tag, api=api)
188 result: list[dict[str, dict[str, Any]]] = []
189 for iov
in cast(Iterable, all_iovs.resources):
191 payload_hash = iov.payload_hash
193 payload = TriggerCrestUtil.getPayloadFromHash(payload_hash, api=api)
194 for channel, data
in payload.items():
195 payload_for_iov[channel] = dict(zip(attr_list, data))
196 entry = {
'since': since,
'payload': payload_for_iov}
198 TriggerCrestUtil._update_with_time_type(entry, time_type, since)
203 def getTagTimeType(tag: str, *, server: str =
"", api: CrestApi |
None =
None) -> str:
204 """ time type of the tag, either 'run-lumi' or 'time'
208 server (str, optional): crest server name. Only needed if api is not provided. Defaults to "".
209 api (CrestApi, optional): Crest API instance. Defaults to None.
212 RuntimeError: if both server and api are missing
215 str: time type of the tag ['run-lumi' or 'time']
217 if server==
"" and api
is None:
218 log.error(
"Need either crest_server name or crest api for retrieving the payload")
219 raise RuntimeError(
"Crest access information missing")
221 api = CrestApi(host=server)
222 tag_info : TagDto | HTTPResponse = api.find_tag(tag)
223 time_type: str = tag_info.time_type
227 def getAttribs(tag: str, *, server: str =
"", api: CrestApi |
None =
None) -> list[str]:
228 """ list of attributes for this tag
232 server (str, optional): crest server name. Only needed if api is not provided. Defaults to "".
233 api (CrestApi, optional): Crest API instance. Defaults to None.
236 RuntimeError: if both server and api are missing
239 dict: list of attributes
241 if server==
"" and api
is None:
242 log.error(
"Need either crest_server name or crest api for retrieving the payload")
243 raise RuntimeError(
"Crest access information missing")
245 api = CrestApi(host=server)
246 attr_list, _ = TriggerCrestUtil._get_payload_spec(tag, api)
251 """ Get payload from hash
254 payload_hash (str): hash of the payload
255 server (str, optional): crest server name. Defaults to "".
256 api (CrestApi, optional): Crest API instance. Defaults to None.
259 RuntimeError: if both server and api are missing
262 dict: payload retrieved from the hash
264 if server ==
"" and api
is None:
265 log.error(
"Need either crest_server name or crest api for retrieving the payload")
266 raise RuntimeError(
"Crest access information missing")
271 return TriggerCrestUtil._get_payload_workaround(payload_hash, server)
274 api = CrestApi(host=server)
275 return TriggerCrestUtil._get_payload(payload_hash, api)
279 def getEORParams(run: int, *, server: str =
"", api: CrestApi |
None =
None) -> dict[str, Any] |
None:
281 api = CrestApi(host=server)
282 start_of_run = (run << 32)
283 cond_data = TriggerCrestUtil.getConditionsForTimestamp(
"TDAQRunCtrlEOR-HEAD", timestamp=start_of_run, api=api, get_time_type=
False)
285 log.error(
"No EOR params found for run %s", run)
287 return cond_data[
'payload'][
'0']
290 def getHLTPrescaleKeys(run: int, *, server: str =
"", api: CrestApi |
None =
None) -> list[dict[str, dict[str, Any]]]:
292 api = CrestApi(host=server)
293 run_start = (run << 32)
294 run_end = ((run + 1) << 32) - 1
295 cond = TriggerCrestUtil.getConditionsInRange(
"TRIGGERHLTPrescaleKey-HEAD", since=run_start, until=run_end, api=api, get_time_type=
True)
297 entry.update(entry.pop(
'payload')[
'0'])
298 entry[
'key'] = entry[
'HltPrescaleKey']
301 cond: list[dict[str, dict[str, Any]]] = [entry
for entry
in cond
if entry[
'since_run']==run]
305 def getHLTPrescaleKey(run: int, lb: int, *, server: str =
"", api: CrestApi |
None =
None) -> int |
None:
307 api = CrestApi(host=server)
308 run_lb = (run << 32) + lb
309 cond_entry: dict[str, Any] = TriggerCrestUtil.getConditionsForTimestamp(
"TRIGGERHLTPrescaleKey-HEAD", timestamp=run_lb, api=api, get_time_type=
True)
310 if cond_entry[
'since_run'] < run:
312 return cond_entry[
'payload'][
'0'][
'HltPrescaleKey']
315 def getL1PrescaleKeys(run: int, *, server: str =
"", api: CrestApi |
None =
None) -> list[dict[str, dict[str, Any]]]:
317 api = CrestApi(host=server)
318 run_start = (run << 32)
319 run_end = ((run + 1) << 32) - 1
320 cond = TriggerCrestUtil.getConditionsInRange(
"TRIGGERLVL1Lvl1ConfigKey-HEAD", since=run_start, until=run_end, api=api, get_time_type=
True)
322 entry.update(entry.pop(
'payload')[
'0'])
323 entry[
'key'] = entry[
'Lvl1PrescaleConfigurationKey']
326 cond: list[dict[str, dict[str, Any]]] = [entry
for entry
in cond
if entry[
'since_run']==run]
330 def getL1PrescaleKey(run: int, lb: int, *, server: str =
"", api: CrestApi |
None =
None) -> int |
None:
332 api = CrestApi(host=server)
333 run_lb = (run << 32) + lb
334 cond_entry: dict[str, Any] = TriggerCrestUtil.getConditionsForTimestamp(
"TRIGGERLVL1Lvl1ConfigKey-HEAD", timestamp=run_lb, api=api, get_time_type=
True)
335 if cond_entry[
'since_run'] < run:
337 return cond_entry[
'payload'][
'0'][
'Lvl1PrescaleConfigurationKey']
340 def getBunchGroupKeys(run: int, *, server: str =
"", api: CrestApi |
None =
None) -> list[dict[str, dict[str, Any]]]:
342 api = CrestApi(host=server)
343 run_start = (run << 32)
344 run_end = ((run+1) << 32)-1
345 cond = TriggerCrestUtil.getConditionsInRange(
"TRIGGERLVL1BunchGroupKey-HEAD", since=run_start, until=run_end, api=api, get_time_type=
True)
347 entry.update(entry.pop(
'payload')[
'0'])
348 entry[
'key'] = entry[
'Lvl1BunchGroupConfigurationKey']
351 cond: list[dict[str, dict[str, Any]]] = [entry
for entry
in cond
if entry[
'since_run']==run]
355 def getBunchGroupKey(run: int, lb: int, *, server: str =
"", api: CrestApi |
None =
None) -> int |
None:
357 api = CrestApi(host=server)
358 run_lb = (run << 32) + lb
359 cond_entry: dict[str, Any] = TriggerCrestUtil.getConditionsForTimestamp(
"TRIGGERLVL1BunchGroupKey-HEAD", timestamp=run_lb, api=api, get_time_type=
True)
360 if cond_entry[
'since_run'] < run:
362 return cond_entry[
'payload'][
'0'][
'Lvl1BunchGroupConfigurationKey']
365 def getMenuConfigKey(run: int, *, server: str =
"", api: CrestApi |
None =
None) -> dict[str, Any] |
None:
367 def _parse_info(run, cond):
374 confsrc = cond[
'ConfigSource'].
split(
';')
377 release = confsrc[2].
split(
',')[0] + f
",{confsrc[1]}"
379 confsrc = cond[
'ConfigSource'].
split(
',', maxsplit=1)
382 release = f
"{confsrc[2]},{confsrc[1]}"
384 "SMK": cond[
'MasterConfigurationKey'],
385 "HLTPSK": cond[
'HltPrescaleConfigurationKey'],
391 api = CrestApi(host=server)
393 run_start = (run << 32) + 1
394 cond: dict[str, Any] = TriggerCrestUtil.getConditionsForTimestamp(
"TRIGGERHLTHltConfigKeys-HEAD", timestamp=run_start, api=api, get_time_type=
True)
395 if cond[
'since_run'] < run:
397 cond.update(cond.pop(
'payload')[
'0'])
398 return _parse_info(run, cond)
401 def getTrigConfKeys(runNumber: int, lumiBlock: int, server: str =
"", api: CrestApi |
None =
None) -> dict[str, Any]:
403 api = CrestApi(host=server)
404 bgkey: int |
None = TriggerCrestUtil.getBunchGroupKey(runNumber, lumiBlock, api=api)
405 l1pskey: int |
None = TriggerCrestUtil.getL1PrescaleKey(runNumber, lumiBlock, api=api)
406 hltpskey: int |
None = TriggerCrestUtil.getHLTPrescaleKey(runNumber, lumiBlock, api=api)
407 menucfg: dict[str, Any] |
None = TriggerCrestUtil.getMenuConfigKey(runNumber, api=api)
409 "SMK": menucfg[
'SMK']
if menucfg
else None,
410 "DB": menucfg[
'DB']
if menucfg
else None,
419 result[
'time_type'] = time_type
420 if time_type ==
'run-lumi':
422 'since_run': since >> 32,
423 'since_lb': since & 0xFFFFFFFF
425 elif time_type ==
'time':
427 'since_formatted': dt.fromtimestamp(since / 1e9)
433 return api.get_payload(hash=payload_hash).decode(
'utf-8')
439 url = f
"{server}/payloads/data"
444 preq = requests.Request(method=
'GET', url=url, params=params).prepare()
445 with requests.Session()
as session:
447 resp = session.send(preq)
448 except requests.ConnectionError
as exc:
449 raise RuntimeError(f
"Could not connect to CREST server {server}")
from exc
451 if resp.status_code != 200:
452 raise RuntimeError(f
"Query {payload_hash} to crest failed with status code {resp.status_code}")
454 return json.loads(resp.content)
458 """Helper to retrieve all IOVs for a run"""
459 run_start = (run << 32) + 1
460 run_end = ((run + 1) << 32) - 1
461 return TriggerCrestUtil._get_iovs_range(since=run_start, until=run_end, api=api, tag=tag)
465 """Helper to retrieve the IOV for a given timestamp
469 timestamp (int): time-stamp in format run<32+lb or time in nanoseconds since 1.1.1970
470 api (CrestApi): Crest API instance.
473 IovSetDto: set of IOVs (of size 1)
476 return api.select_iovs(tag,
"0", str(timestamp+1), sort=
'id.since:DESC', size=1, snapshot=0)
479 def _get_iovs_range(*, since: int, until: int, api: CrestApi, tag: str) -> IovSetDto | HTTPResponse:
480 """Helper to retrieve the IOV in a given range
484 since (int): start of the range (either run<32+lb or time in nanoseconds since 1.1.1970, inclusive)
485 until (int): end of the range (either run<32+lb or time in nanoseconds since 1.1.1970, exclusive)
486 api (CrestApi): Crest API instance.
489 IovSetDto: set of IOVs
492 iovs = api.select_iovs(tag,
"0", str(since+1), sort=
'id.since:DESC', size=1, snapshot=0)
493 if cast(int, iovs.size) < 1:
494 raise RuntimeError(f
"Did not get an iov which includes the start of run {since}")
495 firstiov = cast(list, iovs.resources)[0]
496 all_iovs = api.select_iovs(tag, str(firstiov.since), str(until), sort=
'id.since:ASC', snapshot=0)
501 """Helper to retrieve the payload spec for a given tag"""
502 meta: TagMetaDto | HTTPResponse = api.find_tag_meta(tag)
504 raise RuntimeError(f
"Could not retrieve tag info for tag {tag}")
505 if isinstance(meta, HTTPResponse):
506 raise RuntimeError(f
"Failed to retrieve tag metadata for tag {tag}: HTTP response received instead of tag metadata")
508 tag_info = meta.tag_info
509 tag_info_dict = json.loads(tag_info)
512 for d
in tag_info_dict[
'payload_spec']:
515 return attr_list, type_dict
518if __name__ ==
"__main__":
522 crest_server =
"https://crest.cern.ch/api-v5.0"
523 api = CrestApi(crest_server)
525 print(f
"run {testrun}:")
526 print(
"\nSuper master key, etc:")
527 pprint(TriggerCrestUtil.getMenuConfigKey(testrun, api=api))
528 print(
"\nL1 prescale keys:")
529 pprint(TriggerCrestUtil.getL1PrescaleKeys(testrun, api=api))
530 pprint(TriggerCrestUtil.getL1PrescaleKey(testrun, 1, api=api))
531 print(
"\nL1 bunchgroup keys:")
532 pprint(TriggerCrestUtil.getBunchGroupKeys(testrun, api=api))
533 pprint(TriggerCrestUtil.getBunchGroupKey(testrun, 1, api=api))
534 print(
"\nHLT prescale keys:")
535 pprint(TriggerCrestUtil.getHLTPrescaleKeys(testrun, api=api))
536 pprint(TriggerCrestUtil.getHLTPrescaleKey(testrun, 506, api=api))
537 pprint(TriggerCrestUtil.getHLTPrescaleKey(testrun, 507, api=api))
538 print(
"\nEOR params:")
539 pprint(TriggerCrestUtil.getEORParams(testrun, api=api),sort_dicts=
False)
541 print(
"\nNon existing lb:")
542 pprint(TriggerCrestUtil.getHLTPrescaleKey(testrun, 1_000_000, api=api))
544 print(
"\nNon existing run:")
545 pprint(TriggerCrestUtil.getHLTPrescaleKey(1_000_000, 1, api=api))
546 pprint(TriggerCrestUtil.getHLTPrescaleKeys(1_000_000, api=api))
void print(char *figname, TCanvas *c1)
CrestApi getCrestApi(str server)
dict getPayloadFromHash(str payload_hash, *, str server="", CrestApi|None api=None)
int|None getBunchGroupKey(int run, int lb, *, str server="", CrestApi|None api=None)
dict[str, Any]|None getEORParams(int run, *, str server="", CrestApi|None api=None)
dict[str, Any]|None getMenuConfigKey(int run, *, str server="", CrestApi|None api=None)
int|None getHLTPrescaleKey(int run, int lb, *, str server="", CrestApi|None api=None)
dict _get_payload_workaround(payload_hash, server)
_get_iovs_for_run(str tag, *, int run, CrestApi api)
list[str] allCrestConnections()
str getTagTimeType(str tag, *, str server="", CrestApi|None api=None)
dict[str, dict[str, Any]] getConditionsForTimestamp(str tag, *, int timestamp, str server="", CrestApi|None api=None, bool get_time_type=False)
list[dict[str, dict[str, Any]]] getConditionsInRange(str tag, *, int since, int until, str server="", CrestApi|None api=None, bool get_time_type=False)
tuple[list[Any], dict[Any, Any]] _get_payload_spec(str tag, CrestApi api)
IovSetDto|HTTPResponse _get_iov_for_timestamp(str tag, *, int timestamp, CrestApi api)
list[dict[str, dict[str, Any]]] getHLTPrescaleKeys(int run, *, str server="", CrestApi|None api=None)
IovSetDto|HTTPResponse _get_iovs_range(*, int since, int until, CrestApi api, str tag)
int|None getL1PrescaleKey(int run, int lb, *, str server="", CrestApi|None api=None)
dict[str, Any] getTrigConfKeys(int runNumber, int lumiBlock, str server="", CrestApi|None api=None)
list[dict[str, dict[str, Any]]] getBunchGroupKeys(int run, *, str server="", CrestApi|None api=None)
list[dict[str, dict[str, Any]]] getL1PrescaleKeys(int run, *, str server="", CrestApi|None api=None)
dict[str, str] getDBNameMapping()
list[str] getAttribs(str tag, *, str server="", CrestApi|None api=None)
dict _get_payload(payload_hash, api)
str|None getCrestConnection(str dbname)
None _update_with_time_type(dict[str, Any] result, str time_type, int since)
std::vector< std::string > split(const std::string &s, const std::string &t=":")