ATLAS Offline Software
CrestFunctions.cxx
Go to the documentation of this file.
1 /*
2  Copyright (C) 2002-2023 CERN for the benefit of the ATLAS collaboration
3 */
4 // @file CrestFunctions.cxx
5 // Implementation for CrestFunctions utilities
6 // @author Shaun Roe
7 // @date 1 July 2019
8 
9 #include "CrestFunctions.h"
10 #include "CrestApi/CrestApi.h"
11 #include <iostream>
12 #include <exception>
13 #include <regex>
14 #include "IOVDbStringFunctions.h"
15 #include <string>
16 #include <algorithm>
17 #include <map>
18 
19 namespace IOVDbNamespace{
20 
21  CrestFunctions::CrestFunctions(const std::string & crest_path){
22  setURLBase(crest_path);
23  }
24 
25  const std::string &
26  CrestFunctions::getURLBase(){
27  return m_CREST_PATH;
28  }
29 
30  void
31  CrestFunctions::setURLBase(const std::string & crest_path){
32  m_CREST_PATH = crest_path;
33  }
34 
35  std::vector<IovHashPair>
36  CrestFunctions::extractIovAndHash(const std::string_view jsonReply){
37  std::vector<IovHashPair> iovHashPairs;
38  bool all_ok = true;
39  std::string_view iovSignature = "since\":";
40  std::string_view hashSignature = "payloadHash\":\"";
41  size_t startpoint = jsonReply.find(hashSignature);
42  size_t endpoint = 0;
43 
44  while(startpoint!=std::string::npos) {
45  startpoint+=hashSignature.size();
46  endpoint = jsonReply.find('\"',startpoint);
47  if(endpoint==std::string::npos) {
48  all_ok = false;
49  break;
50  }
51  std::string_view hashString = jsonReply.substr(startpoint,endpoint-startpoint);
52  startpoint= jsonReply.find(iovSignature,endpoint);
53  if(startpoint==std::string::npos) {
54  all_ok = false;
55  break;
56  }
57  startpoint+=iovSignature.size();
58  endpoint = jsonReply.find(',',startpoint);
59  if(endpoint==std::string::npos) {
60  all_ok = false;
61  break;
62  }
63  std::string_view iovString = jsonReply.substr(startpoint,endpoint-startpoint);
64  iovHashPairs.emplace_back(iovString,hashString);
65  startpoint= jsonReply.find(hashSignature,endpoint);
66  }
67  if(!all_ok) {
68  std::cerr<<__FILE__<<":"<<__LINE__<< ": Formatting error found while trying to extract IOVs and Hashes from "<<jsonReply<<std::endl;
69  iovHashPairs.clear();
70  }
71  return iovHashPairs;
72  }
73 
74  std::string
75  CrestFunctions::extractHashFromJson(const std::string & jsonReply){
76  std::string hash{};
77  try{
78  std::string_view signature="payloadHash\":\"";
79  auto signaturePosition=jsonReply.rfind(signature);
80  if (signaturePosition == std::string::npos) throw std::runtime_error("signature "+std::string(signature)+" not found");
81  auto startOfHash=signaturePosition + signature.size();
82  auto endOfHash=jsonReply.find('\"',startOfHash);
83  auto len=endOfHash-startOfHash;
84  if (startOfHash > jsonReply.size()) throw std::runtime_error("Hash start is beyond end of string");
85  hash=jsonReply.substr(startOfHash, len);
86  } catch (std::exception & e){
87  std::cout<<__FILE__<<":"<<__LINE__<< ": "<<e.what()<<" while trying to find the hash in "<<jsonReply<<std::endl;
88  }
89  return hash;
90  }
91 
92  std::vector<IovHashPair>
93  CrestFunctions::getIovsForTag(const std::string & tag, const bool testing){
94  std::string reply{R"delim([{"insertionTime":"2022-05-26T12:10:58+0000","payloadHash":"99331506eefbe6783a8d5d5bc8b9a44828a325adfcaac32f62af212e9642db71","since":0,"tagName":"LARIdentifierFebRodMap-RUN2-000"}])delim"};
95  if (not testing){
96  //...CrestApi returns Iovs as a json object
97  auto myCrestClient = Crest::CrestClient(getURLBase());
98  try{
99  reply = myCrestClient.findAllIovs(tag).dump();
100  } catch (std::exception & e){
101  std::cout<<__FILE__<<":"<<__LINE__<< ": "<<e.what()<<" while trying to find the IOVs"<<std::endl;
102  return {};
103  }
104  }
105  return extractIovAndHash(reply);
106  }
107 
108  std::string
109  CrestFunctions::getLastHashForTag(const std::string & tag, const bool testing){
110  char tu[] = "";
111  strfry(tu);
112  std::string reply{R"delim([{"insertionTime":"2022-05-26T12:10:58+0000","payloadHash":"99331506eefbe6783a8d5d5bc8b9a44828a325adfcaac32f62af212e9642db71","since":0,"tagName":"LARIdentifierFebRodMap-RUN2-000"}])delim"};
113  if (not testing){
114  //...CrestApi returns Iovs as a json object
115  auto myCrestClient = Crest::CrestClient(getURLBase());
116  try{
117  reply = myCrestClient.findAllIovs(tag).dump();
118  } catch (std::exception & e){
119  std::cout<<__FILE__<<":"<<__LINE__<< ": "<<e.what()<<" while trying to find the IOVs"<<std::endl;
120  return "";
121  }
122  }
123  return extractHashFromJson(reply);
124  }
125 
126 
127  std::string
128  CrestFunctions::getPayloadForHash(const std::string & hash, const bool testing){
129  std::string reply{R"delim({"data":{"0":["[DB=B2E3B2B6-B76C-DF11-A505-000423D5ADDA][CNT=CollectionTree(LArTTCell_P/LArTTCellMapAtlas)][CLID=DF8C509C-A91A-40B5-B76C-5B57EEE21EC3][TECH=00000202][OID=00000003-00000000]"]}})delim"};
130  if (not testing){
131  //CrestApi method:
132  try{
133  auto myCrestClient = Crest::CrestClient(getURLBase());
134  reply = myCrestClient.getPayloadAsString(hash);
135  } catch (std::exception & e){
136  std::cout<<__FILE__<<":"<<__LINE__<< ": "<<e.what()<<" while trying to find the payload"<<std::endl;
137  return "";
138  }
139  }
140  return reply;
141  }
142 
143  std::string
144  CrestFunctions::getPayloadForTag(const std::string & tag, const bool testing){
145  return getPayloadForHash(getLastHashForTag(tag, testing), testing);
146  }
147 
148  std::string
149  CrestFunctions::extractDescriptionFromJson(const std::string & jsonReply){
150  std::string description{};
151  try{
152  const std::string_view signature="node_description\\\":\\\"";
153  const auto signaturePosition = jsonReply.find(signature);
154  if (signaturePosition == std::string::npos) throw std::runtime_error("signature "+std::string(signature)+" not found");
155  const auto startOfDescription= signaturePosition + signature.size();
156  const std::string_view endSignature = "\\\",\\\"payload_spec";
157  const auto endOfDescription=jsonReply.find(endSignature);
158  if (endOfDescription == std::string::npos) throw std::runtime_error("end signature "+std::string(endSignature)+" not found");
159  const auto len=endOfDescription-startOfDescription;
160  description=jsonReply.substr(startOfDescription, len);
161  } catch (std::exception & e){
162  std::cout<<__FILE__<<":"<<__LINE__<< ": "<<e.what()<<" while trying to find the description in "<<jsonReply<<std::endl;
163  }
164 
165  return unescapeQuotes(unescapeBackslash(description));
166  }
167 
168  std::string
169  CrestFunctions::extractSpecificationFromJson(const std::string & jsonReply){
170  std::string spec{};
171  try{
172  const std::string_view signature="payload_spec\\\":\\\"";
173  const auto signaturePosition = jsonReply.find(signature);
174  if (signaturePosition == std::string::npos) throw std::runtime_error("signature "+std::string(signature)+" not found");
175  const auto startOfSpec= signaturePosition + signature.size();
176  const auto endOfSpec=jsonReply.find("\\\"}\"",startOfSpec);
177  const auto len=endOfSpec-startOfSpec;
178  spec=jsonReply.substr(startOfSpec, len);
179  } catch (std::exception & e){
180  std::cout<<__FILE__<<":"<<__LINE__<< ": "<<e.what()<<" while trying to find the payload spec in "<<jsonReply<<std::endl;
181  }
182  return spec;
183  }
184 
185  std::pair<std::vector<cool::ChannelId> , std::vector<std::string>>
186  CrestFunctions::extractChannelListFromJson(const std::string & jsonReply){
187  std::vector<cool::ChannelId> list;
188  std::vector<std::string> names;
189  std::string textRep;
190  try{
191  const std::string_view signature="channel_list\\\":[";
192  const auto startOfList=jsonReply.find(signature) + signature.size();
193  const auto endOfList=jsonReply.find(']', startOfList);
194  const auto len=endOfList-startOfList;
195  textRep=jsonReply.substr(startOfList, len);
196  } catch (std::exception & e){
197  std::cout<<__FILE__<<":"<<__LINE__<< ": "<<e.what()<<"\n while trying to find the description in "<<jsonReply<<std::endl;
198  }
199  //channel list is of format [{\"956301312\":\"barrel A 01L PS\"},{\"956334080\":\"barrel A 01L F0\"}]
200  std::string s=R"d(\{\\\"([0-9]+)\\\":\\\"([^\"]*)\"},?)d";
201  std::regex r(s);
202  std::sregex_iterator it(textRep.begin(), textRep.end(), r);
203  std::sregex_iterator end;
204  for (;it!=end;++it){
205  const std::smatch& m= *it;
206  if (not m.empty()){
207  list.push_back(std::stoll(m[1].str()));
208  //chomp the last backslash
209  std::string s = m[2].str();
210  s.pop_back();
211  names.emplace_back(std::move(s));
212  }
213  }
214  // if all the names are empty, these are unnamed channels, and can just return an empty vector for the names
215  auto isEmpty=[](const std::string & s){return s.empty();};
216  if ( std::all_of(names.begin(), names.end(), isEmpty)) names.clear();
217  return std::make_pair(std::move(list), std::move(names));
218  }
219 
220  std::string
221  CrestFunctions::folderDescriptionForTag(const std::string & tag, const bool testing){
222  std::string jsonReply{R"delim({"format":"TagMetaSetDto","resources":[{"tagName":"LARAlign-RUN2-UPD4-03","description":"{\"dbname\":\"CONDBR2\",\"nodeFullpath\":\"/LAR/Align\",\"schemaName\":\"COOLONL_LAR\"}","chansize":1,"colsize":1,"tagInfo":"{\"channel_list\":[{\"0\":\"\"}],\"node_description\":\"<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type=\\\"256\\\" clid=\\\"1238547719\\\" /></addrHeader><typeName>CondAttrListCollection</typeName><updateMode>UPD1</updateMode>\",\"payload_spec\":\"PoolRef:String4k\"}","insertionTime":"2022-05-26T12:10:38+0000"}],"size":1,"datatype":"tagmetas","format":null,"page":null,"filter":null})delim"};
223  if (not testing){
224  auto myCrestClient = Crest::CrestClient(getURLBase());
225  jsonReply= myCrestClient.getTagMetaInfo(tag).dump();
226  }
227  return extractDescriptionFromJson(jsonReply);
228  }
229 
230  std::string
231  CrestFunctions::payloadSpecificationForTag(const std::string & specTag, const bool testing){
232  std::string jsonReply{R"delim({"folder_payloadspec": "PoolRef: String4k"})delim"};
233  if (not testing){
234  auto myCrestClient = Crest::CrestClient(getURLBase());
235  jsonReply= myCrestClient.getTagMetaInfo(specTag).dump();
236  }
237  return extractSpecificationFromJson(jsonReply);
238  }
239 
240  std::pair<std::vector<cool::ChannelId> , std::vector<std::string>>
241  CrestFunctions::channelListForTag(const std::string & tag, const bool testing){
242  std::string reply{R"delim([{"chansize":8,"colsize":5,"description":"{\"dbname\":\"CONDBR2\",\"nodeFullpath\":\"/LAR/BadChannelsOfl/BadChannels\",\"schemaName\":\"COOLOFL_LAR\"}","insertionTime":"2022-05-26T16:40:32+0000","tagInfo":"{\"channel_list\":[{\"0\":\"\"},{\"1\":\"\"},{\"2\":\"\"},{\"3\":\"\"},{\"4\":\"\"},{\"5\":\"\"},{\"6\":\"\"},{\"7\":\"\"}],\"node_description\":\"<timeStamp>run-lumi</timeStamp><addrHeader><address_header service_type=\\\"71\\\" clid=\\\"1238547719\\\" /></addrHeader><typeName>CondAttrListCollection</typeName>\",\"payload_spec\":\"ChannelSize:UInt32,StatusWordSize:UInt32,Endianness:UInt32,Version:UInt32,Blob:Blob64k\"}","tagName":"LARBadChannelsOflBadChannels-RUN2-UPD4-21"}])delim"};
243  if (not testing){
244  auto myCrestClient = Crest::CrestClient(getURLBase());
245  reply= myCrestClient.getTagMetaInfo(tag).dump();
246  }
247  return extractChannelListFromJson(reply);
248  }
249 
250  std::string
251  CrestFunctions::resolveCrestTag(const std::string & globalTagName, const std::string & folderName, const std::string & forceTag, const bool testing){
252  std::string result{};
253  if (not forceTag.empty()) return forceTag;
254  if (testing) return "LARAlign-RUN2-UPD4-03";
255  auto crestClient = Crest::CrestClient(getURLBase());
256  auto j = crestClient.findGlobalTagMap(globalTagName);
257  for (const auto &i:j){
258  if (i["label"] == folderName){
259  result=static_cast<std::string>(i["tagName"]);
260  break;
261  }
262  }
263  return result;
264  }
265 
266  std::string
267  CrestFunctions::jsonTagName(const std::string &globalTag, const std::string & folderName){
268  return resolveCrestTag(globalTag,folderName);
269  }
270 
271  std::map<std::string, std::string>
272  CrestFunctions::getGlobalTagMap(const std::string& globaltag){
273  std::map<std::string, std::string> tagmap;
274  try{
275  auto crestClient = Crest::CrestClient(getURLBase());
276  nlohmann::json j = crestClient.findGlobalTagMap(globaltag);
277  int n = j.size();
278  for (int i = 0; i < n; i++ ){
279  nlohmann::json j_item = j[i];
280  if (j_item.contains("label") && j_item.contains("tagName") ){
281  tagmap[j_item["label"]] = j_item["tagName"];
282  }
283  }
284  } catch (std::exception & e){
285  std::cout<<__FILE__<<":"<<__LINE__<< ": " << e.what() << " Cannot get a global tag map for " << globaltag << std::endl;
286  }
287 
288  return tagmap;
289  }
290 
291 
292  nlohmann::json CrestFunctions::getTagInfo(const std::string & tag){
293  try{
294  auto crestClient = Crest::CrestClient(getURLBase());
295  nlohmann::json meta_info = crestClient.getTagMetaInfo(tag)[0];
296 
297  if (meta_info.contains("tagInfo")){
298  return crestClient.getJson(meta_info["tagInfo"]);
299  }
300 
301  } catch (std::exception & e){
302  std::cout<<__FILE__<<":"<<__LINE__<< ": " << e.what() << " Cannot get a tag meta info " << tag << std::endl;
303  }
304  return nullptr;
305  }
306 
307  nlohmann::json CrestFunctions::getTagProperties(const std::string & tag){
308  try{
309  auto crestClient = Crest::CrestClient(getURLBase());
310  return crestClient.findTag(tag)[0];
311 
312  } catch (std::exception & e){
313  std::cout<<__FILE__<<":"<<__LINE__<< ": " << e.what() << " Cannot get a tag Properties of " << tag << std::endl;
314  }
315  return nullptr;
316  }
317 
318  std::string CrestFunctions::getTagInfoElement(nlohmann::json tag_info, const std::string & key){
319  if (tag_info.contains(key)){
320  if (key == "channel_list"){
321  return tag_info[key].dump();
322  }
323  else{
324  return tag_info[key];
325  }
326  }
327  return "";
328  }
329 
330  std::pair<std::vector<cool::ChannelId> , std::vector<std::string>>
331  CrestFunctions::extractChannelListFromString(const std::string & chanString){
332  std::vector<cool::ChannelId> list;
333  std::vector<std::string> names;
334  nlohmann::json js = nlohmann::json::parse(chanString);
335  int n = js.size();
336 
337  for (int i = 0; i <= n; i++) {
338  nlohmann::json j_object = js[i];
339  for (auto& [key, val] : j_object.items()){
340  list.push_back(std::stoll(key));
341  names.push_back(val);
342  }
343  }
344 
345  // if all the names are empty, these are unnamed channels, and can just return an empty vector for the names
346  auto isEmpty=[](const std::string & s){return s.empty();};
347  if ( std::all_of(names.begin(), names.end(), isEmpty)) names.clear();
348  return std::make_pair(std::move(list), std::move(names));
349  }
350 
351 }
beamspotman.r
def r
Definition: beamspotman.py:676
IOVDbNamespace::CrestFunctions::getGlobalTagMap
std::map< std::string, std::string > getGlobalTagMap(const std::string &globaltag)
IOVDbNamespace::CrestFunctions::m_CREST_PATH
std::string m_CREST_PATH
Definition: CrestFunctions.h:103
python.SystemOfUnits.s
int s
Definition: SystemOfUnits.py:131
get_generator_info.result
result
Definition: get_generator_info.py:21
python.SystemOfUnits.m
int m
Definition: SystemOfUnits.py:91
IOVDbNamespace::CrestFunctions::payloadSpecificationForTag
std::string payloadSpecificationForTag(const std::string &tag, const bool testing=false)
json
nlohmann::json json
Definition: HistogramDef.cxx:9
CaloCondBlobAlgs_fillNoiseFromASCII.spec
spec
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:47
IOVDbNamespace::CrestFunctions::setURLBase
void setURLBase(const std::string &crest_path)
IOVDbNamespace::CrestFunctions::getURLBase
const std::string & getURLBase()
parse
std::map< std::string, std::string > parse(const std::string &list)
Definition: egammaLayerRecalibTool.cxx:983
IOVDbNamespace::CrestFunctions::extractIovAndHash
std::vector< IovHashPair > extractIovAndHash(const std::string_view jsonReply)
skel.it
it
Definition: skel.GENtoEVGEN.py:423
IOVDbNamespace::CrestFunctions::extractSpecificationFromJson
std::string extractSpecificationFromJson(const std::string &jsonReply)
IOVDbNamespace::CrestFunctions::getTagProperties
nlohmann::json getTagProperties(const std::string &tag)
IOVDbNamespace::CrestFunctions::CrestFunctions
CrestFunctions(const std::string &crest_path)
IOVDbNamespace::CrestFunctions::resolveCrestTag
std::string resolveCrestTag(const std::string &globalTagName, const std::string &folderName, const std::string &forceTag="", const bool testing=false)
CrestApi.h
Header file for CREST C++ Client Library.
mergePhysValFiles.end
end
Definition: DataQuality/DataQualityUtils/scripts/mergePhysValFiles.py:93
IOVDbNamespace::CrestFunctions::extractHashFromJson
std::string extractHashFromJson(const std::string &jsonReply)
IOVDbNamespace::CrestFunctions::channelListForTag
std::pair< std::vector< cool::ChannelId >, std::vector< std::string > > channelListForTag(const std::string &tag, const bool testing=false)
PrepareReferenceFile.regex
regex
Definition: PrepareReferenceFile.py:43
IOVDbStringFunctions.h
lumiFormat.i
int i
Definition: lumiFormat.py:92
beamspotman.n
n
Definition: beamspotman.py:731
IOVDbNamespace::CrestFunctions::folderDescriptionForTag
std::string folderDescriptionForTag(const std::string &tag, const bool testing=false)
python.subdetectors.mmg.names
names
Definition: mmg.py:8
IOVDbNamespace::CrestFunctions::getLastHashForTag
std::string getLastHashForTag(const std::string &tag, const bool testing=false)
calibdata.exception
exception
Definition: calibdata.py:496
histSizes.list
def list(name, path='/')
Definition: histSizes.py:38
CheckTagAssociation.globaltag
globaltag
Definition: CheckTagAssociation.py:18
IOVDbNamespace::CrestFunctions::getPayloadForTag
std::string getPayloadForTag(const std::string &tag, const bool testing=false)
CaloCellTimeCorrFiller.folderName
string folderName
Definition: CaloCellTimeCorrFiller.py:20
IOVDbNamespace::unescapeBackslash
std::string unescapeBackslash(const std::string &original)
Definition: IOVDbStringFunctions.cxx:102
IOVDbNamespace::CrestFunctions::getTagInfoElement
std::string getTagInfoElement(nlohmann::json tag_info, const std::string &key)
IOVDbNamespace::CrestFunctions::getTagInfo
nlohmann::json getTagInfo(const std::string &tag)
IOVDbNamespace::CrestFunctions::extractChannelListFromString
std::pair< std::vector< cool::ChannelId >, std::vector< std::string > > extractChannelListFromString(const std::string &chanString)
DiTauMassTools::MaxHistStrategyV2::e
e
Definition: PhysicsAnalysis/TauID/DiTauMassTools/DiTauMassTools/HelperFunctions.h:26
CaloCondBlobAlgs_fillNoiseFromASCII.hash
dictionary hash
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:109
python.CaloScaleNoiseConfig.str
str
Definition: CaloScaleNoiseConfig.py:78
IOVDbNamespace::CrestFunctions::jsonTagName
std::string jsonTagName(const std::string &globalTag, const std::string &folderName)
Pythia8_RapidityOrderMPI.val
val
Definition: Pythia8_RapidityOrderMPI.py:14
IOVDbNamespace::unescapeQuotes
std::string unescapeQuotes(const std::string &original)
Definition: IOVDbStringFunctions.cxx:95
Crest::CrestClient
Definition: CrestApi.h:62
IOVDbNamespace::CrestFunctions::getIovsForTag
std::vector< IovHashPair > getIovsForTag(const std::string &tag, const bool testing=false)
IOVDbNamespace::CrestFunctions::extractChannelListFromJson
std::pair< std::vector< cool::ChannelId >, std::vector< std::string > > extractChannelListFromJson(const std::string &jsonReply)
Example_ReadSampleNoise.globalTag
globalTag
Definition: Example_ReadSampleNoise.py:15
IOVDbNamespace::CrestFunctions::getPayloadForHash
std::string getPayloadForHash(const std::string &hash, const bool testing=false)
CaloCondBlobAlgs_fillNoiseFromASCII.tag
string tag
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:24
CrestFunctions.h
Header for CrestFunctions utilities.
IOVDbNamespace::CrestFunctions::extractDescriptionFromJson
std::string extractDescriptionFromJson(const std::string &jsonReply)
description
std::string description
glabal timer - how long have I taken so far?
Definition: hcg.cxx:88
IOVDbNamespace
Definition: Base64Codec.cxx:16
mapkey::key
key
Definition: TElectronEfficiencyCorrectionTool.cxx:37
Generated on Tue Jul 2 2024 21:13:04 for ATLAS Offline Software by   doxygen 1.8.18