Loading [MathJax]/extensions/tex2jax.js
ATLAS Offline Software
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros Modules Pages
CrestFunctions.cxx
Go to the documentation of this file.
1 /*
2  Copyright (C) 2002-2024 CERN for the benefit of the ATLAS collaboration
3 */
4 // @file CrestFunctions.cxx
5 // Implementation for CrestFunctions utilities
6 // @author Shaun Roe
7 // @date 1 July 2019
8 
9 #include "CrestFunctions.h"
10 #include "CrestApi/CrestApi.h"
11 #include "CrestApi/CrestApiFs.h"
12 #include <iostream>
13 #include <exception>
14 #include <regex>
15 #include "IOVDbStringFunctions.h"
16 #include <string>
17 #include <algorithm>
18 #include <map>
19 
20 namespace IOVDbNamespace{
21 
22  using namespace Crest;
23 
24  CrestFunctions::CrestFunctions(const std::string & crest_path){
25  setURLBase(crest_path);
26 
27  const std::string prefix1 = "http://";
28  const std::string prefix2 = "https://";
29  if (crest_path.starts_with(prefix1) || crest_path.starts_with(prefix2)){
30  m_crestCl = std::unique_ptr<Crest::CrestApi>(new Crest::CrestApi(getURLBase()));
31  }
32  else{
33  m_crestCl = std::unique_ptr<Crest::CrestApiFs>(new Crest::CrestApiFs(true,getURLBase()));
34  }
35  }
36 
37  const std::string &
39  return m_CREST_PATH;
40  }
41 
42  void
43  CrestFunctions::setURLBase(const std::string & crest_path){
44  m_CREST_PATH = crest_path;
45  }
46 
47 
48  std::string
49  CrestFunctions::extractHashFromJson(const std::string & jsonReply){
50  std::string hash{};
51  try{
52  std::string_view signature="payloadHash\":\"";
53  auto signaturePosition=jsonReply.rfind(signature);
54  if (signaturePosition == std::string::npos) throw std::runtime_error("signature "+std::string(signature)+" not found");
55  auto startOfHash=signaturePosition + signature.size();
56  auto endOfHash=jsonReply.find('\"',startOfHash);
57  auto len=endOfHash-startOfHash;
58  if (startOfHash > jsonReply.size()) throw std::runtime_error("Hash start is beyond end of string");
59  hash=jsonReply.substr(startOfHash, len);
60  } catch (std::exception & e){
61  std::cerr<<__FILE__<<":"<<__LINE__<< ": "<<e.what()<<" while trying to find the hash in "<<jsonReply<<std::endl;
62  }
63  return hash;
64  }
65 
66 
67  std::string
68  CrestFunctions::getLastHashForTag(const std::string & tag){
69  char tu[] = "";
70  strfry(tu);
71  std::string reply = "";
72 
73  try{
74  IovSetDto dto = m_crestCl->selectIovs(tag, 0, -1, 0, 10000, 0, "id.since:ASC");
75 
76  nlohmann::json iov_data = dto.toJson();
77  nlohmann::json iov_list = getResources(iov_data);
78  reply = iov_list.dump();
79  } catch (std::exception & e){
80  std::cerr<<__FILE__<<":"<<__LINE__<< ": "<<e.what()<<" while trying to find the IOVs"<<std::endl;
81  return "";
82  }
83 
84  return extractHashFromJson(reply);
85  }
86 
87  std::string
89  std::string reply = "";
90 
91  try{
92  reply = m_crestCl->getPayload(hash);
93  } catch (std::exception & e){
94  std::cerr<<__FILE__<<":"<<__LINE__<< ": "<<e.what()<<" while trying to find the payload"<<std::endl;
95  return "";
96  }
97 
98  return reply;
99  }
100 
101 
102  std::string
103  CrestFunctions::extractDescriptionFromJson(const std::string & jsonReply){
104  std::string description{};
105  try{
106  const std::string_view signature="node_description\\\":\\\"";
107  const auto signaturePosition = jsonReply.find(signature);
108  if (signaturePosition == std::string::npos) throw std::runtime_error("signature "+std::string(signature)+" not found");
109  const auto startOfDescription= signaturePosition + signature.size();
110  const std::string_view endSignature = "\\\",\\\"payload_spec";
111  const auto endOfDescription=jsonReply.find(endSignature);
112  if (endOfDescription == std::string::npos) throw std::runtime_error("end signature "+std::string(endSignature)+" not found");
113  const auto len=endOfDescription-startOfDescription;
114  description=jsonReply.substr(startOfDescription, len);
115  } catch (std::exception & e){
116  std::cerr<<__FILE__<<":"<<__LINE__<< ": "<<e.what()<<" while trying to find the description in "<<jsonReply<<std::endl;
117  }
118 
120  }
121 
122 
123  std::string
125 
126  std::string jsonReply = "";
127 
128  TagMetaDto dto = m_crestCl->findTagMeta(tag);
129  TagInfoDto tag_info_dto = dto.getTagInfoDto();
130  jsonReply = tag_info_dto.getFolderDescription();
131  return jsonReply;
132  }
133 
135  try{
136  TagMetaDto dto = m_crestCl->findTagMeta(tag);
137  TagInfoDto tag_info_dto = dto.getTagInfoDto();
138  nlohmann::json tag_info = tag_info_dto.toJson();
139  return tag_info;
140  } catch (std::exception & e){
141  std::cerr<<__FILE__<<":"<<__LINE__<< ": " << e.what() << " Cannot get a tag meta info " << tag << std::endl;
142  }
143  return nullptr;
144  }
145 
147  CrestFunctions::getTagProperties(const std::string & tag){
148  try{
149  TagDto dto = m_crestCl->findTag(tag);
150  return dto.toJson();
151  } catch (std::exception & e){
152  std::cerr<<__FILE__<<":"<<__LINE__<< ": " << e.what() << " Cannot get a tag Properties of " << tag << std::endl;
153  }
154  return nullptr;
155  }
156 
157  std::string
158  CrestFunctions::getTagInfoElement(nlohmann::json tag_info, const std::string & key){
159  if (tag_info.contains(key)){
160  if (key == "channel_list"){
161  return tag_info[key].dump();
162  } else if (key== "node_description"){
163  std::string v;
164  tag_info[key].get_to(v);
165  return v;
166  } else{
167  return nlohmann::to_string(tag_info[key]);
168  }
169  }
170  return "";
171  }
172 
173  std::pair<std::vector<cool::ChannelId> , std::vector<std::string>>
174  CrestFunctions::extractChannelListFromString(const std::string & chanString){
175  std::vector<cool::ChannelId> list;
176  std::vector<std::string> names;
177  nlohmann::json js = nlohmann::json::parse(chanString);
178  int n = js.size();
179 
180  for (int i = 0; i <= n; i++) {
181  nlohmann::json j_object = js[i];
182  for (auto& [key, val] : j_object.items()){
183  list.push_back(std::stoll(key));
184  names.push_back(val);
185  }
186  }
187 
188  // if all the names are empty, these are unnamed channels, and can just return an empty vector for the names
189  auto isEmpty=[](const std::string & s){return s.empty();};
190  if ( std::all_of(names.begin(), names.end(), isEmpty)) names.clear();
191  return std::make_pair(std::move(list), std::move(names));
192  }
193 
196  nlohmann::json js2 = json::array();
197  nlohmann::json result = js.value("resources", js2);
198  return result;
199  }
200 
201  std::vector<uint64_t>
202  CrestFunctions::getIovGroups(const std::string & tag){
203  std::vector<uint64_t> v;
204  try{
205  IovSetDto dto = m_crestCl->selectGroups(tag, 0, 10000, 0, "id.since:ASC");
206  const std::vector<IovDto> & res = dto.getResources();
207  for (const IovDto & item_iov: res){
208  v.emplace_back(item_iov.getSince());
209  }
210  } catch (std::exception & e){
211  std::cerr<<__FILE__<<":"<<__LINE__<< ": "<<e.what()<<" while trying to find the IOVs"<<std::endl;
212  return {};
213  }
214  return v;
215  }
216 
217 
218  std::pair<uint64_t,uint64_t>
219  CrestFunctions::getSinceUntilPair(const std::vector<uint64_t>& v, const uint64_t since, const uint64_t until){
220  uint64_t new_since = 0;
221  uint64_t new_until = 0;
222  std::pair<uint64_t,uint64_t> answer = std::make_pair(0,0);
223 
224  if (until < since){
225  std::cerr << "Wrong since/until." << std::endl;
226  return answer;
227  }
228 
229  int N = v.size();
230  for (int i = 0; i < N; i++) {
231  if(v[i] <= since && since < v[i+1]){
232  new_since = v[i];
233  break;
234  }
235  }
236 
237  for (int i = 0; i < N; i++) {
238  if(v[i] < until && until <= v[i+1]){
239  new_until = v[i+1];
240  break;
241  }
242  }
243 
244  answer = std::make_pair(new_since,new_until);
245  return answer;
246  }
247 
248 
249  int CrestFunctions::getTagSize(const std::string& tagname){
250  int res = 0;
251  try{
252  res = m_crestCl->getSize(tagname);
253  } catch (std::exception & e){
254  std::cerr<<__FILE__<<":"<<__LINE__<< ": " << e.what() << " Cannot get the tag size for " << tagname << std::endl;
255  }
256  return res;
257  }
258 
259  std::pair<uint64_t,uint64_t>
260  CrestFunctions::getIovInterval(const std::string& tag, const uint64_t since, const uint64_t until){
261  std::vector<uint64_t> v = getIovGroups(tag);
262  v.push_back(std::numeric_limits<uint64_t>::max()); // added "infinity" as the last item
263  return getSinceUntilPair(std::move(v), since, until);
264  }
265 
266 
267  std::vector<IovHashPair>
269  std::vector<IovHashPair> iovHashPairs;
270  int iovNumber = getTagSize(tag);
271  try{
272  IovSetDto dto;
273  if (iovNumber <=1000) {
274  dto = m_crestCl->selectIovs(tag, 0, -1, 0, 10000, 0, "id.since:ASC");
275  } else {
276  const auto &[s_time, u_time] = getIovInterval(tag, since, until);
277  if (s_time == 0 && u_time == 0){ // data out of range
278  return iovHashPairs;
279  } else {
280  dto = m_crestCl->selectIovs(tag, s_time, u_time, 0, 10000, 0, "id.since:ASC");
281  }
282  }
283  std::vector<IovDto> res = dto.getResources();
284  std::map<uint64_t, std::string> hashmap;
285  for (const IovDto & item: res) {
286  hashmap[item.getSince()] = item.getPayloadHash();
287  }
288  for (auto& t : hashmap){
289  iovHashPairs.emplace_back(std::to_string(t.first),t.second);
290  }
291  } catch (std::exception & e){
292  std::cerr<<__FILE__<<":"<<__LINE__<< ": "<<e.what()<<" while trying to find the IOVs"<<std::endl;
293  return {};
294  }
295  return iovHashPairs;
296  }
297 }
AllowedVariables::e
e
Definition: AsgElectronSelectorTool.cxx:37
IOVDbNamespace::CrestFunctions::getIovsForTag
std::vector< IovHashPair > getIovsForTag(const std::string &tag, uint64_t since, uint64_t until)
Definition: CrestFunctions.cxx:268
get_generator_info.result
result
Definition: get_generator_info.py:21
IOVDbNamespace::CrestFunctions::getIovInterval
std::pair< uint64_t, uint64_t > getIovInterval(const std::string &tag, const uint64_t since, const uint64_t until)
Definition: CrestFunctions.cxx:260
json
nlohmann::json json
Definition: HistogramDef.cxx:9
IOVDbNamespace::CrestFunctions::getSinceUntilPair
std::pair< uint64_t, uint64_t > getSinceUntilPair(const std::vector< uint64_t > &v, const uint64_t since, const uint64_t until)
Definition: CrestFunctions.cxx:219
max
constexpr double max()
Definition: ap_fixedTest.cxx:33
IOVDbNamespace::CrestFunctions::setURLBase
void setURLBase(const std::string &crest_path)
Definition: CrestFunctions.cxx:43
IOVDbNamespace::CrestFunctions::extractChannelListFromString
std::pair< std::vector< cool::ChannelId >, std::vector< std::string > > extractChannelListFromString(const std::string &chanString)
Definition: CrestFunctions.cxx:174
parse
std::map< std::string, std::string > parse(const std::string &list)
Definition: egammaLayerRecalibTool.cxx:1080
IOVDbNamespace::CrestFunctions::getTagProperties
nlohmann::json getTagProperties(const std::string &tag)
Definition: CrestFunctions.cxx:147
IOVDbNamespace::CrestFunctions::CrestFunctions
CrestFunctions(const std::string &crest_path)
Definition: CrestFunctions.cxx:24
JetTiledMap::N
@ N
Definition: TiledEtaPhiMap.h:44
read_hist_ntuple.t
t
Definition: read_hist_ntuple.py:5
IOVDbNamespace::CrestFunctions::extractHashFromJson
std::string extractHashFromJson(const std::string &jsonReply)
Definition: CrestFunctions.cxx:49
dq_defect_copy_defect_database.since
def since
Definition: dq_defect_copy_defect_database.py:54
dq_defect_copy_defect_database.until
def until
Definition: dq_defect_copy_defect_database.py:55
IOVDbStringFunctions.h
lumiFormat.i
int i
Definition: lumiFormat.py:85
beamspotman.n
n
Definition: beamspotman.py:731
python.subdetectors.mmg.names
names
Definition: mmg.py:8
calibdata.exception
exception
Definition: calibdata.py:496
res
std::pair< std::vector< unsigned int >, bool > res
Definition: JetGroupProductTest.cxx:11
xAOD::uint64_t
uint64_t
Definition: EventInfo_v1.cxx:123
histSizes.list
def list(name, path='/')
Definition: histSizes.py:38
lumiFormat.array
array
Definition: lumiFormat.py:91
ActsTrk::to_string
std::string to_string(const DetectorType &type)
Definition: GeometryDefs.h:34
IOVDbNamespace::unescapeBackslash
std::string unescapeBackslash(const std::string &original)
Definition: IOVDbStringFunctions.cxx:102
item
Definition: ItemListSvc.h:43
IOVDbNamespace::CrestFunctions::getTagInfoElement
std::string getTagInfoElement(nlohmann::json tag_info, const std::string &key)
Definition: CrestFunctions.cxx:158
IOVDbNamespace::CrestFunctions::getTagInfo
nlohmann::json getTagInfo(const std::string &tag)
Definition: CrestFunctions.cxx:134
python.PyAthena.v
v
Definition: PyAthena.py:154
IOVDbNamespace::CrestFunctions::getResources
nlohmann::json getResources(nlohmann::json &js)
Definition: CrestFunctions.cxx:195
IOVDbNamespace::CrestFunctions::getURLBase
const std::string & getURLBase()
Definition: CrestFunctions.cxx:38
CaloCondBlobAlgs_fillNoiseFromASCII.hash
dictionary hash
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:109
IOVDbNamespace::CrestFunctions::getPayloadForHash
std::string getPayloadForHash(const std::string &hash)
Definition: CrestFunctions.cxx:88
IOVDbNamespace::CrestFunctions::getIovGroups
std::vector< uint64_t > getIovGroups(const std::string &tag)
Definition: CrestFunctions.cxx:202
Pythia8_RapidityOrderMPI.val
val
Definition: Pythia8_RapidityOrderMPI.py:14
IOVDbNamespace::unescapeQuotes
std::string unescapeQuotes(const std::string &original)
Definition: IOVDbStringFunctions.cxx:95
IOVDbNamespace::CrestFunctions::folderDescriptionForTag
std::string folderDescriptionForTag(const std::string &tag)
Definition: CrestFunctions.cxx:124
python.SystemOfUnits.s
float s
Definition: SystemOfUnits.py:146
tagname
Definition: tagname.h:29
IOVDbNamespace::CrestFunctions::getTagSize
int getTagSize(const std::string &tagname)
Definition: CrestFunctions.cxx:249
CaloCondBlobAlgs_fillNoiseFromASCII.tag
string tag
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:24
IOVDbNamespace::CrestFunctions::getLastHashForTag
std::string getLastHashForTag(const std::string &tag)
Definition: CrestFunctions.cxx:68
CrestFunctions.h
Header for CrestFunctions utilities.
IOVDbNamespace::CrestFunctions::extractDescriptionFromJson
std::string extractDescriptionFromJson(const std::string &jsonReply)
Definition: CrestFunctions.cxx:103
description
std::string description
glabal timer - how long have I taken so far?
Definition: hcg.cxx:88
IOVDbNamespace
Definition: Base64Codec.cxx:16
mapkey::key
key
Definition: TElectronEfficiencyCorrectionTool.cxx:37