ATLAS Offline Software
CrestApiFs.cxx
Go to the documentation of this file.
1 /*
2  Copyright (C) 2020-2024 CERN for the benefit of the ATLAS collaboration
3 */
4 
5 #include <CrestApi/CrestApiFs.h>
7 #include <CrestApi/CrestModel.h>
8 
9 #include <boost/uuid/uuid.hpp> // uuid class
10 #include <boost/uuid/uuid_generators.hpp> // generators
11 #include <boost/uuid/uuid_io.hpp>
12 #include <boost/asio.hpp>
13 
14 #include <fstream>
15 #include <filesystem>
16 #include <iostream>
17 
18 #include <CrestApi/picosha2.h>
19 
20 #include <cstdio>
21 #include <ctime>
22 #include <cstdlib>
23 
24 #include <algorithm>
25 
26 namespace Crest
27 {
28 
35  CrestFsClient::CrestFsClient(bool rewriteIfExists, const std::string &root_folder)
36  : m_root_folder(root_folder), m_isRewrite(rewriteIfExists)
37  {
38  if (m_root_folder == "") {
39  m_root_folder = std::filesystem::current_path();
40  m_root_folder += "/crest";
41  }
43  {
44  std::filesystem::create_directory(std::filesystem::path(m_root_folder));
45  }
46  m_data_folder = m_root_folder + "/data";
48  {
49  std::filesystem::create_directory(std::filesystem::path(m_data_folder));
50  }
51  }
52 
57 
58  // UTILITIES Methods
59 
60  std::string CrestFsClient::getFileString(const std::string& path) {
61  std::ifstream ifs(path);
62  std::stringstream buf;
63  buf << ifs.rdbuf();
64  return buf.str();
65  }
66 
67  std::string CrestFsClient::buildPath(const std::string &path, const std::string &name)
68  {
69  std::string p = m_root_folder;
71  {
72  std::filesystem::create_directory(std::filesystem::path(m_root_folder));
73  }
74  p += path;
76  {
77  std::filesystem::create_directory(std::filesystem::path(p));
78  }
79  p += '/';
80  p += name;
82  {
83  std::filesystem::create_directory(std::filesystem::path(p));
84  }
85  return p;
86  }
87 
88  void CrestFsClient::getFileList(const std::string &path)
89  {
91  for (auto i = std::filesystem::directory_iterator(p); i != std::filesystem::directory_iterator(); i++)
92  {
93  std::string file = i->path().filename().string();
94  if (file != "data")
95  {
96  std::cout << file << std::endl;
97  }
98  else{
99  continue;
100  }
101  }
102  }
103 
104  // Global tag methods:
105 
107  {
108  std::string name = "";
109  if (globalTag.name != "")
110  {
111  name = globalTag.name;
112  }
113  else
114  {
115  throw CrestException("ERROR in CrestFsClient::createGlobalTag: global tag name is not set.");
116  }
117 
118  std::string workDir = buildPath(s_FS_GLOBALTAG_PATH, name);
119  std::string globalTagFile = workDir + s_FS_GLOBALTAG_FILE;
120 
121  if (m_isRewrite)
122  {
124  {
126  }
127 
128  std::ofstream outFile;
129 
130  outFile.open(globalTagFile.c_str());
131  outFile << globalTag.to_json();
132  outFile.close();
133  }
134 
135  return;
136  }
137 
139  {
140  nlohmann::json js = nullptr;
141  GlobalTagDto dto;
142 
143  std::string workDir = buildPath(s_FS_GLOBALTAG_PATH, name);
144  std::string file_path = workDir + s_FS_GLOBALTAG_FILE;
145 
146  try
147  {
148  std::string tag = getFileString(file_path);
150  dto = GlobalTagDto::from_json(js);
151  }
152  catch (...)
153  {
154  throw CrestException(
155  "ERROR in CrestFsClient::findGlobalTag: cannot get the global tag " + name + " form the file storage.");
156  }
157 
158  return dto;
159  }
160 
161  void CrestFsClient::removeGlobalTag(const std::string &)
162  {
163  checkFsException("CrestFsClient::removeGlobalTag");
164  }
165 
166  GlobalTagSetDto CrestFsClient::listGlobalTags(const std::string &name, int size, int page, const std::string &sort)
167  {
168 
169  std::string folder = m_root_folder + s_FS_GLOBALTAG_PATH;
170 
171  GlobalTagSetDto tagSet;
172 
173  bool ascending = true;
174  if (sort == "name:ASC")
175  {
176  ascending = true;
177  }
178  else if (sort == "name:DESC")
179  {
180  ascending = false;
181  }
182  else
183  {
184  throw CrestException(
185  "ERROR in CrestFsClient::listTags: wrong sort parameter." + sort);
186  }
187 
188  try
189  {
190  std::vector<std::string> taglist = nameList(folder, ascending);
191  std::vector<std::string> clearedTaglist;
192 
193  for (const std::string& tag : taglist)
194  {
195  std::string file_name = folder + "/" + tag + "/" + s_FS_GLOBALTAG_FILE;
196 
198  {
199  if (name != "")
200  {
201  if (isMatch(tag, name))
202  {
203  clearedTaglist.push_back(tag);
204  }
205  }
206  else
207  {
208  clearedTaglist.push_back(tag);
209  }
210  }
211  }
212 
213  taglist = getVectorPage(clearedTaglist, size, page);
214  for (const std::string& tag : taglist)
215  {
216  std::string file_name = folder + "/" + tag + "/" + s_FS_GLOBALTAG_FILE;
218  tagSet.resources.push_back(dto);
219  }
220  tagSet.datatype = "";
221  }
222  catch (const std::exception &e)
223  {
224  throw CrestException(
225  "ERROR in CrestFsClient::listGlobalTags: cannot get the tag list.");
226  }
227 
228  return tagSet;
229  }
230 
231  // Tag methods:
232 
234  {
235  std::string name = tag.name;
236  std::string workDir = buildPath(s_FS_TAG_PATH, name);
237 
238  if (name.compare(m_currentTag) != 0 && m_isRewrite)
239  {
240  flush();
241  }
242  m_currentTag = name;
243  std::string tagFile = workDir + s_FS_TAG_FILE;
244  std::string iovFile = workDir + s_FS_IOV_FILE;
245 
246  if (m_isRewrite)
247  {
249  {
251  }
253  {
255  }
256  std::ofstream outFile;
257  outFile.open(tagFile.c_str());
258  outFile << tag.to_json();
259  outFile.close();
260  }
261  if (m_data.find(name) == m_data.end())
262  {
263  m_data.insert(std::pair<std::string, nlohmann::json>(name, nlohmann::json(nlohmann::json::value_t::array)));
264  }
265  }
266 
267  TagDto CrestFsClient::findTag(const std::string &name)
268  {
269  nlohmann::json js = nullptr;
270  TagDto dto;
271 
272  std::string workDir = buildPath(s_FS_TAG_PATH, name);
273  std::string file_path = workDir + s_FS_TAG_FILE;
274 
275  try
276  {
277  std::string tag = getFileString(file_path);
279  dto = TagDto::from_json(js);
280  }
281  catch (...)
282  {
283  throw CrestException(
284  "ERROR in CrestFsClient::findTag: cannot get the tag " + name + " form the file storage.");
285  }
286 
287  return dto;
288  }
289 
290  void CrestFsClient::removeTag(const std::string &)
291  {
292  checkFsException("CrestFsClient::removeTag");
293  }
294 
295  TagSetDto CrestFsClient::listTags(const std::string &name, int size, int page, const std::string &sort)
296  {
297 
298  std::string folder = m_root_folder + s_FS_TAG_PATH;
299 
300  TagSetDto tagSet;
301 
302  bool ascending = true;
303  if (sort == "name:ASC")
304  {
305  ascending = true;
306  }
307  else if (sort == "name:DESC")
308  {
309  ascending = false;
310  }
311  else
312  {
313  throw CrestException(
314  "ERROR in CrestFsClient::listTags: wrong sort parameter." + sort);
315  }
316 
317  try
318  {
319  std::vector<std::string> taglist = nameList(folder, ascending);
320  std::vector<std::string> clearedTaglist;
321 
322  for (const std::string& tag : taglist)
323  {
324  std::string file_name = folder + "/" + tag + "/" + s_FS_TAG_FILE;
325 
327  {
328  if (name != "")
329  {
330  if (isMatch(tag, name))
331  {
332  clearedTaglist.push_back(tag);
333  }
334  }
335  else
336  {
337  clearedTaglist.push_back(tag);
338  }
339  }
340  }
341 
342  taglist = getVectorPage(clearedTaglist, size, page);
343  for (const std::string& tag : taglist)
344  {
345  std::string file_name = folder + "/" + tag + "/" + s_FS_TAG_FILE;
346  TagDto dto = findTag(tag);
347  tagSet.resources.push_back(dto);
348  }
349  tagSet.datatype = "";
350  }
351  catch (const std::exception &e)
352  {
353  throw CrestException(
354  "ERROR in CrestFsClient::listTags: cannot get the tag list.");
355  }
356 
357  return tagSet;
358  }
359 
361  {
362 
363  for (auto &item : m_data)
364  {
365  nlohmann::json iov = item.second;
366  std::string name = item.first;
367  std::string workDir = m_root_folder + s_FS_TAG_PATH + '/' + name;
368  std::ofstream outFile;
369  std::string tagFile = workDir + s_FS_IOV_FILE;
370  outFile.open(tagFile.c_str());
371  outFile << iov;
372  outFile.close();
373  }
374 
375  m_data.clear();
376  }
377 
378  // TagMeta methods
380  {
381  std::string name = tag.tagName;
382 
383  std::string workDir = buildPath(s_FS_TAG_PATH, name);
384  std::string tagMetaFile = workDir + s_FS_TAGMETAINFO_FILE;
385 
386  if (m_isRewrite)
387  {
389  {
391  }
392 
393  std::ofstream outFile;
394 
395  outFile.open(tagMetaFile.c_str());
396  outFile << tag.to_json();
397  outFile.close();
398  }
399  }
400 
402  {
403  checkFsException("CrestFsClient::updateTagMeta");
404  }
405 
407  {
408  nlohmann::json js = nullptr;
409  TagMetaDto dto;
410 
411  std::string workDir = buildPath(s_FS_TAG_PATH, name);
412  std::string file_path = workDir + s_FS_TAGMETAINFO_FILE;
413 
414  try
415  {
416  std::string tag = getFileString(file_path);
418  dto = TagMetaDto::from_json(js);
419  }
420  catch (...)
421  {
422  throw CrestException(
423  "ERROR in CrestFsClient::findTagMeta: cannot get the tag " + name + " form the file storage.");
424  }
425 
426  return dto;
427  }
428 
429  // GlobalTagMap methods
430 
432  {
433  nlohmann::json js = globalTagMap.to_json();
434 
435  // global tag name:
436  std::string name = "";
437  try
438  {
439  name = static_cast<std::string>(js["globalTagName"]);
440  }
441  catch (...)
442  {
443  throw CrestException("ERROR in CrestClient::createGlobalTagMap: cannot get the global tag name from JSON.");
444  }
445 
446  // tag name:
447  std::string tagname = "";
448  try
449  {
450  tagname = static_cast<std::string>(js["tagName"]);
451  }
452  catch (...)
453  {
454  throw CrestException("ERROR in CrestClient::createGlobalTagMap: cannot get the tag name from JSON.");
455  }
456 
457  std::string workDir = buildPath(s_FS_GLOBALTAG_PATH, name);
458  std::string catalogFile = workDir + s_FS_MAP_FILE;
459 
461  {
462  // cathalogue file exists:
463 
464  std::string array_lst = getFileString(catalogFile);
465 
466  nlohmann::json cathalogue;
467  try
468  {
469  cathalogue = nlohmann::json::parse(array_lst);
470  }
471  catch (...)
472  {
473  throw CrestException("ERROR in CrestFsClient::createGlobalTagMap: global tag map file corrupted.");
474  }
475 
477  {
478  // the file storage contains the record of the global tag map:
479  int m = cathalogue.size();
480  for (int i = 0; i < m; i++)
481  {
482  const std::string &tn = cathalogue[i]["tagName"];
483  if (tn == tagname)
484  {
485  cathalogue.erase(i);
486  }
487  }
488  }
489  else
490  {
491  // the file storage does not contain the record of the global tag map:
492  }
494  cathalogue.push_back(js);
495  std::ofstream outFile;
496 
497  outFile.open(catalogFile.c_str());
498  outFile << cathalogue;
499  outFile.close();
500  }
501  else
502  {
503  // cathalogue file does not exist (creation):
504 
505  nlohmann::json cathalogue = nlohmann::json::array();
506  cathalogue.push_back(js);
507 
508  std::ofstream outFile;
509 
510  outFile.open(catalogFile.c_str());
511  outFile << cathalogue;
512  outFile.close();
513  }
514 
515  return;
516  }
517 
518  void CrestFsClient::removeGlobalTagMap(const std::string &, const std::string &, const std::string &, const std::string &)
519  {
520  checkFsException("CrestFsClient::removeGlobalTagMap");
521  }
522 
523  void CrestFsClient::checkFsException(const char *method_name)
524  {
525  throw CrestException("ERROR in " + std::string(method_name) + " This methods is unsupported for FILESYSTEM mode");
526  }
527 
528  GlobalTagMapSetDto CrestFsClient::findGlobalTagMap(const std::string &name, const std::string &xCrestMapMode)
529  {
530 
531  nlohmann::json js = nullptr;
532 
533  if (xCrestMapMode != "Trace")
534  {
535  throw CrestException(
536  "ERROR in CrestFsClient::getGlobalTagMap: not supported value for the parameter xCrestMapMode = " + xCrestMapMode);
537  }
538 
539  std::string workDir = m_root_folder + s_FS_GLOBALTAG_PATH;
540  std::string file_path = workDir;
541  file_path += '/';
542  file_path += name;
544 
545  try
546  {
547  std::string tag = getFileString(file_path);
549  }
550  catch (...)
551  {
552  throw CrestException(
553  "ERROR in CrestFsClient::getGlobalTagMap: cannot get the global tag map " + name +
554  " form the file storage.");
555  }
556 
557  GlobalTagMapSetDto dto;
559  return dto;
560  }
561 
562  // Iovs methods
563 
564  IovSetDto CrestFsClient::selectIovs(const std::string &name, uint64_t since, uint64_t until, long , int size, int page, const std::string &sort)
565  {
566  IovSetDto dto;
567 
569 
570  try
571  {
573  int niovs = iovList.size();
574 
575  for (int i = 0; i < niovs; i++)
576  {
577  if (iovList[i].find("since") != iovList[i].end())
578  {
579  uint64_t currentS = iovList[i]["since"];
580 
581  if (until != static_cast<uint64_t>(-1))
582  {
583  if (currentS >= since && currentS <= until)
584  {
585  js.push_back(iovList[i]);
586  }
587  }
588  else
589  { // until == -1, Infinity
590  if (currentS >= since)
591  {
592  js.push_back(iovList[i]);
593  }
594  } // until == -1
595  }
596  } // for
597  }
598  catch (...)
599  {
600  throw CrestException("ERROR in CrestClient::selectIovsFS : cannot get the iov list form file storage");
601  }
602 
603  bool ascending = true;
604  if (sort == "id.since:ASC")
605  ascending = true;
606  else if (sort == "id.since:DESC")
607  ascending = false;
608  else
609  {
610  throw CrestException(
611  "ERROR in CrestFsClient::selectIovs: wrong sort parameter." + sort);
612  }
613 
617 
618  return dto;
619  }
620 
622  {
623  nlohmann::json js = nullptr;
624  std::string file_path = m_root_folder;
625  file_path += '/';
627  file_path += '/';
628  file_path += tagname;
630 
631  try
632  {
633  std::string tag = getFileString(file_path);
635  }
636  catch (const std::exception &e)
637  {
638  throw CrestException("ERROR in CrestFsClient::findAllIovs : cannot get the iov information form file storage ");
639  }
640 
641  return js;
642  }
643 
644  int CrestFsClient::getSize(const std::string &tagname)
645  {
646  int res = 0;
647 
648  try
649  {
651  int length = iovs.size();
652  return length;
653  }
654  catch (...)
655  {
656  return res;
657  }
658 
659  return res;
660  }
661 
662  IovSetDto CrestFsClient::selectGroups(const std::string &, long , int , int , const std::string &)
663  {
664  checkFsException("CrestFsClient::selectGroups");
665  return IovSetDto();
666  }
667 
668  // Payload methods
669  void CrestFsClient::storeData(const std::string &tag,
670  const StoreSetDto &storeSetJson,
671  const std::string &payloadFormat,
672  const std::string &objectType,
673  const std::string &compressionType,
674  const std::string &version,
675  uint64_t)
676  {
677  std::string name = tag;
678 
679  std::string workDir = buildPath(s_FS_TAG_PATH, name);
680  std::string tagFile = workDir + s_FS_TAG_FILE;
681  std::string iovFile = workDir + s_FS_IOV_FILE;
682 
683  nlohmann::json js_data = storeSetJson.to_json();
684 
686  auto it = js_data.find("resources");
687  if (it != js_data.end())
688  {
689  res = js_data["resources"];
690  }
691 
692  try
693  {
694  for (auto &kvp : res)
695  {
696  std::string payload = kvp.value("data", "");
697  int since = kvp.value("since", 0);
698  std::string streamer = kvp.value("streamerInfo", "");
699  // Register everything on the file system.
700  storePayloadDump(tag, since, payload, payloadFormat,
701  objectType, compressionType, version, streamer);
702  }
703  } // end of try
704  catch (...)
705  {
706  throw CrestException("ERROR in CrestFsClient::storeData cannot store the data in a file");
707  } // end of catch
708  flush();
709  }
710 
712  {
713  time_t now = time(0);
714  struct tm tstruct;
715  char buf[80];
716 
717  localtime_r(&now, &tstruct);
718  strftime(buf, sizeof(buf), "%Y-%m-%d %X", &tstruct);
719  return buf;
720  }
721 
722  std::string CrestFsClient::getFirstLetters(const std::string &str)
723  {
724  std::string result = str.substr(0, s_FS_PREFIX_LENGTH);
725  return result;
726  }
727 
728  void CrestFsClient::storePayloadDump(const std::string &tag,
729  uint64_t since,
730  const std::string &js,
731  const std::string &payloadFormat,
732  const std::string &objectType,
733  const std::string &compressionType,
734  const std::string &version,
735  const std::string &streamerInfo)
736  {
737 
738  std::ofstream outFile;
739 
740  std::string hashCode;
741  std::string payloadLocalFile;
742 
743  // payload file:
744  if (payloadFormat == "JSON")
745  {
746  hashCode = getHash(js);
747  }
748  else
749  {
750 
751  int found_dots = js.find_first_of(':');
752  int word_size = js.size();
753  payloadLocalFile = js.substr(found_dots + 3, word_size);
754 
755  hashCode = getHashForFile(payloadLocalFile);
756  }
757 
758  std::string workDir = m_data_folder;
759  workDir += '/';
760  workDir += getFirstLetters(hashCode);
762  {
763  std::filesystem::create_directory(std::filesystem::path(workDir));
764  }
765  workDir += '/';
766  workDir += hashCode;
768  {
769  std::filesystem::create_directory(std::filesystem::path(workDir));
770  }
771  std::string tagFile = workDir + "/payload.json";
772 
773  if (payloadFormat == "JSON")
774  {
775  outFile.open(tagFile);
776  outFile << js;
777  outFile.close();
778  }
779  else
780  {
781  try
782  {
783  std::filesystem::copy_file(payloadLocalFile, tagFile);
784  }
785  catch (std::filesystem::filesystem_error &e)
786  {
787  throw CrestException("ERROR in CrestFsClient::storePayloadDump cannot not save payload file: " + tagFile + e.what());
788  }
789  }
790 
791  // Define the meta info for the payload:
792  nlohmann::json jsn =
793  {
794  {"hash", hashCode},
795  {"checkSum", "SHA-256"},
796  {"objectType", objectType},
797  {"version", version},
798  {"size", js.size()},
799  {"streamerInfo", streamerInfo},
800  {"compressionType", compressionType},
801  {"insertionTime", getDateAndTime()}};
802 
803  // payload meta info file:
804  std::string metaFile = workDir + "/meta.json";
805 
806  outFile.open(metaFile);
807  outFile << jsn.dump();
808  outFile.close();
809 
810  // check if data exists
811 
812  if (m_data.find(tag) == m_data.end())
813  {
814  try
815  {
817  m_data.insert(std::pair<std::string, nlohmann::json>(tag, jsi));
818  }
819  catch (...)
820  {
821  try
822  { // tag exists, but there are no IOVs
823  nlohmann::json tagJS = findTag(tag).to_json();
825  m_data.insert(std::pair<std::string, nlohmann::json>(tag, jsFree));
826  }
827  catch (...)
828  {
829  throw CrestException(
830  "ERROR in CrestFsClient::storePayloadDump cannot get data for tag \"" + tag + "\" from file storage.");
831  }
832  }
833  }
834 
836  if (it != m_data.end())
837  {
838  std::string link = hashCode;
839  nlohmann::json iovs = it->second;
841  obj["tagName"] = tag;
842  obj["since"] = since;
843  obj["insertionTime"] = getDateAndTime();
844  obj["payloadHash"] = link;
845  iovs.push_back(obj);
846  m_data[it->first] = iovs;
847  }
848  }
849 
850 
851  // Payload methods
852 
853  std::string CrestFsClient::getPayload(const std::string &hash)
854  {
855  std::string workDir = m_data_folder;
856  workDir += '/';
857  workDir += getFirstLetters(hash);
858  workDir += '/';
859  workDir += hash;
860  std::string filePath = workDir + "/payload.json";
861  std::string res = "";
862 
863  try
864  {
866  {
868  }
869  else
870  {
871  throw CrestException("payload with hash " + hash + " does not exist.");
872  }
873  }
874  catch (const std::exception &e)
875  {
876  std::string message = e.what();
877  throw CrestException("ERROR in CrestFsClient::getPayload cannot get the payload form file storage, " + message);
878  }
879 
880  return res;
881  }
882 
884  {
885  nlohmann::json js = nullptr;
886  PayloadDto dto;
887 
888  std::string workDir = m_data_folder;
889  workDir += '/';
890  workDir += getFirstLetters(hash);
891  workDir += '/';
892  workDir += hash;
893 
894  std::string filePath = workDir;
895  filePath += "/meta.json";
896 
897  std::string res = "";
898 
899  try
900  {
902  {
905  dto = PayloadDto::from_json(js);
906  }
907  else
908  {
909  throw CrestException("payload meta info with hash " + hash + " does not exist.");
910  }
911  }
912  catch (const std::exception &e)
913  {
914  std::string message = e.what();
915  throw CrestException("ERROR in CrestClient::getPayloadMeta cannot get the payload meta info form file storage, " + message);
916  }
917 
918  return dto;
919  }
920 
922  {
924  int dataSize = data.size();
925 
926  if (dataSize == 0)
927  return js; // the data is absent
928 
929  // index interval to load the data from JSON array:
930  int kmin = size * page;
931  int kmax = size * (page + 1);
932 
933  // check if the interval is correct:
934  if (kmin > dataSize - 1)
935  return js; // out of range
936 
937  if (kmax > dataSize - 1)
938  { // this step is not full
939  kmax = dataSize;
940  }
941 
942  for (int i = kmin; i < kmax; i++)
943  {
944  js.push_back(data[i]);
945  }
946  return js;
947  }
948 
949  std::vector<std::string> CrestFsClient::getVectorPage(const std::vector<std::string>& data, int size, int page)
950  {
951  std::vector<std::string> res;
952  int dataSize = data.size();
953 
954  if (dataSize == 0)
955  return res; // the data is absent
956 
957  // index interval to load the data from JSON array:
958  int kmin = size * page;
959  int kmax = size * (page + 1);
960 
961  // check if the interval is correct:
962  if (kmin > dataSize - 1)
963  return res; // out of range
964 
965  if (kmax > dataSize - 1)
966  { // this step is not full
967  kmax = dataSize;
968  }
969 
970  for (int i = kmin; i < kmax; i++)
971  {
972  res.push_back(data[i]);
973  }
974  return res;
975  }
976 
977  std::vector<std::string> CrestFsClient::nameList(std::string &folder, bool ascending)
978  {
979  std::vector<std::string> tag_list;
981 
982  for (auto i = std::filesystem::directory_iterator(p); i != std::filesystem::directory_iterator(); i++)
983  {
984  std::string file = i->path().filename().string();
985  tag_list.push_back(file);
986  }
987 
988  std::sort(tag_list.begin(), tag_list.end());
989 
990  if (ascending == false)
991  {
992  std::reverse(tag_list.begin(), tag_list.end());
993  }
994 
995  return tag_list;
996  }
997 
998  // method to sort JSON array by the key (par) value
1000  {
1002  std::vector<std::string> parlist;
1003  std::map<std::string, nlohmann::json> m;
1004 
1005  int size = js.size();
1006  for (int i = 0; i < size; i++)
1007  {
1008  nlohmann::json elem = js[i];
1009 
1010  auto res = elem.find(par);
1011 
1012  if (res != elem.end())
1013  {
1014  std::string par_val = elem[par];
1015  parlist.push_back(par_val);
1016  m[par_val] = elem;
1017  }
1018  }
1019 
1020  std::sort(parlist.begin(), parlist.end());
1021 
1022  if (order == false)
1023  {
1024  std::reverse(parlist.begin(), parlist.end());
1025  }
1026 
1027  for (std::string item : parlist)
1028  {
1029  respond.push_back(m[item]);
1030  }
1031 
1032  return respond;
1033  }
1034 
1035  // method to sort JSON array with IOVs by the since value
1037  {
1038  std::string par = "since";
1040  std::vector<double> parlist;
1041  std::map<double, nlohmann::json> m;
1042 
1043  int size = js.size();
1044  for (int i = 0; i < size; i++)
1045  {
1046  nlohmann::json elem = js[i];
1047 
1048  auto res = elem.find(par);
1049 
1050  if (res != elem.end())
1051  {
1052  double par_val = elem[par];
1053  parlist.push_back(par_val);
1054  m[par_val] = elem;
1055  }
1056  }
1057 
1058  std::sort(parlist.begin(), parlist.end());
1059 
1060  if (order == false)
1061  {
1062  std::reverse(parlist.begin(), parlist.end());
1063  }
1064 
1065  for (double item : parlist)
1066  {
1067  respond.push_back(m[item]);
1068  }
1069 
1070  return respond;
1071  }
1072 
1073  bool CrestFsClient::isMatch(const std::string& word, long unsigned int n, const std::string& pattern, long unsigned int m)
1074  {
1075  if (m == pattern.size())
1076  {
1077  return n == word.size();
1078  }
1079 
1080  if (n == word.size())
1081  {
1082  for (long unsigned int i = m; i < pattern.size(); i++)
1083  {
1084  if (pattern[i] != '%')
1085  {
1086  return false;
1087  }
1088  }
1089 
1090  return true;
1091  }
1092 
1093  if (pattern[m] == '?' || pattern[m] == word[n])
1094  {
1095  return isMatch(word, n + 1, pattern, m + 1);
1096  }
1097 
1098  if (pattern[m] == '%')
1099  {
1100  return isMatch(word, n + 1, pattern, m) || isMatch(word, n, pattern, m + 1);
1101  }
1102 
1103  return false;
1104  }
1105 
1106  bool CrestFsClient::isMatch(const std::string& word, const std::string& pattern)
1107  {
1108  return isMatch(word, 0, pattern, 0);
1109  }
1110 
1112  {
1113  throw CrestException(
1114  "ERROR in CrestFsClient::getCrestVersion: cannot get the CREST server version for file storage.");
1115  }
1116 
1117 } // namespace Crest
1118 
xAOD::iterator
JetConstituentVector::iterator iterator
Definition: JetConstituentVector.cxx:68
mergePhysValFiles.pattern
pattern
Definition: DataQuality/DataQualityUtils/scripts/mergePhysValFiles.py:26
AllowedVariables::e
e
Definition: AsgElectronSelectorTool.cxx:37
Crest::CrestFsClient::s_FS_GLOBALTAG_PATH
static const std::string s_FS_GLOBALTAG_PATH
Definition: CrestApiFs.h:42
Crest::CrestFsClient::s_FS_TAG_PATH
static const std::string s_FS_TAG_PATH
Definition: CrestApiFs.h:41
IovSetDto::from_fs_json
static IovSetDto from_fs_json(const json &j)
Definition: CrestModel.cxx:531
GlobalTagMapSetDto::from_fs_json
static GlobalTagMapSetDto from_fs_json(const json &j)
Definition: CrestModel.cxx:277
data
char data[hepevt_bytes_allocation_ATLAS]
Definition: HepEvt.cxx:11
Crest::CrestFsClient::~CrestFsClient
~CrestFsClient()
CrestClient destructor.
Definition: CrestApiFs.cxx:56
GlobalTagMapDto::to_json
json to_json() const
Definition: CrestModel.cxx:235
get_generator_info.result
result
Definition: get_generator_info.py:21
dqt_zlumi_alleff_HIST.iov
iov
Definition: dqt_zlumi_alleff_HIST.py:119
python.SystemOfUnits.m
int m
Definition: SystemOfUnits.py:91
Crest::CrestFsClient::updateTagMeta
void updateTagMeta(TagMetaDto &tag) override
This method updates a tag meta info on the file storage.
Definition: CrestApiFs.cxx:401
athena.path
path
python interpreter configuration --------------------------------------—
Definition: athena.py:128
Crest::CrestFsClient::createTagMeta
void createTagMeta(TagMetaDto &tag) override
This method creates a tag meta info on the file storage.
Definition: CrestApiFs.cxx:379
Crest::CrestFsClient::listGlobalTags
GlobalTagSetDto listGlobalTags(const std::string &name, int size, int page, const std::string &sort) override
This method finds the global tags on the file storage.
Definition: CrestApiFs.cxx:166
Crest::CrestFsClient::m_data_folder
std::string m_data_folder
Definition: CrestApiFs.h:35
find
std::string find(const std::string &s)
return a remapped string
Definition: hcg.cxx:135
Crest::CrestFsClient::listTags
TagSetDto listTags(const std::string &name, int size, int page, const std::string &sort) override
This method returns the tag list as TagSetDto.
Definition: CrestApiFs.cxx:295
json
nlohmann::json json
Definition: HistogramDef.cxx:9
Crest::CrestFsClient::getFirstLetters
std::string getFirstLetters(const std::string &str)
This is an auxillary method extract first letters from the string (hash).
Definition: CrestApiFs.cxx:722
GlobalTagSetDto::resources
std::vector< GlobalTagDto > resources
Definition: CrestModel.h:83
find_tgc_unfilled_channelids.iovs
iovs
Definition: find_tgc_unfilled_channelids.py:12
TagDto::to_json
json to_json() const
Definition: CrestModel.cxx:172
Crest::CrestFsClient::sortIOVJson
nlohmann::json sortIOVJson(nlohmann::json js, bool order)
Definition: CrestApiFs.cxx:1036
parse
std::map< std::string, std::string > parse(const std::string &list)
Definition: egammaLayerRecalibTool.cxx:1054
CrestModel.h
skel.it
it
Definition: skel.GENtoEVGEN.py:396
Crest::CrestFsClient::getFileList
void getFileList(const std::string &path)
Auxiliary method to get a file list in the given directory.
Definition: CrestApiFs.cxx:88
Crest::CrestFsClient::getSize
int getSize(const std::string &tagname) override
This method gets the number of IOVs for the given tag.
Definition: CrestApiFs.cxx:644
CrestApiFs.h
Header file for CREST C++ Client Library.
Crest::CrestFsClient::s_FS_IOV_FILE
static const std::string s_FS_IOV_FILE
Definition: CrestApiFs.h:46
Crest::CrestFsClient::selectIovs
IovSetDto selectIovs(const std::string &name, uint64_t since, uint64_t until, long snapshot, int size, int page, const std::string &sort) override
This method selects IOVs for a given tagname from the file storage.
Definition: CrestApiFs.cxx:564
TagSetDto
Definition: CrestModel.h:126
CheckTagAssociation.taglist
taglist
Definition: CheckTagAssociation.py:103
Crest::CrestFsClient::findTagMeta
TagMetaDto findTagMeta(const std::string &name) override
This method gets a tag meta info by the tag name from the file storage.
Definition: CrestApiFs.cxx:406
LArConditionsTestConfig.catalogFile
catalogFile
Definition: LArConditionsTestConfig.py:83
WriteCaloSwCorrections.tag_list
tag_list
Definition: WriteCaloSwCorrections.py:28
TagDto::from_json
static TagDto from_json(const json &j)
Definition: CrestModel.cxx:188
Crest::CrestFsClient::flush
void flush()
Auxiliary method flush the data to the file system.
Definition: CrestApiFs.cxx:360
GlobalTagSetDto
Definition: CrestModel.h:81
Crest
Definition: CrestApi.h:30
Crest::CrestApiBase::getHash
std::string getHash(std::string_view str)
This method method calculates the hash for string.
Definition: CrestApiBase.cxx:32
ReweightUtils.message
message
Definition: ReweightUtils.py:15
makeDTCalibBlob_pickPhase.globalTag
globalTag
Definition: makeDTCalibBlob_pickPhase.py:398
Crest::CrestFsClient::findGlobalTagMap
GlobalTagMapSetDto findGlobalTagMap(const std::string &name, const std::string &xCrestMapMode) override
This method searches for tag mappings using the global tag name or tag name on the file storage.
Definition: CrestApiFs.cxx:528
Crest::CrestFsClient::createTag
void createTag(TagDto &tag) override
This method creates a tag on the file storage.
Definition: CrestApiFs.cxx:233
TagMetaDto
Definition: CrestModel.h:238
mergePhysValFiles.end
end
Definition: DataQuality/DataQualityUtils/scripts/mergePhysValFiles.py:93
hotSpotInTAG.objectType
objectType
Definition: hotSpotInTAG.py:107
DeMoUpdate.reverse
reverse
Definition: DeMoUpdate.py:563
PayloadDto
Definition: CrestModel.h:345
dq_defect_copy_defect_database.since
def since
Definition: dq_defect_copy_defect_database.py:54
dq_defect_copy_defect_database.until
def until
Definition: dq_defect_copy_defect_database.py:55
Crest::CrestFsClient::s_FS_MAP_FILE
static const std::string s_FS_MAP_FILE
Definition: CrestApiFs.h:53
PixelModuleFeMask_create_db.remove
string remove
Definition: PixelModuleFeMask_create_db.py:83
Crest::CrestFsClient::isMatch
bool isMatch(const std::string &word, long unsigned int n, const std::string &pattern, long unsigned int m)
Definition: CrestApiFs.cxx:1073
Crest::CrestFsClient::nameList
std::vector< std::string > nameList(std::string &folder, bool ascending=true)
Definition: CrestApiFs.cxx:977
python.setupRTTAlg.size
int size
Definition: setupRTTAlg.py:39
physics_parameters.file_name
string file_name
Definition: physics_parameters.py:32
TagSetDto::resources
std::vector< TagDto > resources
Definition: CrestModel.h:128
GlobalTagDto::from_json
static GlobalTagDto from_json(const json &j)
Definition: CrestModel.cxx:155
Crest::CrestFsClient::getFileString
std::string getFileString(const std::string &path)
Auxiliary method to get a file as a string.
Definition: CrestApiFs.cxx:60
Crest::CrestFsClient::m_root_folder
std::string m_root_folder
Definition: CrestApiFs.h:34
python.handimod.now
now
Definition: handimod.py:675
python.utils.AtlRunQueryDQUtils.p
p
Definition: AtlRunQueryDQUtils.py:210
defineDB.tagname
string tagname
Definition: JetTagCalibration/share/defineDB.py:19
Crest::CrestFsClient::s_FS_TAG_FILE
static const std::string s_FS_TAG_FILE
Definition: CrestApiFs.h:45
lumiFormat.i
int i
Definition: lumiFormat.py:85
GlobalTagDto
Definition: CrestModel.h:55
CrestRequest.h
Crest::CrestFsClient::getDateAndTime
std::string getDateAndTime()
Definition: CrestApiFs.cxx:711
beamspotman.n
n
Definition: beamspotman.py:731
PayloadDto::from_json
static PayloadDto from_json(const json &j)
Definition: CrestModel.cxx:627
TagDto
The TagDto class It contains all fields of the TagDto class from the CREST API.
Definition: CrestModel.h:98
mc.order
order
Configure Herwig7.
Definition: mc.Herwig7_Dijet.py:12
Crest::CrestFsClient::CrestFsClient
CrestFsClient(bool rewriteIfExists, const std::string &root_folder)
CrestFsClient constructor.
Definition: CrestApiFs.cxx:35
calibdata.exception
exception
Definition: calibdata.py:496
res
std::pair< std::vector< unsigned int >, bool > res
Definition: JetGroupProductTest.cxx:14
athena.file_path
file_path
Definition: athena.py:94
file
TFile * file
Definition: tile_monitor.h:29
Crest::CrestFsClient::s_FS_PREFIX_LENGTH
static const int s_FS_PREFIX_LENGTH
Definition: CrestApiFs.h:57
xAOD::uint64_t
uint64_t
Definition: EventInfo_v1.cxx:123
Crest::CrestFsClient::createGlobalTagMap
void createGlobalTagMap(GlobalTagMapDto &globalTagMap) override
This method creates a global tag map on the file storage.
Definition: CrestApiFs.cxx:431
Crest::CrestFsClient::checkFsException
void checkFsException(const char *method_name)
Auxiliary method to check if the CrestApi library initialized to work with the file system.
Definition: CrestApiFs.cxx:523
MakeFileForMJB.ext
string ext
Definition: Moriond2016/MakeFileForMJB.py:41
python.TriggerAPI.TriggerAPISession.ascending
ascending
Definition: TriggerAPISession.py:435
Crest::CrestFsClient::sortJson
nlohmann::json sortJson(nlohmann::json js, const std::string &par, bool order)
Definition: CrestApiFs.cxx:999
GlobalTagMapSetDto
Definition: CrestModel.h:160
Crest::CrestFsClient::getCrestVersion
std::string getCrestVersion() override
This method returns the full CREST Server version.
Definition: CrestApiFs.cxx:1111
CrestBaseResponse::datatype
std::optional< std::string > datatype
Definition: CrestModel.h:45
DerivationFramework::TriggerMatchingUtils::sorted
std::vector< typename T::value_type > sorted(T begin, T end)
Helper function to create a sorted vector from an unsorted one.
DQPostProcessTest.outFile
outFile
Comment Out Those You do not wish to run.
Definition: DQPostProcessTest.py:37
Crest::CrestFsClient::findGlobalTag
GlobalTagDto findGlobalTag(const std::string &name) override
This method finds a global tag by name on the file storage.
Definition: CrestApiFs.cxx:138
lumiFormat.array
array
Definition: lumiFormat.py:91
Crest::CrestFsClient::findTag
TagDto findTag(const std::string &name) override
This method finds a tag by the name on the file storage.
Definition: CrestApiFs.cxx:267
Crest::CrestFsClient::createGlobalTag
void createGlobalTag(GlobalTagDto &globalTag) override
This method creates a global tag on the file storage.
Definition: CrestApiFs.cxx:106
Crest::CrestFsClient::s_FS_TAGMETAINFO_FILE
static const std::string s_FS_TAGMETAINFO_FILE
Definition: CrestApiFs.h:47
hancool.filePath
string filePath
Definition: hancool.py:28
name
std::string name
Definition: Control/AthContainers/Root/debug.cxx:228
createCoolChannelIdFile.par
par
Definition: createCoolChannelIdFile.py:29
Crest::CrestFsClient::getVectorPage
std::vector< std::string > getVectorPage(const std::vector< std::string > &data, int size, int page)
Definition: CrestApiFs.cxx:949
PayloadHelpers::dataSize
size_t dataSize(TDA::PayloadIterator start)
Size in bytes of the buffer that is needed to decode next fragment data content.
Definition: TriggerEDMDeserialiserAlg.cxx:188
Crest::CrestFsClient::removeGlobalTagMap
void removeGlobalTagMap(const std::string &name, const std::string &record, const std::string &label, const std::string &tagname) override
This method removes a global tag map from the file storage.
Definition: CrestApiFs.cxx:518
item
Definition: ItemListSvc.h:43
Crest::CrestFsClient::s_FS_GLOBALTAG_FILE
static const std::string s_FS_GLOBALTAG_FILE
Definition: CrestApiFs.h:52
PixelModuleFeMask_create_db.payload
string payload
Definition: PixelModuleFeMask_create_db.py:69
Crest::CrestFsClient::selectGroups
IovSetDto selectGroups(const std::string &name, long snapshot, int size, int page, const std::string &sort) override
This method returns IOV groups for a given tagname.
Definition: CrestApiFs.cxx:662
Crest::CrestFsClient::getPage
nlohmann::json getPage(nlohmann::json data, int size, int page)
Definition: CrestApiFs.cxx:921
MyPlots.page
page
Definition: MyPlots.py:234
get_generator_info.version
version
Definition: get_generator_info.py:33
TagMetaDto::from_json
static TagMetaDto from_json(const json &j)
Definition: CrestModel.cxx:401
Crest::CrestFsClient::m_data
std::map< std::string, nlohmann::json > m_data
Definition: CrestApiFs.h:36
Crest::CrestFsClient::m_currentTag
std::string m_currentTag
Definition: CrestApiFs.h:58
Crest::CrestFsClient::removeGlobalTag
void removeGlobalTag(const std::string &name) override
This method removes a global tag.
Definition: CrestApiFs.cxx:161
CaloCondBlobAlgs_fillNoiseFromASCII.hash
dictionary hash
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:109
Crest::CrestFsClient::buildPath
std::string buildPath(const std::string &path, const std::string &file)
Definition: CrestApiFs.cxx:67
StoreSetDto::to_json
json to_json() const
Definition: CrestModel.cxx:576
IovSetDto
Definition: CrestModel.h:287
Crest::CrestFsClient::storePayloadDump
void storePayloadDump(const std::string &tag, uint64_t since, const std::string &js, const std::string &payloadFormat, const std::string &objectType, const std::string &compressionType, const std::string &version, const std::string &streamerInfo)
This auxiliary method stores a single payload (with since time) on the file system.
Definition: CrestApiFs.cxx:728
picosha2.h
tagname
Definition: tagname.h:29
CaloCondBlobAlgs_fillNoiseFromASCII.folder
folder
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:56
pickleTool.object
object
Definition: pickleTool.py:30
PlotCalibFromCool.iovList
iovList
Definition: PlotCalibFromCool.py:351
str
Definition: BTagTrackIpAccessor.cxx:11
Crest::CrestApiBase::getHashForFile
std::string getHashForFile(const std::string &file)
This method method calculates the hash for a file.
Definition: CrestApiBase.cxx:8
Crest::CrestFsClient::getPayloadMeta
PayloadDto getPayloadMeta(const std::string &hash) override
This method finds a payload meta info for the hash on the file storage.
Definition: CrestApiFs.cxx:883
CaloCondBlobAlgs_fillNoiseFromASCII.tag
string tag
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:24
python.dummyaccess.exists
def exists(filename)
Definition: dummyaccess.py:9
StoreSetDto
Definition: CrestModel.h:331
Crest::CrestFsClient::getPayload
std::string getPayload(const std::string &hash) override
This method finds a payload resource associated to the hash on the file storage.
Definition: CrestApiFs.cxx:853
Crest::CrestFsClient::storeData
void storeData(const std::string &tag, const StoreSetDto &storeSetJson, const std::string &payloadFormat="JSON", const std::string &objectType="none", const std::string &compressionType="none", const std::string &version="1.0", uint64_t endTime=-1) override
This method stores several payloads in batch mode on the file storage.
Definition: CrestApiFs.cxx:669
python.PyAthena.obj
obj
Definition: PyAthena.py:132
Crest::CrestFsClient::m_isRewrite
bool m_isRewrite
Definition: CrestApiFs.h:37
length
double length(const pvec &v)
Definition: FPGATrackSimLLPDoubletHoughTransformTool.cxx:26
Crest::CrestFsClient::removeTag
void removeTag(const std::string &name) override
This method removes a tag from the file storage.
Definition: CrestApiFs.cxx:290
Crest::CrestException
Definition: CrestException.h:9
GlobalTagMapDto
Definition: CrestModel.h:139
Crest::CrestFsClient::findAllIovs
nlohmann::json findAllIovs(const std::string &tagname)
This auxiliary method finds all iovs for a given tag name.
Definition: CrestApiFs.cxx:621