CrestApi.h

Go to the documentation of this file.

/*
   Copyright (C) 2019-2023 CERN for the benefit of the ATLAS collaboration
 */
 
 
#ifndef CRESTAPI_CRESAPI_H
#define CRESTAPI_CRESAPI_H
 
#include <string>
#include <map>
#include <list>
#include <iosfwd>
#include <cstdint>
#include "nlohmann/json.hpp"
#include <curl/curl.h>
#include <iostream>
 
 
namespace Crest {
 
// AUXILIARY CLASS to store URL request parameters
  class urlParameters
  {
  private:
    std::string m_str {};
  public:
    void add(const std::string& key, const std::string& value) {
      if (m_str.empty()) {
        m_str = key;
        m_str += '=';
        m_str += value;
      } else {
        m_str +='&';
        m_str += key;
        m_str += '=';
        m_str += value;
      }
    }
 
    const std::string &getParams() const {
      return m_str;
    }
 
    void reset() {
      m_str.clear();
    }
  };
 
 
 
  class CrestClient
  {
  private:
    std::string make_url(const std::string &address) const;
    std::string m_host {};
    std::string m_port {};
    int m_mode {};
    std::string m_root_folder {};
    std::string m_data_folder {};
    std::map<std::string, nlohmann::json> m_data {};
    bool m_isRewrite {};
    std::string m_currentTag {};
 
    std::string m_PATH = "/api-v4.0";
    char* m_CREST_PROXY = NULL;
    const char* m_CREST_PROXY_VAR = "SOCKS_PROXY";
    inline static const std::string s_TAG_PATH = "/tags";
    inline static const std::string s_ADMIN_PATH = "/admin";
    inline static const std::string s_IOV_PATH = "/iovs";
 
    inline static const std::string s_IOV_SIZE_PATH = "/size";
 
    inline static const std::string s_GLOBALTAG_PATH = "/globaltags";
    inline static const std::string s_GLOBALTAG_MAP_PATH = "/globaltagmaps";
    inline static const std::string s_PAYLOAD_PATH = "/payloads";
    inline static const std::string s_MONITORING_PAYLOAD_PATH = "/monitoring/payloads";
    inline static const std::string s_META_PATH = "/meta";
    inline static const std::string s_DATA_PATH = "/data";
 
    inline static const std::string s_FOLDER_PATH = "/folders";
    inline static const std::string s_RUNINFO_PATH = "/runinfo";
    inline static const std::string s_RUNINFO_LIST_PATH = "/list";
 
    inline static const std::string s_METHOD_IOVS = "IOVS";
    inline static const std::string s_METHOD_GROUPS = "GROUPS";
    inline static const std::string s_METHOD_MONITOR = "MONITOR";
 
 
    // parameters for CREST file storage:
 
    inline static const std::string s_FS_TAG_PATH = "/tags";
    inline static const std::string s_FS_GLOBALTAG_PATH = "/globaltags";
    inline static const std::string s_FS_DATA_PATH = "/data";
 
    inline static const std::string s_FS_TAG_FILE = "/tag.json";
    inline static const std::string s_FS_IOV_FILE = "/iovs.json";
    inline static const std::string s_FS_TAGMETAINFO_FILE = "/tagmetainfo.json";
 
    inline static const std::string s_FS_META_FILE = "/meta.json";
    inline static const std::string s_FS_PALOAD_FILE = "/payload.json";
 
    inline static const std::string s_FS_GLOBALTAG_FILE = "/globaltag.json";
    inline static const std::string s_FS_MAP_FILE = "/maps.json";
 
    inline static const std::string s_FS_PATH = "";
 
    inline static const int s_FS_PREFIX_LENGTH = 3;
 
    // CREST Server data paths:
    inline static const std::string s_MGMT_PATH = "/mgmt";
    inline static const std::string s_MGMT_INFO_PATH = "/info";
    inline static const std::string s_CREST_CLIENT_VERSION = "4.0";
    inline static const std::string s_MGMT_INFO_PATH_2 = "/actuator/info";
 
    enum CrestMode {
      SERVER_MODE = 1, FILESYSTEM_MODE = 2
    };
    enum Action {
      GET = 0, POST = 1, PUT = 2, DELETE = 3
    };
 
 
// ====================================================
// REQUEST METHODS
 
    std::string performRequest(const std::string& current_path, Action action, nlohmann::json& js);
 
 
    std::string performRequest(const std::string& current_path, Action action, nlohmann::json& js, std::ofstream& out);
 
 
    std::string performRequest(const std::string& current_path, Action action, nlohmann::json& js,
                               const char* method_name);
 
    std::string storeBatchPayloadRequest(const std::string& tag, uint64_t endtime, const std::string& js);
 
 
// ====================================================
// AUXILLARY METHODS
 
    nlohmann::json getResources(nlohmann::json& js);
 
 
    nlohmann::json getResFirst(nlohmann::json& js);
 
 
    void checkFsException(const char* method_name);
 
 
    int checkErrors(const nlohmann::json& js, const char* method);
 
    std::string ParseXMLOutput(std::string_view xmlBuffer);
 
    std::string removeCR(const std::string& str);
 
 
    void checkResult(CURLcode res, long response_code, const std::string& st, const char* method_name);
 
    // NEW:
 
void getProxyPath();
 
 
  public:
 
    inline static const bool s_CREST_CLIENT_CHECK = false;
 
// ===================================
// CONSTRUCTORS
 
    CrestClient(bool rewriteIfExists, const std::string& root_folder = "/tmp/crest_dump");
 
 
    CrestClient(const std::string& _host, const std::string& _port, bool check_version = s_CREST_CLIENT_CHECK);
 
    CrestClient(std::string_view url, bool check_version = s_CREST_CLIENT_CHECK);
 
    ~CrestClient();
 
 
 
// ==========================================
// Utilities
 
    
    nlohmann::json getJson(const std::string& str); // string to json
 
    bool isJson(const std::string& str);
 
 
  private:
 
    nlohmann::json getJson(const std::string& str, const char* method); // string to json
 
    std::string getFileString(const std::string& path); // method to read file as a string
 
    void getFileList(const std::string& path);
 
    
  public:
 
//=====================================================
// AdminApi
//
 
    void removeGlobalTag(const std::string& name);
 
 
    void removeTag(const std::string& tagName);
 
 
    void updateGlobalTag(const std::string& name, nlohmann::json body);
 
//=====================================================
 
//=====================================================
// FoldersApi
//
 
    std::string createFolder(nlohmann::json& body);  //method does not work, maybe the parameters were wrong
 
 
    nlohmann::json listFolders();
 
 
//=====================================================
 
//=====================================================
// GlobaltagmapsApi
//
 
    void createGlobalTagMap(nlohmann::json& gt);
 
 
 
    void createGlobalTagMap(const std::string& globaltag, const std::string& tagname,
                            const std::string& record, const std::string& label);
 
    nlohmann::json findGlobalTagMap(const std::string& name);
 
 
//=====================================================
 
//=====================================================
// GlobaltagsApi
//
 
    void createGlobalTag(nlohmann::json& js);
 
    void createGlobalTag(const std::string& tag);
 
    void createGlobalTag(const std::string& tagname, const std::string& description);
 
 
    std::string findGlobalTagAsString(const std::string& name);
 
    nlohmann::json findGlobalTag(const std::string& name);
 
 
    nlohmann::json listGlobalTags();
 
 
 
    nlohmann::json listGlobalTags(const std::string& name, int size, int page, const std::string& sort);
 
    std::string listGlobalTagsAsString();
 
 
//=====================================================
 
// ====================================================
// IovsApi
 
 
    void createIov(nlohmann::json& iov);
 
 
    nlohmann::json findAllIovs(const std::string& tagname);
 
    nlohmann::json findAllIovs(const std::string& tagname, int size, int page, const std::string& sort,
                               const std::string& dateformat);
 
 
    int getSize(const std::string& tagname);
 
 
 
    nlohmann::json selectGroups(const std::string& tagname);
 
 
    nlohmann::json selectGroups(const std::string& tagname, long snapshot);
 
 
    nlohmann::json selectIovs(const std::string& tagname);
 
 
    nlohmann::json selectIovs(const std::string& tagname, long snapshot);
 
 
    void storeBatchIOVs(nlohmann::json& js);
 
 
 
//=======================================================
 
//=======================================================
// MonitoringApi
//
 
    nlohmann::json listPayloadTagInfo();
 
    nlohmann::json listPayloadTagInfo(const std::string& tagname);
 
//=======================================================
 
//=======================================================
// PayloadsApi
//
 
 
    void createPayload(nlohmann::json& js);
 
 
    std::string getBlob(const std::string& hash);
 
    std::string getBlobInStream(const std::string& hash, std::ofstream& out);
 
 
    nlohmann::json getPayloadAsJson(const std::string& hash);
 
 
    std::string getPayloadAsString(const std::string& hash);
 
 
    nlohmann::json getPayloadMetaInfo(const std::string& hash);
 
    std::string getPayloadMetaInfoAsString(const std::string& hash);
 
 
 
    void storePayload(const std::string& tag, uint64_t since, const std::string& js);
 
// Stores payloads in batch mode
 
    void storeBatchPayloads(const std::string& tag_name, uint64_t endtime, const std::string& iovsetupload);
 
 
    void storeBatchPayloads(const std::string& tag_name, uint64_t endtime, nlohmann::json& js);
 
    void storeBatchPayloads(const std::string& tag_name, const std::string& iovsetupload, uint64_t endtime = 0);
 
    void storeBatchPayloads(const std::string& tag_name, nlohmann::json& js, uint64_t endtime = 0);
 
//=======================================================
 
//=======================================================
// RuninfoApi
//
 
 
    void createRunLumiInfo(nlohmann::json& body);
 
 
    nlohmann::json findRunLumiInfo(const urlParameters& params);
 
 
    nlohmann::json listRunLumiInfo();
 
//=======================================================
 
//=======================================================
// TagsApi
 
// CREATE TAG
 
 
    void createTag(nlohmann::json& js);
 
 
    void createTag(const std::string& name, const std::string& desc, const std::string& timeType);
 
    void createTag(const std::string& name, const std::string& desc, const std::string& timeType,
                   const std::string& payloadSpec);
 
//std::string uploadPayload(std::string  tag,std::string  since, std::string& js);
 
 
    nlohmann::json findTag(const std::string& name);
 
 
// TAG LIST methods:
 
    nlohmann::json listTags(void);
 
    nlohmann::json listTags(int size, int page);
 
 
    nlohmann::json listTags(const std::string& name, int size, int page, const std::string& sort);
 
 
// UPDATE TAG
 
    void updateTag(const std::string& tagname, nlohmann::json body);
 
 
    void updateTagSpecification(const std::string& tagname, const std::string& objectType);
 
// FILE SYSTEM METHODS
 
//Dump methods
 
    void createTagDump(nlohmann::json& js);
 
    void flush();
 
    void storePayloadDump(const std::string& tag, uint64_t since, const std::string& js);
 
 
    std::list<std::string> shortListTagsFs();
 
    nlohmann::json findTagFs(const std::string& name);
 
    nlohmann::json findAllIovsFs(const std::string& tagname);
 
    nlohmann::json findAllIovsFs(const std::string& tagname, int size, int page);
 
    std::string getFileName(const std::string& path);
 
 
    std::string getBlobFs(const std::string& hash);
 
    std::string getBlobInStreamFs(const std::string& hash, std::ofstream& out);
 
 
    void storeBatchPayloadsFs(const std::string& tag_name, std::string& iovsetupload);
 
    void storeBatchPayloadsFs(const std::string& tag_name, nlohmann::json& js);
 
 
    void streamTest();
 
  private:
    
    std::string getEnvA(const char* varname);
 
    std::string getDataPath();
 
  public:
    
// Tag Meta Info Methods
    
    void createTagMetaInfo(nlohmann::json& js);
 
    void createTagMetaInfo(const std::string& tagname, nlohmann::json& js);
 
 
    nlohmann::json getTagMetaInfo(const std::string& tagname);
 
    void updateTagMetaInfo(nlohmann::json& js);
 
 
    void updateTagMetaInfo(const std::string& tagname, nlohmann::json& js);
 
 
    void createTagMetaInfoFs(nlohmann::json& js);
 
    nlohmann::json getTagMetaInfoFs(const std::string& name);
 
 
    void updateTagMetaInfoFs(const std::string& tagname, nlohmann::json& js);
 
    std::string getTagMetaInfoElement(const std::string& name, nlohmann::json& js);
 
 
    nlohmann::json convertTagMetaInfo2IOVDbSvc(nlohmann::json& js);
 
 
    nlohmann::json convertTagMetaInfo2CREST(nlohmann::json& js);
 
  private:
    
    std::vector<std::string> split(std::string_view str, char delim);
 
  public:
 
    nlohmann::json getTagMetaInfoIOVDbSvc(const std::string& tagname);
 
 
    void createTagMetaInfoIOVDbSvc(const std::string& tagname, nlohmann::json& js);
 
    void createTagMetaInfoIOVDbSvc(const std::string& tagname, nlohmann::json& js, const std::string& description);
 
//=====================================================
// Global Tag and Global Tag Map methods for the file storage
//
 
    void createGlobalTagFs(nlohmann::json& js);
 
 
    nlohmann::json findGlobalTagFs(const std::string& name);
 
 
    void createGlobalTagMapFs(nlohmann::json& js);
 
    nlohmann::json findGlobalTagMapFs(const std::string& name);
 
 
//=====================================================
// Payloads methods for the file storage
//
 
    std::string getPayloadAsStringFS(const std::string& hash);
 
    nlohmann::json getPayloadAsJsonFS(const std::string& hash);
 
    std::string getPayloadMetaInfoAsStringFS(const std::string& hash);
 
    nlohmann::json getPayloadMetaInfoAsJsonFS(const std::string& hash);
 
 
 
//=====================================================
// IOV number methods for the file storage
//
 
    int getSizeFS(const std::string& tagname);
 
 
    nlohmann::json getSizeByTagFS(const std::string& tagname);
 
 
//=====================================================
// select IOV list methods
//
 
 
    nlohmann::json selectIovs(const std::string& tagname, long since, long until, long snapshot = 0);
 
 
    nlohmann::json selectIovsFS(const std::string& tagname, long since, long until);
 
 
//=====================================================
// Auxillary methods
//
 
    std::string getHash(std::string_view str);
 
    std::string getDateAndTime();
 
    nlohmann::json returnJArray(nlohmann::json js);
 
    void getTagDataInfo(const std::string& tagname);
 
    std::string getFirstLetters(const std::string& str);
 
 
//=====================================================
// Methods to check the CREST Server and CrestApi versions
//
 
    nlohmann::json getMgmtInfo();
 
    std::string getCrestVersion();
 
    std::string getClientVersion();
 
    int getMajorVersion(std::string& str);
 
    void checkCrestVersion();
 
//===================================
// Second version of the version checking:
//
 
    nlohmann::json getMgmtInfo2();
 
    std::string getCrestVersion2();
 
    void checkCrestVersion2();
 
// NEW:
 
  void removeTagFromGlobalTagMap(const std::string& global_tag,const std::string& tag,const std::string& label);
 
 
  void storeBatchPayloadsFiles(const std::string& tag_name, uint64_t endtime, nlohmann::json& js);
 
 
 std::string storeBatchPayloadFilesRequest(const std::string& tag, uint64_t endtime, const std::string& js);
 
 
  void storeData(const std::string& tag, uint64_t endtime, uint64_t since, const std::string& file);
 
 
  void storeDataArray(const std::string& tag, uint64_t endtime, std::map<uint64_t, std::string> m);
 
  void replaceSymbols(std::string& str, const std::string& from, const std::string& to);
 
  };
  
} // namespace
 
 
#endif // not CRESTAPI_CRESAPI_H