ATLAS Offline Software
Loading...
Searching...
No Matches
Classes | Functions
AtlCoolCopy.cxx File Reference
#include <algorithm>
#include <cmath>
#include <ctime>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include <nlohmann/json.hpp>
#include <curl/curl.h>
#include "CoolKernel/DatabaseId.h"
#include "CoolKernel/Exception.h"
#include "CoolKernel/IDatabaseSvc.h"
#include "CoolKernel/IDatabase.h"
#include "CoolKernel/IFolder.h"
#include "CoolKernel/FolderSpecification.h"
#include "CoolKernel/IFolderSet.h"
#include "CoolKernel/IObject.h"
#include "CoolKernel/IObjectIterator.h"
#include "CoolKernel/IRecordIterator.h"
#include "CoolApplication/Application.h"
#include "CoralBase/AttributeListException.h"
#include "FileCatalog/IFileCatalog.h"
#include "PersistencySvc/IDatabase.h"
#include "PersistencySvc/ISession.h"
#include "PersistencySvc/ITransaction.h"
#include "PersistencySvc/SimpleUtilityBase.h"
#include "RelationalAccess/ConnectionService.h"
#include "RelationalAccess/IConnectionServiceConfiguration.h"
#include "RelationalAccess/ISessionProxy.h"
#include "RelationalAccess/ITransaction.h"
#include "RelationalAccess/ISchema.h"
#include "RelationalAccess/ITable.h"
#include "RelationalAccess/IQuery.h"
#include "RelationalAccess/ICursor.h"
#include "CoraCool/CoraCoolDatabase.h"
#include "CoraCool/CoraCoolFolder.h"
#include "CoraCool/CoraCoolObjectIter.h"
#include "CoraCool/CoraCoolObject.h"
#include "CxxUtils/checker_macros.h"
#include "PoolMapElement.h"
#include "ReplicaSorter.h"
#include "CoolTagInfo.h"
#include "TFile.h"
#include "TTree.h"
#include "TH1.h"
#include "TObjString.h"

Go to the source code of this file.

Classes

class  AtlCoolCopy

Functions

size_t WriteCallback (void *contents, size_t size, size_t nmemb, std::string *s)
void printHelp ()
int AtlCoolCopy::copyFolder ATLAS_NOT_THREAD_SAFE (const std::string &folder, const std::vector< std::string > &taglist)
int AtlCoolCopy::copyIOVs ATLAS_NOT_THREAD_SAFE (const std::string &folder, const std::string &destfolder, const cool::IFolderPtr &sourcefl, const CoraCoolFolderPtr &sourceflc, const cool::IFolderPtr &destfl, const CoraCoolFolderPtr &destflc, const std::string &sourcetag, const std::string &desttag, const cool::ValidityKey since, const cool::ValidityKey until, bool timestamp, bool checkrefs, bool iscora, const cool::PayloadMode::Mode paymode, bool created)
int AtlCoolCopy::doCopy ATLAS_NOT_THREAD_SAFE ()
 Install fatal handler with default options.
int main ATLAS_NOT_THREAD_SAFE (int argc, const char *argv[])

Function Documentation

◆ ATLAS_NOT_THREAD_SAFE() [1/4]

int AtlCoolCopy::doCopy ATLAS_NOT_THREAD_SAFE ( void )
inline

Install fatal handler with default options.

This is meant to be easy to call from python via ctypes.

Definition at line 2097 of file AtlCoolCopy.cxx.

2097 {
2098 if (isOpen()) {
2099 int retcode=0;
2100 // execute copy for all defined folders
2101 for (std::vector<std::string>::const_iterator ifolder=
2102 m_folderlist.begin();ifolder!=m_folderlist.end();++ifolder) {
2103 int code=copyFolder(*ifolder,m_tags);
2104 if (code>retcode) retcode=code;
2105 }
2106 if (!m_hiparent.empty()) {
2107 // fill in any hierarchical tags for parent folders
2108 int code=tagParents();
2109 if (code>retcode) retcode=code;
2110 }
2111 if (m_copytaginfo) {
2112 // write information (tag description, lock status) to dest tags
2113 int code=writeTagInfo();
2114 if (code>retcode) retcode=code;
2115 }
2116 if (m_checkrefs) {
2117 int code=0;
2118 if (m_poolcat.empty()) {
2119 code=listPoolRefs();
2120 if (!m_checkoutputfile.empty()) filePoolRefs();
2121 } else {
2122 code=resolvePoolRefs();
2123 }
2124 if (code>retcode) retcode=code;
2125 }
2126 if (p_rootfile) {
2127 p_rootfile->Write();
2128 delete p_rootfile;
2129 }
2130 return retcode;
2131 } else {
2132 return 10;
2133 }
2134}
python.TileCalibTools.copyFolder
copyFolder(dbr, dbw, folder, tagr, tagw, chanNum, pointInTime1, pointInTime2)
Definition TileCalibTools.py:494

◆ ATLAS_NOT_THREAD_SAFE() [2/4]

int AtlCoolCopy::copyIOVs ATLAS_NOT_THREAD_SAFE ( const std::string & folder,
const std::string & destfolder,
const cool::IFolderPtr & sourcefl,
const CoraCoolFolderPtr & sourceflc,
const cool::IFolderPtr & destfl,
const CoraCoolFolderPtr & destflc,
const std::string & sourcetag,
const std::string & desttag,
const cool::ValidityKey since,
const cool::ValidityKey until,
bool timestamp,
bool checkrefs,
bool iscora,
const cool::PayloadMode::Mode paymode,
bool created )

Definition at line 882 of file AtlCoolCopy.cxx.

890 {
891
892 std::cout << "Copying tag " << sourcetag << " of folder " << folder <<
893 " to destination tag " << desttag << std::endl;
894 cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
895 cool::FolderVersioning::Mode dvermode=destfl->versioningMode();
896 // check for online mode
897 int updatemode=getUpdateMode(destfl->description(),desttag);
898 std::vector<std::string> dtaglist=destfl->listTags();
899 // if online mode, must have --getonline and either -truncate or -alliov
900 // only if destination DB is oracle
901 if (updatemode==1 && m_destdb.find("oracle")!=std::string::npos &&
902 (!m_getonline || (!m_truncate && !m_alliov))) {
903 if (m_ignoremode) {
904 std::cout << "Folder is online (UPD1) mode but IGNORING PROTECTION"
905 << std::endl;
906 } else {
907 if(find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
908 std::cout << "Folder is online mode (UPD1) and tag already exist - -getonline and -truncate or -alliov options MUST be used" << std::endl;
909 return 35;
910 }
911 else {
912 std::cout << "Folder is online mode (UPD1), new tag will be created" << std::endl;
913 }
914
915 }
916 }
917 // if UPD4 mode, must have -getbulk and either -truncate or -alliov
918 if (updatemode==4 && m_destdb.find("oracle")!=std::string::npos &&
919 (!m_getbulk || (!m_truncate && !m_alliov))) {
920 if (m_ignoremode) {
921 std::cout << "Folder is bulkreco (UPD4) mode but IGNORING PROTECTION"
922 << std::endl;
923 } else {
924 if(find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
925 std::cout << "Folder is bulkreco mode (UPD4) - -getbulk and -truncate or -alliov options MUST be used" << std::endl;
926 return 35;
927 } else {
928 std::cout << "Folder is bulkreco mode (UPD4), new tag will be created" << std::endl;
929 }
930
931 }
932 }
933
934 // if destination tags are being checked, check if this tag already
935 // exists in destination folder, if so skip copy
936 // also skip if tag is HEAD, and folder was not newly created
937 if (m_checkdesttag) {
938 if (find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()
939 || (desttag=="HEAD" && !created)) {
940 std::cout << "Destination tag " << desttag <<
941 " already exists in folder " << folder << " - skip copy" << std::endl;
942 return 0;
943 }
944 }
945 bool relock=false; // tag must be relocked afterwards
946 bool prot=false; //update is protected, irrespective of if tag must be locked
947 if ((m_applock || (updatemode==1 && m_getonline) || updatemode==2 ||
948 (updatemode==4 && m_getbulk)) &&
949 dvermode==cool::FolderVersioning::MULTI_VERSION) {
950 // check if the destination tag exists and is locked
951 std::vector<std::string> dtaglist=destfl->listTags();
952 for (std::vector<std::string>::const_iterator itr=dtaglist.begin();
953 itr!=dtaglist.end();++itr) {
954 if (desttag==*itr) {
955 // tag exists - must protect it unless ignoremode
956 prot=!m_ignoremode;
957 // check lock status
958 if (destfl->tagLockStatus(desttag)==cool::HvsTagLock::LOCKED) {
959 std::cout << "Unlocking destination tag " << desttag <<
960 " for append or UPDx access" << std::endl;
961 if (m_applocksv) {
962 std::cout << "Appending according to SV folder rules" << std::endl;
963 if (updatemode!=3) {
964 std::cout <<
965 "ERROR: Only allowed for UPD3 mode tags" << std::endl;
966 return 36;
967 }
968 }
969 destfl->setTagLockStatus(desttag,cool::HvsTagLock::UNLOCKED);
970 relock=true;
971 }
972 }
973 }
974 }
975 int nobj=0;
976 int nbuf=0;
977 int nskip=0;
978 int nbad=0;
979 // set up tag to be used at point of insertion
980 std::string localtag="";
981 if (dvermode==cool::FolderVersioning::MULTI_VERSION &&
982 desttag!="HEAD" && m_usertags) localtag=desttag;
983 // now loop over IOVs - separate for CoraCool and COOL
984 try {
985 if (iscora) {
986 // branch for CoraCool
987 sourceflc->setPrefetchAll(false);
988 CoraCoolObjectIterPtr sourceitr;
989 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
990 sourceitr=sourceflc->browseObjects(since,until,m_chansel);
991 } else {
992 sourceitr=sourceflc->browseObjects(since,until,m_chansel,sourcetag);
993 }
994 // keep track of new FKs of inserted objects
995 std::map<int,int> insertkeymap;
996 unsigned int nref=0;
997 while (sourceitr->hasNext()) {
998 CoraCoolObjectPtr obj=sourceitr->next();
999 // check object should not be skipped as spreading outside select-IOV
1000 if (m_skipout &&
1001 (obj->since()<m_srunemin || obj->until()>m_srunemax)) {
1002 ++nskip;
1003 continue;
1004 }
1005 // check object should not be skipped as excluded channel
1006 if (!m_excludechans.empty()) {
1007 if (find(m_excludechans.begin(),m_excludechans.end(),
1008 obj->channelId())!=m_excludechans.end()) {
1009 ++nskip;
1010 continue;
1011 }
1012 }
1013 if (nbuf==0) {
1014 if (m_debug) std::cout <<
1015 "Setup new CoraCool storage buffer at object " << nobj << std::endl;
1016 destflc->setupStorageBuffer();
1017 }
1018 cool::ValidityKey newsince,newuntil;
1019 adjustIOVs(obj->since(),obj->until(),since,until,newsince,newuntil,
1020 timestamp);
1021 // skip negative/zero IOV lengths
1022 if (newsince>=newuntil) {
1023 std::cout << "WARNING: Skipping IOV with since " << newsince <<
1024 ">= until" << newuntil << std::endl;
1025 ++nbad;
1026 continue;
1027 }
1028 // find the old FK, check if we have already inserted this
1029 // in which case a reference can be added
1030 int oldfk=0;
1031 std::map<int,int>::const_iterator ikey=insertkeymap.end();
1032 bool foundkey=false;
1033 // ensure returned data is not of zero size - if so cannot extract
1034 // and remember the FK
1035 if (obj->size()>0) {
1036 oldfk=sourceflc->getAttrKey(
1037 (*obj->begin())[sourceflc->coralFKey()]);
1038 ikey=insertkeymap.find(oldfk);
1039 foundkey=true;
1040 }
1041 if (ikey==insertkeymap.end()) {
1042 int newfk=
1043 destflc->storeObject(newsince,newuntil,obj->begin(),obj->end(),
1044 obj->channelId(),localtag,
1045 (!m_userupdatehead && !localtag.empty()));
1046 if (foundkey) insertkeymap[oldfk]=newfk;
1047 } else {
1048 destflc->referenceObject(newsince,newuntil,ikey->second,
1049 obj->channelId(),localtag,
1050 (!m_userupdatehead && !localtag.empty()));
1051 ++nref;
1052 }
1053 ++nbuf;
1054 // flush buffer every m_bufsize objects
1055 if (nbuf==m_bufsize) {
1056 if (m_debug) std::cout << "Flush buffer after " << nobj << "," <<
1057 nbuf << " objects " << std::endl;
1058 // note CoraCool requires explicit re-setup of buffer after a flush
1059 destflc->flushStorageBuffer();
1060 destflc->setupStorageBuffer();
1061 nbuf=0;
1062 }
1063 ++nobj;
1064 }
1065 sourceitr->close();
1066 if (nref>0) std::cout << "Reference-to-existing used for " << nref
1067 << " payload objects" << std::endl;
1068 if (nbuf>0) {
1069 if (m_debug) std::cout << "Final buffer flush at " << nobj <<
1070 "," << nbuf << std::endl;
1071 destflc->flushStorageBuffer();
1072 }
1073 } else {
1074 // branch for pure COOL objects
1075 cool::IObjectIteratorPtr sourceitr;
1076 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1077 sourceitr=sourcefl->browseObjects(since,until,m_chansel);
1078 } else {
1079 sourceitr=sourcefl->browseObjects(since,until,m_chansel,sourcetag);
1080 }
1081 while (sourceitr->goToNext()) {
1082 const cool::IObject& obj=sourceitr->currentRef();
1083 // check object should not be skipped as spreading outside select-IOV
1084 if (m_skipout && (obj.since()<m_srunemin || obj.until()>m_srunemax)) {
1085 ++nskip;
1086 continue;
1087 }
1088 // check object should not be skipped as excluded channel
1089 if (!m_excludechans.empty()) {
1090 if (find(m_excludechans.begin(),m_excludechans.end(),
1091 obj.channelId())!=m_excludechans.end()) {
1092 ++nskip;
1093 continue;
1094 }
1095 }
1096 if (nbuf==0) {
1097 if (m_debug) std::cout << "Setup new storage buffer at object " <<
1098 nobj << std::endl;
1099 destfl->setupStorageBuffer();
1100 }
1101 cool::ValidityKey newsince;
1102 cool::ValidityKey newuntil;
1103 adjustIOVs(obj.since(),obj.until(),since,until,newsince,newuntil,
1104 timestamp);
1105 // skip negative/zero IOV lengths
1106 if (newsince>=newuntil) {
1107 std::cout << "WARNING: Skipping IOV with since " << newsince <<
1108 ">= until" << newuntil << std::endl;
1109 ++nbad;
1110 continue;
1111 }
1112 if (checkrefs) checkRef(obj.payload(),folder,sourcetag);
1113 if (prot) {
1114 // check existing data in destination DB to ensure no clash
1115 if (m_applocksv) {
1116 // appendlockedsv mode - all IOV ends must be infinite
1117 // and new since must be greater than old since
1118 if (newuntil!=cool::ValidityKeyMax) {
1119 std::cout << "New IOVs must have until=cool::ValidityKeyMax" << std::endl;
1120 throw cool::Exception("Illegal insert over locked IOV",
1121 "AtlCoolCopy");
1122 }
1123 cool::IObjectIteratorPtr checkitr=destfl->browseObjects(newsince,
1124 newuntil,obj.channelId(),desttag);
1125 while (checkitr->goToNext()) {
1126 const cool::IObject& checkobj=checkitr->currentRef();
1127 if (checkobj.since()>=newsince) {
1128 std::cout << "ERROR:: Attempt to insert SV overlapping IOV whilst appending to locked tag" << std::endl;
1129 throw cool::Exception("Illegal insert over locked IOV",
1130 "AtlCoolCopy");
1131 }
1132 if (checkobj.until()!=cool::ValidityKeyMax) {
1133 std::cout << "Existing IOVs must have until=cool::ValidityKeyMax" << std::endl;
1134 throw cool::Exception("Illegal insert over locked IOV",
1135 "AtlCoolCopy");
1136 }
1137 }
1138 checkitr->close();
1139 } else if (m_applock) {
1140 // appendlocked mode - just make sure no objects in this IOV
1141 const unsigned int nexist=
1142 destfl->countObjects(newsince,newuntil,obj.channelId(),desttag);
1143 if (nexist>0) {
1144 std::cout << "ERROR: Attempt to insert IOV over " << nexist <<
1145 " objects whilst appending to locked tag" << std::endl;
1146 throw cool::Exception("Illegal insert over locked IOV",
1147 "AtlCoolCopy");
1148 }
1149 } else if (updatemode==2) {
1150 // UPD2 tag - make sure run is on list of runs open for update
1151 // only for run/lumi based folders
1152 if (timestamp)
1153 throw cool::Exception(
1154 "Attempt to insert into locked UPD2 tag with timestamp format",
1155 "AtlCoolCopy");
1156 unsigned int run1=newsince >> 32;
1157 unsigned int run2=(newuntil-1) >> 32;
1158 for (unsigned int irun=run1;irun<=run2;++irun) {
1159 if (!std::binary_search(m_runlist.begin(),m_runlist.end(),irun))
1160 {
1161 std::cout << "Run " << irun << " from range [" << run1 << ","
1162 << run2 << "] not found in runfile list" << std::endl;
1163 throw cool::Exception("Illegal insert over locked IOV",
1164 "AtlCoolCopy");
1165 }
1166 }
1167 }
1168 // updatemode==1 will fall through, but no checks are done here
1169 }
1170 if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
1171 // cool vector payload object copy
1172 cool::IRecordIterator& pitr=obj.payloadIterator();
1173 const cool::IRecordVectorPtr vptr=pitr.fetchAllAsVector();
1174 destfl->storeObject(newsince,newuntil,
1175 *vptr,obj.channelId(),localtag,
1176 (!m_userupdatehead && !localtag.empty()));
1177 nbuf+=vptr->size();
1178 pitr.close();
1179 } else {
1180 // standard COOL object copy
1181 destfl->storeObject(newsince,newuntil,
1182 obj.payload(),obj.channelId(),localtag,
1183 (!m_userupdatehead && !localtag.empty()));
1184 ++nbuf;
1185 }
1186 // flush buffer every m_bufsize objects
1187 if (nbuf>=m_bufsize) {
1188 if (m_debug) std::cout << "Flush buffer after " << nobj << "," <<
1189 nbuf << " objects " << std::endl;
1190 destfl->flushStorageBuffer();
1191 nbuf=0;
1192 }
1193 ++nobj;
1194 }
1195 // finished with the iterator
1196 sourceitr->close();
1197 if (nbuf>0) {
1198 if (m_debug) std::cout << "Final buffer flush at " << nobj <<
1199 "," << nbuf << std::endl;
1200 destfl->flushStorageBuffer();
1201 } // end of COOL-specific part
1202 }
1203 std::cout << "Folder copied with " << nobj << " objects" << std::endl;
1204 if (nskip>0) std::cout << nskip <<
1205 " objects were skipped extending outside IOV selection or excluded channel"
1206 << std::endl;
1207 if (nbad>0) std::cout << nbad <<
1208 " objects were skipped having zero or negative IOV lengths" << std::endl;
1209 }
1210 // exceptions thrown from insert loop (both COOL or CORACOOL)
1211 catch (cool::Exception& e) {
1212 std::cout << "Exception thrown from copy loop: " << e.what() <<
1213 std::endl;
1214 if (relock) {
1215 std::cout << "Relocking destination tag " << desttag << std::endl;
1216 destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1217 }
1218 return 31;
1219 }
1220 // check if tag needs to be relocked
1221 if (relock) {
1222 std::cout << "Relocking destination tag " << desttag << std::endl;
1223 destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1224 }
1225
1226 if (dvermode==cool::FolderVersioning::MULTI_VERSION && desttag!="HEAD") {
1227 if (!m_usertags) {
1228 // apply HEAD style tagging
1229 std::cout << "Tag folder with HEAD-style tagging for tag: " << desttag
1230 << std::endl;
1231 try {
1232 destfl->tagCurrentHead(desttag,sourcefl->tagDescription(sourcetag));
1233 }
1234 catch (cool::Exception& e) {
1235 std::cout << "Exception thrown in HEAD-style folder-tag: " <<
1236 e.what() << std::endl;
1237 return 32;
1238 }
1239 }
1240 // record information about this tag for later
1241 if (m_copytaginfo) m_cooltagmap.insert(
1242 CoolTagMap::value_type(desttag,CoolTagInfo(m_sourceDbPtr,folder,
1243 destfolder,sourcetag,desttag,m_taglabel)));
1244 // check if any hierarchical tags reference the just-copied tag
1245 if (!m_hitagmap.empty()) {
1246 for (HiTagMap::const_iterator imap=m_hitagmap.begin();
1247 imap!=m_hitagmap.end();++imap) {
1248 try {
1249 if (sourcefl->findTagRelation(imap->first)==sourcetag) {
1250 std::cout << "Create hierarchical tag between " << desttag <<
1251 " and " << imap->first << " in folder " << imap->second <<
1252 std::endl;
1253 // first check this relation has not already been created
1254 try {
1255 std::string etag=destfl->resolveTag(imap->first);
1256 if (etag==desttag) {
1257 std::cout << "This relation has already been created" << std::endl;
1258 } else {
1259 std::cout << "ERROR: Tag in parent already related to " <<
1260 desttag << std::endl;
1261 }
1262 }
1263 catch (cool::Exception& e ) {
1264 // only do creation if does not already exist - via exception
1265 try {
1266 destfl->createTagRelation(imap->first,desttag);
1267 // check if this implicates a new parent folder
1268 // to be checked for further tags up the tree
1269 if (find(m_hiparent.begin(),m_hiparent.end(),imap->second)==
1270 m_hiparent.end()) m_hiparent.push_back(imap->second);
1271 // store the original tag information to transfer
1272 // properties later
1273 if (m_copytaginfo &&
1274 m_cooltagmap.find(imap->first)==m_cooltagmap.end())
1275 m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
1276 CoolTagInfo(m_sourceDbPtr,imap->second,imap->second,
1277 imap->first,imap->first)));
1278 }
1279 catch (cool::Exception& e) {
1280 std::cout << "Cool exception " << e.what() <<
1281 "thrown in hierarchical tag creation" << std::endl;
1282 return 34;
1283 }
1284 }
1285 }
1286 }
1287 catch (cool::Exception& e) {
1288 }
1289 }
1290 }
1291 }
1292 return 0;
1293}
CoraCoolObjectPtr
boost::shared_ptr< CoraCoolObject > CoraCoolObjectPtr
Definition CoraCoolTypes.h:19
CoraCoolObjectIterPtr
boost::shared_ptr< CoraCoolObjectIter > CoraCoolObjectIterPtr
Definition CoraCoolTypes.h:22
find
std::string find(const std::string &s)
return a remapped string
Definition hcg.cxx:138
CaloCondBlobAlgs_fillNoiseFromASCII.folder
folder
Definition CaloCondBlobAlgs_fillNoiseFromASCII.py:55

◆ ATLAS_NOT_THREAD_SAFE() [3/4]

int AtlCoolCopy::copyFolder ATLAS_NOT_THREAD_SAFE ( const std::string & folder,
const std::vector< std::string > & taglist )

Definition at line 534 of file AtlCoolCopy.cxx.

535 {
536 // get source folder
537 cool::IFolderPtr sourcefl,destfl;
538 CoraCoolFolderPtr sourceflc,destflc;
539 try {
540 sourcefl=m_sourceDbPtr->getFolder(folder);
541 }
542 catch (cool::Exception& e) {
543 std::cout << "Could not get source folder: " << folder << std::endl;
544 return 20;
545 }
546 const std::string& sourcedesc=sourcefl->description();
547 // check if folder should be skipped from replication
548 if (m_skiprep && sourcedesc.find("<norep/>")!=std::string::npos) {
549 std::cout << "Folder " << folder <<
550 " skipped due to <norep/> metadata" << std::endl;
551 return 0;
552 }
553 const bool iscora=(m_coracool &&
554 sourcedesc.find("<coracool")!=std::string::npos);
555 if (iscora) {
556 // activate CoraCool and get source folder pointer
557 openCoraCool();
558 sourceflc=m_sourceCoraPtr->getFolder(folder);
559 }
560 // setup the channel-range
561 setChannelRange(sourcefl);
562
563 cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
564 cool::FolderVersioning::Mode dvermode=vermode;
565 if (m_forcesingle) {
566 dvermode=cool::FolderVersioning::SINGLE_VERSION;
567 std::cout << "Forcing destination folder to singleversion" << std::endl;
568 }
569 if (m_forcemulti) {
570 dvermode=cool::FolderVersioning::MULTI_VERSION;
571 std::cout << "Forcing destination folder to multiversion" << std::endl;
572 }
573 // setup type of folder (payload mode, inline, separate, or vector)
574 const cool::PayloadMode::Mode spaymode=
575 sourcefl->folderSpecification().payloadMode();
576 cool::PayloadMode::Mode dpaymode=spaymode;
577
578 // set name for destination folder
579 std::string destfolder=folder;
580 if (!m_outfolder.empty()) {
581 destfolder=m_outfolder;
582 std::cout << "Destination folder will be renamed to " << destfolder <<
583 std::endl;
584 }
585
586// get destination folder, try to create if not there (and not verifying!)
587 bool created=false;
588 if (!m_nocopy && !m_root && !m_analyse) {
589 if (!m_destDbPtr->existsFolder(destfolder) && !m_verify) {
590 std::cout << "Creating folder " << destfolder << " payload-type " <<
591 dpaymode << " on destination" << std::endl;
592 created=true;
593 std::string metadata=sourcedesc;
594 if (m_forcerune || m_forcetime) {
595 std::string newmeta=m_forcerune ? "run-lumi" : "time";
596 std::string::size_type p1,p2;
597 p1=metadata.find("<timeStamp>");
598 p2=metadata.find("</timeStamp>");
599 if (p1!=std::string::npos && p2!=std::string::npos) {
600 metadata.replace(0,p2,"<timeStamp>"+newmeta);
601 std::cout << "Forced destination folder to " << newmeta << " : "
602 << metadata << std::endl;
603 } else {
604 std::cout <<
605 "ERROR: Could not parse metadata string to force timestamp type"
606 << std::endl;
607 }
608 }
609 // force separate or inline payload, but not if input is vector
610 if (m_forcepay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
611 dpaymode=cool::PayloadMode::SEPARATEPAYLOAD;
612 if (m_forcenopay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
613 dpaymode=cool::PayloadMode::INLINEPAYLOAD;
614
615 try {
616 if (iscora) {
617 destflc=m_destCoraPtr->createFolder(destfolder,
618 sourceflc->coralTableName(),
619 sourceflc->fkSpecification(),
620 sourceflc->payloadSpecification(),
621 sourceflc->coralFKey(),
622 sourceflc->coralPKey(),
623 metadata,dvermode,true);
624 std::cout << "Created CoraCool folder" << std::endl;
625 destfl=m_destDbPtr->getFolder(destfolder);
626 } else {
627 destfl=m_destDbPtr->createFolder(destfolder,
628 cool::FolderSpecification(dvermode,
629 sourcefl->payloadSpecification(),dpaymode),
630 metadata,true);
631 }
632 }
633 catch (cool::Exception&e ) {
634 std::cout << "Create folder failed - aborting" << std::endl;
635 return 30;
636 }
637 }
638 // now get the pointer to destination folder if needed
639 // note both COOL and CoraCool pointers are set if needed
640 try {
641 if (iscora) destflc=m_destCoraPtr->getFolder(destfolder);
642 destfl=m_destDbPtr->getFolder(destfolder);
643 }
644 catch (cool::Exception& e) {
645 std::cout << "Could not get destination folder: " << destfolder
646 << std::endl;
647 return 21;
648 }
649 // check payload specifications of folders are the same
650 const cool::IRecordSpecification& sourcespec=
651 sourcefl->payloadSpecification();
652 const cool::IRecordSpecification& destspec=
653 destfl->payloadSpecification();
654 if (!(sourcespec==destspec)) {
655 bool badspec=false;
656 if (m_ignorespec) {
657 // specifications differ - check names are same
658 std::cout <<
659 "WARNING Source and destination folder specifications differ" <<
660 std::endl;
661 for (unsigned int i=0;i<sourcespec.size();++i) {
662 const std::string& sname=sourcespec[i].name();
663 if (!destspec.exists(sname)) {
664 std::cout << "ERROR: Field " << sname << " absent from destination"
665 << std::endl;
666 badspec=true;
667 }
668 }
669 } else {
670 badspec=true;
671 }
672 if (badspec) {
673 std::cout <<
674 "ERROR Source and destination folder specifications differ"
675 << std::endl;
676 return 22;
677 }
678 }
679 // check folder descriptions are the same - just print WARNING if not
680 const std::string& destdesc=destfl->description();
681 if (sourcedesc!=destdesc) {
682 std::cout << "WARNING: Source and destination folder descriptions (meta-data) differ" << std::endl;
683 std::cout << "Source folder: " << sourcedesc << std::endl;
684 std::cout << "Destn folder: " << destdesc << std::endl;
685 }
686 // check payload modes are same - print warning if not
687 const cool::PayloadMode::Mode dpaymode=
688 destfl->folderSpecification().payloadMode();
689 if (spaymode!=dpaymode) {
690 std::cout << "WARNING: Source (" << spaymode << ") and destination (" <<
691 dpaymode << " folder payload modes differ" << std::endl;
692 }
693
694 // check/set channels table if requested
695 if (!m_nochannel) {
696 if (!checkChannels(folder,sourcefl,destfl,created)) return 23;
697 }
698 }
699 // if only copying structures, stop here
700 if (m_nodata) return 0;
701 // extract folder/range information
702 std::cout << "Start to process folder: " << folder;
703 // check for timestamp XML
704 bool timestamp=(sourcefl->description().find("<timeStamp>time")!=
705 std::string::npos);
706 cool::ValidityKey since,until;
707 if (timestamp) {
708 since=m_timemin;
709 until=m_timemax;
710 std::cout << " (timestamp)" << std::endl;
711 } else {
712 since=m_runemin;
713 until=m_runemax;
714 std::cout << " (run/lumi)" << std::endl;
715 }
716 if (m_alliov || m_truncate)
717 std::cout << "Output IOVs will be modified" << std::endl;
718 if (m_skipout)
719 std::cout << "IOVs extending outside selection range will be skipped"
720 << std::endl;
721
722 bool checkrefs=false;
723 const std::string name0=sourcefl->payloadSpecification()[0].name();
724 if (m_checkrefs && (name0=="PoolRef" || name0=="fileGUID")) {
725 checkrefs=true;
726 std::cout << "Check POOL references in folder " << folder << std::endl;
727 }
728 // if nocopy, and checking POOL references, and folder is not a POOL ref one
729 // can skip the data access
730 if (m_checkrefs && m_nocopy && !checkrefs) return 0;
731
732 // check for <fullrep/> metadata indicating copy all tags if -skiprep option
733 bool copyall=
734 (m_skiprep && (sourcedesc.find("<fullrep/>")!=std::string::npos));
735 if (copyall) std::cout <<
736 "All tags in folder will be copied due to <fullrep/> metadata" << std::endl;
737
738 // set up true list of tags to copy
739 std::vector<std::string> tags;
740 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
741 std::cout << "Single version folder" << std::endl;
742 tags.emplace_back("HEAD");
743 } else {
744 std::cout << "Multi version folder: consider tags [ ";
745 // get list of tags in this node which are requested
746 const std::vector<std::string> foldertags=sourcefl->listTags();
747 for (std::vector<std::string>::const_iterator itag=foldertags.begin();
748 itag!=foldertags.end();++itag) {
749 bool copyit=copyall;
750 // if input taglist is empty, take all tags in folder
751 if (taglist.empty() ||
752 find(taglist.begin(),taglist.end(),*itag)!=taglist.end())
753 copyit=true;
754 for (std::vector<std::string>::const_iterator imtag=m_magic.begin();
755 imtag!=m_magic.end();++imtag) {
756 if (itag->find(*imtag)!=std::string::npos) copyit=true;
757 }
758 if (copyit) {
759 tags.push_back(*itag);
760 std::cout << *itag << " ";
761 }
762 }
763 // if no tags were found, or doing inclusive hierarchical tag copying
764 // try resolving input tags hierarchically
765 if (tags.empty() || m_hitag) {
766 for (std::vector<std::string>::const_iterator itag=taglist.begin();
767 itag!=taglist.end();++itag) {
768 try {
769 std::string htag=sourcefl->resolveTag(*itag);
770 if (find(tags.begin(),tags.end(),htag)==tags.end()) {
771 std::cout << *itag << "=" << htag << " ";
772 tags.push_back(std::move(htag));
773 }
774 }
775 // ignore exceptions indicating tag not defined here
776 // note std::exception rather than COOL exception to cover case
777 // when trying to resolve a leaf tag in the '/' folder, which
778 // throws a coral AttributeException
779 catch (std::exception& e) { }
780 }
781 }
782 std::cout << "]" << std::endl;
783 // if still no tags were found, or forced HEAD tag copying, or no
784 // tags in the folder at all (=MV folder being used as SV), add HEAD
785 if (((tags.empty() || m_includehead) && !m_excludehead) ||
786 foldertags.empty())
787 tags.emplace_back("HEAD");
788 }
789
790 sourcefl->setPrefetchAll(false);
791 for (std::vector<std::string>::const_iterator itag=tags.begin();
792 itag!=tags.end();++itag) {
793 std::string outtag=*itag;
794 if (!m_outtag.empty()) outtag=m_outtag;
795 int retcode;
796 // verify or copy folder
797 if (m_verify) {
798 if (!destfl) {
799 retcode = 1;
800 }
801 else {
802 retcode=verifyIOVs(folder,sourcefl,sourceflc,destfl,destflc,
803 *itag,since,until,checkrefs,iscora,spaymode);
804 }
805 } else if (m_nocopy) {
806 retcode=nocopyIOVs(folder,sourcefl,*itag,since,until,checkrefs);
807 } else if (m_root) {
808 retcode=rootIOVs(folder,sourcefl,*itag,since,until,timestamp);
809 } else if (m_analyse) {
810 retcode=analyseIOVs(folder,sourcefl,*itag,since,until,timestamp);
811 } else {
812 retcode=copyIOVs(folder,destfolder,sourcefl,sourceflc,destfl,destflc,
813 *itag,outtag,since,until,timestamp,checkrefs,iscora,
814 spaymode,created);
815 }
816 if (retcode!=0) {
817 std::cout << "ERROR operation failed for folder " << folder << " tag " <<
818 *itag << std::endl;
819 return retcode;
820 }
821 }
822 return 0;
823}
CoraCoolFolderPtr
boost::shared_ptr< CoraCoolFolder > CoraCoolFolderPtr
Definition CoraCoolTypes.h:16
tags
std::vector< std::string > tags
Definition hcg.cxx:105
TRTCalib_cfilter.p2
p2
Definition TRTCalib_cfilter.py:131
TRTCalib_cfilter.p1
p1
Definition TRTCalib_cfilter.py:130
dq_defect_copy_defect_database.since
since
Definition dq_defect_copy_defect_database.py:54
dq_defect_copy_defect_database.until
until
Definition dq_defect_copy_defect_database.py:55
makeDTCalibBlob_pickPhase.outtag
str outtag
Definition makeDTCalibBlob_pickPhase.py:286

◆ ATLAS_NOT_THREAD_SAFE() [4/4]

int main ATLAS_NOT_THREAD_SAFE ( int argc,
const char * argv[] )

Definition at line 3484 of file AtlCoolCopy.cxx.

3484 {
3485 int retcode=0;
3486 if (argc<3) {
3487 printHelp();
3488 retcode=1;
3489 } else {
3490 AtlCoolCopy mycopy(argv[1],argv[2]);
3491 retcode=mycopy.setOpts(argc-3,&argv[3]);
3492 if (retcode==999) {
3493 printHelp();
3494 return 0;
3495 }
3496 if (retcode==0) retcode=mycopy.doCopy();
3497 }
3498 if (retcode>0) std::cout << "ERROR AtlCoolCopy.exe fails with exit code " <<
3499 retcode << std::endl;
3500 return retcode;
3501}
printHelp
void printHelp()
Definition AtlCoolCopy.cxx:3354

◆ printHelp()

void printHelp ( )

Definition at line 3354 of file AtlCoolCopy.cxx.

3354 {
3355 std::cout << "usage: AtlCoolCopy.exe <sourceCoolDB> <destinationCoolDB> { <options> }" << std::endl;
3356 std::cout << "Options are (see doc/mainpage.h for more):" << std::endl <<
3357 "-a, -alliov : set IOVs on destination to [ValidityKeyMin, ValidityKeyMax]" << std::endl <<" or following settings of -nrls,-nrlu,-nts,-ntu" <<
3358 std::endl <<
3359 "-ag, -addguid <guid> : Add GUID to list accessed for POOL checks" <<
3360 std::endl <<
3361 "-alf, -addlfn <LFN> : Add LFN to list accessed for POOL checks" <<
3362 std::endl <<
3363 "-al, -appendlocked : Allow locked tags to be updated if no overlap" <<
3364 std::endl <<
3365 "-alsv, -appendlockedsv : Allow locked tag update for openended IOVs" <<
3366 std::endl <<
3367 "-ana, -analyse <delta_t>: produce analysis ROOT file (filename = dest DB argument)" << std::endl <<
3368"-bs, -buffersize <size> : set size of bulkstorage output buf to <size> objs"
3369 << std::endl <<
3370 "-c, -create : create destination DB if not already existing"
3371 << std::endl <<
3372 "-cd, -checkdest : Check destination DB and skip already existing tags"
3373 << std::endl <<
3374 "-ch, -channel : restrict selection to given channel or range specified as c1:c2" << std::endl <<
3375 "-ch1, -channel1 : specify start of a range of channels" << std::endl <<
3376 "-ch2, -channel2 : specify end of a range of channels" << std::endl <<
3377 "-cf, -checkfiles : check POOL files can be opened and have correct GUID"
3378 << std::endl <<
3379 "-co, -checkoutput <file> : write POOL file check output on file"
3380 << std::endl << "-cti, -copytaginfo : Copy tag descriptions"
3381 << std::endl << "-ctl, -copytaglock : Copy tag locked status and descriptions"
3382 << std::endl << "-cr, -checkrefs : check POOL references"
3383 << std::endl << "-d, -debug : produce debug output" << std::endl;
3384 std::cout <<
3385 "-ds, -dataset : output register.sh for creating DQ2 datasets "
3386 << std::endl <<
3387 "-ec, -excludechannel : exclude given channel from copy" << std::endl <<
3388 "-eh, -excludehead : exclude HEAD tag even if no tags found in MV folders"
3389 << std::endl <<
3390 "-ih, -includehead : include HEAD as well as any tags in MV folders" <<
3391 std::endl <<
3392 "-e, -exclude <pattern> : exclude folders" << std::endl <<
3393 "-f, -folder <pattern> : include folders" << std::endl <<
3394 "-fs, -forcesingle : force destination folder to be singleversion"
3395 << std::endl <<
3396 "-fm, -forcemulti : force destination folder to be multiversion"
3397 << std::endl <<
3398 "-frl, -forcerunlumi : force destination folder to be run/LB indexed"
3399 << std::endl <<
3400 "-ftm, -forcetime : force destination folder to be timestamp indexed"
3401 << std::endl <<
3402 "-fp, -forcepayload : Force destn folders to be created with payload tbl" << std::endl <<
3403 "-fnp, -forcenopayload : Force destn folders to be created without payload tbl" << std::endl <<
3404 "-forcerecreate : delete and recreate destination database" << std::endl;
3405 std::cout << "-ht, -headtag : Use HEAD-style tagging"
3406 << std::endl <<
3407 "-go, -getonline : Set minimum run number (-rls setting) to next ONLINE run"
3408 << std::endl <<
3409 "-gb, -getbulk : Set minimum run number (-rls setting) to next bulk reco run"
3410 << std::endl <<
3411 "-gt, -gettime : Extract timestamp information for given run number range"
3412 << std::endl <<
3413 "-h, -help : print help and exit" << std::endl <<
3414 "-hi, -hitag : Copy hierrchical tags inclusively" << std::endl <<
3415 "-ignoremode <pwd> : Ignore UPDx mode protection (experts only)"
3416 << std::endl <<
3417 "-is, -ignorespec : Ignore differences in folder spec if names are all equal" << std::endl <<
3418 "-lo, -lockedonly : Only copy locked/partially-locked top-level tags"
3419 << std::endl <<
3420 "-nc, -nocopy : Do not actually copy, just read source DB" << std::endl <<
3421 "-nch, -nochannel : Do not check or copy channel information" << std::endl
3422<< "-ncr, -noclobroot: Do not copy CLOB data into ROOT files" << std::endl
3423<< "-noc, -nocoracool : Do not copy Coracool structure payloads " << std::endl <<
3424 "-nd, -nodata : Copy only folder structures, not data" << std::endl <<
3425 "-nh, -nohitag : Do not follow hierarchical tag relations" << std::endl <<
3426"-mc, -mergecat <catfile> : specify POOL file catalogue for new dataset making"
3427 << std::endl;
3428 std::cout << "-of, -outfolder <folder> : rename folder on output"
3429 << std::endl <<
3430 "-onr, -onlinerun : Retrieve run number from online server (not replica)" <<std::endl <<
3431 "-ot, -outtag : " << "Rename tag on output" << std::endl;
3432 std::cout <<
3433 "-pa, -poolall : Output all POOL files (incl good) when checking files"
3434 << std::endl <<
3435"-pc, -poolcat <catfile> : specify POOL file catalogue for ref checking"
3436 << std::endl <<
3437 "-pf, -parfile <file> : Read additional options/parameters from file" <<
3438 std::endl <<
3439"-pt, -prunetags : Copy only hierarchical tags descending from specified toptags"
3440 << std::endl <<
3441"-rdo, -readoracle : force data to be read from Oracle, using dbreplica.config"
3442 << std::endl <<
3443 "-sl, -seal <val>: Set SEAL (hence COOL/POOL) output level to <val>"
3444 << std::endl <<
3445 "-sr, -skiprep : Skip folders having <norep/> folder metadata tag" <<
3446 std::endl <<
3447 "-t, -tag <tag> : Copy multiversion data with tag <tag>" << std::endl <<
3448 "-mt, -magic <tag> : Include magic tags involving the given pattern " << std::endl <<
3449"-tr, -truncate : Set destination IOVs outside query interval to query interval"
3450 << std::endl <<
3451"-tl, -taglabel : Specify tag description to enter in destination DB"
3452 << std::endl <<
3453 "-uht, -userheadtag : Also copy user tag data to the HEAD" << std::endl <<
3454 "-v, -verify : Verify data present on destination rather than copying" <<
3455 std::endl <<
3456 "-ro, -root : Produce ROOT output file instead of copying to COOL"
3457 << std::endl <<
3458 "-zn, -zeronull : Zero NULLs in ROOT file instead of skipping them" <<
3459 std::endl <<
3460 "-rls, -runlumisince <run> <LB> : Set minimum IOV interval" << std::endl <<
3461 "-rlu, -runlumiuntil <run> <LB> : Set maximum IOV interval" << std::endl <<
3462 "-rs, -runsince <run> : Set minimum IOV interval" << std::endl <<
3463 "-ru, -rununtil <run> : Set maximum IOV interval" << std::endl <<
3464 "-r, -run <run> : Copy only run <run>" << std::endl <<
3465"-srls, -skiprunlumisince <run> <LB> : Set minimum IOV for skipping IOV in copy"
3466 << std::endl <<
3467"-srlu, -skiprunlumiuntil <run> <LB> : Set maximum IOV for skipping IOV in copy"
3468 << std::endl <<
3469 "-tdb, -timedb <dbconn> : Set database connection for time information" <<
3470 "-ts, -timesince <time> : Set minimum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3471 std::endl <<
3472 "-tu, -timeuntil <time> : Set maximum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3473 std::endl;
3474 std::cout <<
3475 "-nrls, -newrunlumisince <run> <LB> : Set minimum of output IOV interval (use with -alliov)" << std::endl <<
3476 "-nrlu, -newrunlumiuntil <run> <LB> : Set maximum of output IOV interval (use with --aliov)" << std::endl <<
3477 "-nts, -newtimesince <time> : Set minimum of outputIOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3478 std::endl <<
3479 "-ntu, -newtimeuntil <time> : Set maximum of output IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3480 std::endl;
3481 std::cout << "See http://twiki.cern.ch/twiki/bin/view/Atlas/AtlCoolCopy for more details" << std::endl;
3482}

◆ WriteCallback()

size_t WriteCallback ( void * contents,
size_t size,
size_t nmemb,
std::string * s )

Definition at line 64 of file AtlCoolCopy.cxx.

64 {
65 size_t newLength = size * nmemb;
66
67 try
68 {
69 s->append((char *)contents, newLength);
70 }
71 catch (std::bad_alloc &e)
72 {
73 // handle memory problem
74 return 0;
75 }
76 return newLength;
77}
contents
void contents(std::vector< std::string > &keys, TDirectory *td, const std::string &directory, const std::string &pattern, const std::string &path)
Definition computils.cxx:321
Generated on for ATLAS Offline Software by doxygen 1.14.0