18 #include <nlohmann/json.hpp>
19 #include <curl/curl.h>
21 #include "CoolKernel/DatabaseId.h"
22 #include "CoolKernel/Exception.h"
23 #include "CoolKernel/IDatabaseSvc.h"
24 #include "CoolKernel/IDatabase.h"
25 #include "CoolKernel/IFolder.h"
26 #include "CoolKernel/FolderSpecification.h"
27 #include "CoolKernel/IFolderSet.h"
28 #include "CoolKernel/IObject.h"
29 #include "CoolKernel/IObjectIterator.h"
30 #include "CoolKernel/IRecordIterator.h"
31 #include "CoolApplication/Application.h"
32 #include "CoralBase/AttributeListException.h"
39 #include "RelationalAccess/ConnectionService.h"
40 #include "RelationalAccess/IConnectionServiceConfiguration.h"
41 #include "RelationalAccess/ISessionProxy.h"
42 #include "RelationalAccess/ITransaction.h"
43 #include "RelationalAccess/ISchema.h"
44 #include "RelationalAccess/ITable.h"
45 #include "RelationalAccess/IQuery.h"
46 #include "RelationalAccess/ICursor.h"
62 #include "TObjString.h"
65 size_t newLength =
size * nmemb;
71 catch (std::bad_alloc &
e)
82 AtlCoolCopy(
const std::string& sourcedb,
const std::string& destdb,
83 bool allowcreate=
false);
88 (
const std::string&
folder,
const std::vector<std::string>&
taglist);
94 bool openConnections(
const std::string& sourcedb,
const std::string& destdb,
99 std::vector<std::string>&
folders);
100 static std::string
transConn(
const std::string& inconn);
103 const std::string& chanstring);
105 (
const std::string&
folder,
const std::string& destfolder,
108 const std::string& sourcetag,
const std::string& desttag,
109 const cool::ValidityKey
since,
const cool::ValidityKey
until,
110 bool timestamp,
bool checkrefs,
bool iscora,
113 const cool::ValidityKey&
until,
114 const cool::ValidityKey& qsince,
115 const cool::ValidityKey& quntil,
116 cool::ValidityKey& newsince,
117 cool::ValidityKey& newuntil,
118 const bool timestamp)
const;
120 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
121 const cool::ValidityKey
since,
const cool::ValidityKey
until,
126 const std::string& sourcetag,
127 const cool::ValidityKey
since,
const cool::ValidityKey
until,
128 const bool checkrefs,
const bool iscora,
131 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
132 const cool::ValidityKey
since,
const cool::ValidityKey
until,
133 const bool timestamp);
134 std::string
rootDirs(
const std::string&
folder,
const std::string& toproot);
136 void*& sptr,
char& rootID)
const;
140 const cool::IRecord& rhs);
142 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
143 const cool::ValidityKey
since,
const cool::ValidityKey
until,
144 const bool timestamp);
146 const int chan,
const float xlow,
const float xhigh);
148 static std::string
timeString(
const cool::ValidityKey iovtime);
149 static cool::ValidityKey
runLBVal(
const char*
input1,
const char* input2);
158 const cool::IFolderPtr& sourcefl,
const cool::IFolderPtr& destfl,
161 const std::string&
folder,
const std::string&
tag);
268 typedef std::map<std::string,PoolMapElement>
PoolMap;
272 using HiTagMap = std::map<std::string, std::string>;
302 m_sourcedb(sourcedb),m_destdb(destdb),m_allowcreate(allowcreate),
304 m_includehead(false),m_excludehead(false),
305 m_usertags(true),m_userupdatehead(false),m_debug(false),m_alliov(false),
306 m_verify(false),m_root(false),m_zeronull(false),m_analyse(false),
307 m_checkrefs(false),m_listpfn(false),m_poolopen(false),m_poolall(false),
308 m_nocopy(false),m_nodata(false),m_nochannel(false),m_chdesc(false),
309 m_hitag(false),m_nohitag(false),m_forcesingle(false),m_forcemulti(false),
310 m_forcerune(false),m_forcetime(false),m_forcepay(false),m_forcenopay(false),
311 m_sourceread(true),m_truncate(false),m_skipout(false),m_skiprep(false),
312 m_applock(false),m_applocksv(false),
313 m_readoracle(false),m_gettime(false),m_getonline(false),m_onlinerun(false),
315 m_prunetags(false),m_lockedonly(false),m_copytaginfo(false),
316 m_copytaglock(false),m_coracool(true),m_ignoremode(false),
317 m_ignorespec(false),m_checkdesttag(false),m_noclobroot(false),
318 m_runemin(
cool::ValidityKeyMin),m_runemax(
cool::ValidityKeyMax),
319 m_timemin(
cool::ValidityKeyMin),m_timemax(
cool::ValidityKeyMax),
320 m_newrunemin(
cool::ValidityKeyMin),m_newrunemax(
cool::ValidityKeyMax),
321 m_newtimemin(
cool::ValidityKeyMin),m_newtimemax(
cool::ValidityKeyMax),
322 m_srunemin(
cool::ValidityKeyMin),m_srunemax(
cool::ValidityKeyMax),
323 m_channel1(
""),m_channel2(
""),m_bufsize(1000),m_sealmsg(5),
324 m_anadelt(-1),m_outfolder(
""),m_outtag(
""),m_newdataset(
""),
325 m_checkoutputfile(
""),m_timedb(
""),m_taglabel(
""),
326 m_runinfohost(
"http://atlas-run-info-api.web.cern.ch/api"),
327 m_coolapp(&m_coralsvc),
328 m_dbSvc(&(m_coolapp.databaseService())),m_repsort(nullptr),
329 m_open(false),m_chansel(
cool::ChannelSelection::
all()),p_rootfile(nullptr)
334 coral::IConnectionServiceConfiguration& csconfig=
m_coralsvc.configuration();
335 csconfig.disablePoolAutomaticCleanUp();
336 csconfig.setConnectionTimeOut(0);
340 const std::string& destdb,
bool allowcreate) {
346 coral::IConnectionServiceConfiguration& csconfig=
m_coralsvc.configuration();
347 csconfig.setReplicaSortingAlgorithm(*
m_repsort);
351 std::cout <<
"Open source database: " << sourcedb << std::endl;
352 if (!
m_sourceread) std::cout <<
"... in UPDATE mode" << std::endl;
357 std::cout <<
"Cool exception caught: " <<
e.what() << std::endl;
365 std::cout <<
"Open destination ROOT file: " << destdb << std::endl;
366 p_rootfile=
new TFile(destdb.c_str(),
"RECREATE");
367 if (
p_rootfile==
nullptr) std::cout <<
"ERROR: Could not open ROOT file" <<
373 std::cout <<
"Open destination database: " << tdestdb << std::endl;
380 "Forcing recreation of destination database - deleting existing data!"
383 m_dbSvc->dropDatabase(tdestdb);
385 throw cool::DatabaseDoesNotExist(
"old database deleted");
389 std::cout <<
"COOL exception caught: " <<
e.what() << std::endl;
392 std::cout <<
"Try to create new conditions DB" << std::endl;
395 std::cout <<
"Creation succeeded" << std::endl;
397 catch (cool::Exception&
e) {
398 std::cout <<
"Creation failed" << std::endl;
411 std::cout <<
"Attempt to open source CoraCool DB " <<
m_sourcedb <<
415 std::cout <<
"Opened CoraCool source DB" << std::endl;
420 std::cout <<
"Attempt to open destination CoraCool DB " <<
m_destdb <<
424 std::cout <<
"Opened CoraCool dest DB" << std::endl;
432 cool::IFolderSetPtr topfolder=
m_sourceDbPtr->getFolderSet(
"/");
433 const std::vector<std::string>& toptaglist=topfolder->listTags();
434 for (std::vector<std::string>::const_iterator toptag=toptaglist.begin();
435 toptag!=toptaglist.end();++toptag) {
437 if (tstat==cool::HvsTagLock::LOCKED ||
438 tstat==cool::HvsTagLock::PARTIALLYLOCKED) {
439 std::cout <<
"Top-level tag " << *toptag <<
" will be copied" <<
441 m_tags.push_back(*toptag);
444 std::cout <<
"Total of " <<
m_tags.size() <<
" top-level tags to be copied"
452 if (inconn.find(
'/')==std::string::npos) {
453 return "sqlite://X;schema=mycool.db;dbname="+inconn;
461 std::cout <<
"Add folders in path:" <<
folder <<
" [ ";
462 const std::vector<std::string> nodelist=
m_sourceDbPtr->listAllNodes();
464 for (std::vector<std::string>::const_iterator nodeitr=nodelist.begin();
465 nodeitr!=nodelist.end();++nodeitr) {
471 nodeitr->size()>
folder.size() && nodeitr->compare(
folder.size(),1,
"/")==0)) {
474 for (std::vector<std::string>::const_iterator iexcl=
m_folderexcl.begin();
476 if (iexcl->compare(0,1,
"/")==0) {
489 std::cout << *nodeitr <<
" ";
496 const std::vector<std::string> foldersettags=
498 if (!foldersettags.empty()) {
499 for (std::vector<std::string>::const_iterator
500 itr=foldersettags.begin();itr!=foldersettags.end();++itr) {
507 cool::IFolderSetPtr sfolder=
m_sourceDbPtr->getFolderSet(*nodeitr);
508 for (std::vector<std::string>::const_iterator toptag=
m_tags.begin();
509 toptag!=
m_tags.end();++toptag) {
511 std::string rtag=sfolder->resolveTag(*toptag);
524 std::cout <<
"]" << std::endl;
529 std::cout <<
"Adding folder to exclude list: " <<
folder << std::endl;
535 (
const std::string&
folder,
const std::vector<std::string>&
taglist) {
537 cool::IFolderPtr sourcefl,destfl;
540 sourcefl=m_sourceDbPtr->getFolder(
folder);
542 catch (cool::Exception&
e) {
543 std::cout <<
"Could not get source folder: " <<
folder << std::endl;
546 const std::string& sourcedesc=sourcefl->description();
548 if (m_skiprep && sourcedesc.find(
"<norep/>")!=std::string::npos) {
549 std::cout <<
"Folder " <<
folder <<
550 " skipped due to <norep/> metadata" << std::endl;
553 const bool iscora=(m_coracool &&
554 sourcedesc.find(
"<coracool")!=std::string::npos);
558 sourceflc=m_sourceCoraPtr->getFolder(
folder);
561 setChannelRange(sourcefl);
566 dvermode=cool::FolderVersioning::SINGLE_VERSION;
567 std::cout <<
"Forcing destination folder to singleversion" << std::endl;
570 dvermode=cool::FolderVersioning::MULTI_VERSION;
571 std::cout <<
"Forcing destination folder to multiversion" << std::endl;
575 sourcefl->folderSpecification().payloadMode();
579 std::string destfolder=
folder;
580 if (!m_outfolder.empty()) {
581 destfolder=m_outfolder;
582 std::cout <<
"Destination folder will be renamed to " << destfolder <<
588 if (!m_nocopy && !m_root && !m_analyse) {
589 if (!m_destDbPtr->existsFolder(destfolder) && !m_verify) {
590 std::cout <<
"Creating folder " << destfolder <<
" payload-type " <<
591 dpaymode <<
" on destination" << std::endl;
594 if (m_forcerune || m_forcetime) {
595 std::string newmeta=m_forcerune ?
"run-lumi" :
"time";
596 std::string::size_type
p1,
p2;
599 if (
p1!=std::string::npos &&
p2!=std::string::npos) {
601 std::cout <<
"Forced destination folder to " << newmeta <<
" : "
605 "ERROR: Could not parse metadata string to force timestamp type"
610 if (m_forcepay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
611 dpaymode=cool::PayloadMode::SEPARATEPAYLOAD;
612 if (m_forcenopay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
613 dpaymode=cool::PayloadMode::INLINEPAYLOAD;
617 destflc=m_destCoraPtr->createFolder(destfolder,
618 sourceflc->coralTableName(),
619 sourceflc->fkSpecification(),
620 sourceflc->payloadSpecification(),
621 sourceflc->coralFKey(),
622 sourceflc->coralPKey(),
624 std::cout <<
"Created CoraCool folder" << std::endl;
625 destfl=m_destDbPtr->getFolder(destfolder);
627 destfl=m_destDbPtr->createFolder(destfolder,
628 cool::FolderSpecification(dvermode,
629 sourcefl->payloadSpecification(),dpaymode),
633 catch (cool::Exception&
e ) {
634 std::cout <<
"Create folder failed - aborting" << std::endl;
641 if (iscora) destflc=m_destCoraPtr->getFolder(destfolder);
642 destfl=m_destDbPtr->getFolder(destfolder);
644 catch (cool::Exception&
e) {
645 std::cout <<
"Could not get destination folder: " << destfolder
650 const cool::IRecordSpecification& sourcespec=
651 sourcefl->payloadSpecification();
652 const cool::IRecordSpecification& destspec=
653 destfl->payloadSpecification();
654 if (!(sourcespec==destspec)) {
659 "WARNING Source and destination folder specifications differ" <<
661 for (
unsigned int i=0;
i<sourcespec.size();++
i) {
662 const std::string& sname=sourcespec[
i].name();
663 if (!destspec.exists(sname)) {
664 std::cout <<
"ERROR: Field " << sname <<
" absent from destination"
674 "ERROR Source and destination folder specifications differ"
680 const std::string& destdesc=destfl->description();
681 if (sourcedesc!=destdesc) {
682 std::cout <<
"WARNING: Source and destination folder descriptions (meta-data) differ" << std::endl;
683 std::cout <<
"Source folder: " << sourcedesc << std::endl;
684 std::cout <<
"Destn folder: " << destdesc << std::endl;
688 destfl->folderSpecification().payloadMode();
689 if (spaymode!=dpaymode) {
690 std::cout <<
"WARNING: Source (" << spaymode <<
") and destination (" <<
691 dpaymode <<
" folder payload modes differ" << std::endl;
696 if (!checkChannels(
folder,sourcefl,destfl,created))
return 23;
700 if (m_nodata)
return 0;
702 std::cout <<
"Start to process folder: " <<
folder;
704 bool timestamp=(sourcefl->description().find(
"<timeStamp>time")!=
710 std::cout <<
" (timestamp)" << std::endl;
714 std::cout <<
" (run/lumi)" << std::endl;
716 if (m_alliov || m_truncate)
717 std::cout <<
"Output IOVs will be modified" << std::endl;
719 std::cout <<
"IOVs extending outside selection range will be skipped"
722 bool checkrefs=
false;
723 const std::string name0=sourcefl->payloadSpecification()[0].name();
724 if (m_checkrefs && (name0==
"PoolRef" || name0==
"fileGUID")) {
726 std::cout <<
"Check POOL references in folder " <<
folder << std::endl;
730 if (m_checkrefs && m_nocopy && !checkrefs)
return 0;
734 (m_skiprep && (sourcedesc.find(
"<fullrep/>")!=std::string::npos));
736 "All tags in folder will be copied due to <fullrep/> metadata" << std::endl;
739 std::vector<std::string>
tags;
740 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
741 std::cout <<
"Single version folder" << std::endl;
742 tags.emplace_back(
"HEAD");
744 std::cout <<
"Multi version folder: consider tags [ ";
746 const std::vector<std::string> foldertags=sourcefl->listTags();
747 for (std::vector<std::string>::const_iterator itag=foldertags.begin();
748 itag!=foldertags.end();++itag) {
754 for (std::vector<std::string>::const_iterator imtag=m_magic.begin();
755 imtag!=m_magic.end();++imtag) {
756 if (itag->find(*imtag)!=std::string::npos) copyit=
true;
759 tags.push_back(*itag);
760 std::cout << *itag <<
" ";
765 if (
tags.empty() || m_hitag) {
766 for (std::vector<std::string>::const_iterator itag=
taglist.begin();
769 std::string htag=sourcefl->resolveTag(*itag);
771 std::cout << *itag <<
"=" << htag <<
" ";
772 tags.push_back(std::move(htag));
782 std::cout <<
"]" << std::endl;
785 if (((
tags.empty() || m_includehead) && !m_excludehead) ||
787 tags.emplace_back(
"HEAD");
790 sourcefl->setPrefetchAll(
false);
791 for (std::vector<std::string>::const_iterator itag=
tags.begin();
792 itag!=
tags.end();++itag) {
794 if (!m_outtag.empty())
outtag=m_outtag;
802 retcode=verifyIOVs(
folder,sourcefl,sourceflc,destfl,destflc,
805 }
else if (m_nocopy) {
809 }
else if (m_analyse) {
812 retcode=copyIOVs(
folder,destfolder,sourcefl,sourceflc,destfl,destflc,
817 std::cout <<
"ERROR operation failed for folder " <<
folder <<
" tag " <<
833 for (
size_t i=0;
i<nChanRange;
i++) {
838 " to channel selection" << std::endl;
849 std::cout <<
"Adding channel range " <<
c1 <<
" to " <<
c2 <<
850 " to channel selection" << std::endl;
861 const std::string& chanstring) {
862 const char* cstr=chanstring.c_str();
865 return cool::ChannelId(strtoul(cstr,
nullptr,10));
867 cool::ChannelId
chan=0;
870 std::cout <<
"Channel name " << chanstring <<
" maps to channelID "
871 <<
chan << std::endl;
873 catch (cool::Exception&
e) {
874 std::cout <<
"ERROR: Channel name " << chanstring <<
875 " not defined in folder " << std::endl;
883 (
const std::string&
folder,
884 const std::string& destfolder,
887 const std::string& sourcetag,
const std::string& desttag,
888 const cool::ValidityKey
since,
const cool::ValidityKey
until,
889 bool timestamp,
bool checkrefs,
bool iscora,
892 std::cout <<
"Copying tag " << sourcetag <<
" of folder " <<
folder <<
893 " to destination tag " << desttag << std::endl;
897 int updatemode=getUpdateMode(destfl->description(),desttag);
898 std::vector<std::string> dtaglist=destfl->listTags();
901 if (updatemode==1 && m_destdb.find(
"oracle")!=std::string::npos &&
902 (!m_getonline || (!m_truncate && !m_alliov))) {
904 std::cout <<
"Folder is online (UPD1) mode but IGNORING PROTECTION"
907 if(
find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
908 std::cout <<
"Folder is online mode (UPD1) and tag already exist - -getonline and -truncate or -alliov options MUST be used" << std::endl;
912 std::cout <<
"Folder is online mode (UPD1), new tag will be created" << std::endl;
918 if (updatemode==4 && m_destdb.find(
"oracle")!=std::string::npos &&
919 (!m_getbulk || (!m_truncate && !m_alliov))) {
921 std::cout <<
"Folder is bulkreco (UPD4) mode but IGNORING PROTECTION"
924 if(
find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
925 std::cout <<
"Folder is bulkreco mode (UPD4) - -getbulk and -truncate or -alliov options MUST be used" << std::endl;
928 std::cout <<
"Folder is bulkreco mode (UPD4), new tag will be created" << std::endl;
937 if (m_checkdesttag) {
938 if (
find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()
939 || (desttag==
"HEAD" && !created)) {
940 std::cout <<
"Destination tag " << desttag <<
941 " already exists in folder " <<
folder <<
" - skip copy" << std::endl;
947 if ((m_applock || (updatemode==1 && m_getonline) || updatemode==2 ||
948 (updatemode==4 && m_getbulk)) &&
949 dvermode==cool::FolderVersioning::MULTI_VERSION) {
951 std::vector<std::string> dtaglist=destfl->listTags();
952 for (std::vector<std::string>::const_iterator itr=dtaglist.begin();
953 itr!=dtaglist.end();++itr) {
958 if (destfl->tagLockStatus(desttag)==cool::HvsTagLock::LOCKED) {
959 std::cout <<
"Unlocking destination tag " << desttag <<
960 " for append or UPDx access" << std::endl;
962 std::cout <<
"Appending according to SV folder rules" << std::endl;
965 "ERROR: Only allowed for UPD3 mode tags" << std::endl;
969 destfl->setTagLockStatus(desttag,cool::HvsTagLock::UNLOCKED);
981 if (dvermode==cool::FolderVersioning::MULTI_VERSION &&
982 desttag!=
"HEAD" && m_usertags)
localtag=desttag;
987 sourceflc->setPrefetchAll(
false);
989 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
990 sourceitr=sourceflc->browseObjects(
since,
until,m_chansel);
992 sourceitr=sourceflc->browseObjects(
since,
until,m_chansel,sourcetag);
995 std::map<int,int> insertkeymap;
997 while (sourceitr->hasNext()) {
1001 (
obj->since()<m_srunemin ||
obj->until()>m_srunemax)) {
1006 if (!m_excludechans.empty()) {
1007 if (
find(m_excludechans.begin(),m_excludechans.end(),
1008 obj->channelId())!=m_excludechans.end()) {
1014 if (m_debug) std::cout <<
1015 "Setup new CoraCool storage buffer at object " << nobj << std::endl;
1016 destflc->setupStorageBuffer();
1018 cool::ValidityKey newsince,newuntil;
1022 if (newsince>=newuntil) {
1023 std::cout <<
"WARNING: Skipping IOV with since " << newsince <<
1024 ">= until" << newuntil << std::endl;
1031 std::map<int,int>::const_iterator ikey=insertkeymap.end();
1032 bool foundkey=
false;
1035 if (
obj->size()>0) {
1036 oldfk=sourceflc->getAttrKey(
1037 (*
obj->begin())[sourceflc->coralFKey()]);
1038 ikey=insertkeymap.find(oldfk);
1041 if (ikey==insertkeymap.end()) {
1043 destflc->storeObject(newsince,newuntil,
obj->begin(),
obj->end(),
1045 (!m_userupdatehead && !
localtag.empty()));
1046 if (foundkey) insertkeymap[oldfk]=newfk;
1048 destflc->referenceObject(newsince,newuntil,ikey->second,
1050 (!m_userupdatehead && !
localtag.empty()));
1055 if (nbuf==m_bufsize) {
1056 if (m_debug) std::cout <<
"Flush buffer after " << nobj <<
"," <<
1057 nbuf <<
" objects " << std::endl;
1059 destflc->flushStorageBuffer();
1060 destflc->setupStorageBuffer();
1066 if (nref>0) std::cout <<
"Reference-to-existing used for " << nref
1067 <<
" payload objects" << std::endl;
1069 if (m_debug) std::cout <<
"Final buffer flush at " << nobj <<
1070 "," << nbuf << std::endl;
1071 destflc->flushStorageBuffer();
1075 cool::IObjectIteratorPtr sourceitr;
1076 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1077 sourceitr=sourcefl->browseObjects(
since,
until,m_chansel);
1079 sourceitr=sourcefl->browseObjects(
since,
until,m_chansel,sourcetag);
1081 while (sourceitr->goToNext()) {
1082 const cool::IObject&
obj=sourceitr->currentRef();
1084 if (m_skipout && (
obj.since()<m_srunemin ||
obj.until()>m_srunemax)) {
1089 if (!m_excludechans.empty()) {
1090 if (
find(m_excludechans.begin(),m_excludechans.end(),
1091 obj.channelId())!=m_excludechans.end()) {
1097 if (m_debug) std::cout <<
"Setup new storage buffer at object " <<
1099 destfl->setupStorageBuffer();
1101 cool::ValidityKey newsince;
1102 cool::ValidityKey newuntil;
1106 if (newsince>=newuntil) {
1107 std::cout <<
"WARNING: Skipping IOV with since " << newsince <<
1108 ">= until" << newuntil << std::endl;
1112 if (checkrefs) checkRef(
obj.payload(),
folder,sourcetag);
1118 if (newuntil!=cool::ValidityKeyMax) {
1119 std::cout <<
"New IOVs must have until=cool::ValidityKeyMax" << std::endl;
1120 throw cool::Exception(
"Illegal insert over locked IOV",
1123 cool::IObjectIteratorPtr checkitr=destfl->browseObjects(newsince,
1124 newuntil,
obj.channelId(),desttag);
1125 while (checkitr->goToNext()) {
1126 const cool::IObject& checkobj=checkitr->currentRef();
1127 if (checkobj.since()>=newsince) {
1128 std::cout <<
"ERROR:: Attempt to insert SV overlapping IOV whilst appending to locked tag" << std::endl;
1129 throw cool::Exception(
"Illegal insert over locked IOV",
1132 if (checkobj.until()!=cool::ValidityKeyMax) {
1133 std::cout <<
"Existing IOVs must have until=cool::ValidityKeyMax" << std::endl;
1134 throw cool::Exception(
"Illegal insert over locked IOV",
1139 }
else if (m_applock) {
1141 const unsigned int nexist=
1142 destfl->countObjects(newsince,newuntil,
obj.channelId(),desttag);
1144 std::cout <<
"ERROR: Attempt to insert IOV over " << nexist <<
1145 " objects whilst appending to locked tag" << std::endl;
1146 throw cool::Exception(
"Illegal insert over locked IOV",
1149 }
else if (updatemode==2) {
1153 throw cool::Exception(
1154 "Attempt to insert into locked UPD2 tag with timestamp format",
1156 unsigned int run1=newsince >> 32;
1157 unsigned int run2=(newuntil-1) >> 32;
1158 for (
unsigned int irun=
run1;irun<=
run2;++irun) {
1159 if (!std::binary_search(m_runlist.begin(),m_runlist.end(),irun))
1161 std::cout <<
"Run " << irun <<
" from range [" <<
run1 <<
","
1162 <<
run2 <<
"] not found in runfile list" << std::endl;
1163 throw cool::Exception(
"Illegal insert over locked IOV",
1170 if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
1172 cool::IRecordIterator& pitr=
obj.payloadIterator();
1173 const cool::IRecordVectorPtr vptr=pitr.fetchAllAsVector();
1174 destfl->storeObject(newsince,newuntil,
1176 (!m_userupdatehead && !
localtag.empty()));
1181 destfl->storeObject(newsince,newuntil,
1183 (!m_userupdatehead && !
localtag.empty()));
1187 if (nbuf>=m_bufsize) {
1188 if (m_debug) std::cout <<
"Flush buffer after " << nobj <<
"," <<
1189 nbuf <<
" objects " << std::endl;
1190 destfl->flushStorageBuffer();
1198 if (m_debug) std::cout <<
"Final buffer flush at " << nobj <<
1199 "," << nbuf << std::endl;
1200 destfl->flushStorageBuffer();
1203 std::cout <<
"Folder copied with " << nobj <<
" objects" << std::endl;
1205 " objects were skipped extending outside IOV selection or excluded channel"
1207 if (nbad>0) std::cout << nbad <<
1208 " objects were skipped having zero or negative IOV lengths" << std::endl;
1211 catch (cool::Exception&
e) {
1212 std::cout <<
"Exception thrown from copy loop: " <<
e.what() <<
1215 std::cout <<
"Relocking destination tag " << desttag << std::endl;
1216 destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1222 std::cout <<
"Relocking destination tag " << desttag << std::endl;
1223 destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1226 if (dvermode==cool::FolderVersioning::MULTI_VERSION && desttag!=
"HEAD") {
1229 std::cout <<
"Tag folder with HEAD-style tagging for tag: " << desttag
1232 destfl->tagCurrentHead(desttag,sourcefl->tagDescription(sourcetag));
1234 catch (cool::Exception&
e) {
1235 std::cout <<
"Exception thrown in HEAD-style folder-tag: " <<
1236 e.what() << std::endl;
1241 if (m_copytaginfo) m_cooltagmap.insert(
1243 destfolder,sourcetag,desttag,m_taglabel)));
1245 if (!m_hitagmap.empty()) {
1246 for (HiTagMap::const_iterator imap=m_hitagmap.begin();
1247 imap!=m_hitagmap.end();++imap) {
1249 if (sourcefl->findTagRelation(imap->first)==sourcetag) {
1250 std::cout <<
"Create hierarchical tag between " << desttag <<
1251 " and " << imap->first <<
" in folder " << imap->second <<
1255 std::string etag=destfl->resolveTag(imap->first);
1256 if (etag==desttag) {
1257 std::cout <<
"This relation has already been created" << std::endl;
1259 std::cout <<
"ERROR: Tag in parent already related to " <<
1260 desttag << std::endl;
1263 catch (cool::Exception&
e ) {
1266 destfl->createTagRelation(imap->first,desttag);
1269 if (
find(m_hiparent.begin(),m_hiparent.end(),imap->second)==
1270 m_hiparent.end()) m_hiparent.push_back(imap->second);
1273 if (m_copytaginfo &&
1274 m_cooltagmap.find(imap->first)==m_cooltagmap.end())
1275 m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
1276 CoolTagInfo(m_sourceDbPtr,imap->second,imap->second,
1277 imap->first,imap->first)));
1279 catch (cool::Exception&
e) {
1280 std::cout <<
"Cool exception " <<
e.what() <<
1281 "thrown in hierarchical tag creation" << std::endl;
1287 catch (cool::Exception&
e) {
1296 const cool::ValidityKey&
until,
1297 const cool::ValidityKey& qsince,
1298 const cool::ValidityKey& quntil,
1299 cool::ValidityKey& newsince,
1300 cool::ValidityKey& newuntil,
1301 bool timestamp)
const {
1318 if (newsince<qsince) newsince=qsince;
1319 if (newuntil>quntil) newuntil=quntil;
1322 if (newuntil<newsince)
throw cool::Exception(
1323 "Attempt to insert IOV with -ve length",
"AtlCoolCopy::adjustIOVs");
1327 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
1328 const cool::ValidityKey
since,
const cool::ValidityKey
until,
1331 std::cout <<
"Reading tag " << sourcetag <<
" of folder " <<
folder <<
1332 " (no copy)" << std::endl;
1335 cool::IObjectIteratorPtr sourceitr;
1336 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1342 while (sourceitr->goToNext()) {
1343 const cool::IObject&
obj=sourceitr->currentRef();
1347 std::cout <<
"Folder scanned with " << nobj <<
" objects" << std::endl;
1350 catch (cool::Exception&
e) {
1351 std::cout <<
"Exception thrown from read loop: " <<
e.what() <<
1363 const std::string& sourcetag,
1364 const cool::ValidityKey
since,
const cool::ValidityKey
until,
1365 const bool checkrefs,
const bool iscora,
1368 std::cout <<
"Verifying tag " << sourcetag <<
" of folder " <<
folder <<
1370 destfl->setPrefetchAll(
false);
1373 cool::IObjectIteratorPtr sourceitr,destitr;
1375 std::string
tag=sourcetag;
1376 if (vermode==cool::FolderVersioning::SINGLE_VERSION || sourcetag==
"HEAD")
1380 sourceflc->setPrefetchAll(
false);
1389 std::cout <<
"Exception thrown from verify iterator setup: " <<
e.what() <<
1398 const std::string& cfkey=sourceflc->coralFKey();
1399 const std::string& cpkey=sourceflc->coralPKey();
1400 while (csourceitr->hasNext()) {
1409 if (cdestitr->hasNext()) {
1414 if (sobj->since()!=dobj->since() || sobj->until()!=dobj->until())
1417 if (sobj->channelId()!=dobj->channelId())
1420 if (sobj->size()!=dobj->size()) {
1421 std::cout <<
"ERROR CoraCool object " << nobj <<
1422 " sizes do not match: " << sobj->size() <<
" " << dobj->size()
1429 sitr!=sobj->end();++sitr,++ditr) {
1431 for (coral::AttributeList::const_iterator aitr=sitr->begin();
1432 aitr!=sitr->end();++aitr) {
1433 const std::string&
aname=aitr->specification().name();
1436 if (*aitr!=(*ditr)[
aname]) {
1437 std::cout <<
"ERROR Values of attriute " <<
aname <<
1438 " differ" << std::endl;
1442 catch (coral::AttributeListException&
e) {
1443 std::cout <<
"ERROR: CoraCool attribute " <<
aname <<
1444 " not found in destination!" << std::endl;
1451 std::cout <<
"ERROR database entries do not match: since: " <<
1452 sobj->since() <<
"," << dobj->since() <<
" until: " <<
1453 sobj->until() <<
"," << dobj->until() <<
" channel: " <<
1454 sobj->channelId() <<
"," << dobj->channelId() << std::endl;
1459 "ERROR destination folder no matching CoraCool iterator for object "
1460 << nobj << std::endl;
1466 while (sourceitr->goToNext()) {
1467 const cool::IObject& sobj=sourceitr->currentRef();
1476 if (destitr->goToNext()) {
1477 const cool::IObject& dobj=destitr->currentRef();
1481 if (sobj.since()!=dobj.since() || sobj.until()!=dobj.until())
1484 if (sobj.channelId()!=dobj.channelId())
1489 if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
1490 cool::IRecordIterator& spitr=sobj.payloadIterator();
1491 const cool::IRecordVectorPtr svptr=spitr.fetchAllAsVector();
1492 cool::IRecordIterator& dpitr=dobj.payloadIterator();
1493 const cool::IRecordVectorPtr dvptr=dpitr.fetchAllAsVector();
1494 if (svptr->size()!=dvptr->size()) {
1499 cool::IRecordVector::const_iterator svitr=svptr->begin();
1500 cool::IRecordVector::const_iterator svend=svptr->end();
1501 cool::IRecordVector::const_iterator dvitr=dvptr->begin();
1502 for (;svitr!=svend;++svitr,++dvitr) {
1507 std::cout <<
"ERROR vector payloads do not match (size " << svptr->size() <<
"," << dvptr->size() <<
")" << std::endl;
1512 if (!
equalRecord(sobj.payload(),dobj.payload())) iret=102;
1516 std::cout <<
"ERROR database entries do not match: since: " <<
1517 sobj.since() <<
"," << dobj.since() <<
" until: " <<
1518 sobj.until() <<
"," << dobj.until() <<
" channel: " <<
1519 sobj.channelId() <<
"," << dobj.channelId() << std::endl;
1520 std::cout <<
"Source payload:" << std::endl;
1521 sobj.payload().attributeList().toOutputStream(std::cout);
1522 std::cout << std::endl <<
"Destination payload:" << std::endl;
1523 dobj.payload().attributeList().toOutputStream(std::cout);
1524 std::cout << std::endl;
1529 "ERROR destination folder no matching iterator for object "
1530 << nobj << std::endl;
1537 catch (cool::Exception&
e) {
1538 std::cout <<
"Exception thrown from verify loop: " <<
e.what() <<
1548 std::cout <<
"Verification of folder " <<
folder <<
" tag " <<
1549 sourcetag <<
" OK (" << nobj <<
" objects)" << std::endl;
1556 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
1557 const cool::ValidityKey
since,
const cool::ValidityKey
until,
1558 const bool timestamp) {
1560 std::cout <<
"Write tag " << sourcetag <<
" of folder " <<
folder <<
1561 " to ROOT file" << std::endl;
1564 bool timestamp2=timestamp;
1570 TTree*
tree=
static_cast<TTree*
>(
gDirectory->FindObject(treename.c_str()));
1571 if (
tree==
nullptr) {
1572 std::cout <<
"Book TTree " << treename << std::endl;
1573 tree=
new TTree(treename.c_str(),
"COOL datadump");
1584 if (vermode==cool::FolderVersioning::MULTI_VERSION)
1587 const cool::IRecordSpecification&
spec=
1588 (sourcefl->folderSpecification()).payloadSpecification();
1589 unsigned int ncolumns=
spec.size();
1596 const cool::IFieldSpecification& fieldspec=
spec[
icol];
1600 tree->Branch(fieldspec.name().c_str(),
ptr,
1601 (fieldspec.name()+
"/"+rootID).c_str());
1603 <<
" of type " <<
spec[
icol].storageType().name() << std::endl;
1605 std::cout <<
"Attribute " <<
spec[
icol].name() <<
" of type " <<
1606 spec[
icol].storageType().name() <<
" will be skipped" << std::endl;
1611 std::cout <<
"TTree " << treename <<
" already exists" << std::endl;
1614 (sourcefl->folderSpecification()).payloadSpecification().size();
1616 std::cout <<
"ERROR in tree buffer definition: expect " << treename <<
1625 cool::IObjectIteratorPtr sourceitr;
1627 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1633 catch (cool::Exception&
e) {
1634 std::cout <<
"Exception thrown from ROOT copy iterator setup: " <<
1635 e.what() << std::endl;
1640 while (sourceitr->goToNext()) {
1641 const cool::IObject& sobj=sourceitr->currentRef();
1652 if (vermode==cool::FolderVersioning::MULTI_VERSION) {
1655 std::string sourcetag2=sourcetag.substr(0,255);
1661 const cool::IRecord& record=sobj.payload();
1668 catch (cool::Exception&
e) {
1675 std::cout <<
"Written " << nobj <<
" objects to ROOT TTree with " << nex
1676 <<
" nulls/exceptions" << std::endl;
1678 catch (cool::Exception&
e) {
1679 std::cout <<
"Exception thrown from ROOT file writing loop: " <<
1680 e.what() << std::endl;
1687 const std::string& toproot) {
1692 if (
p_rootfile->FindObject(toproot.c_str())==
nullptr) {
1694 std::cout <<
"Made top directory " << toproot << std::endl;
1699 std::string::size_type iofs1=1;
1700 std::string::size_type iofs2=1;
1701 std::string treename;
1702 while (iofs2!=std::string::npos) {
1703 iofs2=
folder.find(
'/',iofs1);
1704 if (iofs2==std::string::npos) {
1706 treename=
folder.substr(iofs1);
1711 std::cout <<
"Make directory " <<
dirname << std::endl;
1721 void*& sptr,
char& rootID)
const {
1722 const cool::StorageType& stype=
spec.storageType();
1725 if (stype==cool::StorageType::Bool) {
1727 sptr=
static_cast<void*
>(
new unsigned int(0));
1729 }
else if (stype==cool::StorageType::UChar) {
1730 sptr=
static_cast<void*
>(
new unsigned char(
' '));
1732 }
else if (stype==cool::StorageType::Int16) {
1733 sptr=
static_cast<void*
>(
new short(0));
1735 }
else if (stype==cool::StorageType::UInt16) {
1736 sptr=
static_cast<void*
>(
new unsigned short(0));
1738 }
else if (stype==cool::StorageType::Int32) {
1739 sptr=
static_cast<void*
>(
new int(0));
1741 }
else if (stype==cool::StorageType::UInt32) {
1742 sptr=
static_cast<void*
>(
new unsigned int(0));
1744 }
else if (stype==cool::StorageType::Int64) {
1745 sptr=
static_cast<void*
>(
new long long int(0));
1747 }
else if (stype==cool::StorageType::UInt63) {
1748 sptr=
static_cast<void*
>(
new unsigned long long int(0));
1751 sptr=
static_cast<void*
>(
new float(0.));
1753 }
else if (stype==cool::StorageType::Double) {
1754 sptr=
static_cast<void*
>(
new double(0.));
1756 }
else if (stype==cool::StorageType::String4k ||
1757 stype==cool::StorageType::String255) {
1758 sptr=
static_cast<void*
>(
new char[4100]);
1760 }
else if (stype==cool::StorageType::String64k && !
m_noclobroot) {
1761 sptr=
static_cast<void*
>(
new char[65536]);
1763 }
else if (stype==cool::StorageType::String16M && !
m_noclobroot) {
1764 sptr=
static_cast<void*
>(
new char[67108864]);
1766 }
else if (stype==cool::StorageType::Blob64k && !
m_noclobroot) {
1767 sptr=
static_cast<void*
>(
new char[65536]);
1770 std::cout <<
"rootAllocate: Unsupported storage type for attribute: " <<
1771 spec.name() << std::endl;
1773 return (sptr!=
nullptr);
1779 const cool::StorageType& stype=
field.storageType();
1780 if (stype==cool::StorageType::Bool) {
1782 *(
static_cast<int*
>(sptr))=0;
1784 *(
static_cast<int*
>(sptr))=(
field.data<
bool>() ? 1 : 0);
1786 }
else if (stype==cool::StorageType::UChar) {
1788 *(
static_cast<unsigned char*
>(sptr))=0;
1790 *(
static_cast<unsigned char*
>(sptr))=
field.data<
unsigned char>();
1792 }
else if (stype==cool::StorageType::Int16) {
1794 *(
static_cast<short*
>(sptr))=0;
1796 *(
static_cast<short*
>(sptr))=
field.data<
short>();
1798 }
else if (stype==cool::StorageType::UInt16) {
1800 *(
static_cast<unsigned short*
>(sptr))=0;
1802 *(
static_cast<unsigned short*
>(sptr))=
field.data<
unsigned short>();
1804 }
else if (stype==cool::StorageType::Int32) {
1806 *(
static_cast<int*
>(sptr))=0;
1808 *(
static_cast<int*
>(sptr))=
field.data<
int>();
1810 }
else if (stype==cool::StorageType::UInt32) {
1812 *(
static_cast<unsigned int*
>(sptr))=0;
1814 *(
static_cast<unsigned int*
>(sptr))=
field.data<
unsigned int>();
1816 }
else if (stype==cool::StorageType::Int64) {
1818 *(
static_cast<long long*
>(sptr))=0;
1820 *(
static_cast<long long*
>(sptr))=
field.data<
long long>();
1822 }
else if (stype==cool::StorageType::UInt63) {
1824 *(
static_cast<unsigned long long*
>(sptr))=0;
1826 *(
static_cast<unsigned long long*
>(sptr))=
field.data<
unsigned long long>();
1830 *(
static_cast<float*
>(sptr))=0.;
1832 *(
static_cast<float*
>(sptr))=
field.data<
float>();
1834 }
else if (stype==cool::StorageType::Double) {
1836 *(
static_cast<double*
>(sptr))=0.;
1838 *(
static_cast<double*
>(sptr))=
field.data<
double>();
1840 }
else if (stype==cool::StorageType::String255 ||
1841 stype==cool::StorageType::String4k ||
1842 stype==cool::StorageType::String64k ||
1843 stype==cool::StorageType::String16M) {
1845 strcpy(
static_cast<char*
>(sptr),
"NULL");
1847 strcpy(
static_cast<char*
>(sptr),
field.data<std::string>().c_str());
1849 }
else if (stype==cool::StorageType::Blob64k && !
m_noclobroot) {
1851 strcpy(
static_cast<char*
>(sptr),
"NULL");
1854 std::string blobStr((
char*)
blob.startingAddress(),
blob.size());
1855 strcpy(
static_cast<char*
>(sptr), blobStr.c_str());
1858 std::cout <<
"ERROR: Unknown storage type in rootWrite!" << std::endl;
1863 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
1864 const cool::ValidityKey
since,
const cool::ValidityKey
until,
1865 const bool timestamp) {
1867 std::cout <<
"Analyse tag " << sourcetag <<
" of folder " <<
folder <<
1871 std::cout <<
"Analyse time structures with tolerance of " <<
1872 m_anadelt/1.E9 <<
" seconds " << std::endl;
1877 std::cout <<
"Make directory " <<
dirname << std::endl;
1882 std::vector<cool::ChannelId>
channels=sourcefl->listChannels();
1885 long long* lastiov=
new long long[
nchan];
1886 long long* iovtotlen=
new long long[
nchan];
1887 long long* iovtotgap=
new long long[
nchan];
1888 int* iovn=
new int[
nchan];
1889 for (
unsigned int i=0;
i<
nchan;++
i) {
1895 cool::ValidityKey globsince=cool::ValidityKeyMax;
1896 cool::ValidityKey globuntil=cool::ValidityKeyMin;
1902 "IOVs per channel-name (copy of IOVSperChannel)",
1909 TH1F* h_anadelt=
nullptr;
1910 TH1F* h_chandelt=
nullptr;
1912 h_anadelt=
bookOrFindTH1F(
"IOVAlignLogDelT",
"IOV alignment log deltaT",
1916 h_chandelt=
bookOrFindTH1F(
"AlignedChanPerIOV",
"Aligned channels per IOV",
1919 std::cout <<
"Booked histograms for folder with " <<
nchan <<
" channels"
1922 using IOVTimeMap = std::map<cool::ValidityKey, int>;
1923 IOVTimeMap iov_time_map;
1928 cool::IObjectIteratorPtr sourceitr;
1930 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1936 catch (cool::Exception&
e) {
1937 std::cout <<
"Exception thrown from analysis copy iterator setup: " <<
1938 e.what() << std::endl;
1942 std::map<cool::ChannelId,std::string>
1943 chanmap=sourcefl->listChannelsWithNames();
1946 while (sourceitr->goToNext()) {
1947 const cool::IObject& sobj=sourceitr->currentRef();
1949 cool::ChannelId ichanid=sobj.channelId();
1950 const std::string& cname=chanmap[ichanid];
1954 if (chanitr.first!=chanitr.second) {
1957 if (
ichan && !h_iovname->GetBinContent(
bin)) {
1958 h_iovname->GetXaxis()->SetBinLabel(
bin,cname.c_str());
1960 h_iovname->Fill(
ichan);
1962 cool::ValidityKey since2=sobj.since();
1963 cool::ValidityKey until2=sobj.until();
1964 if (since2<globsince) globsince=since2;
1966 if (until2!=cool::ValidityKeyMax) {
1967 if (until2>globuntil) globuntil=until2;
1968 long long len=until2-since2;
1969 h_iovlength->Fill(log10(len));
1971 iovtotlen[
ichan]+=len;
1973 if (lastiov[
ichan]<
static_cast<long long>(since2) &&
1974 lastiov[
ichan]>=0) {
1976 long long gap=since2-lastiov[
ichan];
1978 h_iovgap->Fill(log10(
gap));
1979 std::cout <<
"Gap of " <<
gap << std::endl;
1981 if (until2!=cool::ValidityKeyMax) {
1982 lastiov[
ichan]=until2;
1984 lastiov[
ichan]=since2;
1994 if (hiitr!=iov_time_map.end()) {
1996 del1=hiitr->first-since2;
1998 if (lowitr!=iov_time_map.begin()) {
2001 del2=since2-lowitr->first;
2006 if (del1<=m_anadelt && del1>-1) {
2011 if (del2<=m_anadelt && del2>-1 && (del2<del || del==-1)) {
2019 h_anadelt->Fill(log10(del));
2021 h_anadelt->Fill(0.);
2024 iov_time_map[since2]=0;
2029 std::cout <<
"ERROR :Channel " << ichanid <<
2030 " not found in channel list - ignored" << std::endl;
2034 std::cout <<
"Finished analysis with " << nobj <<
" objects" << std::endl;
2036 std::cout <<
"IOV timestamp range: " << globsince <<
" " <<
2037 timeString(globsince) <<
"to " << globuntil <<
" " <<
2040 std::cout <<
"IOV run/LB range [" << (globsince >> 32) <<
"," <<
2041 (globsince & 0
xFFFFFFFF) <<
"] to [" << (globuntil >> 32) <<
"," <<
2042 (globuntil & 0
xFFFFFFFF) <<
"]" << std::endl;
2048 "Mean IOV length per channel",
nchan,-0.5,
nchan-0.5);
2049 for (
unsigned int i=0;
i<
nchan;++
i) {
2053 occ=1.-
float(iovtotgap[
i])/
float(iovtotlen[
i]+iovtotgap[
i]);
2054 avlen=
float(iovtotlen[
i])/iovn[
i];
2056 h_iovoccchan->Fill(
i,occ);
2057 h_iovlenchan->Fill(
i,avlen);
2060 std::cout <<
"Alignment analysis: " << iov_time_map.size()
2061 <<
" seperate IOV starts (tolerance " <<
m_anadelt/1.E9 <<
2062 " seconds)" << std::endl;
2064 for (IOVTimeMap::const_iterator itr=iov_time_map.begin();
2065 itr!=iov_time_map.end();++itr) {
2066 h_chandelt->Fill(itr->second);
2070 catch (cool::Exception&
e) {
2071 std::cout <<
"Exception thrown from folder analysis reading loop: " <<
2072 e.what() << std::endl;
2077 delete [] iovtotlen;
2078 delete [] iovtotgap;
2079 delete[] iovn; iovn=
nullptr;
2084 const std::string& htitle,
2085 const int nchan,
const float xlow,
const float xhigh) {
2090 return new TH1F(hID.c_str(),htitle.c_str(),
nchan,xlow,xhigh);
2092 return static_cast<TH1F*
>(
obj);
2101 for (std::vector<std::string>::const_iterator ifolder=
2102 m_folderlist.begin();ifolder!=m_folderlist.end();++ifolder) {
2106 if (!m_hiparent.empty()) {
2108 int code=tagParents();
2111 if (m_copytaginfo) {
2113 int code=writeTagInfo();
2118 if (m_poolcat.empty()) {
2119 code=listPoolRefs();
2120 if (!m_checkoutputfile.empty()) filePoolRefs();
2122 code=resolvePoolRefs();
2127 p_rootfile->Write();
2137 std::vector<std::string>&
folders) {
2143 std::string_view par0=
argv[
ic];
2145 if (par0.compare(0,2,
"--")==0) par0=par0.substr(1);
2146 if ((par0==
"-f" || par0==
"-folder") &&
ir>1) {
2149 }
else if ((par0==
"-e" || par0==
"-exclude") &&
ir>1) {
2152 }
else if ((par0==
"-t" || par0==
"-tag") &&
ir>1) {
2155 }
else if ((par0==
"-mt" || par0==
"-magic") &&
ir>1) {
2158 }
else if ((par0==
"-of" || par0==
"-outfolder") &&
ir>1) {
2161 }
else if ((par0==
"-ot" || par0==
"-outtag") &&
ir>1) {
2164 }
else if ((par0==
"-bs" || par0==
"-buffersize") &&
ir>1) {
2167 }
else if ((par0==
"-sl" || par0==
"-seal") &&
ir>1) {
2170 }
else if ((par0==
"-rls" || par0==
"-runlumisince") &&
ir>2) {
2173 }
else if ((par0==
"-rlu" || par0==
"-runlumiuntil") &&
ir>2) {
2176 }
else if ((par0==
"-nrls" || par0==
"-newrunlumisince") &&
ir>2) {
2179 }
else if ((par0==
"-nrlu" || par0==
"-newrunlumiuntil") &&
ir>2) {
2182 }
else if ((par0==
"-srls" || par0==
"-skiprunlumisince") &&
ir>2) {
2186 }
else if ((par0==
"-srlu" || par0==
"-skiprunlumiuntil") &&
ir>2) {
2190 }
else if (par0==
"-ro" || par0==
"-root") {
2192 }
else if (par0==
"-zn" || par0==
"-zeronull") {
2194 }
else if (par0==
"-ana" || par0==
"-analyse") {
2197 m_anadelt=
static_cast<long long>(ttime);
2199 }
else if ((par0==
"-rs" || par0==
"-runsince") &&
ir>1) {
2202 }
else if ((par0==
"-ru" || par0==
"-rununtil") &&
ir>1) {
2205 }
else if ((par0==
"-r" || par0==
"-run") &&
ir>1) {
2209 }
else if ((par0==
"-ts" || par0==
"-timesince") &&
ir>1) {
2212 }
else if ((par0==
"-tu" || par0==
"-timeuntil") &&
ir>1) {
2215 }
else if ((par0==
"-nts" || par0==
"-newtimesince") &&
ir>1) {
2218 }
else if ((par0==
"-ntu" || par0==
"-newtimeuntil") &&
ir>1) {
2221 }
else if (par0==
"-c" || par0==
"-create") {
2223 }
else if ((par0==
"-ch" || par0==
"-channel") &&
ir>1) {
2226 }
else if ((par0==
"-ch1" || par0==
"-channel1") &&
ir>1) {
2229 }
else if ((par0==
"-ch2" || par0==
"-channel2") &&
ir>1) {
2232 }
else if (par0==
"-chd" || par0==
"-channeldesc") {
2235 }
else if (par0==
"-forcerecreate") {
2237 }
else if (par0==
"-d" || par0==
"-debug") {
2239 }
else if (par0==
"-a" || par0==
"-alliov") {
2241 }
else if (par0==
"-ih" || par0==
"-includehead") {
2243 }
else if (par0==
"-eh" || par0==
"-excludehead") {
2245 }
else if (par0==
"-ht" || par0==
"-headtag") {
2247 }
else if (par0==
"-uht" || par0==
"-userheadtag") {
2249 }
else if (par0==
"-v" || par0==
"-verify") {
2251 }
else if (par0==
"-nc" || par0==
"-nocopy") {
2253 }
else if (par0==
"-noc" || par0==
"-nocoracool") {
2255 }
else if (par0==
"-nch" || par0==
"-nochannel") {
2257 }
else if (par0==
"-ncr" || par0==
"-noclobroot") {
2259 }
else if (par0==
"-nd" || par0==
"-nodata") {
2261 }
else if (par0==
"-nh" || par0==
"-nohitag") {
2263 }
else if (par0==
"-hi" || par0==
"-hitag") {
2265 }
else if ((par0==
"-ec" || par0==
"-excludechannel") &&
ir>1) {
2268 }
else if (par0==
"-fs" || par0==
"-forcesingle") {
2270 }
else if (par0==
"-fm" || par0==
"-forcemulti") {
2272 }
else if (par0==
"-frl" || par0==
"-forcerunlumi") {
2274 }
else if (par0==
"-ftm" || par0==
"-forcetime") {
2276 }
else if (par0==
"-fp" || par0==
"-forcepayload") {
2278 }
else if (par0==
"-fnp" || par0==
"-forcenopayload") {
2280 }
else if (par0==
"-cr" || par0==
"-checkrefs") {
2282 }
else if (par0==
"-lp" || par0==
"-listpfn") {
2285 }
else if (par0==
"-cf" || par0==
"-checkfiles") {
2288 }
else if (par0==
"-pa" || par0==
"-poolall") {
2290 }
else if ((par0==
"-co" || par0==
"-checkoutput") &&
ir>1) {
2293 }
else if ((par0==
"-pc" || par0==
"-poolcat") &&
ir>1) {
2296 }
else if ((par0==
"-mc" || par0==
"-mergecat") &&
ir>1) {
2299 }
else if ((par0==
"-ds" || par0==
"-dataset") &&
ir>1) {
2302 }
else if (par0==
"-us" || par0==
"-updatesource") {
2304 }
else if (par0==
"-cd" || par0==
"-checkdest") {
2306 }
else if (par0==
"-tr" || par0==
"-truncate") {
2308 }
else if (par0==
"-al" || par0==
"-appendlocked") {
2310 }
else if (par0==
"-alsv" || par0==
"-appendlockedsv") {
2313 }
else if (par0==
"-rdo" || par0==
"-readoracle") {
2315 }
else if (par0==
"-skiprep" || par0==
"-sr") {
2317 }
else if (par0==
"-go" || par0==
"-getonline") {
2319 }
else if (par0==
"-onr" || par0==
"-onlinerun") {
2321 }
else if (par0==
"-gb" || par0==
"-getbulk") {
2323 }
else if (par0==
"-gt" || par0==
"-gettime") {
2325 }
else if ((par0==
"-tdb" || par0==
"-timedb") &&
ir>1) {
2328 }
else if (par0==
"-ignoremode" &&
ir>1) {
2329 std::cout <<
"Ignoremode password is " <<
argv[
ic+1] <<
":end" << std::endl;
2330 if (strcmp(
argv[
ic+1],
"BackDoor")==0) {
2333 std::cout <<
"ERROR: Incorrect password for -ignoremode" << std::endl;
2337 }
else if (par0==
"-is" || par0==
"-ignorespec") {
2339 }
else if (par0==
"-pt" || par0==
"-prunetags") {
2342 }
else if (par0==
"-lo" || par0==
"-lockedonly") {
2346 }
else if (par0==
"-cti" || par0==
"-copytaginfo") {
2348 }
else if (par0==
"-ctl" || par0==
"-copytaglock") {
2351 }
else if ((par0==
"-tl" || par0==
"-taglabel") &&
ir>1) {
2355 }
else if ((par0==
"-ag" || par0==
"-addguid") &&
ir>1) {
2358 }
else if ((par0==
"-alf" || par0==
"-addlfn") &&
ir>1) {
2361 }
else if ((par0==
"-pf" || par0==
"-parfile") &&
ir>1) {
2364 }
else if ((par0==
"-rf" || par0==
"-runfile") &&
ir>1) {
2367 }
else if ((par0==
"-ws" || par0==
"-runinfohost") &&
ir>1) {
2370 }
else if (par0==
"-h" || par0==
"-help") {
2374 std::cout <<
"Parameter error for argument: " << par0 << std::endl;
2385 std::vector<std::string>
folders;
2390 for (std::vector<std::string>::const_iterator
ifile=
m_parfile.begin();
2392 std::cout <<
"Reading parameters from file " << *
ifile << std::endl;
2393 FILE* p_inp=fopen(
ifile->c_str(),
"r");
2394 if (p_inp==
nullptr) {
2395 std::cout <<
"File not found" << std::endl;
2398 std::vector<char> p_buf (999);
2399 while (!feof(p_inp)) {
2400 char* p_line=fgets(p_buf.data(),p_buf.size(),p_inp);
2401 if (p_line!=
nullptr) {
2403 const char* fargv[99];
2405 char* p_start=
nullptr;
2406 while (*p_line!=
'\0') {
2407 if (*p_line==
' ' || *p_line==
'\n') {
2409 if (p_start!=
nullptr) {
2412 fargv[fargc]=p_start;
2418 if (p_start==
nullptr) p_start=p_line;
2430 std::cout <<
"Close file" << std::endl;
2436 std::cout <<
"Problem opening connections" << std::endl;
2450 for (std::vector<std::string>::const_iterator ifold=
folders.begin();
2470 std::cout <<
"Source run/LB range [" << (
m_runemin >> 32) <<
"," <<
2474 std::cout <<
"Source timestamp range " <<
m_timemin <<
" " <<
2478 std::cout <<
"Change o/p run/LB range [" << (
m_newrunemin >> 32) <<
"," <<
2482 std::cout <<
"Change o/p timestamp range " <<
m_newtimemin <<
" " <<
2486 std::cout <<
"Skip IOVs extending outside run/LB range [" <<
2492 for (std::vector<cool::ChannelId>::const_iterator itr=
2494 std::cout <<
"Channel " << *itr <<
" will be excluded" << std::endl;
2496 if (
m_hitag) std::cout <<
"All hierarchical tags connecting to referenced tags will be copied" << std::endl;
2497 if (
m_nohitag) std::cout <<
"Hierarchical tag relations will not be copied"
2505 const char* cptr=
input;
2506 while (*cptr!=
'\0') {
2507 if (!isdigit(*cptr)) { isnum=
false;
break;}
2514 const cool::IRecord& rhs) {
2517 if (lhs.size()!=rhs.size())
return false;
2518 for (
size_t i=0;
i<lhs.size();++
i) {
2520 if (lhs[
i].specification().storageType()!=
2521 rhs[
i].specification().storageType())
2524 if (lhs[
i]!=rhs[
i]) {
2527 const cool::StorageType& stype=lhs[
i].specification().storageType();
2528 if (stype==cool::StorageType::String255 ||
2529 stype==cool::StorageType::String4k ||
2530 stype==cool::StorageType::String64k ||
2531 stype==cool::StorageType::String16M) {
2533 if (lhs[
i].data<std::string>()!=rhs[
i].data<std::string>())
2551 return static_cast<long long>(atol(
input))*
2552 static_cast<long long>(1.E9);
2554 struct tm mytm{},mytm2{};
2562 cool::ValidityKey itime=
static_cast<cool::ValidityKey
>(mktime(&mytm));
2563 strptime(
"1970-01-02:00:00:00",
"%Y-%m-%d:%T",&mytm2);
2564 cool::ValidityKey caltime=
static_cast<cool::ValidityKey
>(mktime(&mytm2));
2565 if (caltime ==
static_cast<cool::ValidityKey
>(-1)) {
2567 "ERROR in mktime" << std::endl;
2570 itime+=24*60*60-caltime;
2571 return itime*
static_cast<cool::ValidityKey
>(1.E9);
2574 "ERROR in format of time value, use e.g. 2007-05-25:14:01:00" <<
2582 if (iovtime==cool::ValidityKeyMin) {
2583 return "ValidityKeyMin ";
2584 }
else if (iovtime==cool::ValidityKeyMax) {
2585 return "ValidityKeyMax ";
2587 time_t
time=
static_cast<time_t
>(iovtime/1E9);
2590 return "UTC "+std::string(asctime_r(gmtime_r(&
time, &
result), buf));
2597 cool::ValidityKey
val;
2599 val=((1LL << 31)-1) << 32;
2601 val=
static_cast<long long>(atol(
input1)) << 32;
2603 if (input2[0]==
'M' || input2[0]==
'm') {
2608 if (
val>cool::ValidityKeyMax)
val=cool::ValidityKeyMax;
2626 std::string sorfolder=
"/TDAQ/RunCtrl/SOR_Params";
2627 std::string eorfolder=
"/TDAQ/RunCtrl/EOR_Params";
2628 if (
m_timedb.find (
"CONDBR2")!=std::string::npos) {
2629 sorfolder=
"/TDAQ/RunCtrl/SOR";
2630 eorfolder=
"/TDAQ/RunCtrl/EOR";
2632 std::cout <<
"Extracting times for run range [" << (
m_runemin >> 32) <<
2634 " folder " << sorfolder << std::endl;
2636 std::cout <<
"ERROR: Run range not correctly specified" << std::endl;
2640 cool::IDatabasePtr tdaqdb;
2643 std::cout <<
"Opened database connection" << std::endl;
2646 std::cout <<
"COOL exception caught: " <<
e.what() << std::endl;
2651 cool::IFolderPtr
folder=tdaqdb->getFolder(sorfolder);
2656 while (itr->goToNext()) {
2657 const cool::IRecord&
payload=itr->currentRef().payload();
2663 std::cout <<
"ERROR: Found " << nobj <<
" SOR records" << std::endl;
2668 std::cout <<
"Exception accessing SOR information: " <<
e.what() <<
2674 cool::IFolderPtr
folder=tdaqdb->getFolder(eorfolder);
2679 while (itr->goToNext()) {
2680 const cool::IRecord&
payload=itr->currentRef().payload();
2686 std::cout <<
"ERROR: Found " << nobj <<
" EOR records" << std::endl;
2691 std::cout <<
"Exception accessing SOR information: " <<
e.what() <<
2695 tdaqdb->closeDatabase();
2696 std::cout <<
"Timestamp range set to " <<
m_timemin <<
" " <<
2706 std::cout <<
"Extracting current run-number from ATLAS_RUN_NUMBER @ ATONR_ADG ... " <<
2708 curl_global_init(CURL_GLOBAL_ALL);
2709 using uniqueCurl_t = std::unique_ptr<CURL,decltype(&curl_easy_cleanup)>;
2710 uniqueCurl_t curl(curl_easy_init(), curl_easy_cleanup);
2711 CURLcode
res = CURLE_OK;
2715 res = curl_easy_setopt(curl.get(), CURLOPT_URL,
url.c_str());
2716 if (
res != CURLE_OK) {
2717 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2721 res = curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1
L);
2722 if (
res != CURLE_OK) {
2723 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2731 if (
res != CURLE_OK) {
2732 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2735 res = curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &
response);
2736 if (
res != CURLE_OK) {
2737 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2742 res = curl_easy_perform(curl.get());
2743 if (
res != CURLE_OK) {
2744 std::cerr <<
"Failed to perform request: " << curl_easy_strerror(
res) <<
":" <<
url.c_str() << std::endl;
2748 std::cout <<
"Response: " << std::endl;
2749 std::cout <<
response << std::endl;
2755 int runNumber = jsonData[
"resources"][0][
"runnumber"];
2757 std::cout <<
"Next run started will be " << nextrun << std::endl;
2759 std::cout <<
"Epoch time extracted " <<
rtime << std::endl;
2766 m_runemin=(
static_cast<long long>(nextrun)) << 32;
2770 }
catch (nlohmann::json::parse_error&
e) {
2771 std::cerr <<
"Failed to parse JSON response: " <<
e.what() << std::endl;
2773 }
catch (nlohmann::json::type_error&
e) {
2774 std::cerr <<
"Failed to extract data from JSON response: " <<
e.what() << std::endl;
2777 std::cerr <<
"Failed to extract run and timestamp from JSON response: " <<
e.what() << std::endl;
2783 std::cerr <<
"Failed to initialize libcurl." << std::endl;
2787 curl_global_cleanup();
2793 std::cout <<
"ERROR: -getonline and -getbulk cannot be used simultaneously"
2797 std::cout <<
"Call getbulk using URL" << std::endl;
2799 curl_global_init(CURL_GLOBAL_ALL);
2800 using uniqueCurl_t = std::unique_ptr<CURL,decltype(&curl_easy_cleanup)>;
2801 uniqueCurl_t curl(curl_easy_init(), curl_easy_cleanup);
2803 CURLcode
res = CURLE_OK;
2808 res = curl_easy_setopt(curl.get(), CURLOPT_URL,
url.c_str());
2809 if (
res != CURLE_OK) {
2810 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2814 res = curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1
L);
2815 if (
res != CURLE_OK) {
2816 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2824 if (
res != CURLE_OK) {
2825 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2828 res = curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &
response);
2829 if (
res != CURLE_OK) {
2830 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2834 res = curl_easy_perform(curl.get());
2835 if (
res != CURLE_OK) {
2836 std::cerr <<
"Failed to perform request: " << curl_easy_strerror(
res) <<
":" <<
url.c_str() << std::endl;
2840 std::cout <<
"Response: " << std::endl;
2841 std::cout <<
response << std::endl;
2846 int nextrun = jsonData[
"run"];
2847 long long rtime = jsonData[
"timestamp"];
2848 std::cout <<
"Next run started will be " << nextrun << std::endl;
2849 std::cout <<
"Epoch time extracted " <<
rtime << std::endl;
2856 m_runemin=(
static_cast<long long>(nextrun)) << 32;
2860 }
catch (nlohmann::json::parse_error&
e) {
2861 std::cerr <<
"Failed to parse JSON response: " <<
e.what() << std::endl;
2863 }
catch (nlohmann::json::type_error&
e) {
2864 std::cerr <<
"Failed to extract data from JSON response: " <<
e.what() << std::endl;
2867 std::cerr <<
"Failed to extract run and timestamp from JSON response: " <<
e.what() << std::endl;
2873 std::cerr <<
"Failed to initialize libcurl." << std::endl;
2877 curl_global_cleanup();
2884 for (std::vector<std::string>::const_iterator itr=
m_runfile.begin();
2886 std::cout <<
"Reading allowed run numbers from file " << *itr << std::endl;
2887 FILE* p_inp=fopen(itr->c_str(),
"r");
2888 if (p_inp==
nullptr) {
2889 std::cout <<
"File not found" << std::endl;
2892 std::vector<char> p_buf (999);
2893 while (!feof(p_inp)) {
2894 char* p_line=fgets(p_buf.data(),p_buf.size(),p_inp);
2895 if (p_line!=
nullptr) {
2896 unsigned int run=
atoi(p_line);
2903 std::cout <<
"Read list of " <<
m_runlist.size() <<
" runs from " <<
2904 m_runfile.size() <<
" input runlist files" << std::endl;
2905 for (std::vector<unsigned int>::const_iterator itr=
m_runlist.begin();
2906 itr!=
m_runlist.end();++itr) std::cout <<
2907 "Update allowed for run " << *itr << std::endl;
2912 std::string_view
tag) {
2915 std::string_view modestr=
"";
2917 std::string::size_type iofs1=
desc.find(
"<updateMode>");
2918 std::string::size_type iofs2=
desc.find(
"</updateMode>");
2919 if (iofs1!=std::string::npos && iofs2!=std::string::npos && iofs2>iofs1)
2920 modestr=
desc.substr(iofs1+12,iofs2-iofs1-12);
2921 if (modestr==
"UPD1" ||
tag.find(
"UPD1")!=std::string::npos)
mode=1;
2922 if (modestr==
"UPD2" ||
tag.find(
"UPD2")!=std::string::npos)
mode=2;
2923 if (modestr==
"UPD3" ||
tag.find(
"UPD3")!=std::string::npos)
mode=3;
2924 if (modestr==
"UPD4" ||
tag.find(
"UPD4")!=std::string::npos)
mode=4;
2930 const cool::IFolderPtr& sourcefl,
const cool::IFolderPtr& destfl,
2936 const std::map<cool::ChannelId,std::string> chanmap=
2937 sourcefl->listChannelsWithNames();
2938 if (
m_debug) std::cout <<
"Checking channels table for folder " <<
folder
2939 <<
" with " << chanmap.size() <<
" channels" << std::endl;
2943 for (std::map<cool::ChannelId,std::string>::const_iterator
2944 itr=chanmap.begin();itr!=chanmap.end();++itr) {
2946 const cool::ChannelId
chan=itr->first;
2954 const std::string& sourcename=itr->second;
2956 std::string sourcedesc=
"";
2957 if (
m_chdesc) sourcedesc=sourcefl->channelDescription(
chan);
2960 const std::string & destname=destfl->channelName(
chan);
2961 std::string destdesc=
"";
2962 if (
m_chdesc) destdesc=destfl->channelDescription(
chan);
2964 if (sourcename!=destname && !sourcename.empty()) {
2967 std::cout <<
"ERROR: Channel " <<
chan <<
" names differ: " <<
2968 sourcename <<
" " << destname << std::endl;
2971 destfl->setChannelName(
chan,sourcename);
2972 std::cout <<
"Modify channel " <<
chan <<
" name from "
2973 << destname <<
" to " << sourcename << std::endl;
2976 if (sourcedesc!=destdesc && !sourcedesc.empty()) {
2979 std::cout <<
"ERROR: Channel " <<
chan <<
" descriptions differ: "
2980 << sourcedesc <<
" " << destdesc << std::endl;
2983 std::cout <<
"Modify channel " <<
chan <<
" description from "
2984 << sourcedesc <<
" to " << destdesc << std::endl;
2985 destfl->setChannelDescription(
chan,sourcedesc);
2992 if (
m_debug) std::cout <<
"Create new channel " <<
chan <<
" name "
2993 << sourcename <<
" description" << sourcedesc << std::endl;
2994 destfl->createChannel(
chan,sourcename,sourcedesc);
2998 if (ncreate>0) std::cout <<
"Created " << ncreate <<
" new channels for "
3000 if (nmodify>0) std::cout <<
"Modified " << nmodify <<
" channel info for "
3007 const std::string&
folder,
const std::string&
tag) {
3009 if (
name==
"PoolRef" ||
name==
"fileGUID") {
3010 std::string poolref;
3011 std::string addr=
payload[0].data<std::string>();
3012 if (
name==
"PoolRef") {
3013 std::string::size_type iofs1=addr.find(
"[DB=");
3014 std::string::size_type iofs2=addr.find(
']',iofs1);
3015 if (iofs1!=std::string::npos && iofs2!=std::string::npos && iofs2>iofs1) {
3016 poolref=addr.substr(iofs1+4,iofs2-iofs1-4);
3018 std::cout <<
"ERROR: Badly formed POOL object reference " <<
3023 poolref=std::move(addr);
3044 std::cout <<
"Copying additional hierarchical tags of parent folders"
3052 cool::IFolderSetPtr sourcefl=
m_sourceDbPtr->getFolderSet(folderset);
3053 cool::IFolderSetPtr destfl=
m_destDbPtr->getFolderSet(folderset);
3054 std::cout <<
"Processing folderset " << folderset << std::endl;
3055 for (HiTagMap::const_iterator imap=
m_hitagmap.begin();
3059 std::string sourcetag=sourcefl->findTagRelation(imap->first);
3060 std::cout <<
"Create hierarchical tag between " << sourcetag <<
3061 " and " << imap->first <<
" in folder " << imap->second <<
3065 std::string etag=destfl->resolveTag(imap->first);
3066 if (etag==sourcetag) {
3067 std::cout <<
"This relation has already been created" << std::endl;
3069 std::cout <<
"ERROR: Tag in parent already related to " <<
3074 catch (cool::Exception&
e ) {
3077 destfl->createTagRelation(imap->first,sourcetag);
3083 m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
3085 imap->first,imap->first)));
3087 catch (cool::Exception&
e) {
3088 std::cout <<
"Cool exception " <<
e.what() <<
3089 "thrown in hierarchical tag creation" << std::endl;
3095 catch (cool::Exception&
e) { }
3098 std::cout <<
"ERROR Folderset " << folderset <<
" not found" <<
3106 std::cout <<
"Write tag description ";
3108 std::cout <<
"for " <<
m_cooltagmap.size() <<
" tags" << std::endl;
3109 for (CoolTagMap::const_iterator itr=
m_cooltagmap.begin();
3117 std::cout <<
"Total of " <<
m_poolrefs.size() <<
" POOL files referenced"
3119 for (PoolMap::const_iterator ipool=
m_poolrefs.begin();
3121 std::cout <<
"Ref " << ipool->first <<
" (" << ipool->second.count()
3122 <<
")" << std::endl;
3128 std::cout <<
"Total of " << m_poolrefs.size() <<
" POOL Files referenced"
3131 if (catalog==
nullptr)
return 110;
3135 pool_utility.startSession();
3138 if (!m_addguid.empty()) {
3139 for (std::vector<std::string>::const_iterator itr=m_addguid.begin();
3140 itr!=m_addguid.end();++itr) {
3142 std::cout <<
"Added POOL file GUID: " << *itr << std::endl;
3145 if (!m_addlfn.empty()) {
3146 for (std::vector<std::string>::const_iterator itr=m_addlfn.begin();
3147 itr!=m_addlfn.end();++itr) {
3150 std::cout <<
"Add POOL file GUID: " <<
guid <<
" from LFN " << *itr
3157 using LFNGVec = std::vector<std::pair<std::string, std::string> >;
3159 bool dscopy=!m_newdataset.empty();
3164 ipool!=m_poolrefs.end();++ipool) {
3168 if( !lfns.empty() ) {
3170 const std::string
lfn = lfns[0].first;
3171 ipool->second.setlfn(
lfn);
3172 std::cout <<
"LFN: " <<
lfn <<
" (" << ipool->second.count()
3173 <<
")" << std::endl;
3175 dsfound.push_back(std::pair<std::string,std::string>(
lfn,
guid));
3178 if (!m_listpfn) ipool->second.setErrorBit(0);
3180 if (m_listpfn || m_poolopen) {
3181 std::string pfn, tech;
3183 if( !pfn.empty() ) {
3184 ipool->second.setpfn(pfn);
3185 std::cout <<
"PFN: " << pfn <<
" (" << ipool->second.count()
3186 <<
")" << std::endl;
3189 std::string hguid=getCoolHistGUID(pfn);
3190 if (!hguid.empty()) {
3192 if (hguid!=ipool->first) {
3193 std::cout <<
"ERROR File CoolHist GUID " << hguid <<
3194 " inconsistent with catalogue " << ipool->first << std::endl;
3195 ipool->second.setErrorBit(3);
3200 const std::string fid = pool_utility.
readFileGUID( pfn );
3201 if( fid != ipool->first ) {
3202 std::cout <<
"ERROR File GUID " << fid <<
3203 " inconsistent with catalogue " << ipool->first << std::endl;
3205 ipool->second.setErrorBit(3);
3208 catch( std::runtime_error&
e ) {
3209 std::cout <<
"Cannot open file for reading!" << std::endl;
3210 std::cout <<
e.what() << std::endl;
3212 ipool->second.setErrorBit(2);
3218 ipool->second.setErrorBit(1);
3222 if (ipool->second.errcode()>0) ++nbad;
3233 catalog=setupCatalog(m_mergecat);
3234 if (catalog==
nullptr)
return 110;
3235 const std::string dssname=
"register.sh";
3236 std::cout <<
"Write DQ2 registerFileInDataset commands to " << dssname
3237 <<
" for registration in dataset " << m_newdataset << std::endl;
3238 std::ofstream dsstream(dssname.c_str());
3239 for (LFNGVec::const_iterator itr=dsfound.begin();
3240 itr!=dsfound.end();++itr) {
3241 const std::string&
lfn=itr->first;
3242 const std::string&
guid=itr->second;
3245 if( !lfns.empty() ) {
3247 const std::string lfn2 = lfns[0].first;
3248 if (lfn2!=
lfn) std::cout <<
"WARNING: LFNs for GUID " <<
guid <<
3249 " differ in input/merge datasets: " <<
lfn <<
" vs "
3250 << lfn2 << std::endl;
3253 dsstream <<
"dq2-register-files " << m_newdataset <<
" "
3254 <<
lfn <<
" " <<
guid << std::endl;
3264 std::cout <<
"All POOL references were resolved by catalogues"
3267 std::cout <<
"ERROR Could not resolve " << nbad <<
" files" << std::endl;
3270 if (nbad>0 || m_poolall) filePoolRefs();
3276 std::string hguid=
"";
3277 TFile* myfile=TFile::Open(
file.c_str(),
"READ");
3278 if (myfile!=
nullptr) {
3280 myfile->GetObject(
"fileGUID",oguid);
3281 if (oguid!=
nullptr) {
3282 hguid=oguid->GetString();
3283 std::cout <<
"CoolHist GUID found to be " << hguid << std::endl;
3299 std::cout <<
"ErrCode GUID Count LFN PFN Folders ..." << std::endl;
3300 for (PoolMap::const_iterator ipool=
m_poolrefs.begin();
3302 if (
m_poolall || ipool->second.errcode()>0) {
3303 std::ostringstream
line;
3305 line << ipool->second.errcode() <<
" " << ipool->first <<
" " <<
3306 ipool->second.count() <<
" ";
3307 if (!ipool->second.lfn().empty()) {
3308 line << ipool->second.lfn() <<
" ";
3312 if (!ipool->second.pfn().empty()) {
3313 line << ipool->second.pfn() <<
" ";
3317 for (std::vector<std::string>::const_iterator
3318 itr=ipool->second.foldertag().begin();
3319 itr!=ipool->second.foldertag().end();++itr)
3320 line << *itr <<
" ";
3321 std::cout <<
line.str() << std::endl;
3322 chkostream <<
line.str() << std::endl;
3328 const std::vector<std::string>& catvec) {
3332 for (std::vector<std::string>::const_iterator icat=catvec.begin();
3333 icat!=catvec.end();++icat) {
3334 std::cout <<
"Add catalogue: " << *icat << std::endl;
3336 if (icat->find(
':')==std::string::npos) {
3347 std::cout <<
"Could not setup POOL catalogues, exception:" <<
e.what()
3354 std::cout <<
"usage: AtlCoolCopy.exe <sourceCoolDB> <destinationCoolDB> { <options> }" << std::endl;
3355 std::cout <<
"Options are (see doc/mainpage.h for more):" << std::endl <<
3356 "-a, -alliov : set IOVs on destination to [ValidityKeyMin, ValidityKeyMax]" << std::endl <<
" or following settings of -nrls,-nrlu,-nts,-ntu" <<
3358 "-ag, -addguid <guid> : Add GUID to list accessed for POOL checks" <<
3360 "-alf, -addlfn <LFN> : Add LFN to list accessed for POOL checks" <<
3362 "-al, -appendlocked : Allow locked tags to be updated if no overlap" <<
3364 "-alsv, -appendlockedsv : Allow locked tag update for openended IOVs" <<
3366 "-ana, -analyse <delta_t>: produce analysis ROOT file (filename = dest DB argument)" << std::endl <<
3367 "-bs, -buffersize <size> : set size of bulkstorage output buf to <size> objs"
3369 "-c, -create : create destination DB if not already existing"
3371 "-cd, -checkdest : Check destination DB and skip already existing tags"
3373 "-ch, -channel : restrict selection to given channel or range specified as c1:c2" << std::endl <<
3374 "-ch1, -channel1 : specify start of a range of channels" << std::endl <<
3375 "-ch2, -channel2 : specify end of a range of channels" << std::endl <<
3376 "-cf, -checkfiles : check POOL files can be opened and have correct GUID"
3378 "-co, -checkoutput <file> : write POOL file check output on file"
3379 << std::endl <<
"-cti, -copytaginfo : Copy tag descriptions"
3380 << std::endl <<
"-ctl, -copytaglock : Copy tag locked status and descriptions"
3381 << std::endl <<
"-cr, -checkrefs : check POOL references"
3382 << std::endl <<
"-d, -debug : produce debug output" << std::endl;
3384 "-ds, -dataset : output register.sh for creating DQ2 datasets "
3386 "-ec, -excludechannel : exclude given channel from copy" << std::endl <<
3387 "-eh, -excludehead : exclude HEAD tag even if no tags found in MV folders"
3389 "-ih, -includehead : include HEAD as well as any tags in MV folders" <<
3391 "-e, -exclude <pattern> : exclude folders" << std::endl <<
3392 "-f, -folder <pattern> : include folders" << std::endl <<
3393 "-fs, -forcesingle : force destination folder to be singleversion"
3395 "-fm, -forcemulti : force destination folder to be multiversion"
3397 "-frl, -forcerunlumi : force destination folder to be run/LB indexed"
3399 "-ftm, -forcetime : force destination folder to be timestamp indexed"
3401 "-fp, -forcepayload : Force destn folders to be created with payload tbl" << std::endl <<
3402 "-fnp, -forcenopayload : Force destn folders to be created without payload tbl" << std::endl <<
3403 "-forcerecreate : delete and recreate destination database" << std::endl;
3404 std::cout <<
"-ht, -headtag : Use HEAD-style tagging"
3406 "-go, -getonline : Set minimum run number (-rls setting) to next ONLINE run"
3408 "-gb, -getbulk : Set minimum run number (-rls setting) to next bulk reco run"
3410 "-gt, -gettime : Extract timestamp information for given run number range"
3412 "-h, -help : print help and exit" << std::endl <<
3413 "-hi, -hitag : Copy hierrchical tags inclusively" << std::endl <<
3414 "-ignoremode <pwd> : Ignore UPDx mode protection (experts only)"
3416 "-is, -ignorespec : Ignore differences in folder spec if names are all equal" << std::endl <<
3417 "-lo, -lockedonly : Only copy locked/partially-locked top-level tags"
3419 "-nc, -nocopy : Do not actually copy, just read source DB" << std::endl <<
3420 "-nch, -nochannel : Do not check or copy channel information" << std::endl
3421 <<
"-ncr, -noclobroot: Do not copy CLOB data into ROOT files" << std::endl
3422 <<
"-noc, -nocoracool : Do not copy Coracool structure payloads " << std::endl <<
3423 "-nd, -nodata : Copy only folder structures, not data" << std::endl <<
3424 "-nh, -nohitag : Do not follow hierarchical tag relations" << std::endl <<
3425 "-mc, -mergecat <catfile> : specify POOL file catalogue for new dataset making"
3427 std::cout <<
"-of, -outfolder <folder> : rename folder on output"
3429 "-onr, -onlinerun : Retrieve run number from online server (not replica)" <<std::endl <<
3430 "-ot, -outtag : " <<
"Rename tag on output" << std::endl;
3432 "-pa, -poolall : Output all POOL files (incl good) when checking files"
3434 "-pc, -poolcat <catfile> : specify POOL file catalogue for ref checking"
3436 "-pf, -parfile <file> : Read additional options/parameters from file" <<
3438 "-pt, -prunetags : Copy only hierarchical tags descending from specified toptags"
3440 "-rdo, -readoracle : force data to be read from Oracle, using dbreplica.config"
3442 "-sl, -seal <val>: Set SEAL (hence COOL/POOL) output level to <val>"
3444 "-sr, -skiprep : Skip folders having <norep/> folder metadata tag" <<
3446 "-t, -tag <tag> : Copy multiversion data with tag <tag>" << std::endl <<
3447 "-mt, -magic <tag> : Include magic tags involving the given pattern " << std::endl <<
3448 "-tr, -truncate : Set destination IOVs outside query interval to query interval"
3450 "-tl, -taglabel : Specify tag description to enter in destination DB"
3452 "-uht, -userheadtag : Also copy user tag data to the HEAD" << std::endl <<
3453 "-v, -verify : Verify data present on destination rather than copying" <<
3455 "-ro, -root : Produce ROOT output file instead of copying to COOL"
3457 "-zn, -zeronull : Zero NULLs in ROOT file instead of skipping them" <<
3459 "-rls, -runlumisince <run> <LB> : Set minimum IOV interval" << std::endl <<
3460 "-rlu, -runlumiuntil <run> <LB> : Set maximum IOV interval" << std::endl <<
3461 "-rs, -runsince <run> : Set minimum IOV interval" << std::endl <<
3462 "-ru, -rununtil <run> : Set maximum IOV interval" << std::endl <<
3463 "-r, -run <run> : Copy only run <run>" << std::endl <<
3464 "-srls, -skiprunlumisince <run> <LB> : Set minimum IOV for skipping IOV in copy"
3466 "-srlu, -skiprunlumiuntil <run> <LB> : Set maximum IOV for skipping IOV in copy"
3468 "-tdb, -timedb <dbconn> : Set database connection for time information" <<
3469 "-ts, -timesince <time> : Set minimum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3471 "-tu, -timeuntil <time> : Set maximum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3474 "-nrls, -newrunlumisince <run> <LB> : Set minimum of output IOV interval (use with -alliov)" << std::endl <<
3475 "-nrlu, -newrunlumiuntil <run> <LB> : Set maximum of output IOV interval (use with --aliov)" << std::endl <<
3476 "-nts, -newtimesince <time> : Set minimum of outputIOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3478 "-ntu, -newtimeuntil <time> : Set maximum of output IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3480 std::cout <<
"See http://twiki.cern.ch/twiki/bin/view/Atlas/AtlCoolCopy for more details" << std::endl;
3495 if (retcode==0) retcode=mycopy.doCopy();
3497 if (retcode>0) std::cout <<
"ERROR AtlCoolCopy.exe fails with exit code " <<
3498 retcode << std::endl;