18 #include <nlohmann/json.hpp>
19 #include <curl/curl.h>
21 #include "CoolKernel/DatabaseId.h"
22 #include "CoolKernel/Exception.h"
23 #include "CoolKernel/IDatabaseSvc.h"
24 #include "CoolKernel/IDatabase.h"
25 #include "CoolKernel/IFolder.h"
26 #include "CoolKernel/FolderSpecification.h"
27 #include "CoolKernel/IFolderSet.h"
28 #include "CoolKernel/IObject.h"
29 #include "CoolKernel/IObjectIterator.h"
30 #include "CoolKernel/IRecordIterator.h"
31 #include "CoolApplication/Application.h"
32 #include "CoralBase/AttributeListException.h"
40 #include "RelationalAccess/ConnectionService.h"
41 #include "RelationalAccess/IConnectionServiceConfiguration.h"
42 #include "RelationalAccess/ISessionProxy.h"
43 #include "RelationalAccess/ITransaction.h"
44 #include "RelationalAccess/ISchema.h"
45 #include "RelationalAccess/ITable.h"
46 #include "RelationalAccess/IQuery.h"
47 #include "RelationalAccess/ICursor.h"
63 #include "TObjString.h"
66 size_t newLength =
size * nmemb;
72 catch (std::bad_alloc &
e)
83 AtlCoolCopy(
const std::string& sourcedb,
const std::string& destdb,
84 bool allowcreate=
false);
89 (
const std::string&
folder,
const std::vector<std::string>&
taglist);
95 bool openConnections(
const std::string& sourcedb,
const std::string& destdb,
100 std::vector<std::string>&
folders);
101 static std::string
transConn(
const std::string& inconn);
104 const std::string& chanstring);
106 (
const std::string&
folder,
const std::string& destfolder,
109 const std::string& sourcetag,
const std::string& desttag,
110 const cool::ValidityKey
since,
const cool::ValidityKey
until,
111 bool timestamp,
bool checkrefs,
bool iscora,
114 const cool::ValidityKey&
until,
115 const cool::ValidityKey& qsince,
116 const cool::ValidityKey& quntil,
117 cool::ValidityKey& newsince,
118 cool::ValidityKey& newuntil,
119 const bool timestamp)
const;
121 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
122 const cool::ValidityKey
since,
const cool::ValidityKey
until,
127 const std::string& sourcetag,
128 const cool::ValidityKey
since,
const cool::ValidityKey
until,
129 const bool checkrefs,
const bool iscora,
132 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
133 const cool::ValidityKey
since,
const cool::ValidityKey
until,
134 const bool timestamp);
135 std::string
rootDirs(
const std::string&
folder,
const std::string& toproot);
137 void*& sptr,
char& rootID)
const;
141 const cool::IRecord& rhs);
143 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
144 const cool::ValidityKey
since,
const cool::ValidityKey
until,
145 const bool timestamp);
147 const int chan,
const float xlow,
const float xhigh);
149 static std::string
timeString(
const cool::ValidityKey iovtime);
150 static cool::ValidityKey
runLBVal(
const char*
input1,
const char* input2);
159 const cool::IFolderPtr& sourcefl,
const cool::IFolderPtr& destfl,
162 const std::string&
folder,
const std::string&
tag);
269 typedef std::map<std::string,PoolMapElement>
PoolMap;
273 using HiTagMap = std::map<std::string, std::string>;
303 m_sourcedb(sourcedb),m_destdb(destdb),m_allowcreate(allowcreate),
305 m_includehead(false),m_excludehead(false),
306 m_usertags(true),m_userupdatehead(false),m_debug(false),m_alliov(false),
307 m_verify(false),m_root(false),m_zeronull(false),m_analyse(false),
308 m_checkrefs(false),m_listpfn(false),m_poolopen(false),m_poolall(false),
309 m_nocopy(false),m_nodata(false),m_nochannel(false),m_chdesc(false),
310 m_hitag(false),m_nohitag(false),m_forcesingle(false),m_forcemulti(false),
311 m_forcerune(false),m_forcetime(false),m_forcepay(false),m_forcenopay(false),
312 m_sourceread(true),m_truncate(false),m_skipout(false),m_skiprep(false),
313 m_applock(false),m_applocksv(false),
314 m_readoracle(false),m_gettime(false),m_getonline(false),m_onlinerun(false),
316 m_prunetags(false),m_lockedonly(false),m_copytaginfo(false),
317 m_copytaglock(false),m_coracool(true),m_ignoremode(false),
318 m_ignorespec(false),m_checkdesttag(false),m_noclobroot(false),
319 m_runemin(
cool::ValidityKeyMin),m_runemax(
cool::ValidityKeyMax),
320 m_timemin(
cool::ValidityKeyMin),m_timemax(
cool::ValidityKeyMax),
321 m_newrunemin(
cool::ValidityKeyMin),m_newrunemax(
cool::ValidityKeyMax),
322 m_newtimemin(
cool::ValidityKeyMin),m_newtimemax(
cool::ValidityKeyMax),
323 m_srunemin(
cool::ValidityKeyMin),m_srunemax(
cool::ValidityKeyMax),
324 m_channel1(
""),m_channel2(
""),m_bufsize(1000),m_sealmsg(5),
325 m_anadelt(-1),m_outfolder(
""),m_outtag(
""),m_newdataset(
""),
326 m_checkoutputfile(
""),m_timedb(
""),m_taglabel(
""),
327 m_runinfohost(
"http://atlas-run-info-api.web.cern.ch/api"),
328 m_coolapp(&m_coralsvc),
329 m_dbSvc(&(m_coolapp.databaseService())),m_repsort(nullptr),
330 m_open(false),m_chansel(
cool::ChannelSelection::
all()),p_rootfile(nullptr)
335 coral::IConnectionServiceConfiguration& csconfig=
m_coralsvc.configuration();
336 csconfig.disablePoolAutomaticCleanUp();
337 csconfig.setConnectionTimeOut(0);
341 const std::string& destdb,
bool allowcreate) {
347 coral::IConnectionServiceConfiguration& csconfig=
m_coralsvc.configuration();
348 csconfig.setReplicaSortingAlgorithm(*
m_repsort);
352 std::cout <<
"Open source database: " << sourcedb << std::endl;
353 if (!
m_sourceread) std::cout <<
"... in UPDATE mode" << std::endl;
358 std::cout <<
"Cool exception caught: " <<
e.what() << std::endl;
366 std::cout <<
"Open destination ROOT file: " << destdb << std::endl;
367 p_rootfile=
new TFile(destdb.c_str(),
"RECREATE");
368 if (
p_rootfile==
nullptr) std::cout <<
"ERROR: Could not open ROOT file" <<
374 std::cout <<
"Open destination database: " << tdestdb << std::endl;
381 "Forcing recreation of destination database - deleting existing data!"
384 m_dbSvc->dropDatabase(tdestdb);
386 throw cool::DatabaseDoesNotExist(
"old database deleted");
390 std::cout <<
"COOL exception caught: " <<
e.what() << std::endl;
393 std::cout <<
"Try to create new conditions DB" << std::endl;
396 std::cout <<
"Creation succeeded" << std::endl;
398 catch (cool::Exception&
e) {
399 std::cout <<
"Creation failed" << std::endl;
412 std::cout <<
"Attempt to open source CoraCool DB " <<
m_sourcedb <<
416 std::cout <<
"Opened CoraCool source DB" << std::endl;
421 std::cout <<
"Attempt to open destination CoraCool DB " <<
m_destdb <<
425 std::cout <<
"Opened CoraCool dest DB" << std::endl;
433 cool::IFolderSetPtr topfolder=
m_sourceDbPtr->getFolderSet(
"/");
434 const std::vector<std::string>& toptaglist=topfolder->listTags();
435 for (std::vector<std::string>::const_iterator toptag=toptaglist.begin();
436 toptag!=toptaglist.end();++toptag) {
438 if (tstat==cool::HvsTagLock::LOCKED ||
439 tstat==cool::HvsTagLock::PARTIALLYLOCKED) {
440 std::cout <<
"Top-level tag " << *toptag <<
" will be copied" <<
442 m_tags.push_back(*toptag);
445 std::cout <<
"Total of " <<
m_tags.size() <<
" top-level tags to be copied"
453 if (inconn.find(
'/')==std::string::npos) {
454 return "sqlite://X;schema=mycool.db;dbname="+inconn;
462 std::cout <<
"Add folders in path:" <<
folder <<
" [ ";
463 const std::vector<std::string> nodelist=
m_sourceDbPtr->listAllNodes();
465 for (std::vector<std::string>::const_iterator nodeitr=nodelist.begin();
466 nodeitr!=nodelist.end();++nodeitr) {
472 nodeitr->size()>
folder.size() && nodeitr->compare(
folder.size(),1,
"/")==0)) {
475 for (std::vector<std::string>::const_iterator iexcl=
m_folderexcl.begin();
477 if (iexcl->compare(0,1,
"/")==0) {
490 std::cout << *nodeitr <<
" ";
497 const std::vector<std::string> foldersettags=
499 if (!foldersettags.empty()) {
500 for (std::vector<std::string>::const_iterator
501 itr=foldersettags.begin();itr!=foldersettags.end();++itr) {
508 cool::IFolderSetPtr sfolder=
m_sourceDbPtr->getFolderSet(*nodeitr);
509 for (std::vector<std::string>::const_iterator toptag=
m_tags.begin();
510 toptag!=
m_tags.end();++toptag) {
512 std::string rtag=sfolder->resolveTag(*toptag);
525 std::cout <<
"]" << std::endl;
530 std::cout <<
"Adding folder to exclude list: " <<
folder << std::endl;
536 (
const std::string&
folder,
const std::vector<std::string>&
taglist) {
538 cool::IFolderPtr sourcefl,destfl;
541 sourcefl=m_sourceDbPtr->getFolder(
folder);
543 catch (cool::Exception&
e) {
544 std::cout <<
"Could not get source folder: " <<
folder << std::endl;
547 const std::string& sourcedesc=sourcefl->description();
549 if (m_skiprep && sourcedesc.find(
"<norep/>")!=std::string::npos) {
550 std::cout <<
"Folder " <<
folder <<
551 " skipped due to <norep/> metadata" << std::endl;
554 const bool iscora=(m_coracool &&
555 sourcedesc.find(
"<coracool")!=std::string::npos);
559 sourceflc=m_sourceCoraPtr->getFolder(
folder);
562 setChannelRange(sourcefl);
567 dvermode=cool::FolderVersioning::SINGLE_VERSION;
568 std::cout <<
"Forcing destination folder to singleversion" << std::endl;
571 dvermode=cool::FolderVersioning::MULTI_VERSION;
572 std::cout <<
"Forcing destination folder to multiversion" << std::endl;
576 sourcefl->folderSpecification().payloadMode();
580 std::string destfolder=
folder;
581 if (!m_outfolder.empty()) {
582 destfolder=m_outfolder;
583 std::cout <<
"Destination folder will be renamed to " << destfolder <<
589 if (!m_nocopy && !m_root && !m_analyse) {
590 if (!m_destDbPtr->existsFolder(destfolder) && !m_verify) {
591 std::cout <<
"Creating folder " << destfolder <<
" payload-type " <<
592 dpaymode <<
" on destination" << std::endl;
595 if (m_forcerune || m_forcetime) {
596 std::string newmeta=m_forcerune ?
"run-lumi" :
"time";
597 std::string::size_type p1,p2;
600 if (p1!=std::string::npos && p2!=std::string::npos) {
601 metadata.replace(0,p2,
"<timeStamp>"+newmeta);
602 std::cout <<
"Forced destination folder to " << newmeta <<
" : "
606 "ERROR: Could not parse metadata string to force timestamp type"
611 if (m_forcepay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
612 dpaymode=cool::PayloadMode::SEPARATEPAYLOAD;
613 if (m_forcenopay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
614 dpaymode=cool::PayloadMode::INLINEPAYLOAD;
618 destflc=m_destCoraPtr->createFolder(destfolder,
619 sourceflc->coralTableName(),
620 sourceflc->fkSpecification(),
621 sourceflc->payloadSpecification(),
622 sourceflc->coralFKey(),
623 sourceflc->coralPKey(),
625 std::cout <<
"Created CoraCool folder" << std::endl;
626 destfl=m_destDbPtr->getFolder(destfolder);
628 destfl=m_destDbPtr->createFolder(destfolder,
629 cool::FolderSpecification(dvermode,
630 sourcefl->payloadSpecification(),dpaymode),
634 catch (cool::Exception&
e ) {
635 std::cout <<
"Create folder failed - aborting" << std::endl;
642 if (iscora) destflc=m_destCoraPtr->getFolder(destfolder);
643 destfl=m_destDbPtr->getFolder(destfolder);
645 catch (cool::Exception&
e) {
646 std::cout <<
"Could not get destination folder: " << destfolder
651 const cool::IRecordSpecification& sourcespec=
652 sourcefl->payloadSpecification();
653 const cool::IRecordSpecification& destspec=
654 destfl->payloadSpecification();
655 if (!(sourcespec==destspec)) {
660 "WARNING Source and destination folder specifications differ" <<
662 for (
unsigned int i=0;
i<sourcespec.size();++
i) {
663 const std::string& sname=sourcespec[
i].name();
664 if (!destspec.exists(sname)) {
665 std::cout <<
"ERROR: Field " << sname <<
" absent from destination"
675 "ERROR Source and destination folder specifications differ"
681 const std::string& destdesc=destfl->description();
682 if (sourcedesc!=destdesc) {
683 std::cout <<
"WARNING: Source and destination folder descriptions (meta-data) differ" << std::endl;
684 std::cout <<
"Source folder: " << sourcedesc << std::endl;
685 std::cout <<
"Destn folder: " << destdesc << std::endl;
689 destfl->folderSpecification().payloadMode();
690 if (spaymode!=dpaymode) {
691 std::cout <<
"WARNING: Source (" << spaymode <<
") and destination (" <<
692 dpaymode <<
" folder payload modes differ" << std::endl;
697 if (!checkChannels(
folder,sourcefl,destfl,created))
return 23;
701 if (m_nodata)
return 0;
703 std::cout <<
"Start to process folder: " <<
folder;
705 bool timestamp=(sourcefl->description().find(
"<timeStamp>time")!=
711 std::cout <<
" (timestamp)" << std::endl;
715 std::cout <<
" (run/lumi)" << std::endl;
717 if (m_alliov || m_truncate)
718 std::cout <<
"Output IOVs will be modified" << std::endl;
720 std::cout <<
"IOVs extending outside selection range will be skipped"
723 bool checkrefs=
false;
724 const std::string name0=sourcefl->payloadSpecification()[0].name();
725 if (m_checkrefs && (name0==
"PoolRef" || name0==
"fileGUID")) {
727 std::cout <<
"Check POOL references in folder " <<
folder << std::endl;
731 if (m_checkrefs && m_nocopy && !checkrefs)
return 0;
735 (m_skiprep && (sourcedesc.find(
"<fullrep/>")!=std::string::npos));
737 "All tags in folder will be copied due to <fullrep/> metadata" << std::endl;
740 std::vector<std::string>
tags;
741 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
742 std::cout <<
"Single version folder" << std::endl;
743 tags.emplace_back(
"HEAD");
745 std::cout <<
"Multi version folder: consider tags [ ";
747 const std::vector<std::string> foldertags=sourcefl->listTags();
748 for (std::vector<std::string>::const_iterator itag=foldertags.begin();
749 itag!=foldertags.end();++itag) {
755 for (std::vector<std::string>::const_iterator imtag=m_magic.begin();
756 imtag!=m_magic.end();++imtag) {
757 if (itag->find(*imtag)!=std::string::npos) copyit=
true;
760 tags.push_back(*itag);
761 std::cout << *itag <<
" ";
766 if (
tags.empty() || m_hitag) {
767 for (std::vector<std::string>::const_iterator itag=
taglist.begin();
770 std::string htag=sourcefl->resolveTag(*itag);
772 std::cout << *itag <<
"=" << htag <<
" ";
773 tags.push_back(htag);
783 std::cout <<
"]" << std::endl;
786 if (((
tags.empty() || m_includehead) && !m_excludehead) ||
788 tags.emplace_back(
"HEAD");
791 sourcefl->setPrefetchAll(
false);
792 for (std::vector<std::string>::const_iterator itag=
tags.begin();
793 itag!=
tags.end();++itag) {
795 if (!m_outtag.empty())
outtag=m_outtag;
799 retcode=verifyIOVs(
folder,sourcefl,sourceflc,destfl,destflc,
801 }
else if (m_nocopy) {
805 }
else if (m_analyse) {
808 retcode=copyIOVs(
folder,destfolder,sourcefl,sourceflc,destfl,destflc,
813 std::cout <<
"ERROR operation failed for folder " <<
folder <<
" tag " <<
829 for (
size_t i=0;
i<nChanRange;
i++) {
834 " to channel selection" << std::endl;
845 std::cout <<
"Adding channel range " <<
c1 <<
" to " <<
c2 <<
846 " to channel selection" << std::endl;
857 const std::string& chanstring) {
858 const char* cstr=chanstring.c_str();
861 return cool::ChannelId(strtoul(cstr,
nullptr,10));
863 cool::ChannelId
chan=0;
866 std::cout <<
"Channel name " << chanstring <<
" maps to channelID "
867 <<
chan << std::endl;
869 catch (cool::Exception&
e) {
870 std::cout <<
"ERROR: Channel name " << chanstring <<
871 " not defined in folder " << std::endl;
879 (
const std::string&
folder,
880 const std::string& destfolder,
883 const std::string& sourcetag,
const std::string& desttag,
884 const cool::ValidityKey
since,
const cool::ValidityKey
until,
885 bool timestamp,
bool checkrefs,
bool iscora,
888 std::cout <<
"Copying tag " << sourcetag <<
" of folder " <<
folder <<
889 " to destination tag " << desttag << std::endl;
893 int updatemode=getUpdateMode(destfl->description(),desttag);
894 std::vector<std::string> dtaglist=destfl->listTags();
897 if (updatemode==1 && m_destdb.find(
"oracle")!=std::string::npos &&
898 (!m_getonline || (!m_truncate && !m_alliov))) {
900 std::cout <<
"Folder is online (UPD1) mode but IGNORING PROTECTION"
903 if(
find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
904 std::cout <<
"Folder is online mode (UPD1) and tag already exist - -getonline and -truncate or -alliov options MUST be used" << std::endl;
908 std::cout <<
"Folder is online mode (UPD1), new tag will be created" << std::endl;
914 if (updatemode==4 && m_destdb.find(
"oracle")!=std::string::npos &&
915 (!m_getbulk || (!m_truncate && !m_alliov))) {
917 std::cout <<
"Folder is bulkreco (UPD4) mode but IGNORING PROTECTION"
920 if(
find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
921 std::cout <<
"Folder is bulkreco mode (UPD4) - -getbulk and -truncate or -alliov options MUST be used" << std::endl;
924 std::cout <<
"Folder is bulkreco mode (UPD4), new tag will be created" << std::endl;
933 if (m_checkdesttag) {
934 if (
find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()
935 || (desttag==
"HEAD" && !created)) {
936 std::cout <<
"Destination tag " << desttag <<
937 " already exists in folder " <<
folder <<
" - skip copy" << std::endl;
943 if ((m_applock || (updatemode==1 && m_getonline) || updatemode==2 ||
944 (updatemode==4 && m_getbulk)) &&
945 dvermode==cool::FolderVersioning::MULTI_VERSION) {
947 std::vector<std::string> dtaglist=destfl->listTags();
948 for (std::vector<std::string>::const_iterator itr=dtaglist.begin();
949 itr!=dtaglist.end();++itr) {
954 if (destfl->tagLockStatus(desttag)==cool::HvsTagLock::LOCKED) {
955 std::cout <<
"Unlocking destination tag " << desttag <<
956 " for append or UPDx access" << std::endl;
958 std::cout <<
"Appending according to SV folder rules" << std::endl;
961 "ERROR: Only allowed for UPD3 mode tags" << std::endl;
965 destfl->setTagLockStatus(desttag,cool::HvsTagLock::UNLOCKED);
977 if (dvermode==cool::FolderVersioning::MULTI_VERSION &&
978 desttag!=
"HEAD" && m_usertags)
localtag=desttag;
983 sourceflc->setPrefetchAll(
false);
985 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
986 sourceitr=sourceflc->browseObjects(
since,
until,m_chansel);
988 sourceitr=sourceflc->browseObjects(
since,
until,m_chansel,sourcetag);
991 std::map<int,int> insertkeymap;
993 while (sourceitr->hasNext()) {
997 (
obj->since()<m_srunemin ||
obj->until()>m_srunemax)) {
1002 if (!m_excludechans.empty()) {
1003 if (
find(m_excludechans.begin(),m_excludechans.end(),
1004 obj->channelId())!=m_excludechans.end()) {
1010 if (m_debug) std::cout <<
1011 "Setup new CoraCool storage buffer at object " << nobj << std::endl;
1012 destflc->setupStorageBuffer();
1014 cool::ValidityKey newsince,newuntil;
1018 if (newsince>=newuntil) {
1019 std::cout <<
"WARNING: Skipping IOV with since " << newsince <<
1020 ">= until" << newuntil << std::endl;
1027 std::map<int,int>::const_iterator ikey=insertkeymap.end();
1028 bool foundkey=
false;
1031 if (
obj->size()>0) {
1032 oldfk=sourceflc->getAttrKey(
1033 (*
obj->begin())[sourceflc->coralFKey()]);
1034 ikey=insertkeymap.find(oldfk);
1037 if (ikey==insertkeymap.end()) {
1039 destflc->storeObject(newsince,newuntil,
obj->begin(),
obj->end(),
1041 (!m_userupdatehead && !
localtag.empty()));
1042 if (foundkey) insertkeymap[oldfk]=newfk;
1044 destflc->referenceObject(newsince,newuntil,ikey->second,
1046 (!m_userupdatehead && !
localtag.empty()));
1051 if (nbuf==m_bufsize) {
1052 if (m_debug) std::cout <<
"Flush buffer after " << nobj <<
"," <<
1053 nbuf <<
" objects " << std::endl;
1055 destflc->flushStorageBuffer();
1056 destflc->setupStorageBuffer();
1062 if (nref>0) std::cout <<
"Reference-to-existing used for " << nref
1063 <<
" payload objects" << std::endl;
1065 if (m_debug) std::cout <<
"Final buffer flush at " << nobj <<
1066 "," << nbuf << std::endl;
1067 destflc->flushStorageBuffer();
1071 cool::IObjectIteratorPtr sourceitr;
1072 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1073 sourceitr=sourcefl->browseObjects(
since,
until,m_chansel);
1075 sourceitr=sourcefl->browseObjects(
since,
until,m_chansel,sourcetag);
1077 while (sourceitr->goToNext()) {
1078 const cool::IObject&
obj=sourceitr->currentRef();
1080 if (m_skipout && (
obj.since()<m_srunemin ||
obj.until()>m_srunemax)) {
1085 if (!m_excludechans.empty()) {
1086 if (
find(m_excludechans.begin(),m_excludechans.end(),
1087 obj.channelId())!=m_excludechans.end()) {
1093 if (m_debug) std::cout <<
"Setup new storage buffer at object " <<
1095 destfl->setupStorageBuffer();
1097 cool::ValidityKey newsince;
1098 cool::ValidityKey newuntil;
1102 if (newsince>=newuntil) {
1103 std::cout <<
"WARNING: Skipping IOV with since " << newsince <<
1104 ">= until" << newuntil << std::endl;
1108 if (checkrefs) checkRef(
obj.payload(),
folder,sourcetag);
1114 if (newuntil!=cool::ValidityKeyMax) {
1115 std::cout <<
"New IOVs must have until=cool::ValidityKeyMax" << std::endl;
1116 throw cool::Exception(
"Illegal insert over locked IOV",
1119 cool::IObjectIteratorPtr checkitr=destfl->browseObjects(newsince,
1120 newuntil,
obj.channelId(),desttag);
1121 while (checkitr->goToNext()) {
1122 const cool::IObject& checkobj=checkitr->currentRef();
1123 if (checkobj.since()>=newsince) {
1124 std::cout <<
"ERROR:: Attempt to insert SV overlapping IOV whilst appending to locked tag" << std::endl;
1125 throw cool::Exception(
"Illegal insert over locked IOV",
1128 if (checkobj.until()!=cool::ValidityKeyMax) {
1129 std::cout <<
"Existing IOVs must have until=cool::ValidityKeyMax" << std::endl;
1130 throw cool::Exception(
"Illegal insert over locked IOV",
1135 }
else if (m_applock) {
1137 const unsigned int nexist=
1138 destfl->countObjects(newsince,newuntil,
obj.channelId(),desttag);
1140 std::cout <<
"ERROR: Attempt to insert IOV over " << nexist <<
1141 " objects whilst appending to locked tag" << std::endl;
1142 throw cool::Exception(
"Illegal insert over locked IOV",
1145 }
else if (updatemode==2) {
1149 throw cool::Exception(
1150 "Attempt to insert into locked UPD2 tag with timestamp format",
1152 unsigned int run1=newsince >> 32;
1153 unsigned int run2=(newuntil-1) >> 32;
1154 for (
unsigned int irun=
run1;irun<=
run2;++irun) {
1155 if (!std::binary_search(m_runlist.begin(),m_runlist.end(),irun))
1157 std::cout <<
"Run " << irun <<
" from range [" <<
run1 <<
","
1158 <<
run2 <<
"] not found in runfile list" << std::endl;
1159 throw cool::Exception(
"Illegal insert over locked IOV",
1166 if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
1168 cool::IRecordIterator& pitr=
obj.payloadIterator();
1169 const cool::IRecordVectorPtr vptr=pitr.fetchAllAsVector();
1170 destfl->storeObject(newsince,newuntil,
1172 (!m_userupdatehead && !
localtag.empty()));
1177 destfl->storeObject(newsince,newuntil,
1179 (!m_userupdatehead && !
localtag.empty()));
1183 if (nbuf>=m_bufsize) {
1184 if (m_debug) std::cout <<
"Flush buffer after " << nobj <<
"," <<
1185 nbuf <<
" objects " << std::endl;
1186 destfl->flushStorageBuffer();
1194 if (m_debug) std::cout <<
"Final buffer flush at " << nobj <<
1195 "," << nbuf << std::endl;
1196 destfl->flushStorageBuffer();
1199 std::cout <<
"Folder copied with " << nobj <<
" objects" << std::endl;
1201 " objects were skipped extending outside IOV selection or excluded channel"
1203 if (nbad>0) std::cout << nbad <<
1204 " objects were skipped having zero or negative IOV lengths" << std::endl;
1207 catch (cool::Exception&
e) {
1208 std::cout <<
"Exception thrown from copy loop: " <<
e.what() <<
1211 std::cout <<
"Relocking destination tag " << desttag << std::endl;
1212 destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1218 std::cout <<
"Relocking destination tag " << desttag << std::endl;
1219 destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1222 if (dvermode==cool::FolderVersioning::MULTI_VERSION && desttag!=
"HEAD") {
1225 std::cout <<
"Tag folder with HEAD-style tagging for tag: " << desttag
1228 destfl->tagCurrentHead(desttag,sourcefl->tagDescription(sourcetag));
1230 catch (cool::Exception&
e) {
1231 std::cout <<
"Exception thrown in HEAD-style folder-tag: " <<
1232 e.what() << std::endl;
1237 if (m_copytaginfo) m_cooltagmap.insert(
1239 destfolder,sourcetag,desttag,m_taglabel)));
1241 if (!m_hitagmap.empty()) {
1242 for (HiTagMap::const_iterator imap=m_hitagmap.begin();
1243 imap!=m_hitagmap.end();++imap) {
1245 if (sourcefl->findTagRelation(imap->first)==sourcetag) {
1246 std::cout <<
"Create hierarchical tag between " << desttag <<
1247 " and " << imap->first <<
" in folder " << imap->second <<
1251 std::string etag=destfl->resolveTag(imap->first);
1252 if (etag==desttag) {
1253 std::cout <<
"This relation has already been created" << std::endl;
1255 std::cout <<
"ERROR: Tag in parent already related to " <<
1256 desttag << std::endl;
1259 catch (cool::Exception&
e ) {
1262 destfl->createTagRelation(imap->first,desttag);
1265 if (
find(m_hiparent.begin(),m_hiparent.end(),imap->second)==
1266 m_hiparent.end()) m_hiparent.push_back(imap->second);
1269 if (m_copytaginfo &&
1270 m_cooltagmap.find(imap->first)==m_cooltagmap.end())
1271 m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
1272 CoolTagInfo(m_sourceDbPtr,imap->second,imap->second,
1273 imap->first,imap->first)));
1275 catch (cool::Exception&
e) {
1276 std::cout <<
"Cool exception " <<
e.what() <<
1277 "thrown in hierarchical tag creation" << std::endl;
1283 catch (cool::Exception&
e) {
1292 const cool::ValidityKey&
until,
1293 const cool::ValidityKey& qsince,
1294 const cool::ValidityKey& quntil,
1295 cool::ValidityKey& newsince,
1296 cool::ValidityKey& newuntil,
1297 bool timestamp)
const {
1314 if (newsince<qsince) newsince=qsince;
1315 if (newuntil>quntil) newuntil=quntil;
1318 if (newuntil<newsince)
throw cool::Exception(
1319 "Attempt to insert IOV with -ve length",
"AtlCoolCopy::adjustIOVs");
1323 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
1324 const cool::ValidityKey
since,
const cool::ValidityKey
until,
1327 std::cout <<
"Reading tag " << sourcetag <<
" of folder " <<
folder <<
1328 " (no copy)" << std::endl;
1331 cool::IObjectIteratorPtr sourceitr;
1332 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1338 while (sourceitr->goToNext()) {
1339 const cool::IObject&
obj=sourceitr->currentRef();
1343 std::cout <<
"Folder scanned with " << nobj <<
" objects" << std::endl;
1346 catch (cool::Exception&
e) {
1347 std::cout <<
"Exception thrown from read loop: " <<
e.what() <<
1359 const std::string& sourcetag,
1360 const cool::ValidityKey
since,
const cool::ValidityKey
until,
1361 const bool checkrefs,
const bool iscora,
1364 std::cout <<
"Verifying tag " << sourcetag <<
" of folder " <<
folder <<
1366 destfl->setPrefetchAll(
false);
1369 cool::IObjectIteratorPtr sourceitr,destitr;
1371 std::string
tag=sourcetag;
1372 if (vermode==cool::FolderVersioning::SINGLE_VERSION || sourcetag==
"HEAD")
1376 sourceflc->setPrefetchAll(
false);
1385 std::cout <<
"Exception thrown from verify iterator setup: " <<
e.what() <<
1394 const std::string& cfkey=sourceflc->coralFKey();
1395 const std::string& cpkey=sourceflc->coralPKey();
1396 while (csourceitr->hasNext()) {
1405 if (cdestitr->hasNext()) {
1410 if (sobj->since()!=dobj->since() || sobj->until()!=dobj->until())
1413 if (sobj->channelId()!=dobj->channelId())
1416 if (sobj->size()!=dobj->size()) {
1417 std::cout <<
"ERROR CoraCool object " << nobj <<
1418 " sizes do not match: " << sobj->size() <<
" " << dobj->size()
1425 sitr!=sobj->end();++sitr,++ditr) {
1427 for (coral::AttributeList::const_iterator aitr=sitr->begin();
1428 aitr!=sitr->end();++aitr) {
1429 const std::string&
aname=aitr->specification().name();
1432 if (*aitr!=(*ditr)[
aname]) {
1433 std::cout <<
"ERROR Values of attriute " <<
aname <<
1434 " differ" << std::endl;
1438 catch (coral::AttributeListException&
e) {
1439 std::cout <<
"ERROR: CoraCool attribute " <<
aname <<
1440 " not found in destination!" << std::endl;
1447 std::cout <<
"ERROR database entries do not match: since: " <<
1448 sobj->since() <<
"," << dobj->since() <<
" until: " <<
1449 sobj->until() <<
"," << dobj->until() <<
" channel: " <<
1450 sobj->channelId() <<
"," << dobj->channelId() << std::endl;
1455 "ERROR destination folder no matching CoraCool iterator for object "
1456 << nobj << std::endl;
1462 while (sourceitr->goToNext()) {
1463 const cool::IObject& sobj=sourceitr->currentRef();
1472 if (destitr->goToNext()) {
1473 const cool::IObject& dobj=destitr->currentRef();
1477 if (sobj.since()!=dobj.since() || sobj.until()!=dobj.until())
1480 if (sobj.channelId()!=dobj.channelId())
1485 if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
1486 cool::IRecordIterator& spitr=sobj.payloadIterator();
1487 const cool::IRecordVectorPtr svptr=spitr.fetchAllAsVector();
1488 cool::IRecordIterator& dpitr=dobj.payloadIterator();
1489 const cool::IRecordVectorPtr dvptr=dpitr.fetchAllAsVector();
1490 if (svptr->size()!=dvptr->size()) {
1495 cool::IRecordVector::const_iterator svitr=svptr->begin();
1496 cool::IRecordVector::const_iterator svend=svptr->end();
1497 cool::IRecordVector::const_iterator dvitr=dvptr->begin();
1498 for (;svitr!=svend;++svitr,++dvitr) {
1503 std::cout <<
"ERROR vector payloads do not match (size " << svptr->size() <<
"," << dvptr->size() <<
")" << std::endl;
1508 if (!
equalRecord(sobj.payload(),dobj.payload())) iret=102;
1512 std::cout <<
"ERROR database entries do not match: since: " <<
1513 sobj.since() <<
"," << dobj.since() <<
" until: " <<
1514 sobj.until() <<
"," << dobj.until() <<
" channel: " <<
1515 sobj.channelId() <<
"," << dobj.channelId() << std::endl;
1516 std::cout <<
"Source payload:" << std::endl;
1517 sobj.payload().attributeList().toOutputStream(std::cout);
1518 std::cout << std::endl <<
"Destination payload:" << std::endl;
1519 dobj.payload().attributeList().toOutputStream(std::cout);
1520 std::cout << std::endl;
1525 "ERROR destination folder no matching iterator for object "
1526 << nobj << std::endl;
1533 catch (cool::Exception&
e) {
1534 std::cout <<
"Exception thrown from verify loop: " <<
e.what() <<
1544 std::cout <<
"Verification of folder " <<
folder <<
" tag " <<
1545 sourcetag <<
" OK (" << nobj <<
" objects)" << std::endl;
1552 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
1553 const cool::ValidityKey
since,
const cool::ValidityKey
until,
1554 const bool timestamp) {
1556 std::cout <<
"Write tag " << sourcetag <<
" of folder " <<
folder <<
1557 " to ROOT file" << std::endl;
1560 bool timestamp2=timestamp;
1566 TTree*
tree=
static_cast<TTree*
>(
gDirectory->FindObject(treename.c_str()));
1567 if (
tree==
nullptr) {
1568 std::cout <<
"Book TTree " << treename << std::endl;
1569 tree=
new TTree(treename.c_str(),
"COOL datadump");
1580 if (vermode==cool::FolderVersioning::MULTI_VERSION)
1583 const cool::IRecordSpecification&
spec=
1584 (sourcefl->folderSpecification()).payloadSpecification();
1585 unsigned int ncolumns=
spec.size();
1592 const cool::IFieldSpecification& fieldspec=
spec[
icol];
1596 tree->Branch(fieldspec.name().c_str(),ptr,
1597 (fieldspec.name()+
"/"+rootID).c_str());
1599 <<
" of type " <<
spec[
icol].storageType().name() << std::endl;
1601 std::cout <<
"Attribute " <<
spec[
icol].name() <<
" of type " <<
1602 spec[
icol].storageType().name() <<
" will be skipped" << std::endl;
1607 std::cout <<
"TTree " << treename <<
" already exists" << std::endl;
1610 (sourcefl->folderSpecification()).payloadSpecification().size();
1612 std::cout <<
"ERROR in tree buffer definition: expect " << treename <<
1621 cool::IObjectIteratorPtr sourceitr;
1623 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1629 catch (cool::Exception&
e) {
1630 std::cout <<
"Exception thrown from ROOT copy iterator setup: " <<
1631 e.what() << std::endl;
1636 while (sourceitr->goToNext()) {
1637 const cool::IObject& sobj=sourceitr->currentRef();
1648 if (vermode==cool::FolderVersioning::MULTI_VERSION) {
1651 std::string sourcetag2=sourcetag.substr(0,255);
1657 const cool::IRecord& record=sobj.payload();
1664 catch (cool::Exception&
e) {
1671 std::cout <<
"Written " << nobj <<
" objects to ROOT TTree with " << nex
1672 <<
" nulls/exceptions" << std::endl;
1674 catch (cool::Exception&
e) {
1675 std::cout <<
"Exception thrown from ROOT file writing loop: " <<
1676 e.what() << std::endl;
1683 const std::string& toproot) {
1688 if (
p_rootfile->FindObject(toproot.c_str())==
nullptr) {
1690 std::cout <<
"Made top directory " << toproot << std::endl;
1695 std::string::size_type iofs1=1;
1696 std::string::size_type iofs2=1;
1697 std::string treename;
1698 while (iofs2!=std::string::npos) {
1699 iofs2=
folder.find(
'/',iofs1);
1700 if (iofs2==std::string::npos) {
1702 treename=
folder.substr(iofs1);
1707 std::cout <<
"Make directory " <<
dirname << std::endl;
1717 void*& sptr,
char& rootID)
const {
1718 const cool::StorageType& stype=
spec.storageType();
1721 if (stype==cool::StorageType::Bool) {
1723 sptr=
static_cast<void*
>(
new unsigned int(0));
1725 }
else if (stype==cool::StorageType::UChar) {
1726 sptr=
static_cast<void*
>(
new unsigned char(
' '));
1728 }
else if (stype==cool::StorageType::Int16) {
1729 sptr=
static_cast<void*
>(
new short(0));
1731 }
else if (stype==cool::StorageType::UInt16) {
1732 sptr=
static_cast<void*
>(
new unsigned short(0));
1734 }
else if (stype==cool::StorageType::Int32) {
1735 sptr=
static_cast<void*
>(
new int(0));
1737 }
else if (stype==cool::StorageType::UInt32) {
1738 sptr=
static_cast<void*
>(
new unsigned int(0));
1740 }
else if (stype==cool::StorageType::Int64) {
1741 sptr=
static_cast<void*
>(
new long long int(0));
1743 }
else if (stype==cool::StorageType::UInt63) {
1744 sptr=
static_cast<void*
>(
new unsigned long long int(0));
1746 }
else if (stype==cool::StorageType::Float) {
1747 sptr=
static_cast<void*
>(
new float(0.));
1749 }
else if (stype==cool::StorageType::Double) {
1750 sptr=
static_cast<void*
>(
new double(0.));
1752 }
else if (stype==cool::StorageType::String4k ||
1753 stype==cool::StorageType::String255) {
1754 sptr=
static_cast<void*
>(
new char[4100]);
1756 }
else if (stype==cool::StorageType::String64k && !
m_noclobroot) {
1757 sptr=
static_cast<void*
>(
new char[65536]);
1759 }
else if (stype==cool::StorageType::String16M && !
m_noclobroot) {
1760 sptr=
static_cast<void*
>(
new char[67108864]);
1763 std::cout <<
"rootAllocate: Unsupported storage type for attribute: " <<
1764 spec.name() << std::endl;
1766 return (sptr!=
nullptr);
1772 const cool::StorageType& stype=
field.storageType();
1773 if (stype==cool::StorageType::Bool) {
1775 *(
static_cast<int*
>(sptr))=0;
1777 *(
static_cast<int*
>(sptr))=(
field.data<
bool>() ? 1 : 0);
1779 }
else if (stype==cool::StorageType::UChar) {
1781 *(
static_cast<unsigned char*
>(sptr))=0;
1783 *(
static_cast<unsigned char*
>(sptr))=
field.data<
unsigned char>();
1785 }
else if (stype==cool::StorageType::Int16) {
1787 *(
static_cast<short*
>(sptr))=0;
1789 *(
static_cast<short*
>(sptr))=
field.data<
short>();
1791 }
else if (stype==cool::StorageType::UInt16) {
1793 *(
static_cast<unsigned short*
>(sptr))=0;
1795 *(
static_cast<unsigned short*
>(sptr))=
field.data<
unsigned short>();
1797 }
else if (stype==cool::StorageType::Int32) {
1799 *(
static_cast<int*
>(sptr))=0;
1801 *(
static_cast<int*
>(sptr))=
field.data<
int>();
1803 }
else if (stype==cool::StorageType::UInt32) {
1805 *(
static_cast<unsigned int*
>(sptr))=0;
1807 *(
static_cast<unsigned int*
>(sptr))=
field.data<
unsigned int>();
1809 }
else if (stype==cool::StorageType::Int64) {
1811 *(
static_cast<long long*
>(sptr))=0;
1813 *(
static_cast<long long*
>(sptr))=
field.data<
long long>();
1815 }
else if (stype==cool::StorageType::UInt63) {
1817 *(
static_cast<unsigned long long*
>(sptr))=0;
1819 *(
static_cast<unsigned long long*
>(sptr))=
field.data<
unsigned long long>();
1821 }
else if (stype==cool::StorageType::Float) {
1823 *(
static_cast<float*
>(sptr))=0.;
1825 *(
static_cast<float*
>(sptr))=
field.data<
float>();
1827 }
else if (stype==cool::StorageType::Double) {
1829 *(
static_cast<double*
>(sptr))=0.;
1831 *(
static_cast<double*
>(sptr))=
field.data<
double>();
1833 }
else if (stype==cool::StorageType::String255 ||
1834 stype==cool::StorageType::String4k ||
1835 stype==cool::StorageType::String64k ||
1836 stype==cool::StorageType::String16M) {
1838 strcpy(
static_cast<char*
>(sptr),
"NULL");
1840 strcpy(
static_cast<char*
>(sptr),
field.data<std::string>().c_str());
1843 std::cout <<
"ERROR: Unknown storage type in rootWrite!" << std::endl;
1848 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
1849 const cool::ValidityKey
since,
const cool::ValidityKey
until,
1850 const bool timestamp) {
1852 std::cout <<
"Analyse tag " << sourcetag <<
" of folder " <<
folder <<
1856 std::cout <<
"Analyse time structures with tolerance of " <<
1857 m_anadelt/1.E9 <<
" seconds " << std::endl;
1862 std::cout <<
"Make directory " <<
dirname << std::endl;
1867 std::vector<cool::ChannelId>
channels=sourcefl->listChannels();
1870 long long* lastiov=
new long long[
nchan];
1871 long long* iovtotlen=
new long long[
nchan];
1872 long long* iovtotgap=
new long long[
nchan];
1873 int* iovn=
new int[
nchan];
1874 for (
unsigned int i=0;
i<
nchan;++
i) {
1880 cool::ValidityKey globsince=cool::ValidityKeyMax;
1881 cool::ValidityKey globuntil=cool::ValidityKeyMin;
1887 "IOVs per channel-name (copy of IOVSperChannel)",
1894 TH1F* h_anadelt=
nullptr;
1895 TH1F* h_chandelt=
nullptr;
1897 h_anadelt=
bookOrFindTH1F(
"IOVAlignLogDelT",
"IOV alignment log deltaT",
1901 h_chandelt=
bookOrFindTH1F(
"AlignedChanPerIOV",
"Aligned channels per IOV",
1904 std::cout <<
"Booked histograms for folder with " <<
nchan <<
" channels"
1907 using IOVTimeMap = std::map<cool::ValidityKey, int>;
1908 IOVTimeMap iov_time_map;
1913 cool::IObjectIteratorPtr sourceitr;
1915 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1921 catch (cool::Exception&
e) {
1922 std::cout <<
"Exception thrown from analysis copy iterator setup: " <<
1923 e.what() << std::endl;
1927 std::map<cool::ChannelId,std::string>
1928 chanmap=sourcefl->listChannelsWithNames();
1931 while (sourceitr->goToNext()) {
1932 const cool::IObject& sobj=sourceitr->currentRef();
1934 cool::ChannelId ichanid=sobj.channelId();
1935 const std::string& cname=chanmap[ichanid];
1939 if (chanitr.first!=chanitr.second) {
1943 h_iovname->GetXaxis()->SetBinLabel(
bin,cname.c_str());
1945 h_iovname->Fill(
ichan);
1947 cool::ValidityKey since2=sobj.since();
1948 cool::ValidityKey until2=sobj.until();
1949 if (since2<globsince) globsince=since2;
1951 if (until2!=cool::ValidityKeyMax) {
1952 if (until2>globuntil) globuntil=until2;
1953 long long len=until2-since2;
1954 h_iovlength->Fill(log10(len));
1956 iovtotlen[
ichan]+=len;
1958 if (lastiov[
ichan]<
static_cast<long long>(since2) &&
1959 lastiov[
ichan]>=0) {
1961 long long gap=since2-lastiov[
ichan];
1963 h_iovgap->Fill(log10(
gap));
1964 std::cout <<
"Gap of " <<
gap << std::endl;
1966 if (until2!=cool::ValidityKeyMax) {
1967 lastiov[
ichan]=until2;
1969 lastiov[
ichan]=since2;
1979 if (hiitr!=iov_time_map.end()) {
1981 del1=hiitr->first-since2;
1983 if (lowitr!=iov_time_map.begin()) {
1986 del2=since2-lowitr->first;
1991 if (del1<=m_anadelt && del1>-1) {
1996 if (del2<=m_anadelt && del2>-1 && (del2<del || del==-1)) {
2004 h_anadelt->Fill(log10(del));
2006 h_anadelt->Fill(0.);
2009 iov_time_map[since2]=0;
2014 std::cout <<
"ERROR :Channel " << ichanid <<
2015 " not found in channel list - ignored" << std::endl;
2019 std::cout <<
"Finished analysis with " << nobj <<
" objects" << std::endl;
2021 std::cout <<
"IOV timestamp range: " << globsince <<
" " <<
2022 timeString(globsince) <<
"to " << globuntil <<
" " <<
2025 std::cout <<
"IOV run/LB range [" << (globsince >> 32) <<
"," <<
2026 (globsince & 0
xFFFFFFFF) <<
"] to [" << (globuntil >> 32) <<
"," <<
2027 (globuntil & 0
xFFFFFFFF) <<
"]" << std::endl;
2033 "Mean IOV length per channel",
nchan,-0.5,
nchan-0.5);
2034 for (
unsigned int i=0;
i<
nchan;++
i) {
2038 occ=1.-
float(iovtotgap[
i])/
float(iovtotlen[
i]+iovtotgap[
i]);
2039 avlen=
float(iovtotlen[
i])/iovn[
i];
2041 h_iovoccchan->Fill(
i,occ);
2042 h_iovlenchan->Fill(
i,avlen);
2045 std::cout <<
"Alignment analysis: " << iov_time_map.size()
2046 <<
" seperate IOV starts (tolerance " <<
m_anadelt/1.E9 <<
2047 " seconds)" << std::endl;
2049 for (IOVTimeMap::const_iterator itr=iov_time_map.begin();
2050 itr!=iov_time_map.end();++itr) {
2051 h_chandelt->Fill(itr->second);
2055 catch (cool::Exception&
e) {
2056 std::cout <<
"Exception thrown from folder analysis reading loop: " <<
2057 e.what() << std::endl;
2062 delete [] iovtotlen;
2063 delete [] iovtotgap;
2064 delete[] iovn; iovn=
nullptr;
2069 const std::string& htitle,
2070 const int nchan,
const float xlow,
const float xhigh) {
2075 return new TH1F(hID.c_str(),htitle.c_str(),
nchan,xlow,xhigh);
2077 return static_cast<TH1F*
>(
obj);
2086 for (std::vector<std::string>::const_iterator ifolder=
2087 m_folderlist.begin();ifolder!=m_folderlist.end();++ifolder) {
2091 if (!m_hiparent.empty()) {
2093 int code=tagParents();
2096 if (m_copytaginfo) {
2098 int code=writeTagInfo();
2103 if (m_poolcat.empty()) {
2104 code=listPoolRefs();
2105 if (!m_checkoutputfile.empty()) filePoolRefs();
2107 code=resolvePoolRefs();
2112 p_rootfile->Write();
2122 std::vector<std::string>&
folders) {
2128 std::string_view par0=
argv[
ic];
2130 if (par0.compare(0,2,
"--")==0) par0=par0.substr(1);
2131 if ((par0==
"-f" || par0==
"-folder") &&
ir>1) {
2134 }
else if ((par0==
"-e" || par0==
"-exclude") &&
ir>1) {
2137 }
else if ((par0==
"-t" || par0==
"-tag") &&
ir>1) {
2140 }
else if ((par0==
"-mt" || par0==
"-magic") &&
ir>1) {
2143 }
else if ((par0==
"-of" || par0==
"-outfolder") &&
ir>1) {
2146 }
else if ((par0==
"-ot" || par0==
"-outtag") &&
ir>1) {
2149 }
else if ((par0==
"-bs" || par0==
"-buffersize") &&
ir>1) {
2152 }
else if ((par0==
"-sl" || par0==
"-seal") &&
ir>1) {
2155 }
else if ((par0==
"-rls" || par0==
"-runlumisince") &&
ir>2) {
2158 }
else if ((par0==
"-rlu" || par0==
"-runlumiuntil") &&
ir>2) {
2161 }
else if ((par0==
"-nrls" || par0==
"-newrunlumisince") &&
ir>2) {
2164 }
else if ((par0==
"-nrlu" || par0==
"-newrunlumiuntil") &&
ir>2) {
2167 }
else if ((par0==
"-srls" || par0==
"-skiprunlumisince") &&
ir>2) {
2171 }
else if ((par0==
"-srlu" || par0==
"-skiprunlumiuntil") &&
ir>2) {
2175 }
else if (par0==
"-ro" || par0==
"-root") {
2177 }
else if (par0==
"-zn" || par0==
"-zeronull") {
2179 }
else if (par0==
"-ana" || par0==
"-analyse") {
2182 m_anadelt=
static_cast<long long>(ttime);
2184 }
else if ((par0==
"-rs" || par0==
"-runsince") &&
ir>1) {
2187 }
else if ((par0==
"-ru" || par0==
"-rununtil") &&
ir>1) {
2190 }
else if ((par0==
"-r" || par0==
"-run") &&
ir>1) {
2194 }
else if ((par0==
"-ts" || par0==
"-timesince") &&
ir>1) {
2197 }
else if ((par0==
"-tu" || par0==
"-timeuntil") &&
ir>1) {
2200 }
else if ((par0==
"-nts" || par0==
"-newtimesince") &&
ir>1) {
2203 }
else if ((par0==
"-ntu" || par0==
"-newtimeuntil") &&
ir>1) {
2206 }
else if (par0==
"-c" || par0==
"-create") {
2208 }
else if ((par0==
"-ch" || par0==
"-channel") &&
ir>1) {
2211 }
else if ((par0==
"-ch1" || par0==
"-channel1") &&
ir>1) {
2214 }
else if ((par0==
"-ch2" || par0==
"-channel2") &&
ir>1) {
2217 }
else if (par0==
"-chd" || par0==
"-channeldesc") {
2220 }
else if (par0==
"-forcerecreate") {
2222 }
else if (par0==
"-d" || par0==
"-debug") {
2224 }
else if (par0==
"-a" || par0==
"-alliov") {
2226 }
else if (par0==
"-ih" || par0==
"-includehead") {
2228 }
else if (par0==
"-eh" || par0==
"-excludehead") {
2230 }
else if (par0==
"-ht" || par0==
"-headtag") {
2232 }
else if (par0==
"-uht" || par0==
"-userheadtag") {
2234 }
else if (par0==
"-v" || par0==
"-verify") {
2236 }
else if (par0==
"-nc" || par0==
"-nocopy") {
2238 }
else if (par0==
"-noc" || par0==
"-nocoracool") {
2240 }
else if (par0==
"-nch" || par0==
"-nochannel") {
2242 }
else if (par0==
"-ncr" || par0==
"-noclobroot") {
2244 }
else if (par0==
"-nd" || par0==
"-nodata") {
2246 }
else if (par0==
"-nh" || par0==
"-nohitag") {
2248 }
else if (par0==
"-hi" || par0==
"-hitag") {
2250 }
else if ((par0==
"-ec" || par0==
"-excludechannel") &&
ir>1) {
2253 }
else if (par0==
"-fs" || par0==
"-forcesingle") {
2255 }
else if (par0==
"-fm" || par0==
"-forcemulti") {
2257 }
else if (par0==
"-frl" || par0==
"-forcerunlumi") {
2259 }
else if (par0==
"-ftm" || par0==
"-forcetime") {
2261 }
else if (par0==
"-fp" || par0==
"-forcepayload") {
2263 }
else if (par0==
"-fnp" || par0==
"-forcenopayload") {
2265 }
else if (par0==
"-cr" || par0==
"-checkrefs") {
2267 }
else if (par0==
"-lp" || par0==
"-listpfn") {
2270 }
else if (par0==
"-cf" || par0==
"-checkfiles") {
2273 }
else if (par0==
"-pa" || par0==
"-poolall") {
2275 }
else if ((par0==
"-co" || par0==
"-checkoutput") &&
ir>1) {
2278 }
else if ((par0==
"-pc" || par0==
"-poolcat") &&
ir>1) {
2281 }
else if ((par0==
"-mc" || par0==
"-mergecat") &&
ir>1) {
2284 }
else if ((par0==
"-ds" || par0==
"-dataset") &&
ir>1) {
2287 }
else if (par0==
"-us" || par0==
"-updatesource") {
2289 }
else if (par0==
"-cd" || par0==
"-checkdest") {
2291 }
else if (par0==
"-tr" || par0==
"-truncate") {
2293 }
else if (par0==
"-al" || par0==
"-appendlocked") {
2295 }
else if (par0==
"-alsv" || par0==
"-appendlockedsv") {
2298 }
else if (par0==
"-rdo" || par0==
"-readoracle") {
2300 }
else if (par0==
"-skiprep" || par0==
"-sr") {
2302 }
else if (par0==
"-go" || par0==
"-getonline") {
2304 }
else if (par0==
"-onr" || par0==
"-onlinerun") {
2306 }
else if (par0==
"-gb" || par0==
"-getbulk") {
2308 }
else if (par0==
"-gt" || par0==
"-gettime") {
2310 }
else if ((par0==
"-tdb" || par0==
"-timedb") &&
ir>1) {
2313 }
else if (par0==
"-ignoremode" &&
ir>1) {
2314 std::cout <<
"Ignoremode password is " <<
argv[
ic+1] <<
":end" << std::endl;
2315 if (strcmp(
argv[
ic+1],
"BackDoor")==0) {
2318 std::cout <<
"ERROR: Incorrect password for -ignoremode" << std::endl;
2322 }
else if (par0==
"-is" || par0==
"-ignorespec") {
2324 }
else if (par0==
"-pt" || par0==
"-prunetags") {
2327 }
else if (par0==
"-lo" || par0==
"-lockedonly") {
2331 }
else if (par0==
"-cti" || par0==
"-copytaginfo") {
2333 }
else if (par0==
"-ctl" || par0==
"-copytaglock") {
2336 }
else if ((par0==
"-tl" || par0==
"-taglabel") &&
ir>1) {
2340 }
else if ((par0==
"-ag" || par0==
"-addguid") &&
ir>1) {
2343 }
else if ((par0==
"-alf" || par0==
"-addlfn") &&
ir>1) {
2346 }
else if ((par0==
"-pf" || par0==
"-parfile") &&
ir>1) {
2349 }
else if ((par0==
"-rf" || par0==
"-runfile") &&
ir>1) {
2352 }
else if ((par0==
"-ws" || par0==
"-runinfohost") &&
ir>1) {
2355 }
else if (par0==
"-h" || par0==
"-help") {
2359 std::cout <<
"Parameter error for argument: " << par0 << std::endl;
2370 std::vector<std::string>
folders;
2375 for (std::vector<std::string>::const_iterator
ifile=
m_parfile.begin();
2377 std::cout <<
"Reading parameters from file " << *
ifile << std::endl;
2378 FILE* p_inp=fopen(
ifile->c_str(),
"r");
2379 if (p_inp==
nullptr) {
2380 std::cout <<
"File not found" << std::endl;
2383 char* p_buf=
new char[999];
2384 while (!feof(p_inp)) {
2385 char* p_line=fgets(p_buf,999,p_inp);
2386 if (p_line!=
nullptr) {
2388 const char* fargv[99];
2390 char* p_start=
nullptr;
2391 while (*p_line!=
'\0') {
2392 if (*p_line==
' ' || *p_line==
'\n') {
2394 if (p_start!=
nullptr) {
2397 fargv[fargc]=p_start;
2403 if (p_start==
nullptr) p_start=p_line;
2415 std::cout <<
"Close file" << std::endl;
2422 std::cout <<
"Problem opening connections" << std::endl;
2436 for (std::vector<std::string>::const_iterator ifold=
folders.begin();
2456 std::cout <<
"Source run/LB range [" << (
m_runemin >> 32) <<
"," <<
2460 std::cout <<
"Source timestamp range " <<
m_timemin <<
" " <<
2464 std::cout <<
"Change o/p run/LB range [" << (
m_newrunemin >> 32) <<
"," <<
2468 std::cout <<
"Change o/p timestamp range " <<
m_newtimemin <<
" " <<
2472 std::cout <<
"Skip IOVs extending outside run/LB range [" <<
2478 for (std::vector<cool::ChannelId>::const_iterator itr=
2480 std::cout <<
"Channel " << *itr <<
" will be excluded" << std::endl;
2482 if (
m_hitag) std::cout <<
"All hierachical tags connecting to referenced tags will be copied" << std::endl;
2483 if (
m_nohitag) std::cout <<
"Hierarchical tag relations will not be copied"
2491 const char* cptr=
input;
2492 while (*cptr!=
'\0') {
2493 if (!isdigit(*cptr)) { isnum=
false;
break;}
2500 const cool::IRecord& rhs) {
2503 if (lhs.size()!=rhs.size())
return false;
2504 for (
size_t i=0;
i<lhs.size();++
i) {
2506 if (lhs[
i].specification().storageType()!=
2507 rhs[
i].specification().storageType())
2510 if (lhs[
i]!=rhs[
i]) {
2513 const cool::StorageType& stype=lhs[
i].specification().storageType();
2514 if (stype==cool::StorageType::String255 ||
2515 stype==cool::StorageType::String4k ||
2516 stype==cool::StorageType::String64k ||
2517 stype==cool::StorageType::String16M) {
2519 if (lhs[
i].data<std::string>()!=rhs[
i].data<std::string>())
2537 return static_cast<long long>(atol(
input))*
2538 static_cast<long long>(1.E9);
2540 struct tm mytm{},mytm2{};
2548 cool::ValidityKey itime=
static_cast<cool::ValidityKey
>(mktime(&mytm));
2549 strptime(
"1970-01-02:00:00:00",
"%Y-%m-%d:%T",&mytm2);
2550 cool::ValidityKey caltime=
static_cast<cool::ValidityKey
>(mktime(&mytm2));
2551 itime+=24*60*60-caltime;
2552 return itime*
static_cast<cool::ValidityKey
>(1.E9);
2555 "ERROR in format of time value, use e.g. 2007-05-25:14:01:00" <<
2563 if (iovtime==cool::ValidityKeyMin) {
2564 return "ValidityKeyMin ";
2565 }
else if (iovtime==cool::ValidityKeyMax) {
2566 return "ValidityKeyMax ";
2568 time_t
time=
static_cast<time_t
>(iovtime/1E9);
2571 return "UTC "+std::string(asctime_r(gmtime_r(&
time, &
result), buf));
2578 cool::ValidityKey
val;
2580 val=((1LL << 31)-1) << 32;
2582 val=
static_cast<long long>(atol(
input1)) << 32;
2584 if (input2[0]==
'M' || input2[0]==
'm') {
2589 if (
val>cool::ValidityKeyMax)
val=cool::ValidityKeyMax;
2607 std::string sorfolder=
"/TDAQ/RunCtrl/SOR_Params";
2608 std::string eorfolder=
"/TDAQ/RunCtrl/EOR_Params";
2609 if (
m_timedb.find (
"CONDBR2")!=std::string::npos) {
2610 sorfolder=
"/TDAQ/RunCtrl/SOR";
2611 eorfolder=
"/TDAQ/RunCtrl/EOR";
2613 std::cout <<
"Extracting times for run range [" << (
m_runemin >> 32) <<
2615 " folder " << sorfolder << std::endl;
2617 std::cout <<
"ERROR: Run range not correctly specified" << std::endl;
2621 cool::IDatabasePtr tdaqdb;
2624 std::cout <<
"Opened database connection" << std::endl;
2627 std::cout <<
"COOL exception caught: " <<
e.what() << std::endl;
2632 cool::IFolderPtr
folder=tdaqdb->getFolder(sorfolder);
2637 while (itr->goToNext()) {
2638 const cool::IRecord&
payload=itr->currentRef().payload();
2644 std::cout <<
"ERROR: Found " << nobj <<
" SOR records" << std::endl;
2649 std::cout <<
"Exception accessing SOR information: " <<
e.what() <<
2655 cool::IFolderPtr
folder=tdaqdb->getFolder(eorfolder);
2660 while (itr->goToNext()) {
2661 const cool::IRecord&
payload=itr->currentRef().payload();
2667 std::cout <<
"ERROR: Found " << nobj <<
" EOR records" << std::endl;
2672 std::cout <<
"Exception accessing SOR information: " <<
e.what() <<
2676 tdaqdb->closeDatabase();
2677 std::cout <<
"Timestamp range set to " <<
m_timemin <<
" " <<
2687 std::cout <<
"Extracting current run-number from ATLAS_RUN_NUMBER @ ATONR_ADG ... " <<
2689 curl_global_init(CURL_GLOBAL_ALL);
2690 using uniqueCurl_t = std::unique_ptr<CURL,decltype(&curl_easy_cleanup)>;
2691 uniqueCurl_t curl(curl_easy_init(), curl_easy_cleanup);
2692 CURLcode
res = CURLE_OK;
2696 res = curl_easy_setopt(curl.get(), CURLOPT_URL,
url.c_str());
2697 if (
res != CURLE_OK) {
2698 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2702 res = curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1L);
2703 if (
res != CURLE_OK) {
2704 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2712 if (
res != CURLE_OK) {
2713 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2716 res = curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &
response);
2717 if (
res != CURLE_OK) {
2718 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2723 res = curl_easy_perform(curl.get());
2724 if (
res != CURLE_OK) {
2725 std::cerr <<
"Failed to perform request: " << curl_easy_strerror(
res) <<
":" <<
url.c_str() << std::endl;
2729 std::cout <<
"Response: " << std::endl;
2730 std::cout <<
response << std::endl;
2736 int runNumber = jsonData[
"resources"][0][
"runnumber"];
2738 std::cout <<
"Next run started will be " << nextrun << std::endl;
2740 std::cout <<
"Epoch time extracted " <<
rtime << std::endl;
2747 m_runemin=(
static_cast<long long>(nextrun)) << 32;
2751 }
catch (nlohmann::json::parse_error&
e) {
2752 std::cerr <<
"Failed to parse JSON response: " <<
e.what() << std::endl;
2754 }
catch (nlohmann::json::type_error&
e) {
2755 std::cerr <<
"Failed to extract data from JSON response: " <<
e.what() << std::endl;
2758 std::cerr <<
"Failed to extract run and timestamp from JSON response: " <<
e.what() << std::endl;
2764 std::cerr <<
"Failed to initialize libcurl." << std::endl;
2768 curl_global_cleanup();
2774 std::cout <<
"ERROR: -getonline and -getbulk cannot be used simultaneously"
2778 std::cout <<
"Call getbulk using URL" << std::endl;
2780 curl_global_init(CURL_GLOBAL_ALL);
2781 using uniqueCurl_t = std::unique_ptr<CURL,decltype(&curl_easy_cleanup)>;
2782 uniqueCurl_t curl(curl_easy_init(), curl_easy_cleanup);
2784 CURLcode
res = CURLE_OK;
2789 res = curl_easy_setopt(curl.get(), CURLOPT_URL,
url.c_str());
2790 if (
res != CURLE_OK) {
2791 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2795 res = curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1L);
2796 if (
res != CURLE_OK) {
2797 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2805 if (
res != CURLE_OK) {
2806 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2809 res = curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &
response);
2810 if (
res != CURLE_OK) {
2811 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2815 res = curl_easy_perform(curl.get());
2816 if (
res != CURLE_OK) {
2817 std::cerr <<
"Failed to perform request: " << curl_easy_strerror(
res) <<
":" <<
url.c_str() << std::endl;
2821 std::cout <<
"Response: " << std::endl;
2822 std::cout <<
response << std::endl;
2827 int nextrun = jsonData[
"run"];
2828 long long rtime = jsonData[
"timestamp"];
2829 std::cout <<
"Next run started will be " << nextrun << std::endl;
2830 std::cout <<
"Epoch time extracted " <<
rtime << std::endl;
2837 m_runemin=(
static_cast<long long>(nextrun)) << 32;
2841 }
catch (nlohmann::json::parse_error&
e) {
2842 std::cerr <<
"Failed to parse JSON response: " <<
e.what() << std::endl;
2844 }
catch (nlohmann::json::type_error&
e) {
2845 std::cerr <<
"Failed to extract data from JSON response: " <<
e.what() << std::endl;
2848 std::cerr <<
"Failed to extract run and timestamp from JSON response: " <<
e.what() << std::endl;
2854 std::cerr <<
"Failed to initialize libcurl." << std::endl;
2858 curl_global_cleanup();
2865 for (std::vector<std::string>::const_iterator itr=
m_runfile.begin();
2867 std::cout <<
"Reading allowed run numbers from file " << *itr << std::endl;
2868 FILE* p_inp=fopen(itr->c_str(),
"r");
2869 if (p_inp==
nullptr) {
2870 std::cout <<
"File not found" << std::endl;
2873 char* p_buf=
new char[999];
2874 while (!feof(p_inp)) {
2875 char* p_line=fgets(p_buf,999,p_inp);
2876 if (p_line!=
nullptr) {
2877 unsigned int run=
atoi(p_line);
2885 std::cout <<
"Read list of " <<
m_runlist.size() <<
" runs from " <<
2886 m_runfile.size() <<
" input runlist files" << std::endl;
2887 for (std::vector<unsigned int>::const_iterator itr=
m_runlist.begin();
2888 itr!=
m_runlist.end();++itr) std::cout <<
2889 "Update allowed for run " << *itr << std::endl;
2894 std::string_view
tag) {
2897 std::string_view modestr=
"";
2899 std::string::size_type iofs1=
desc.find(
"<updateMode>");
2900 std::string::size_type iofs2=
desc.find(
"</updateMode>");
2901 if (iofs1!=std::string::npos && iofs2!=std::string::npos && iofs2>iofs1)
2902 modestr=
desc.substr(iofs1+12,iofs2-iofs1-12);
2903 if (modestr==
"UPD1" ||
tag.find(
"UPD1")!=std::string::npos)
mode=1;
2904 if (modestr==
"UPD2" ||
tag.find(
"UPD2")!=std::string::npos)
mode=2;
2905 if (modestr==
"UPD3" ||
tag.find(
"UPD3")!=std::string::npos)
mode=3;
2906 if (modestr==
"UPD4" ||
tag.find(
"UPD4")!=std::string::npos)
mode=4;
2912 const cool::IFolderPtr& sourcefl,
const cool::IFolderPtr& destfl,
2918 const std::map<cool::ChannelId,std::string> chanmap=
2919 sourcefl->listChannelsWithNames();
2920 if (
m_debug) std::cout <<
"Checking channels table for folder " <<
folder
2921 <<
" with " << chanmap.size() <<
" channels" << std::endl;
2925 for (std::map<cool::ChannelId,std::string>::const_iterator
2926 itr=chanmap.begin();itr!=chanmap.end();++itr) {
2928 const cool::ChannelId
chan=itr->first;
2936 const std::string& sourcename=itr->second;
2938 std::string sourcedesc=
"";
2939 if (
m_chdesc) sourcedesc=sourcefl->channelDescription(
chan);
2942 const std::string & destname=destfl->channelName(
chan);
2943 std::string destdesc=
"";
2944 if (
m_chdesc) destdesc=destfl->channelDescription(
chan);
2946 if (sourcename!=destname && !sourcename.empty()) {
2949 std::cout <<
"ERROR: Channel " <<
chan <<
" names differ: " <<
2950 sourcename <<
" " << destname << std::endl;
2953 destfl->setChannelName(
chan,sourcename);
2954 std::cout <<
"Modify channel " <<
chan <<
" name from "
2955 << destname <<
" to " << sourcename << std::endl;
2958 if (sourcedesc!=destdesc && !sourcedesc.empty()) {
2961 std::cout <<
"ERROR: Channel " <<
chan <<
" descriptions differ: "
2962 << sourcedesc <<
" " << destdesc << std::endl;
2965 std::cout <<
"Modify channel " <<
chan <<
" description from "
2966 << sourcedesc <<
" to " << destdesc << std::endl;
2967 destfl->setChannelDescription(
chan,sourcedesc);
2974 if (
m_debug) std::cout <<
"Create new channel " <<
chan <<
" name "
2975 << sourcename <<
" description" << sourcedesc << std::endl;
2976 destfl->createChannel(
chan,sourcename,sourcedesc);
2980 if (ncreate>0) std::cout <<
"Created " << ncreate <<
" new channels for "
2982 if (nmodify>0) std::cout <<
"Modified " << nmodify <<
" channel info for "
2989 const std::string&
folder,
const std::string&
tag) {
2991 if (
name==
"PoolRef" ||
name==
"fileGUID") {
2992 std::string poolref;
2993 std::string addr=
payload[0].data<std::string>();
2994 if (
name==
"PoolRef") {
2995 std::string::size_type iofs1=addr.find(
"[DB=");
2996 std::string::size_type iofs2=addr.find(
']',iofs1);
2997 if (iofs1!=std::string::npos && iofs2!=std::string::npos && iofs2>iofs1) {
2998 poolref=addr.substr(iofs1+4,iofs2-iofs1-4);
3000 std::cout <<
"ERROR: Badly formed POOL object reference " <<
3026 std::cout <<
"Copying additional hierarchical tags of parent folders"
3034 cool::IFolderSetPtr sourcefl=
m_sourceDbPtr->getFolderSet(folderset);
3035 cool::IFolderSetPtr destfl=
m_destDbPtr->getFolderSet(folderset);
3036 std::cout <<
"Processing folderset " << folderset << std::endl;
3037 for (HiTagMap::const_iterator imap=
m_hitagmap.begin();
3041 std::string sourcetag=sourcefl->findTagRelation(imap->first);
3042 std::cout <<
"Create hierarchical tag between " << sourcetag <<
3043 " and " << imap->first <<
" in folder " << imap->second <<
3047 std::string etag=destfl->resolveTag(imap->first);
3048 if (etag==sourcetag) {
3049 std::cout <<
"This relation has already been created" << std::endl;
3051 std::cout <<
"ERROR: Tag in parent already related to " <<
3056 catch (cool::Exception&
e ) {
3059 destfl->createTagRelation(imap->first,sourcetag);
3065 m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
3067 imap->first,imap->first)));
3069 catch (cool::Exception&
e) {
3070 std::cout <<
"Cool exception " <<
e.what() <<
3071 "thrown in hierarchical tag creation" << std::endl;
3077 catch (cool::Exception&
e) { }
3080 std::cout <<
"ERROR Folderset " << folderset <<
" not found" <<
3088 std::cout <<
"Write tag description ";
3090 std::cout <<
"for " <<
m_cooltagmap.size() <<
" tags" << std::endl;
3091 for (CoolTagMap::const_iterator itr=
m_cooltagmap.begin();
3099 std::cout <<
"Total of " <<
m_poolrefs.size() <<
" POOL files referenced"
3101 for (PoolMap::const_iterator ipool=
m_poolrefs.begin();
3103 std::cout <<
"Ref " << ipool->first <<
" (" << ipool->second.count()
3104 <<
")" << std::endl;
3110 std::cout <<
"Total of " << m_poolrefs.size() <<
" POOL Files referenced"
3113 if (catalog==
nullptr)
return 110;
3117 pool_utility.startSession();
3120 if (!m_addguid.empty()) {
3121 for (std::vector<std::string>::const_iterator itr=m_addguid.begin();
3122 itr!=m_addguid.end();++itr) {
3124 std::cout <<
"Added POOL file GUID: " << *itr << std::endl;
3127 if (!m_addlfn.empty()) {
3128 for (std::vector<std::string>::const_iterator itr=m_addlfn.begin();
3129 itr!=m_addlfn.end();++itr) {
3132 std::cout <<
"Add POOL file GUID: " <<
guid <<
" from LFN " << *itr
3139 using LFNGVec = std::vector<std::pair<std::string, std::string> >;
3141 bool dscopy=!m_newdataset.empty();
3146 ipool!=m_poolrefs.end();++ipool) {
3150 if( !lfns.empty() ) {
3152 const std::string
lfn = lfns[0].first;
3153 ipool->second.setlfn(
lfn);
3154 std::cout <<
"LFN: " <<
lfn <<
" (" << ipool->second.count()
3155 <<
")" << std::endl;
3157 dsfound.push_back(std::pair<std::string,std::string>(
lfn,
guid));
3160 if (!m_listpfn) ipool->second.setErrorBit(0);
3162 if (m_listpfn || m_poolopen) {
3163 std::string pfn, tech;
3165 if( !pfn.empty() ) {
3166 ipool->second.setpfn(pfn);
3167 std::cout <<
"PFN: " << pfn <<
" (" << ipool->second.count()
3168 <<
")" << std::endl;
3171 std::string hguid=getCoolHistGUID(pfn);
3172 if (!hguid.empty()) {
3174 if (hguid!=ipool->first) {
3175 std::cout <<
"ERROR File CoolHist GUID " << hguid <<
3176 " inconsistent with catalogue " << ipool->first << std::endl;
3177 ipool->second.setErrorBit(3);
3182 const std::string fid = pool_utility.
readFileGUID( pfn );
3183 if( fid != ipool->first ) {
3184 std::cout <<
"ERROR File GUID " << fid <<
3185 " inconsistent with catalogue " << ipool->first << std::endl;
3187 ipool->second.setErrorBit(3);
3190 catch( std::runtime_error&
e ) {
3191 std::cout <<
"Cannot open file for reading!" << std::endl;
3192 std::cout <<
e.what() << std::endl;
3194 ipool->second.setErrorBit(2);
3200 ipool->second.setErrorBit(1);
3204 if (ipool->second.errcode()>0) ++nbad;
3215 catalog=setupCatalog(m_mergecat);
3216 if (catalog==
nullptr)
return 110;
3217 const std::string dssname=
"register.sh";
3218 std::cout <<
"Write DQ2 registerFileInDataset commands to " << dssname
3219 <<
" for registration in dataset " << m_newdataset << std::endl;
3220 std::ofstream dsstream(dssname.c_str());
3221 for (LFNGVec::const_iterator itr=dsfound.begin();
3222 itr!=dsfound.end();++itr) {
3223 const std::string&
lfn=itr->first;
3224 const std::string&
guid=itr->second;
3227 if( !lfns.empty() ) {
3229 const std::string lfn2 = lfns[0].first;
3230 if (lfn2!=
lfn) std::cout <<
"WARNING: LFNs for GUID " <<
guid <<
3231 " differ in input/merge datasets: " <<
lfn <<
" vs "
3232 << lfn2 << std::endl;
3235 dsstream <<
"dq2-register-files " << m_newdataset <<
" "
3236 <<
lfn <<
" " <<
guid << std::endl;
3246 std::cout <<
"All POOL references were resolved by catalogues"
3249 std::cout <<
"ERROR Could not resolve " << nbad <<
" files" << std::endl;
3252 if (nbad>0 || m_poolall) filePoolRefs();
3258 std::string hguid=
"";
3259 TFile* myfile=TFile::Open(
file.c_str(),
"READ");
3260 if (myfile!=
nullptr) {
3262 myfile->GetObject(
"fileGUID",oguid);
3263 if (oguid!=
nullptr) {
3264 hguid=oguid->GetString();
3265 std::cout <<
"CoolHist GUID found to be " << hguid << std::endl;
3281 std::cout <<
"ErrCode GUID Count LFN PFN Folders ..." << std::endl;
3282 for (PoolMap::const_iterator ipool=
m_poolrefs.begin();
3284 if (
m_poolall || ipool->second.errcode()>0) {
3285 std::ostringstream
line;
3287 line << ipool->second.errcode() <<
" " << ipool->first <<
" " <<
3288 ipool->second.count() <<
" ";
3289 if (!ipool->second.lfn().empty()) {
3290 line << ipool->second.lfn() <<
" ";
3294 if (!ipool->second.pfn().empty()) {
3295 line << ipool->second.pfn() <<
" ";
3299 for (std::vector<std::string>::const_iterator
3300 itr=ipool->second.foldertag().begin();
3301 itr!=ipool->second.foldertag().end();++itr)
3302 line << *itr <<
" ";
3303 std::cout <<
line.str() << std::endl;
3304 chkostream <<
line.str() << std::endl;
3310 const std::vector<std::string>& catvec) {
3314 for (std::vector<std::string>::const_iterator icat=catvec.begin();
3315 icat!=catvec.end();++icat) {
3316 std::cout <<
"Add catalogue: " << *icat << std::endl;
3318 if (icat->find(
':')==std::string::npos) {
3329 std::cout <<
"Could not setup POOL catalogues, exception:" <<
e.what()
3336 std::cout <<
"usage: AtlCoolCopy.exe <sourceCoolDB> <destinationCoolDB> { <options> }" << std::endl;
3337 std::cout <<
"Options are (see doc/mainpage.h for more):" << std::endl <<
3338 "-a, -alliov : set IOVs on destination to [ValidityKeyMin, ValidityKeyMax]" << std::endl <<
" or following settings of -nrls,-nrlu,-nts,-ntu" <<
3340 "-ag, -addguid <guid> : Add GUID to list accessed for POOL checks" <<
3342 "-alf, -addlfn <LFN> : Add LFN to list accessed for POOL checks" <<
3344 "-al, -appendlocked : Allow locked tags to be updated if no overlap" <<
3346 "-alsv, -appendlockedsv : Allow locked tag update for openended IOVs" <<
3348 "-ana, -analyse <delta_t>: produce analysis ROOT file (filename = dest DB argument)" << std::endl <<
3349 "-bs, -buffersize <size> : set size of bulkstorage output buf to <size> objs"
3351 "-c, -create : create destination DB if not already existing"
3353 "-cd, -checkdest : Check destination DB and skip already existing tags"
3355 "-ch, -channel : restrict selection to given channel or range specified as c1:c2" << std::endl <<
3356 "-ch1, -channel1 : specify start of a range of channels" << std::endl <<
3357 "-ch2, -channel2 : specify end of a range of channels" << std::endl <<
3358 "-cf, -checkfiles : check POOL files can be opened and have correct GUID"
3360 "-co, -checkoutput <file> : write POOL file check output on file"
3361 << std::endl <<
"-cti, -copytaginfo : Copy tag descriptions"
3362 << std::endl <<
"-ctl, -copytaglock : Copy tag locked status and descriptions"
3363 << std::endl <<
"-cr, -checkrefs : check POOL references"
3364 << std::endl <<
"-d, -debug : produce debug output" << std::endl;
3366 "-ds, -dataset : output register.sh for creating DQ2 datasets "
3368 "-ec, -excludechannel : exclude given channel from copy" << std::endl <<
3369 "-eh, -excludehead : exclude HEAD tag even if no tags found in MV folders"
3371 "-ih, -includehead : include HEAD as well as any tags in MV folders" <<
3373 "-e, -exclude <pattern> : exclude folders" << std::endl <<
3374 "-f, -folder <pattern> : include folders" << std::endl <<
3375 "-fs, -forcesingle : force destination folder to be singleversion"
3377 "-fm, -forcemulti : force destination folder to be multiversion"
3379 "-frl, -forcerunlumi : force destination folder to be run/LB indexed"
3381 "-ftm, -forcetime : force destination folder to be timestamp indexed"
3383 "-fp, -forcepayload : Force destn folders to be created with payload tbl" << std::endl <<
3384 "-fnp, -forcenopayload : Force destn folders to be created without payload tbl" << std::endl <<
3385 "-forcerecreate : delete and recreate destination database" << std::endl;
3386 std::cout <<
"-ht, -headtag : Use HEAD-style tagging"
3388 "-go, -getonline : Set minimum run number (-rls setting) to next ONLINE run"
3390 "-gb, -getbulk : Set minimum run number (-rls setting) to next bulk reco run"
3392 "-gt, -gettime : Extract timestamp information for given run number range"
3394 "-h, -help : print help and exit" << std::endl <<
3395 "-hi, -hitag : Copy hierrchical tags inclusively" << std::endl <<
3396 "-ignoremode <pwd> : Ignore UPDx mode protection (experts only)"
3398 "-is, -ignorespec : Ignore differences in folder spec if names are all equal" << std::endl <<
3399 "-lo, -lockedonly : Only copy locked/partially-locked top-level tags"
3401 "-nc, -nocopy : Do not actually copy, just read source DB" << std::endl <<
3402 "-nch, -nochannel : Do not check or copy channel information" << std::endl
3403 <<
"-ncr, -noclobroot: Do not copy CLOB data into ROOT files" << std::endl
3404 <<
"-noc, -nocoracool : Do not copy Coracool structure payloads " << std::endl <<
3405 "-nd, -nodata : Copy only folder structures, not data" << std::endl <<
3406 "-nh, -nohitag : Do not follow hierarchical tag relations" << std::endl <<
3407 "-mc, -mergecat <catfile> : specify POOL file catalogue for new dataset making"
3409 std::cout <<
"-of, -outfolder <folder> : rename folder on output"
3411 "-onr, -onlinerun : Retrieve run number from online server (not replica)" <<std::endl <<
3412 "-ot, -outtag : " <<
"Rename tag on output" << std::endl;
3414 "-pa, -poolall : Output all POOL files (incl good) when checking files"
3416 "-pc, -poolcat <catfile> : specify POOL file catalogue for ref checking"
3418 "-pf, -parfile <file> : Read additional options/parameters from file" <<
3420 "-pt, -prunetags : Copy only hierarchical tags descending from specified toptags"
3422 "-rdo, -readoracle : force data to be read from Oracle, using dbreplica.config"
3424 "-sl, -seal <val>: Set SEAL (hence COOL/POOL) output level to <val>"
3426 "-sr, -skiprep : Skip folders having <norep/> folder metadata tag" <<
3428 "-t, -tag <tag> : Copy multiversion data with tag <tag>" << std::endl <<
3429 "-mt, -magic <tag> : Include magic tags involving the given pattern " << std::endl <<
3430 "-tr, -truncate : Set destination IOVs outside query interval to query interval"
3432 "-tl, -taglabel : Specify tag description to enter in destination DB"
3434 "-uht, -userheadtag : Also copy user tag data to the HEAD" << std::endl <<
3435 "-v, -verify : Verify data present on destination rather than copying" <<
3437 "-ro, -root : Produce ROOT output file instead of copying to COOL"
3439 "-zn, -zeronull : Zero NULLs in ROOT file instead of skipping them" <<
3441 "-rls, -runlumisince <run> <LB> : Set minimum IOV interval" << std::endl <<
3442 "-rlu, -runlumiuntil <run> <LB> : Set maximum IOV interval" << std::endl <<
3443 "-rs, -runsince <run> : Set minimum IOV interval" << std::endl <<
3444 "-ru, -rununtil <run> : Set maximum IOV interval" << std::endl <<
3445 "-r, -run <run> : Copy only run <run>" << std::endl <<
3446 "-srls, -skiprunlumisince <run> <LB> : Set minimum IOV for skipping IOV in copy"
3448 "-srlu, -skiprunlumiuntil <run> <LB> : Set maximum IOV for skipping IOV in copy"
3450 "-tdb, -timedb <dbconn> : Set database connection for time information" <<
3451 "-ts, -timesince <time> : Set minimum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3453 "-tu, -timeuntil <time> : Set maximum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3456 "-nrls, -newrunlumisince <run> <LB> : Set minimum of output IOV interval (use with -alliov)" << std::endl <<
3457 "-nrlu, -newrunlumiuntil <run> <LB> : Set maximum of output IOV interval (use with --aliov)" << std::endl <<
3458 "-nts, -newtimesince <time> : Set minimum of outputIOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3460 "-ntu, -newtimeuntil <time> : Set maximum of output IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3462 std::cout <<
"See http://twiki.cern.ch/twiki/bin/view/Atlas/AtlCoolCopy for more details" << std::endl;
3477 if (retcode==0) retcode=mycopy.doCopy();
3479 if (retcode>0) std::cout <<
"ERROR AtlCoolCopy.exe fails with exit code " <<
3480 retcode << std::endl;