18 #include <nlohmann/json.hpp>
19 #include <curl/curl.h>
21 #include "CoolKernel/DatabaseId.h"
22 #include "CoolKernel/Exception.h"
23 #include "CoolKernel/IDatabaseSvc.h"
24 #include "CoolKernel/IDatabase.h"
25 #include "CoolKernel/IFolder.h"
26 #include "CoolKernel/FolderSpecification.h"
27 #include "CoolKernel/IFolderSet.h"
28 #include "CoolKernel/IObject.h"
29 #include "CoolKernel/IObjectIterator.h"
30 #include "CoolKernel/IRecordIterator.h"
31 #include "CoolApplication/Application.h"
32 #include "CoralBase/AttributeListException.h"
39 #include "RelationalAccess/ConnectionService.h"
40 #include "RelationalAccess/IConnectionServiceConfiguration.h"
41 #include "RelationalAccess/ISessionProxy.h"
42 #include "RelationalAccess/ITransaction.h"
43 #include "RelationalAccess/ISchema.h"
44 #include "RelationalAccess/ITable.h"
45 #include "RelationalAccess/IQuery.h"
46 #include "RelationalAccess/ICursor.h"
62 #include "TObjString.h"
65 size_t newLength =
size * nmemb;
71 catch (std::bad_alloc &
e)
82 AtlCoolCopy(
const std::string& sourcedb,
const std::string& destdb,
83 bool allowcreate=
false);
88 (
const std::string&
folder,
const std::vector<std::string>&
taglist);
94 bool openConnections(
const std::string& sourcedb,
const std::string& destdb,
99 std::vector<std::string>&
folders);
100 static std::string
transConn(
const std::string& inconn);
103 const std::string& chanstring);
105 (
const std::string&
folder,
const std::string& destfolder,
108 const std::string& sourcetag,
const std::string& desttag,
109 const cool::ValidityKey
since,
const cool::ValidityKey
until,
110 bool timestamp,
bool checkrefs,
bool iscora,
113 const cool::ValidityKey&
until,
114 const cool::ValidityKey& qsince,
115 const cool::ValidityKey& quntil,
116 cool::ValidityKey& newsince,
117 cool::ValidityKey& newuntil,
118 const bool timestamp)
const;
120 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
121 const cool::ValidityKey
since,
const cool::ValidityKey
until,
126 const std::string& sourcetag,
127 const cool::ValidityKey
since,
const cool::ValidityKey
until,
128 const bool checkrefs,
const bool iscora,
131 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
132 const cool::ValidityKey
since,
const cool::ValidityKey
until,
133 const bool timestamp);
134 std::string
rootDirs(
const std::string&
folder,
const std::string& toproot);
136 void*& sptr,
char& rootID)
const;
140 const cool::IRecord& rhs);
142 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
143 const cool::ValidityKey
since,
const cool::ValidityKey
until,
144 const bool timestamp);
146 const int chan,
const float xlow,
const float xhigh);
148 static std::string
timeString(
const cool::ValidityKey iovtime);
149 static cool::ValidityKey
runLBVal(
const char*
input1,
const char* input2);
158 const cool::IFolderPtr& sourcefl,
const cool::IFolderPtr& destfl,
161 const std::string&
folder,
const std::string&
tag);
268 typedef std::map<std::string,PoolMapElement>
PoolMap;
272 using HiTagMap = std::map<std::string, std::string>;
302 m_sourcedb(sourcedb),m_destdb(destdb),m_allowcreate(allowcreate),
304 m_includehead(false),m_excludehead(false),
305 m_usertags(true),m_userupdatehead(false),m_debug(false),m_alliov(false),
306 m_verify(false),m_root(false),m_zeronull(false),m_analyse(false),
307 m_checkrefs(false),m_listpfn(false),m_poolopen(false),m_poolall(false),
308 m_nocopy(false),m_nodata(false),m_nochannel(false),m_chdesc(false),
309 m_hitag(false),m_nohitag(false),m_forcesingle(false),m_forcemulti(false),
310 m_forcerune(false),m_forcetime(false),m_forcepay(false),m_forcenopay(false),
311 m_sourceread(true),m_truncate(false),m_skipout(false),m_skiprep(false),
312 m_applock(false),m_applocksv(false),
313 m_readoracle(false),m_gettime(false),m_getonline(false),m_onlinerun(false),
315 m_prunetags(false),m_lockedonly(false),m_copytaginfo(false),
316 m_copytaglock(false),m_coracool(true),m_ignoremode(false),
317 m_ignorespec(false),m_checkdesttag(false),m_noclobroot(false),
318 m_runemin(
cool::ValidityKeyMin),m_runemax(
cool::ValidityKeyMax),
319 m_timemin(
cool::ValidityKeyMin),m_timemax(
cool::ValidityKeyMax),
320 m_newrunemin(
cool::ValidityKeyMin),m_newrunemax(
cool::ValidityKeyMax),
321 m_newtimemin(
cool::ValidityKeyMin),m_newtimemax(
cool::ValidityKeyMax),
322 m_srunemin(
cool::ValidityKeyMin),m_srunemax(
cool::ValidityKeyMax),
323 m_channel1(
""),m_channel2(
""),m_bufsize(1000),m_sealmsg(5),
324 m_anadelt(-1),m_outfolder(
""),m_outtag(
""),m_newdataset(
""),
325 m_checkoutputfile(
""),m_timedb(
""),m_taglabel(
""),
326 m_runinfohost(
"http://atlas-run-info-api.web.cern.ch/api"),
327 m_coolapp(&m_coralsvc),
328 m_dbSvc(&(m_coolapp.databaseService())),m_repsort(nullptr),
329 m_open(false),m_chansel(
cool::ChannelSelection::
all()),p_rootfile(nullptr)
334 coral::IConnectionServiceConfiguration& csconfig=
m_coralsvc.configuration();
335 csconfig.disablePoolAutomaticCleanUp();
336 csconfig.setConnectionTimeOut(0);
340 const std::string& destdb,
bool allowcreate) {
346 coral::IConnectionServiceConfiguration& csconfig=
m_coralsvc.configuration();
347 csconfig.setReplicaSortingAlgorithm(*
m_repsort);
351 std::cout <<
"Open source database: " << sourcedb << std::endl;
352 if (!
m_sourceread) std::cout <<
"... in UPDATE mode" << std::endl;
357 std::cout <<
"Cool exception caught: " <<
e.what() << std::endl;
365 std::cout <<
"Open destination ROOT file: " << destdb << std::endl;
366 p_rootfile=
new TFile(destdb.c_str(),
"RECREATE");
367 if (
p_rootfile==
nullptr) std::cout <<
"ERROR: Could not open ROOT file" <<
373 std::cout <<
"Open destination database: " << tdestdb << std::endl;
380 "Forcing recreation of destination database - deleting existing data!"
383 m_dbSvc->dropDatabase(tdestdb);
385 throw cool::DatabaseDoesNotExist(
"old database deleted");
389 std::cout <<
"COOL exception caught: " <<
e.what() << std::endl;
392 std::cout <<
"Try to create new conditions DB" << std::endl;
395 std::cout <<
"Creation succeeded" << std::endl;
397 catch (cool::Exception&
e) {
398 std::cout <<
"Creation failed" << std::endl;
411 std::cout <<
"Attempt to open source CoraCool DB " <<
m_sourcedb <<
415 std::cout <<
"Opened CoraCool source DB" << std::endl;
420 std::cout <<
"Attempt to open destination CoraCool DB " <<
m_destdb <<
424 std::cout <<
"Opened CoraCool dest DB" << std::endl;
432 cool::IFolderSetPtr topfolder=
m_sourceDbPtr->getFolderSet(
"/");
433 const std::vector<std::string>& toptaglist=topfolder->listTags();
434 for (std::vector<std::string>::const_iterator toptag=toptaglist.begin();
435 toptag!=toptaglist.end();++toptag) {
437 if (tstat==cool::HvsTagLock::LOCKED ||
438 tstat==cool::HvsTagLock::PARTIALLYLOCKED) {
439 std::cout <<
"Top-level tag " << *toptag <<
" will be copied" <<
441 m_tags.push_back(*toptag);
444 std::cout <<
"Total of " <<
m_tags.size() <<
" top-level tags to be copied"
452 if (inconn.find(
'/')==std::string::npos) {
453 return "sqlite://X;schema=mycool.db;dbname="+inconn;
461 std::cout <<
"Add folders in path:" <<
folder <<
" [ ";
462 const std::vector<std::string> nodelist=
m_sourceDbPtr->listAllNodes();
464 for (std::vector<std::string>::const_iterator nodeitr=nodelist.begin();
465 nodeitr!=nodelist.end();++nodeitr) {
471 nodeitr->size()>
folder.size() && nodeitr->compare(
folder.size(),1,
"/")==0)) {
474 for (std::vector<std::string>::const_iterator iexcl=
m_folderexcl.begin();
476 if (iexcl->compare(0,1,
"/")==0) {
489 std::cout << *nodeitr <<
" ";
496 const std::vector<std::string> foldersettags=
498 if (!foldersettags.empty()) {
499 for (std::vector<std::string>::const_iterator
500 itr=foldersettags.begin();itr!=foldersettags.end();++itr) {
507 cool::IFolderSetPtr sfolder=
m_sourceDbPtr->getFolderSet(*nodeitr);
508 for (std::vector<std::string>::const_iterator toptag=
m_tags.begin();
509 toptag!=
m_tags.end();++toptag) {
511 std::string rtag=sfolder->resolveTag(*toptag);
524 std::cout <<
"]" << std::endl;
529 std::cout <<
"Adding folder to exclude list: " <<
folder << std::endl;
535 (
const std::string&
folder,
const std::vector<std::string>&
taglist) {
537 cool::IFolderPtr sourcefl,destfl;
540 sourcefl=m_sourceDbPtr->getFolder(
folder);
542 catch (cool::Exception&
e) {
543 std::cout <<
"Could not get source folder: " <<
folder << std::endl;
546 const std::string& sourcedesc=sourcefl->description();
548 if (m_skiprep && sourcedesc.find(
"<norep/>")!=std::string::npos) {
549 std::cout <<
"Folder " <<
folder <<
550 " skipped due to <norep/> metadata" << std::endl;
553 const bool iscora=(m_coracool &&
554 sourcedesc.find(
"<coracool")!=std::string::npos);
558 sourceflc=m_sourceCoraPtr->getFolder(
folder);
561 setChannelRange(sourcefl);
566 dvermode=cool::FolderVersioning::SINGLE_VERSION;
567 std::cout <<
"Forcing destination folder to singleversion" << std::endl;
570 dvermode=cool::FolderVersioning::MULTI_VERSION;
571 std::cout <<
"Forcing destination folder to multiversion" << std::endl;
575 sourcefl->folderSpecification().payloadMode();
579 std::string destfolder=
folder;
580 if (!m_outfolder.empty()) {
581 destfolder=m_outfolder;
582 std::cout <<
"Destination folder will be renamed to " << destfolder <<
588 if (!m_nocopy && !m_root && !m_analyse) {
589 if (!m_destDbPtr->existsFolder(destfolder) && !m_verify) {
590 std::cout <<
"Creating folder " << destfolder <<
" payload-type " <<
591 dpaymode <<
" on destination" << std::endl;
594 if (m_forcerune || m_forcetime) {
595 std::string newmeta=m_forcerune ?
"run-lumi" :
"time";
596 std::string::size_type
p1,
p2;
599 if (
p1!=std::string::npos &&
p2!=std::string::npos) {
601 std::cout <<
"Forced destination folder to " << newmeta <<
" : "
605 "ERROR: Could not parse metadata string to force timestamp type"
610 if (m_forcepay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
611 dpaymode=cool::PayloadMode::SEPARATEPAYLOAD;
612 if (m_forcenopay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
613 dpaymode=cool::PayloadMode::INLINEPAYLOAD;
617 destflc=m_destCoraPtr->createFolder(destfolder,
618 sourceflc->coralTableName(),
619 sourceflc->fkSpecification(),
620 sourceflc->payloadSpecification(),
621 sourceflc->coralFKey(),
622 sourceflc->coralPKey(),
624 std::cout <<
"Created CoraCool folder" << std::endl;
625 destfl=m_destDbPtr->getFolder(destfolder);
627 destfl=m_destDbPtr->createFolder(destfolder,
628 cool::FolderSpecification(dvermode,
629 sourcefl->payloadSpecification(),dpaymode),
633 catch (cool::Exception&
e ) {
634 std::cout <<
"Create folder failed - aborting" << std::endl;
641 if (iscora) destflc=m_destCoraPtr->getFolder(destfolder);
642 destfl=m_destDbPtr->getFolder(destfolder);
644 catch (cool::Exception&
e) {
645 std::cout <<
"Could not get destination folder: " << destfolder
650 const cool::IRecordSpecification& sourcespec=
651 sourcefl->payloadSpecification();
652 const cool::IRecordSpecification& destspec=
653 destfl->payloadSpecification();
654 if (!(sourcespec==destspec)) {
659 "WARNING Source and destination folder specifications differ" <<
661 for (
unsigned int i=0;
i<sourcespec.size();++
i) {
662 const std::string& sname=sourcespec[
i].name();
663 if (!destspec.exists(sname)) {
664 std::cout <<
"ERROR: Field " << sname <<
" absent from destination"
674 "ERROR Source and destination folder specifications differ"
680 const std::string& destdesc=destfl->description();
681 if (sourcedesc!=destdesc) {
682 std::cout <<
"WARNING: Source and destination folder descriptions (meta-data) differ" << std::endl;
683 std::cout <<
"Source folder: " << sourcedesc << std::endl;
684 std::cout <<
"Destn folder: " << destdesc << std::endl;
688 destfl->folderSpecification().payloadMode();
689 if (spaymode!=dpaymode) {
690 std::cout <<
"WARNING: Source (" << spaymode <<
") and destination (" <<
691 dpaymode <<
" folder payload modes differ" << std::endl;
696 if (!checkChannels(
folder,sourcefl,destfl,created))
return 23;
700 if (m_nodata)
return 0;
702 std::cout <<
"Start to process folder: " <<
folder;
704 bool timestamp=(sourcefl->description().find(
"<timeStamp>time")!=
710 std::cout <<
" (timestamp)" << std::endl;
714 std::cout <<
" (run/lumi)" << std::endl;
716 if (m_alliov || m_truncate)
717 std::cout <<
"Output IOVs will be modified" << std::endl;
719 std::cout <<
"IOVs extending outside selection range will be skipped"
722 bool checkrefs=
false;
723 const std::string name0=sourcefl->payloadSpecification()[0].name();
724 if (m_checkrefs && (name0==
"PoolRef" || name0==
"fileGUID")) {
726 std::cout <<
"Check POOL references in folder " <<
folder << std::endl;
730 if (m_checkrefs && m_nocopy && !checkrefs)
return 0;
734 (m_skiprep && (sourcedesc.find(
"<fullrep/>")!=std::string::npos));
736 "All tags in folder will be copied due to <fullrep/> metadata" << std::endl;
739 std::vector<std::string>
tags;
740 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
741 std::cout <<
"Single version folder" << std::endl;
742 tags.emplace_back(
"HEAD");
744 std::cout <<
"Multi version folder: consider tags [ ";
746 const std::vector<std::string> foldertags=sourcefl->listTags();
747 for (std::vector<std::string>::const_iterator itag=foldertags.begin();
748 itag!=foldertags.end();++itag) {
754 for (std::vector<std::string>::const_iterator imtag=m_magic.begin();
755 imtag!=m_magic.end();++imtag) {
756 if (itag->find(*imtag)!=std::string::npos) copyit=
true;
759 tags.push_back(*itag);
760 std::cout << *itag <<
" ";
765 if (
tags.empty() || m_hitag) {
766 for (std::vector<std::string>::const_iterator itag=
taglist.begin();
769 std::string htag=sourcefl->resolveTag(*itag);
771 std::cout << *itag <<
"=" << htag <<
" ";
772 tags.push_back(htag);
782 std::cout <<
"]" << std::endl;
785 if (((
tags.empty() || m_includehead) && !m_excludehead) ||
787 tags.emplace_back(
"HEAD");
790 sourcefl->setPrefetchAll(
false);
791 for (std::vector<std::string>::const_iterator itag=
tags.begin();
792 itag!=
tags.end();++itag) {
794 if (!m_outtag.empty())
outtag=m_outtag;
798 retcode=verifyIOVs(
folder,sourcefl,sourceflc,destfl,destflc,
800 }
else if (m_nocopy) {
804 }
else if (m_analyse) {
807 retcode=copyIOVs(
folder,destfolder,sourcefl,sourceflc,destfl,destflc,
812 std::cout <<
"ERROR operation failed for folder " <<
folder <<
" tag " <<
828 for (
size_t i=0;
i<nChanRange;
i++) {
833 " to channel selection" << std::endl;
844 std::cout <<
"Adding channel range " <<
c1 <<
" to " <<
c2 <<
845 " to channel selection" << std::endl;
856 const std::string& chanstring) {
857 const char* cstr=chanstring.c_str();
860 return cool::ChannelId(strtoul(cstr,
nullptr,10));
862 cool::ChannelId
chan=0;
865 std::cout <<
"Channel name " << chanstring <<
" maps to channelID "
866 <<
chan << std::endl;
868 catch (cool::Exception&
e) {
869 std::cout <<
"ERROR: Channel name " << chanstring <<
870 " not defined in folder " << std::endl;
878 (
const std::string&
folder,
879 const std::string& destfolder,
882 const std::string& sourcetag,
const std::string& desttag,
883 const cool::ValidityKey
since,
const cool::ValidityKey
until,
884 bool timestamp,
bool checkrefs,
bool iscora,
887 std::cout <<
"Copying tag " << sourcetag <<
" of folder " <<
folder <<
888 " to destination tag " << desttag << std::endl;
892 int updatemode=getUpdateMode(destfl->description(),desttag);
893 std::vector<std::string> dtaglist=destfl->listTags();
896 if (updatemode==1 && m_destdb.find(
"oracle")!=std::string::npos &&
897 (!m_getonline || (!m_truncate && !m_alliov))) {
899 std::cout <<
"Folder is online (UPD1) mode but IGNORING PROTECTION"
902 if(
find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
903 std::cout <<
"Folder is online mode (UPD1) and tag already exist - -getonline and -truncate or -alliov options MUST be used" << std::endl;
907 std::cout <<
"Folder is online mode (UPD1), new tag will be created" << std::endl;
913 if (updatemode==4 && m_destdb.find(
"oracle")!=std::string::npos &&
914 (!m_getbulk || (!m_truncate && !m_alliov))) {
916 std::cout <<
"Folder is bulkreco (UPD4) mode but IGNORING PROTECTION"
919 if(
find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
920 std::cout <<
"Folder is bulkreco mode (UPD4) - -getbulk and -truncate or -alliov options MUST be used" << std::endl;
923 std::cout <<
"Folder is bulkreco mode (UPD4), new tag will be created" << std::endl;
932 if (m_checkdesttag) {
933 if (
find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()
934 || (desttag==
"HEAD" && !created)) {
935 std::cout <<
"Destination tag " << desttag <<
936 " already exists in folder " <<
folder <<
" - skip copy" << std::endl;
942 if ((m_applock || (updatemode==1 && m_getonline) || updatemode==2 ||
943 (updatemode==4 && m_getbulk)) &&
944 dvermode==cool::FolderVersioning::MULTI_VERSION) {
946 std::vector<std::string> dtaglist=destfl->listTags();
947 for (std::vector<std::string>::const_iterator itr=dtaglist.begin();
948 itr!=dtaglist.end();++itr) {
953 if (destfl->tagLockStatus(desttag)==cool::HvsTagLock::LOCKED) {
954 std::cout <<
"Unlocking destination tag " << desttag <<
955 " for append or UPDx access" << std::endl;
957 std::cout <<
"Appending according to SV folder rules" << std::endl;
960 "ERROR: Only allowed for UPD3 mode tags" << std::endl;
964 destfl->setTagLockStatus(desttag,cool::HvsTagLock::UNLOCKED);
976 if (dvermode==cool::FolderVersioning::MULTI_VERSION &&
977 desttag!=
"HEAD" && m_usertags)
localtag=desttag;
982 sourceflc->setPrefetchAll(
false);
984 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
985 sourceitr=sourceflc->browseObjects(
since,
until,m_chansel);
987 sourceitr=sourceflc->browseObjects(
since,
until,m_chansel,sourcetag);
990 std::map<int,int> insertkeymap;
992 while (sourceitr->hasNext()) {
996 (
obj->since()<m_srunemin ||
obj->until()>m_srunemax)) {
1001 if (!m_excludechans.empty()) {
1002 if (
find(m_excludechans.begin(),m_excludechans.end(),
1003 obj->channelId())!=m_excludechans.end()) {
1009 if (m_debug) std::cout <<
1010 "Setup new CoraCool storage buffer at object " << nobj << std::endl;
1011 destflc->setupStorageBuffer();
1013 cool::ValidityKey newsince,newuntil;
1017 if (newsince>=newuntil) {
1018 std::cout <<
"WARNING: Skipping IOV with since " << newsince <<
1019 ">= until" << newuntil << std::endl;
1026 std::map<int,int>::const_iterator ikey=insertkeymap.end();
1027 bool foundkey=
false;
1030 if (
obj->size()>0) {
1031 oldfk=sourceflc->getAttrKey(
1032 (*
obj->begin())[sourceflc->coralFKey()]);
1033 ikey=insertkeymap.find(oldfk);
1036 if (ikey==insertkeymap.end()) {
1038 destflc->storeObject(newsince,newuntil,
obj->begin(),
obj->end(),
1040 (!m_userupdatehead && !
localtag.empty()));
1041 if (foundkey) insertkeymap[oldfk]=newfk;
1043 destflc->referenceObject(newsince,newuntil,ikey->second,
1045 (!m_userupdatehead && !
localtag.empty()));
1050 if (nbuf==m_bufsize) {
1051 if (m_debug) std::cout <<
"Flush buffer after " << nobj <<
"," <<
1052 nbuf <<
" objects " << std::endl;
1054 destflc->flushStorageBuffer();
1055 destflc->setupStorageBuffer();
1061 if (nref>0) std::cout <<
"Reference-to-existing used for " << nref
1062 <<
" payload objects" << std::endl;
1064 if (m_debug) std::cout <<
"Final buffer flush at " << nobj <<
1065 "," << nbuf << std::endl;
1066 destflc->flushStorageBuffer();
1070 cool::IObjectIteratorPtr sourceitr;
1071 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1072 sourceitr=sourcefl->browseObjects(
since,
until,m_chansel);
1074 sourceitr=sourcefl->browseObjects(
since,
until,m_chansel,sourcetag);
1076 while (sourceitr->goToNext()) {
1077 const cool::IObject&
obj=sourceitr->currentRef();
1079 if (m_skipout && (
obj.since()<m_srunemin ||
obj.until()>m_srunemax)) {
1084 if (!m_excludechans.empty()) {
1085 if (
find(m_excludechans.begin(),m_excludechans.end(),
1086 obj.channelId())!=m_excludechans.end()) {
1092 if (m_debug) std::cout <<
"Setup new storage buffer at object " <<
1094 destfl->setupStorageBuffer();
1096 cool::ValidityKey newsince;
1097 cool::ValidityKey newuntil;
1101 if (newsince>=newuntil) {
1102 std::cout <<
"WARNING: Skipping IOV with since " << newsince <<
1103 ">= until" << newuntil << std::endl;
1107 if (checkrefs) checkRef(
obj.payload(),
folder,sourcetag);
1113 if (newuntil!=cool::ValidityKeyMax) {
1114 std::cout <<
"New IOVs must have until=cool::ValidityKeyMax" << std::endl;
1115 throw cool::Exception(
"Illegal insert over locked IOV",
1118 cool::IObjectIteratorPtr checkitr=destfl->browseObjects(newsince,
1119 newuntil,
obj.channelId(),desttag);
1120 while (checkitr->goToNext()) {
1121 const cool::IObject& checkobj=checkitr->currentRef();
1122 if (checkobj.since()>=newsince) {
1123 std::cout <<
"ERROR:: Attempt to insert SV overlapping IOV whilst appending to locked tag" << std::endl;
1124 throw cool::Exception(
"Illegal insert over locked IOV",
1127 if (checkobj.until()!=cool::ValidityKeyMax) {
1128 std::cout <<
"Existing IOVs must have until=cool::ValidityKeyMax" << std::endl;
1129 throw cool::Exception(
"Illegal insert over locked IOV",
1134 }
else if (m_applock) {
1136 const unsigned int nexist=
1137 destfl->countObjects(newsince,newuntil,
obj.channelId(),desttag);
1139 std::cout <<
"ERROR: Attempt to insert IOV over " << nexist <<
1140 " objects whilst appending to locked tag" << std::endl;
1141 throw cool::Exception(
"Illegal insert over locked IOV",
1144 }
else if (updatemode==2) {
1148 throw cool::Exception(
1149 "Attempt to insert into locked UPD2 tag with timestamp format",
1151 unsigned int run1=newsince >> 32;
1152 unsigned int run2=(newuntil-1) >> 32;
1153 for (
unsigned int irun=
run1;irun<=
run2;++irun) {
1154 if (!std::binary_search(m_runlist.begin(),m_runlist.end(),irun))
1156 std::cout <<
"Run " << irun <<
" from range [" <<
run1 <<
","
1157 <<
run2 <<
"] not found in runfile list" << std::endl;
1158 throw cool::Exception(
"Illegal insert over locked IOV",
1165 if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
1167 cool::IRecordIterator& pitr=
obj.payloadIterator();
1168 const cool::IRecordVectorPtr vptr=pitr.fetchAllAsVector();
1169 destfl->storeObject(newsince,newuntil,
1171 (!m_userupdatehead && !
localtag.empty()));
1176 destfl->storeObject(newsince,newuntil,
1178 (!m_userupdatehead && !
localtag.empty()));
1182 if (nbuf>=m_bufsize) {
1183 if (m_debug) std::cout <<
"Flush buffer after " << nobj <<
"," <<
1184 nbuf <<
" objects " << std::endl;
1185 destfl->flushStorageBuffer();
1193 if (m_debug) std::cout <<
"Final buffer flush at " << nobj <<
1194 "," << nbuf << std::endl;
1195 destfl->flushStorageBuffer();
1198 std::cout <<
"Folder copied with " << nobj <<
" objects" << std::endl;
1200 " objects were skipped extending outside IOV selection or excluded channel"
1202 if (nbad>0) std::cout << nbad <<
1203 " objects were skipped having zero or negative IOV lengths" << std::endl;
1206 catch (cool::Exception&
e) {
1207 std::cout <<
"Exception thrown from copy loop: " <<
e.what() <<
1210 std::cout <<
"Relocking destination tag " << desttag << std::endl;
1211 destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1217 std::cout <<
"Relocking destination tag " << desttag << std::endl;
1218 destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1221 if (dvermode==cool::FolderVersioning::MULTI_VERSION && desttag!=
"HEAD") {
1224 std::cout <<
"Tag folder with HEAD-style tagging for tag: " << desttag
1227 destfl->tagCurrentHead(desttag,sourcefl->tagDescription(sourcetag));
1229 catch (cool::Exception&
e) {
1230 std::cout <<
"Exception thrown in HEAD-style folder-tag: " <<
1231 e.what() << std::endl;
1236 if (m_copytaginfo) m_cooltagmap.insert(
1238 destfolder,sourcetag,desttag,m_taglabel)));
1240 if (!m_hitagmap.empty()) {
1241 for (HiTagMap::const_iterator imap=m_hitagmap.begin();
1242 imap!=m_hitagmap.end();++imap) {
1244 if (sourcefl->findTagRelation(imap->first)==sourcetag) {
1245 std::cout <<
"Create hierarchical tag between " << desttag <<
1246 " and " << imap->first <<
" in folder " << imap->second <<
1250 std::string etag=destfl->resolveTag(imap->first);
1251 if (etag==desttag) {
1252 std::cout <<
"This relation has already been created" << std::endl;
1254 std::cout <<
"ERROR: Tag in parent already related to " <<
1255 desttag << std::endl;
1258 catch (cool::Exception&
e ) {
1261 destfl->createTagRelation(imap->first,desttag);
1264 if (
find(m_hiparent.begin(),m_hiparent.end(),imap->second)==
1265 m_hiparent.end()) m_hiparent.push_back(imap->second);
1268 if (m_copytaginfo &&
1269 m_cooltagmap.find(imap->first)==m_cooltagmap.end())
1270 m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
1271 CoolTagInfo(m_sourceDbPtr,imap->second,imap->second,
1272 imap->first,imap->first)));
1274 catch (cool::Exception&
e) {
1275 std::cout <<
"Cool exception " <<
e.what() <<
1276 "thrown in hierarchical tag creation" << std::endl;
1282 catch (cool::Exception&
e) {
1291 const cool::ValidityKey&
until,
1292 const cool::ValidityKey& qsince,
1293 const cool::ValidityKey& quntil,
1294 cool::ValidityKey& newsince,
1295 cool::ValidityKey& newuntil,
1296 bool timestamp)
const {
1313 if (newsince<qsince) newsince=qsince;
1314 if (newuntil>quntil) newuntil=quntil;
1317 if (newuntil<newsince)
throw cool::Exception(
1318 "Attempt to insert IOV with -ve length",
"AtlCoolCopy::adjustIOVs");
1322 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
1323 const cool::ValidityKey
since,
const cool::ValidityKey
until,
1326 std::cout <<
"Reading tag " << sourcetag <<
" of folder " <<
folder <<
1327 " (no copy)" << std::endl;
1330 cool::IObjectIteratorPtr sourceitr;
1331 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1337 while (sourceitr->goToNext()) {
1338 const cool::IObject&
obj=sourceitr->currentRef();
1342 std::cout <<
"Folder scanned with " << nobj <<
" objects" << std::endl;
1345 catch (cool::Exception&
e) {
1346 std::cout <<
"Exception thrown from read loop: " <<
e.what() <<
1358 const std::string& sourcetag,
1359 const cool::ValidityKey
since,
const cool::ValidityKey
until,
1360 const bool checkrefs,
const bool iscora,
1363 std::cout <<
"Verifying tag " << sourcetag <<
" of folder " <<
folder <<
1365 destfl->setPrefetchAll(
false);
1368 cool::IObjectIteratorPtr sourceitr,destitr;
1370 std::string
tag=sourcetag;
1371 if (vermode==cool::FolderVersioning::SINGLE_VERSION || sourcetag==
"HEAD")
1375 sourceflc->setPrefetchAll(
false);
1384 std::cout <<
"Exception thrown from verify iterator setup: " <<
e.what() <<
1393 const std::string& cfkey=sourceflc->coralFKey();
1394 const std::string& cpkey=sourceflc->coralPKey();
1395 while (csourceitr->hasNext()) {
1404 if (cdestitr->hasNext()) {
1409 if (sobj->since()!=dobj->since() || sobj->until()!=dobj->until())
1412 if (sobj->channelId()!=dobj->channelId())
1415 if (sobj->size()!=dobj->size()) {
1416 std::cout <<
"ERROR CoraCool object " << nobj <<
1417 " sizes do not match: " << sobj->size() <<
" " << dobj->size()
1424 sitr!=sobj->end();++sitr,++ditr) {
1426 for (coral::AttributeList::const_iterator aitr=sitr->begin();
1427 aitr!=sitr->end();++aitr) {
1428 const std::string&
aname=aitr->specification().name();
1431 if (*aitr!=(*ditr)[
aname]) {
1432 std::cout <<
"ERROR Values of attriute " <<
aname <<
1433 " differ" << std::endl;
1437 catch (coral::AttributeListException&
e) {
1438 std::cout <<
"ERROR: CoraCool attribute " <<
aname <<
1439 " not found in destination!" << std::endl;
1446 std::cout <<
"ERROR database entries do not match: since: " <<
1447 sobj->since() <<
"," << dobj->since() <<
" until: " <<
1448 sobj->until() <<
"," << dobj->until() <<
" channel: " <<
1449 sobj->channelId() <<
"," << dobj->channelId() << std::endl;
1454 "ERROR destination folder no matching CoraCool iterator for object "
1455 << nobj << std::endl;
1461 while (sourceitr->goToNext()) {
1462 const cool::IObject& sobj=sourceitr->currentRef();
1471 if (destitr->goToNext()) {
1472 const cool::IObject& dobj=destitr->currentRef();
1476 if (sobj.since()!=dobj.since() || sobj.until()!=dobj.until())
1479 if (sobj.channelId()!=dobj.channelId())
1484 if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
1485 cool::IRecordIterator& spitr=sobj.payloadIterator();
1486 const cool::IRecordVectorPtr svptr=spitr.fetchAllAsVector();
1487 cool::IRecordIterator& dpitr=dobj.payloadIterator();
1488 const cool::IRecordVectorPtr dvptr=dpitr.fetchAllAsVector();
1489 if (svptr->size()!=dvptr->size()) {
1494 cool::IRecordVector::const_iterator svitr=svptr->begin();
1495 cool::IRecordVector::const_iterator svend=svptr->end();
1496 cool::IRecordVector::const_iterator dvitr=dvptr->begin();
1497 for (;svitr!=svend;++svitr,++dvitr) {
1502 std::cout <<
"ERROR vector payloads do not match (size " << svptr->size() <<
"," << dvptr->size() <<
")" << std::endl;
1507 if (!
equalRecord(sobj.payload(),dobj.payload())) iret=102;
1511 std::cout <<
"ERROR database entries do not match: since: " <<
1512 sobj.since() <<
"," << dobj.since() <<
" until: " <<
1513 sobj.until() <<
"," << dobj.until() <<
" channel: " <<
1514 sobj.channelId() <<
"," << dobj.channelId() << std::endl;
1515 std::cout <<
"Source payload:" << std::endl;
1516 sobj.payload().attributeList().toOutputStream(std::cout);
1517 std::cout << std::endl <<
"Destination payload:" << std::endl;
1518 dobj.payload().attributeList().toOutputStream(std::cout);
1519 std::cout << std::endl;
1524 "ERROR destination folder no matching iterator for object "
1525 << nobj << std::endl;
1532 catch (cool::Exception&
e) {
1533 std::cout <<
"Exception thrown from verify loop: " <<
e.what() <<
1543 std::cout <<
"Verification of folder " <<
folder <<
" tag " <<
1544 sourcetag <<
" OK (" << nobj <<
" objects)" << std::endl;
1551 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
1552 const cool::ValidityKey
since,
const cool::ValidityKey
until,
1553 const bool timestamp) {
1555 std::cout <<
"Write tag " << sourcetag <<
" of folder " <<
folder <<
1556 " to ROOT file" << std::endl;
1559 bool timestamp2=timestamp;
1565 TTree*
tree=
static_cast<TTree*
>(
gDirectory->FindObject(treename.c_str()));
1566 if (
tree==
nullptr) {
1567 std::cout <<
"Book TTree " << treename << std::endl;
1568 tree=
new TTree(treename.c_str(),
"COOL datadump");
1579 if (vermode==cool::FolderVersioning::MULTI_VERSION)
1582 const cool::IRecordSpecification&
spec=
1583 (sourcefl->folderSpecification()).payloadSpecification();
1584 unsigned int ncolumns=
spec.size();
1591 const cool::IFieldSpecification& fieldspec=
spec[
icol];
1595 tree->Branch(fieldspec.name().c_str(),
ptr,
1596 (fieldspec.name()+
"/"+rootID).c_str());
1598 <<
" of type " <<
spec[
icol].storageType().name() << std::endl;
1600 std::cout <<
"Attribute " <<
spec[
icol].name() <<
" of type " <<
1601 spec[
icol].storageType().name() <<
" will be skipped" << std::endl;
1606 std::cout <<
"TTree " << treename <<
" already exists" << std::endl;
1609 (sourcefl->folderSpecification()).payloadSpecification().size();
1611 std::cout <<
"ERROR in tree buffer definition: expect " << treename <<
1620 cool::IObjectIteratorPtr sourceitr;
1622 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1628 catch (cool::Exception&
e) {
1629 std::cout <<
"Exception thrown from ROOT copy iterator setup: " <<
1630 e.what() << std::endl;
1635 while (sourceitr->goToNext()) {
1636 const cool::IObject& sobj=sourceitr->currentRef();
1647 if (vermode==cool::FolderVersioning::MULTI_VERSION) {
1650 std::string sourcetag2=sourcetag.substr(0,255);
1656 const cool::IRecord& record=sobj.payload();
1663 catch (cool::Exception&
e) {
1670 std::cout <<
"Written " << nobj <<
" objects to ROOT TTree with " << nex
1671 <<
" nulls/exceptions" << std::endl;
1673 catch (cool::Exception&
e) {
1674 std::cout <<
"Exception thrown from ROOT file writing loop: " <<
1675 e.what() << std::endl;
1682 const std::string& toproot) {
1687 if (
p_rootfile->FindObject(toproot.c_str())==
nullptr) {
1689 std::cout <<
"Made top directory " << toproot << std::endl;
1694 std::string::size_type iofs1=1;
1695 std::string::size_type iofs2=1;
1696 std::string treename;
1697 while (iofs2!=std::string::npos) {
1698 iofs2=
folder.find(
'/',iofs1);
1699 if (iofs2==std::string::npos) {
1701 treename=
folder.substr(iofs1);
1706 std::cout <<
"Make directory " <<
dirname << std::endl;
1716 void*& sptr,
char& rootID)
const {
1717 const cool::StorageType& stype=
spec.storageType();
1720 if (stype==cool::StorageType::Bool) {
1722 sptr=
static_cast<void*
>(
new unsigned int(0));
1724 }
else if (stype==cool::StorageType::UChar) {
1725 sptr=
static_cast<void*
>(
new unsigned char(
' '));
1727 }
else if (stype==cool::StorageType::Int16) {
1728 sptr=
static_cast<void*
>(
new short(0));
1730 }
else if (stype==cool::StorageType::UInt16) {
1731 sptr=
static_cast<void*
>(
new unsigned short(0));
1733 }
else if (stype==cool::StorageType::Int32) {
1734 sptr=
static_cast<void*
>(
new int(0));
1736 }
else if (stype==cool::StorageType::UInt32) {
1737 sptr=
static_cast<void*
>(
new unsigned int(0));
1739 }
else if (stype==cool::StorageType::Int64) {
1740 sptr=
static_cast<void*
>(
new long long int(0));
1742 }
else if (stype==cool::StorageType::UInt63) {
1743 sptr=
static_cast<void*
>(
new unsigned long long int(0));
1745 }
else if (stype==cool::StorageType::Float) {
1746 sptr=
static_cast<void*
>(
new float(0.));
1748 }
else if (stype==cool::StorageType::Double) {
1749 sptr=
static_cast<void*
>(
new double(0.));
1751 }
else if (stype==cool::StorageType::String4k ||
1752 stype==cool::StorageType::String255) {
1753 sptr=
static_cast<void*
>(
new char[4100]);
1755 }
else if (stype==cool::StorageType::String64k && !
m_noclobroot) {
1756 sptr=
static_cast<void*
>(
new char[65536]);
1758 }
else if (stype==cool::StorageType::String16M && !
m_noclobroot) {
1759 sptr=
static_cast<void*
>(
new char[67108864]);
1761 }
else if (stype==cool::StorageType::Blob64k && !
m_noclobroot) {
1762 sptr=
static_cast<void*
>(
new char[65536]);
1765 std::cout <<
"rootAllocate: Unsupported storage type for attribute: " <<
1766 spec.name() << std::endl;
1768 return (sptr!=
nullptr);
1774 const cool::StorageType& stype=
field.storageType();
1775 if (stype==cool::StorageType::Bool) {
1777 *(
static_cast<int*
>(sptr))=0;
1779 *(
static_cast<int*
>(sptr))=(
field.data<
bool>() ? 1 : 0);
1781 }
else if (stype==cool::StorageType::UChar) {
1783 *(
static_cast<unsigned char*
>(sptr))=0;
1785 *(
static_cast<unsigned char*
>(sptr))=
field.data<
unsigned char>();
1787 }
else if (stype==cool::StorageType::Int16) {
1789 *(
static_cast<short*
>(sptr))=0;
1791 *(
static_cast<short*
>(sptr))=
field.data<
short>();
1793 }
else if (stype==cool::StorageType::UInt16) {
1795 *(
static_cast<unsigned short*
>(sptr))=0;
1797 *(
static_cast<unsigned short*
>(sptr))=
field.data<
unsigned short>();
1799 }
else if (stype==cool::StorageType::Int32) {
1801 *(
static_cast<int*
>(sptr))=0;
1803 *(
static_cast<int*
>(sptr))=
field.data<
int>();
1805 }
else if (stype==cool::StorageType::UInt32) {
1807 *(
static_cast<unsigned int*
>(sptr))=0;
1809 *(
static_cast<unsigned int*
>(sptr))=
field.data<
unsigned int>();
1811 }
else if (stype==cool::StorageType::Int64) {
1813 *(
static_cast<long long*
>(sptr))=0;
1815 *(
static_cast<long long*
>(sptr))=
field.data<
long long>();
1817 }
else if (stype==cool::StorageType::UInt63) {
1819 *(
static_cast<unsigned long long*
>(sptr))=0;
1821 *(
static_cast<unsigned long long*
>(sptr))=
field.data<
unsigned long long>();
1823 }
else if (stype==cool::StorageType::Float) {
1825 *(
static_cast<float*
>(sptr))=0.;
1827 *(
static_cast<float*
>(sptr))=
field.data<
float>();
1829 }
else if (stype==cool::StorageType::Double) {
1831 *(
static_cast<double*
>(sptr))=0.;
1833 *(
static_cast<double*
>(sptr))=
field.data<
double>();
1835 }
else if (stype==cool::StorageType::String255 ||
1836 stype==cool::StorageType::String4k ||
1837 stype==cool::StorageType::String64k ||
1838 stype==cool::StorageType::String16M) {
1840 strcpy(
static_cast<char*
>(sptr),
"NULL");
1842 strcpy(
static_cast<char*
>(sptr),
field.data<std::string>().c_str());
1844 }
else if (stype==cool::StorageType::Blob64k && !
m_noclobroot) {
1846 strcpy(
static_cast<char*
>(sptr),
"NULL");
1849 std::string blobStr((
char*)
blob.startingAddress(),
blob.size());
1850 strcpy(
static_cast<char*
>(sptr), blobStr.c_str());
1853 std::cout <<
"ERROR: Unknown storage type in rootWrite!" << std::endl;
1858 const cool::IFolderPtr& sourcefl,
const std::string& sourcetag,
1859 const cool::ValidityKey
since,
const cool::ValidityKey
until,
1860 const bool timestamp) {
1862 std::cout <<
"Analyse tag " << sourcetag <<
" of folder " <<
folder <<
1866 std::cout <<
"Analyse time structures with tolerance of " <<
1867 m_anadelt/1.E9 <<
" seconds " << std::endl;
1872 std::cout <<
"Make directory " <<
dirname << std::endl;
1877 std::vector<cool::ChannelId>
channels=sourcefl->listChannels();
1880 long long* lastiov=
new long long[
nchan];
1881 long long* iovtotlen=
new long long[
nchan];
1882 long long* iovtotgap=
new long long[
nchan];
1883 int* iovn=
new int[
nchan];
1884 for (
unsigned int i=0;
i<
nchan;++
i) {
1890 cool::ValidityKey globsince=cool::ValidityKeyMax;
1891 cool::ValidityKey globuntil=cool::ValidityKeyMin;
1897 "IOVs per channel-name (copy of IOVSperChannel)",
1904 TH1F* h_anadelt=
nullptr;
1905 TH1F* h_chandelt=
nullptr;
1907 h_anadelt=
bookOrFindTH1F(
"IOVAlignLogDelT",
"IOV alignment log deltaT",
1911 h_chandelt=
bookOrFindTH1F(
"AlignedChanPerIOV",
"Aligned channels per IOV",
1914 std::cout <<
"Booked histograms for folder with " <<
nchan <<
" channels"
1917 using IOVTimeMap = std::map<cool::ValidityKey, int>;
1918 IOVTimeMap iov_time_map;
1923 cool::IObjectIteratorPtr sourceitr;
1925 if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1931 catch (cool::Exception&
e) {
1932 std::cout <<
"Exception thrown from analysis copy iterator setup: " <<
1933 e.what() << std::endl;
1937 std::map<cool::ChannelId,std::string>
1938 chanmap=sourcefl->listChannelsWithNames();
1941 while (sourceitr->goToNext()) {
1942 const cool::IObject& sobj=sourceitr->currentRef();
1944 cool::ChannelId ichanid=sobj.channelId();
1945 const std::string& cname=chanmap[ichanid];
1949 if (chanitr.first!=chanitr.second) {
1952 if (
ichan && !h_iovname->GetBinContent(
bin)) {
1953 h_iovname->GetXaxis()->SetBinLabel(
bin,cname.c_str());
1955 h_iovname->Fill(
ichan);
1957 cool::ValidityKey since2=sobj.since();
1958 cool::ValidityKey until2=sobj.until();
1959 if (since2<globsince) globsince=since2;
1961 if (until2!=cool::ValidityKeyMax) {
1962 if (until2>globuntil) globuntil=until2;
1963 long long len=until2-since2;
1964 h_iovlength->Fill(log10(len));
1966 iovtotlen[
ichan]+=len;
1968 if (lastiov[
ichan]<
static_cast<long long>(since2) &&
1969 lastiov[
ichan]>=0) {
1971 long long gap=since2-lastiov[
ichan];
1973 h_iovgap->Fill(log10(
gap));
1974 std::cout <<
"Gap of " <<
gap << std::endl;
1976 if (until2!=cool::ValidityKeyMax) {
1977 lastiov[
ichan]=until2;
1979 lastiov[
ichan]=since2;
1989 if (hiitr!=iov_time_map.end()) {
1991 del1=hiitr->first-since2;
1993 if (lowitr!=iov_time_map.begin()) {
1996 del2=since2-lowitr->first;
2001 if (del1<=m_anadelt && del1>-1) {
2006 if (del2<=m_anadelt && del2>-1 && (del2<del || del==-1)) {
2014 h_anadelt->Fill(log10(del));
2016 h_anadelt->Fill(0.);
2019 iov_time_map[since2]=0;
2024 std::cout <<
"ERROR :Channel " << ichanid <<
2025 " not found in channel list - ignored" << std::endl;
2029 std::cout <<
"Finished analysis with " << nobj <<
" objects" << std::endl;
2031 std::cout <<
"IOV timestamp range: " << globsince <<
" " <<
2032 timeString(globsince) <<
"to " << globuntil <<
" " <<
2035 std::cout <<
"IOV run/LB range [" << (globsince >> 32) <<
"," <<
2036 (globsince & 0
xFFFFFFFF) <<
"] to [" << (globuntil >> 32) <<
"," <<
2037 (globuntil & 0
xFFFFFFFF) <<
"]" << std::endl;
2043 "Mean IOV length per channel",
nchan,-0.5,
nchan-0.5);
2044 for (
unsigned int i=0;
i<
nchan;++
i) {
2048 occ=1.-
float(iovtotgap[
i])/
float(iovtotlen[
i]+iovtotgap[
i]);
2049 avlen=
float(iovtotlen[
i])/iovn[
i];
2051 h_iovoccchan->Fill(
i,occ);
2052 h_iovlenchan->Fill(
i,avlen);
2055 std::cout <<
"Alignment analysis: " << iov_time_map.size()
2056 <<
" seperate IOV starts (tolerance " <<
m_anadelt/1.E9 <<
2057 " seconds)" << std::endl;
2059 for (IOVTimeMap::const_iterator itr=iov_time_map.begin();
2060 itr!=iov_time_map.end();++itr) {
2061 h_chandelt->Fill(itr->second);
2065 catch (cool::Exception&
e) {
2066 std::cout <<
"Exception thrown from folder analysis reading loop: " <<
2067 e.what() << std::endl;
2072 delete [] iovtotlen;
2073 delete [] iovtotgap;
2074 delete[] iovn; iovn=
nullptr;
2079 const std::string& htitle,
2080 const int nchan,
const float xlow,
const float xhigh) {
2085 return new TH1F(hID.c_str(),htitle.c_str(),
nchan,xlow,xhigh);
2087 return static_cast<TH1F*
>(
obj);
2096 for (std::vector<std::string>::const_iterator ifolder=
2097 m_folderlist.begin();ifolder!=m_folderlist.end();++ifolder) {
2101 if (!m_hiparent.empty()) {
2103 int code=tagParents();
2106 if (m_copytaginfo) {
2108 int code=writeTagInfo();
2113 if (m_poolcat.empty()) {
2114 code=listPoolRefs();
2115 if (!m_checkoutputfile.empty()) filePoolRefs();
2117 code=resolvePoolRefs();
2122 p_rootfile->Write();
2132 std::vector<std::string>&
folders) {
2138 std::string_view par0=
argv[
ic];
2140 if (par0.compare(0,2,
"--")==0) par0=par0.substr(1);
2141 if ((par0==
"-f" || par0==
"-folder") &&
ir>1) {
2144 }
else if ((par0==
"-e" || par0==
"-exclude") &&
ir>1) {
2147 }
else if ((par0==
"-t" || par0==
"-tag") &&
ir>1) {
2150 }
else if ((par0==
"-mt" || par0==
"-magic") &&
ir>1) {
2153 }
else if ((par0==
"-of" || par0==
"-outfolder") &&
ir>1) {
2156 }
else if ((par0==
"-ot" || par0==
"-outtag") &&
ir>1) {
2159 }
else if ((par0==
"-bs" || par0==
"-buffersize") &&
ir>1) {
2162 }
else if ((par0==
"-sl" || par0==
"-seal") &&
ir>1) {
2165 }
else if ((par0==
"-rls" || par0==
"-runlumisince") &&
ir>2) {
2168 }
else if ((par0==
"-rlu" || par0==
"-runlumiuntil") &&
ir>2) {
2171 }
else if ((par0==
"-nrls" || par0==
"-newrunlumisince") &&
ir>2) {
2174 }
else if ((par0==
"-nrlu" || par0==
"-newrunlumiuntil") &&
ir>2) {
2177 }
else if ((par0==
"-srls" || par0==
"-skiprunlumisince") &&
ir>2) {
2181 }
else if ((par0==
"-srlu" || par0==
"-skiprunlumiuntil") &&
ir>2) {
2185 }
else if (par0==
"-ro" || par0==
"-root") {
2187 }
else if (par0==
"-zn" || par0==
"-zeronull") {
2189 }
else if (par0==
"-ana" || par0==
"-analyse") {
2192 m_anadelt=
static_cast<long long>(ttime);
2194 }
else if ((par0==
"-rs" || par0==
"-runsince") &&
ir>1) {
2197 }
else if ((par0==
"-ru" || par0==
"-rununtil") &&
ir>1) {
2200 }
else if ((par0==
"-r" || par0==
"-run") &&
ir>1) {
2204 }
else if ((par0==
"-ts" || par0==
"-timesince") &&
ir>1) {
2207 }
else if ((par0==
"-tu" || par0==
"-timeuntil") &&
ir>1) {
2210 }
else if ((par0==
"-nts" || par0==
"-newtimesince") &&
ir>1) {
2213 }
else if ((par0==
"-ntu" || par0==
"-newtimeuntil") &&
ir>1) {
2216 }
else if (par0==
"-c" || par0==
"-create") {
2218 }
else if ((par0==
"-ch" || par0==
"-channel") &&
ir>1) {
2221 }
else if ((par0==
"-ch1" || par0==
"-channel1") &&
ir>1) {
2224 }
else if ((par0==
"-ch2" || par0==
"-channel2") &&
ir>1) {
2227 }
else if (par0==
"-chd" || par0==
"-channeldesc") {
2230 }
else if (par0==
"-forcerecreate") {
2232 }
else if (par0==
"-d" || par0==
"-debug") {
2234 }
else if (par0==
"-a" || par0==
"-alliov") {
2236 }
else if (par0==
"-ih" || par0==
"-includehead") {
2238 }
else if (par0==
"-eh" || par0==
"-excludehead") {
2240 }
else if (par0==
"-ht" || par0==
"-headtag") {
2242 }
else if (par0==
"-uht" || par0==
"-userheadtag") {
2244 }
else if (par0==
"-v" || par0==
"-verify") {
2246 }
else if (par0==
"-nc" || par0==
"-nocopy") {
2248 }
else if (par0==
"-noc" || par0==
"-nocoracool") {
2250 }
else if (par0==
"-nch" || par0==
"-nochannel") {
2252 }
else if (par0==
"-ncr" || par0==
"-noclobroot") {
2254 }
else if (par0==
"-nd" || par0==
"-nodata") {
2256 }
else if (par0==
"-nh" || par0==
"-nohitag") {
2258 }
else if (par0==
"-hi" || par0==
"-hitag") {
2260 }
else if ((par0==
"-ec" || par0==
"-excludechannel") &&
ir>1) {
2263 }
else if (par0==
"-fs" || par0==
"-forcesingle") {
2265 }
else if (par0==
"-fm" || par0==
"-forcemulti") {
2267 }
else if (par0==
"-frl" || par0==
"-forcerunlumi") {
2269 }
else if (par0==
"-ftm" || par0==
"-forcetime") {
2271 }
else if (par0==
"-fp" || par0==
"-forcepayload") {
2273 }
else if (par0==
"-fnp" || par0==
"-forcenopayload") {
2275 }
else if (par0==
"-cr" || par0==
"-checkrefs") {
2277 }
else if (par0==
"-lp" || par0==
"-listpfn") {
2280 }
else if (par0==
"-cf" || par0==
"-checkfiles") {
2283 }
else if (par0==
"-pa" || par0==
"-poolall") {
2285 }
else if ((par0==
"-co" || par0==
"-checkoutput") &&
ir>1) {
2288 }
else if ((par0==
"-pc" || par0==
"-poolcat") &&
ir>1) {
2291 }
else if ((par0==
"-mc" || par0==
"-mergecat") &&
ir>1) {
2294 }
else if ((par0==
"-ds" || par0==
"-dataset") &&
ir>1) {
2297 }
else if (par0==
"-us" || par0==
"-updatesource") {
2299 }
else if (par0==
"-cd" || par0==
"-checkdest") {
2301 }
else if (par0==
"-tr" || par0==
"-truncate") {
2303 }
else if (par0==
"-al" || par0==
"-appendlocked") {
2305 }
else if (par0==
"-alsv" || par0==
"-appendlockedsv") {
2308 }
else if (par0==
"-rdo" || par0==
"-readoracle") {
2310 }
else if (par0==
"-skiprep" || par0==
"-sr") {
2312 }
else if (par0==
"-go" || par0==
"-getonline") {
2314 }
else if (par0==
"-onr" || par0==
"-onlinerun") {
2316 }
else if (par0==
"-gb" || par0==
"-getbulk") {
2318 }
else if (par0==
"-gt" || par0==
"-gettime") {
2320 }
else if ((par0==
"-tdb" || par0==
"-timedb") &&
ir>1) {
2323 }
else if (par0==
"-ignoremode" &&
ir>1) {
2324 std::cout <<
"Ignoremode password is " <<
argv[
ic+1] <<
":end" << std::endl;
2325 if (strcmp(
argv[
ic+1],
"BackDoor")==0) {
2328 std::cout <<
"ERROR: Incorrect password for -ignoremode" << std::endl;
2332 }
else if (par0==
"-is" || par0==
"-ignorespec") {
2334 }
else if (par0==
"-pt" || par0==
"-prunetags") {
2337 }
else if (par0==
"-lo" || par0==
"-lockedonly") {
2341 }
else if (par0==
"-cti" || par0==
"-copytaginfo") {
2343 }
else if (par0==
"-ctl" || par0==
"-copytaglock") {
2346 }
else if ((par0==
"-tl" || par0==
"-taglabel") &&
ir>1) {
2350 }
else if ((par0==
"-ag" || par0==
"-addguid") &&
ir>1) {
2353 }
else if ((par0==
"-alf" || par0==
"-addlfn") &&
ir>1) {
2356 }
else if ((par0==
"-pf" || par0==
"-parfile") &&
ir>1) {
2359 }
else if ((par0==
"-rf" || par0==
"-runfile") &&
ir>1) {
2362 }
else if ((par0==
"-ws" || par0==
"-runinfohost") &&
ir>1) {
2365 }
else if (par0==
"-h" || par0==
"-help") {
2369 std::cout <<
"Parameter error for argument: " << par0 << std::endl;
2380 std::vector<std::string>
folders;
2385 for (std::vector<std::string>::const_iterator
ifile=
m_parfile.begin();
2387 std::cout <<
"Reading parameters from file " << *
ifile << std::endl;
2388 FILE* p_inp=fopen(
ifile->c_str(),
"r");
2389 if (p_inp==
nullptr) {
2390 std::cout <<
"File not found" << std::endl;
2393 char* p_buf=
new char[999];
2394 while (!feof(p_inp)) {
2395 char* p_line=fgets(p_buf,999,p_inp);
2396 if (p_line!=
nullptr) {
2398 const char* fargv[99];
2400 char* p_start=
nullptr;
2401 while (*p_line!=
'\0') {
2402 if (*p_line==
' ' || *p_line==
'\n') {
2404 if (p_start!=
nullptr) {
2407 fargv[fargc]=p_start;
2413 if (p_start==
nullptr) p_start=p_line;
2425 std::cout <<
"Close file" << std::endl;
2432 std::cout <<
"Problem opening connections" << std::endl;
2446 for (std::vector<std::string>::const_iterator ifold=
folders.begin();
2466 std::cout <<
"Source run/LB range [" << (
m_runemin >> 32) <<
"," <<
2470 std::cout <<
"Source timestamp range " <<
m_timemin <<
" " <<
2474 std::cout <<
"Change o/p run/LB range [" << (
m_newrunemin >> 32) <<
"," <<
2478 std::cout <<
"Change o/p timestamp range " <<
m_newtimemin <<
" " <<
2482 std::cout <<
"Skip IOVs extending outside run/LB range [" <<
2488 for (std::vector<cool::ChannelId>::const_iterator itr=
2490 std::cout <<
"Channel " << *itr <<
" will be excluded" << std::endl;
2492 if (
m_hitag) std::cout <<
"All hierarchical tags connecting to referenced tags will be copied" << std::endl;
2493 if (
m_nohitag) std::cout <<
"Hierarchical tag relations will not be copied"
2501 const char* cptr=
input;
2502 while (*cptr!=
'\0') {
2503 if (!isdigit(*cptr)) { isnum=
false;
break;}
2510 const cool::IRecord& rhs) {
2513 if (lhs.size()!=rhs.size())
return false;
2514 for (
size_t i=0;
i<lhs.size();++
i) {
2516 if (lhs[
i].specification().storageType()!=
2517 rhs[
i].specification().storageType())
2520 if (lhs[
i]!=rhs[
i]) {
2523 const cool::StorageType& stype=lhs[
i].specification().storageType();
2524 if (stype==cool::StorageType::String255 ||
2525 stype==cool::StorageType::String4k ||
2526 stype==cool::StorageType::String64k ||
2527 stype==cool::StorageType::String16M) {
2529 if (lhs[
i].data<std::string>()!=rhs[
i].data<std::string>())
2547 return static_cast<long long>(atol(
input))*
2548 static_cast<long long>(1.E9);
2550 struct tm mytm{},mytm2{};
2558 cool::ValidityKey itime=
static_cast<cool::ValidityKey
>(mktime(&mytm));
2559 strptime(
"1970-01-02:00:00:00",
"%Y-%m-%d:%T",&mytm2);
2560 cool::ValidityKey caltime=
static_cast<cool::ValidityKey
>(mktime(&mytm2));
2561 itime+=24*60*60-caltime;
2562 return itime*
static_cast<cool::ValidityKey
>(1.E9);
2565 "ERROR in format of time value, use e.g. 2007-05-25:14:01:00" <<
2573 if (iovtime==cool::ValidityKeyMin) {
2574 return "ValidityKeyMin ";
2575 }
else if (iovtime==cool::ValidityKeyMax) {
2576 return "ValidityKeyMax ";
2578 time_t
time=
static_cast<time_t
>(iovtime/1E9);
2581 return "UTC "+std::string(asctime_r(gmtime_r(&
time, &
result), buf));
2588 cool::ValidityKey
val;
2590 val=((1LL << 31)-1) << 32;
2592 val=
static_cast<long long>(atol(
input1)) << 32;
2594 if (input2[0]==
'M' || input2[0]==
'm') {
2599 if (
val>cool::ValidityKeyMax)
val=cool::ValidityKeyMax;
2617 std::string sorfolder=
"/TDAQ/RunCtrl/SOR_Params";
2618 std::string eorfolder=
"/TDAQ/RunCtrl/EOR_Params";
2619 if (
m_timedb.find (
"CONDBR2")!=std::string::npos) {
2620 sorfolder=
"/TDAQ/RunCtrl/SOR";
2621 eorfolder=
"/TDAQ/RunCtrl/EOR";
2623 std::cout <<
"Extracting times for run range [" << (
m_runemin >> 32) <<
2625 " folder " << sorfolder << std::endl;
2627 std::cout <<
"ERROR: Run range not correctly specified" << std::endl;
2631 cool::IDatabasePtr tdaqdb;
2634 std::cout <<
"Opened database connection" << std::endl;
2637 std::cout <<
"COOL exception caught: " <<
e.what() << std::endl;
2642 cool::IFolderPtr
folder=tdaqdb->getFolder(sorfolder);
2647 while (itr->goToNext()) {
2648 const cool::IRecord&
payload=itr->currentRef().payload();
2654 std::cout <<
"ERROR: Found " << nobj <<
" SOR records" << std::endl;
2659 std::cout <<
"Exception accessing SOR information: " <<
e.what() <<
2665 cool::IFolderPtr
folder=tdaqdb->getFolder(eorfolder);
2670 while (itr->goToNext()) {
2671 const cool::IRecord&
payload=itr->currentRef().payload();
2677 std::cout <<
"ERROR: Found " << nobj <<
" EOR records" << std::endl;
2682 std::cout <<
"Exception accessing SOR information: " <<
e.what() <<
2686 tdaqdb->closeDatabase();
2687 std::cout <<
"Timestamp range set to " <<
m_timemin <<
" " <<
2697 std::cout <<
"Extracting current run-number from ATLAS_RUN_NUMBER @ ATONR_ADG ... " <<
2699 curl_global_init(CURL_GLOBAL_ALL);
2700 using uniqueCurl_t = std::unique_ptr<CURL,decltype(&curl_easy_cleanup)>;
2701 uniqueCurl_t curl(curl_easy_init(), curl_easy_cleanup);
2702 CURLcode
res = CURLE_OK;
2706 res = curl_easy_setopt(curl.get(), CURLOPT_URL,
url.c_str());
2707 if (
res != CURLE_OK) {
2708 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2712 res = curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1
L);
2713 if (
res != CURLE_OK) {
2714 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2722 if (
res != CURLE_OK) {
2723 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2726 res = curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &
response);
2727 if (
res != CURLE_OK) {
2728 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2733 res = curl_easy_perform(curl.get());
2734 if (
res != CURLE_OK) {
2735 std::cerr <<
"Failed to perform request: " << curl_easy_strerror(
res) <<
":" <<
url.c_str() << std::endl;
2739 std::cout <<
"Response: " << std::endl;
2740 std::cout <<
response << std::endl;
2746 int runNumber = jsonData[
"resources"][0][
"runnumber"];
2748 std::cout <<
"Next run started will be " << nextrun << std::endl;
2750 std::cout <<
"Epoch time extracted " <<
rtime << std::endl;
2757 m_runemin=(
static_cast<long long>(nextrun)) << 32;
2761 }
catch (nlohmann::json::parse_error&
e) {
2762 std::cerr <<
"Failed to parse JSON response: " <<
e.what() << std::endl;
2764 }
catch (nlohmann::json::type_error&
e) {
2765 std::cerr <<
"Failed to extract data from JSON response: " <<
e.what() << std::endl;
2768 std::cerr <<
"Failed to extract run and timestamp from JSON response: " <<
e.what() << std::endl;
2774 std::cerr <<
"Failed to initialize libcurl." << std::endl;
2778 curl_global_cleanup();
2784 std::cout <<
"ERROR: -getonline and -getbulk cannot be used simultaneously"
2788 std::cout <<
"Call getbulk using URL" << std::endl;
2790 curl_global_init(CURL_GLOBAL_ALL);
2791 using uniqueCurl_t = std::unique_ptr<CURL,decltype(&curl_easy_cleanup)>;
2792 uniqueCurl_t curl(curl_easy_init(), curl_easy_cleanup);
2794 CURLcode
res = CURLE_OK;
2799 res = curl_easy_setopt(curl.get(), CURLOPT_URL,
url.c_str());
2800 if (
res != CURLE_OK) {
2801 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2805 res = curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1
L);
2806 if (
res != CURLE_OK) {
2807 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2815 if (
res != CURLE_OK) {
2816 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2819 res = curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &
response);
2820 if (
res != CURLE_OK) {
2821 std::cerr <<
"Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2825 res = curl_easy_perform(curl.get());
2826 if (
res != CURLE_OK) {
2827 std::cerr <<
"Failed to perform request: " << curl_easy_strerror(
res) <<
":" <<
url.c_str() << std::endl;
2831 std::cout <<
"Response: " << std::endl;
2832 std::cout <<
response << std::endl;
2837 int nextrun = jsonData[
"run"];
2838 long long rtime = jsonData[
"timestamp"];
2839 std::cout <<
"Next run started will be " << nextrun << std::endl;
2840 std::cout <<
"Epoch time extracted " <<
rtime << std::endl;
2847 m_runemin=(
static_cast<long long>(nextrun)) << 32;
2851 }
catch (nlohmann::json::parse_error&
e) {
2852 std::cerr <<
"Failed to parse JSON response: " <<
e.what() << std::endl;
2854 }
catch (nlohmann::json::type_error&
e) {
2855 std::cerr <<
"Failed to extract data from JSON response: " <<
e.what() << std::endl;
2858 std::cerr <<
"Failed to extract run and timestamp from JSON response: " <<
e.what() << std::endl;
2864 std::cerr <<
"Failed to initialize libcurl." << std::endl;
2868 curl_global_cleanup();
2875 for (std::vector<std::string>::const_iterator itr=
m_runfile.begin();
2877 std::cout <<
"Reading allowed run numbers from file " << *itr << std::endl;
2878 FILE* p_inp=fopen(itr->c_str(),
"r");
2879 if (p_inp==
nullptr) {
2880 std::cout <<
"File not found" << std::endl;
2883 char* p_buf=
new char[999];
2884 while (!feof(p_inp)) {
2885 char* p_line=fgets(p_buf,999,p_inp);
2886 if (p_line!=
nullptr) {
2887 unsigned int run=
atoi(p_line);
2895 std::cout <<
"Read list of " <<
m_runlist.size() <<
" runs from " <<
2896 m_runfile.size() <<
" input runlist files" << std::endl;
2897 for (std::vector<unsigned int>::const_iterator itr=
m_runlist.begin();
2898 itr!=
m_runlist.end();++itr) std::cout <<
2899 "Update allowed for run " << *itr << std::endl;
2904 std::string_view
tag) {
2907 std::string_view modestr=
"";
2909 std::string::size_type iofs1=
desc.find(
"<updateMode>");
2910 std::string::size_type iofs2=
desc.find(
"</updateMode>");
2911 if (iofs1!=std::string::npos && iofs2!=std::string::npos && iofs2>iofs1)
2912 modestr=
desc.substr(iofs1+12,iofs2-iofs1-12);
2913 if (modestr==
"UPD1" ||
tag.find(
"UPD1")!=std::string::npos)
mode=1;
2914 if (modestr==
"UPD2" ||
tag.find(
"UPD2")!=std::string::npos)
mode=2;
2915 if (modestr==
"UPD3" ||
tag.find(
"UPD3")!=std::string::npos)
mode=3;
2916 if (modestr==
"UPD4" ||
tag.find(
"UPD4")!=std::string::npos)
mode=4;
2922 const cool::IFolderPtr& sourcefl,
const cool::IFolderPtr& destfl,
2928 const std::map<cool::ChannelId,std::string> chanmap=
2929 sourcefl->listChannelsWithNames();
2930 if (
m_debug) std::cout <<
"Checking channels table for folder " <<
folder
2931 <<
" with " << chanmap.size() <<
" channels" << std::endl;
2935 for (std::map<cool::ChannelId,std::string>::const_iterator
2936 itr=chanmap.begin();itr!=chanmap.end();++itr) {
2938 const cool::ChannelId
chan=itr->first;
2946 const std::string& sourcename=itr->second;
2948 std::string sourcedesc=
"";
2949 if (
m_chdesc) sourcedesc=sourcefl->channelDescription(
chan);
2952 const std::string & destname=destfl->channelName(
chan);
2953 std::string destdesc=
"";
2954 if (
m_chdesc) destdesc=destfl->channelDescription(
chan);
2956 if (sourcename!=destname && !sourcename.empty()) {
2959 std::cout <<
"ERROR: Channel " <<
chan <<
" names differ: " <<
2960 sourcename <<
" " << destname << std::endl;
2963 destfl->setChannelName(
chan,sourcename);
2964 std::cout <<
"Modify channel " <<
chan <<
" name from "
2965 << destname <<
" to " << sourcename << std::endl;
2968 if (sourcedesc!=destdesc && !sourcedesc.empty()) {
2971 std::cout <<
"ERROR: Channel " <<
chan <<
" descriptions differ: "
2972 << sourcedesc <<
" " << destdesc << std::endl;
2975 std::cout <<
"Modify channel " <<
chan <<
" description from "
2976 << sourcedesc <<
" to " << destdesc << std::endl;
2977 destfl->setChannelDescription(
chan,sourcedesc);
2984 if (
m_debug) std::cout <<
"Create new channel " <<
chan <<
" name "
2985 << sourcename <<
" description" << sourcedesc << std::endl;
2986 destfl->createChannel(
chan,sourcename,sourcedesc);
2990 if (ncreate>0) std::cout <<
"Created " << ncreate <<
" new channels for "
2992 if (nmodify>0) std::cout <<
"Modified " << nmodify <<
" channel info for "
2999 const std::string&
folder,
const std::string&
tag) {
3001 if (
name==
"PoolRef" ||
name==
"fileGUID") {
3002 std::string poolref;
3003 std::string addr=
payload[0].data<std::string>();
3004 if (
name==
"PoolRef") {
3005 std::string::size_type iofs1=addr.find(
"[DB=");
3006 std::string::size_type iofs2=addr.find(
']',iofs1);
3007 if (iofs1!=std::string::npos && iofs2!=std::string::npos && iofs2>iofs1) {
3008 poolref=addr.substr(iofs1+4,iofs2-iofs1-4);
3010 std::cout <<
"ERROR: Badly formed POOL object reference " <<
3036 std::cout <<
"Copying additional hierarchical tags of parent folders"
3044 cool::IFolderSetPtr sourcefl=
m_sourceDbPtr->getFolderSet(folderset);
3045 cool::IFolderSetPtr destfl=
m_destDbPtr->getFolderSet(folderset);
3046 std::cout <<
"Processing folderset " << folderset << std::endl;
3047 for (HiTagMap::const_iterator imap=
m_hitagmap.begin();
3051 std::string sourcetag=sourcefl->findTagRelation(imap->first);
3052 std::cout <<
"Create hierarchical tag between " << sourcetag <<
3053 " and " << imap->first <<
" in folder " << imap->second <<
3057 std::string etag=destfl->resolveTag(imap->first);
3058 if (etag==sourcetag) {
3059 std::cout <<
"This relation has already been created" << std::endl;
3061 std::cout <<
"ERROR: Tag in parent already related to " <<
3066 catch (cool::Exception&
e ) {
3069 destfl->createTagRelation(imap->first,sourcetag);
3075 m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
3077 imap->first,imap->first)));
3079 catch (cool::Exception&
e) {
3080 std::cout <<
"Cool exception " <<
e.what() <<
3081 "thrown in hierarchical tag creation" << std::endl;
3087 catch (cool::Exception&
e) { }
3090 std::cout <<
"ERROR Folderset " << folderset <<
" not found" <<
3098 std::cout <<
"Write tag description ";
3100 std::cout <<
"for " <<
m_cooltagmap.size() <<
" tags" << std::endl;
3101 for (CoolTagMap::const_iterator itr=
m_cooltagmap.begin();
3109 std::cout <<
"Total of " <<
m_poolrefs.size() <<
" POOL files referenced"
3111 for (PoolMap::const_iterator ipool=
m_poolrefs.begin();
3113 std::cout <<
"Ref " << ipool->first <<
" (" << ipool->second.count()
3114 <<
")" << std::endl;
3120 std::cout <<
"Total of " << m_poolrefs.size() <<
" POOL Files referenced"
3123 if (catalog==
nullptr)
return 110;
3127 pool_utility.startSession();
3130 if (!m_addguid.empty()) {
3131 for (std::vector<std::string>::const_iterator itr=m_addguid.begin();
3132 itr!=m_addguid.end();++itr) {
3134 std::cout <<
"Added POOL file GUID: " << *itr << std::endl;
3137 if (!m_addlfn.empty()) {
3138 for (std::vector<std::string>::const_iterator itr=m_addlfn.begin();
3139 itr!=m_addlfn.end();++itr) {
3142 std::cout <<
"Add POOL file GUID: " <<
guid <<
" from LFN " << *itr
3149 using LFNGVec = std::vector<std::pair<std::string, std::string> >;
3151 bool dscopy=!m_newdataset.empty();
3156 ipool!=m_poolrefs.end();++ipool) {
3160 if( !lfns.empty() ) {
3162 const std::string
lfn = lfns[0].first;
3163 ipool->second.setlfn(
lfn);
3164 std::cout <<
"LFN: " <<
lfn <<
" (" << ipool->second.count()
3165 <<
")" << std::endl;
3167 dsfound.push_back(std::pair<std::string,std::string>(
lfn,
guid));
3170 if (!m_listpfn) ipool->second.setErrorBit(0);
3172 if (m_listpfn || m_poolopen) {
3173 std::string pfn, tech;
3175 if( !pfn.empty() ) {
3176 ipool->second.setpfn(pfn);
3177 std::cout <<
"PFN: " << pfn <<
" (" << ipool->second.count()
3178 <<
")" << std::endl;
3181 std::string hguid=getCoolHistGUID(pfn);
3182 if (!hguid.empty()) {
3184 if (hguid!=ipool->first) {
3185 std::cout <<
"ERROR File CoolHist GUID " << hguid <<
3186 " inconsistent with catalogue " << ipool->first << std::endl;
3187 ipool->second.setErrorBit(3);
3192 const std::string fid = pool_utility.
readFileGUID( pfn );
3193 if( fid != ipool->first ) {
3194 std::cout <<
"ERROR File GUID " << fid <<
3195 " inconsistent with catalogue " << ipool->first << std::endl;
3197 ipool->second.setErrorBit(3);
3200 catch( std::runtime_error&
e ) {
3201 std::cout <<
"Cannot open file for reading!" << std::endl;
3202 std::cout <<
e.what() << std::endl;
3204 ipool->second.setErrorBit(2);
3210 ipool->second.setErrorBit(1);
3214 if (ipool->second.errcode()>0) ++nbad;
3225 catalog=setupCatalog(m_mergecat);
3226 if (catalog==
nullptr)
return 110;
3227 const std::string dssname=
"register.sh";
3228 std::cout <<
"Write DQ2 registerFileInDataset commands to " << dssname
3229 <<
" for registration in dataset " << m_newdataset << std::endl;
3230 std::ofstream dsstream(dssname.c_str());
3231 for (LFNGVec::const_iterator itr=dsfound.begin();
3232 itr!=dsfound.end();++itr) {
3233 const std::string&
lfn=itr->first;
3234 const std::string&
guid=itr->second;
3237 if( !lfns.empty() ) {
3239 const std::string lfn2 = lfns[0].first;
3240 if (lfn2!=
lfn) std::cout <<
"WARNING: LFNs for GUID " <<
guid <<
3241 " differ in input/merge datasets: " <<
lfn <<
" vs "
3242 << lfn2 << std::endl;
3245 dsstream <<
"dq2-register-files " << m_newdataset <<
" "
3246 <<
lfn <<
" " <<
guid << std::endl;
3256 std::cout <<
"All POOL references were resolved by catalogues"
3259 std::cout <<
"ERROR Could not resolve " << nbad <<
" files" << std::endl;
3262 if (nbad>0 || m_poolall) filePoolRefs();
3268 std::string hguid=
"";
3269 TFile* myfile=TFile::Open(
file.c_str(),
"READ");
3270 if (myfile!=
nullptr) {
3272 myfile->GetObject(
"fileGUID",oguid);
3273 if (oguid!=
nullptr) {
3274 hguid=oguid->GetString();
3275 std::cout <<
"CoolHist GUID found to be " << hguid << std::endl;
3291 std::cout <<
"ErrCode GUID Count LFN PFN Folders ..." << std::endl;
3292 for (PoolMap::const_iterator ipool=
m_poolrefs.begin();
3294 if (
m_poolall || ipool->second.errcode()>0) {
3295 std::ostringstream
line;
3297 line << ipool->second.errcode() <<
" " << ipool->first <<
" " <<
3298 ipool->second.count() <<
" ";
3299 if (!ipool->second.lfn().empty()) {
3300 line << ipool->second.lfn() <<
" ";
3304 if (!ipool->second.pfn().empty()) {
3305 line << ipool->second.pfn() <<
" ";
3309 for (std::vector<std::string>::const_iterator
3310 itr=ipool->second.foldertag().begin();
3311 itr!=ipool->second.foldertag().end();++itr)
3312 line << *itr <<
" ";
3313 std::cout <<
line.str() << std::endl;
3314 chkostream <<
line.str() << std::endl;
3320 const std::vector<std::string>& catvec) {
3324 for (std::vector<std::string>::const_iterator icat=catvec.begin();
3325 icat!=catvec.end();++icat) {
3326 std::cout <<
"Add catalogue: " << *icat << std::endl;
3328 if (icat->find(
':')==std::string::npos) {
3339 std::cout <<
"Could not setup POOL catalogues, exception:" <<
e.what()
3346 std::cout <<
"usage: AtlCoolCopy.exe <sourceCoolDB> <destinationCoolDB> { <options> }" << std::endl;
3347 std::cout <<
"Options are (see doc/mainpage.h for more):" << std::endl <<
3348 "-a, -alliov : set IOVs on destination to [ValidityKeyMin, ValidityKeyMax]" << std::endl <<
" or following settings of -nrls,-nrlu,-nts,-ntu" <<
3350 "-ag, -addguid <guid> : Add GUID to list accessed for POOL checks" <<
3352 "-alf, -addlfn <LFN> : Add LFN to list accessed for POOL checks" <<
3354 "-al, -appendlocked : Allow locked tags to be updated if no overlap" <<
3356 "-alsv, -appendlockedsv : Allow locked tag update for openended IOVs" <<
3358 "-ana, -analyse <delta_t>: produce analysis ROOT file (filename = dest DB argument)" << std::endl <<
3359 "-bs, -buffersize <size> : set size of bulkstorage output buf to <size> objs"
3361 "-c, -create : create destination DB if not already existing"
3363 "-cd, -checkdest : Check destination DB and skip already existing tags"
3365 "-ch, -channel : restrict selection to given channel or range specified as c1:c2" << std::endl <<
3366 "-ch1, -channel1 : specify start of a range of channels" << std::endl <<
3367 "-ch2, -channel2 : specify end of a range of channels" << std::endl <<
3368 "-cf, -checkfiles : check POOL files can be opened and have correct GUID"
3370 "-co, -checkoutput <file> : write POOL file check output on file"
3371 << std::endl <<
"-cti, -copytaginfo : Copy tag descriptions"
3372 << std::endl <<
"-ctl, -copytaglock : Copy tag locked status and descriptions"
3373 << std::endl <<
"-cr, -checkrefs : check POOL references"
3374 << std::endl <<
"-d, -debug : produce debug output" << std::endl;
3376 "-ds, -dataset : output register.sh for creating DQ2 datasets "
3378 "-ec, -excludechannel : exclude given channel from copy" << std::endl <<
3379 "-eh, -excludehead : exclude HEAD tag even if no tags found in MV folders"
3381 "-ih, -includehead : include HEAD as well as any tags in MV folders" <<
3383 "-e, -exclude <pattern> : exclude folders" << std::endl <<
3384 "-f, -folder <pattern> : include folders" << std::endl <<
3385 "-fs, -forcesingle : force destination folder to be singleversion"
3387 "-fm, -forcemulti : force destination folder to be multiversion"
3389 "-frl, -forcerunlumi : force destination folder to be run/LB indexed"
3391 "-ftm, -forcetime : force destination folder to be timestamp indexed"
3393 "-fp, -forcepayload : Force destn folders to be created with payload tbl" << std::endl <<
3394 "-fnp, -forcenopayload : Force destn folders to be created without payload tbl" << std::endl <<
3395 "-forcerecreate : delete and recreate destination database" << std::endl;
3396 std::cout <<
"-ht, -headtag : Use HEAD-style tagging"
3398 "-go, -getonline : Set minimum run number (-rls setting) to next ONLINE run"
3400 "-gb, -getbulk : Set minimum run number (-rls setting) to next bulk reco run"
3402 "-gt, -gettime : Extract timestamp information for given run number range"
3404 "-h, -help : print help and exit" << std::endl <<
3405 "-hi, -hitag : Copy hierrchical tags inclusively" << std::endl <<
3406 "-ignoremode <pwd> : Ignore UPDx mode protection (experts only)"
3408 "-is, -ignorespec : Ignore differences in folder spec if names are all equal" << std::endl <<
3409 "-lo, -lockedonly : Only copy locked/partially-locked top-level tags"
3411 "-nc, -nocopy : Do not actually copy, just read source DB" << std::endl <<
3412 "-nch, -nochannel : Do not check or copy channel information" << std::endl
3413 <<
"-ncr, -noclobroot: Do not copy CLOB data into ROOT files" << std::endl
3414 <<
"-noc, -nocoracool : Do not copy Coracool structure payloads " << std::endl <<
3415 "-nd, -nodata : Copy only folder structures, not data" << std::endl <<
3416 "-nh, -nohitag : Do not follow hierarchical tag relations" << std::endl <<
3417 "-mc, -mergecat <catfile> : specify POOL file catalogue for new dataset making"
3419 std::cout <<
"-of, -outfolder <folder> : rename folder on output"
3421 "-onr, -onlinerun : Retrieve run number from online server (not replica)" <<std::endl <<
3422 "-ot, -outtag : " <<
"Rename tag on output" << std::endl;
3424 "-pa, -poolall : Output all POOL files (incl good) when checking files"
3426 "-pc, -poolcat <catfile> : specify POOL file catalogue for ref checking"
3428 "-pf, -parfile <file> : Read additional options/parameters from file" <<
3430 "-pt, -prunetags : Copy only hierarchical tags descending from specified toptags"
3432 "-rdo, -readoracle : force data to be read from Oracle, using dbreplica.config"
3434 "-sl, -seal <val>: Set SEAL (hence COOL/POOL) output level to <val>"
3436 "-sr, -skiprep : Skip folders having <norep/> folder metadata tag" <<
3438 "-t, -tag <tag> : Copy multiversion data with tag <tag>" << std::endl <<
3439 "-mt, -magic <tag> : Include magic tags involving the given pattern " << std::endl <<
3440 "-tr, -truncate : Set destination IOVs outside query interval to query interval"
3442 "-tl, -taglabel : Specify tag description to enter in destination DB"
3444 "-uht, -userheadtag : Also copy user tag data to the HEAD" << std::endl <<
3445 "-v, -verify : Verify data present on destination rather than copying" <<
3447 "-ro, -root : Produce ROOT output file instead of copying to COOL"
3449 "-zn, -zeronull : Zero NULLs in ROOT file instead of skipping them" <<
3451 "-rls, -runlumisince <run> <LB> : Set minimum IOV interval" << std::endl <<
3452 "-rlu, -runlumiuntil <run> <LB> : Set maximum IOV interval" << std::endl <<
3453 "-rs, -runsince <run> : Set minimum IOV interval" << std::endl <<
3454 "-ru, -rununtil <run> : Set maximum IOV interval" << std::endl <<
3455 "-r, -run <run> : Copy only run <run>" << std::endl <<
3456 "-srls, -skiprunlumisince <run> <LB> : Set minimum IOV for skipping IOV in copy"
3458 "-srlu, -skiprunlumiuntil <run> <LB> : Set maximum IOV for skipping IOV in copy"
3460 "-tdb, -timedb <dbconn> : Set database connection for time information" <<
3461 "-ts, -timesince <time> : Set minimum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3463 "-tu, -timeuntil <time> : Set maximum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3466 "-nrls, -newrunlumisince <run> <LB> : Set minimum of output IOV interval (use with -alliov)" << std::endl <<
3467 "-nrlu, -newrunlumiuntil <run> <LB> : Set maximum of output IOV interval (use with --aliov)" << std::endl <<
3468 "-nts, -newtimesince <time> : Set minimum of outputIOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3470 "-ntu, -newtimeuntil <time> : Set maximum of output IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3472 std::cout <<
"See http://twiki.cern.ch/twiki/bin/view/Atlas/AtlCoolCopy for more details" << std::endl;
3487 if (retcode==0) retcode=mycopy.doCopy();
3489 if (retcode>0) std::cout <<
"ERROR AtlCoolCopy.exe fails with exit code " <<
3490 retcode << std::endl;