ATLAS Offline Software
Classes | Functions
AtlCoolCopy.cxx File Reference
#include <algorithm>
#include <cmath>
#include <ctime>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include <nlohmann/json.hpp>
#include <curl/curl.h>
#include "CoolKernel/DatabaseId.h"
#include "CoolKernel/Exception.h"
#include "CoolKernel/IDatabaseSvc.h"
#include "CoolKernel/IDatabase.h"
#include "CoolKernel/IFolder.h"
#include "CoolKernel/FolderSpecification.h"
#include "CoolKernel/IFolderSet.h"
#include "CoolKernel/IObject.h"
#include "CoolKernel/IObjectIterator.h"
#include "CoolKernel/IRecordIterator.h"
#include "CoolApplication/Application.h"
#include "CoralBase/AttributeListException.h"
#include "FileCatalog/IFileCatalog.h"
#include "PersistencySvc/IDatabase.h"
#include "PersistencySvc/ISession.h"
#include "PersistencySvc/ITransaction.h"
#include "PersistencySvc/SimpleUtilityBase.h"
#include "RelationalAccess/ConnectionService.h"
#include "RelationalAccess/IConnectionServiceConfiguration.h"
#include "RelationalAccess/ISessionProxy.h"
#include "RelationalAccess/ITransaction.h"
#include "RelationalAccess/ISchema.h"
#include "RelationalAccess/ITable.h"
#include "RelationalAccess/IQuery.h"
#include "RelationalAccess/ICursor.h"
#include "CoraCool/CoraCoolDatabase.h"
#include "CoraCool/CoraCoolFolder.h"
#include "CoraCool/CoraCoolObjectIter.h"
#include "CoraCool/CoraCoolObject.h"
#include "CxxUtils/checker_macros.h"
#include "PoolMapElement.h"
#include "ReplicaSorter.h"
#include "CoolTagInfo.h"
#include "TFile.h"
#include "TTree.h"
#include "TH1.h"
#include "TObjString.h"

Go to the source code of this file.

Classes

class  AtlCoolCopy
 

Functions

size_t WriteCallback (void *contents, size_t size, size_t nmemb, std::string *s)
 
void printHelp ()
 
int AtlCoolCopy::copyFolder ATLAS_NOT_THREAD_SAFE (const std::string &folder, const std::vector< std::string > &taglist)
 
int AtlCoolCopy::copyIOVs ATLAS_NOT_THREAD_SAFE (const std::string &folder, const std::string &destfolder, const cool::IFolderPtr &sourcefl, const CoraCoolFolderPtr &sourceflc, const cool::IFolderPtr &destfl, const CoraCoolFolderPtr &destflc, const std::string &sourcetag, const std::string &desttag, const cool::ValidityKey since, const cool::ValidityKey until, bool timestamp, bool checkrefs, bool iscora, const cool::PayloadMode::Mode paymode, bool created)
 
int AtlCoolCopy::doCopy ATLAS_NOT_THREAD_SAFE ()
 Install fatal handler with default options. More...
 
int main ATLAS_NOT_THREAD_SAFE (int argc, const char *argv[])
 

Function Documentation

◆ ATLAS_NOT_THREAD_SAFE() [1/4]

int AtlCoolCopy::doCopy ATLAS_NOT_THREAD_SAFE ( )
inline

Install fatal handler with default options.

This is meant to be easy to call from python via ctypes.

Definition at line 2097 of file AtlCoolCopy.cxx.

2097  {
2098  if (isOpen()) {
2099  int retcode=0;
2100  // execute copy for all defined folders
2101  for (std::vector<std::string>::const_iterator ifolder=
2102  m_folderlist.begin();ifolder!=m_folderlist.end();++ifolder) {
2103  int code=copyFolder(*ifolder,m_tags);
2104  if (code>retcode) retcode=code;
2105  }
2106  if (!m_hiparent.empty()) {
2107  // fill in any hierarchical tags for parent folders
2108  int code=tagParents();
2109  if (code>retcode) retcode=code;
2110  }
2111  if (m_copytaginfo) {
2112  // write information (tag description, lock status) to dest tags
2113  int code=writeTagInfo();
2114  if (code>retcode) retcode=code;
2115  }
2116  if (m_checkrefs) {
2117  int code=0;
2118  if (m_poolcat.empty()) {
2119  code=listPoolRefs();
2120  if (!m_checkoutputfile.empty()) filePoolRefs();
2121  } else {
2122  code=resolvePoolRefs();
2123  }
2124  if (code>retcode) retcode=code;
2125  }
2126  if (p_rootfile) {
2127  p_rootfile->Write();
2128  delete p_rootfile;
2129  }
2130  return retcode;
2131  } else {
2132  return 10;
2133  }
2134 }

◆ ATLAS_NOT_THREAD_SAFE() [2/4]

int AtlCoolCopy::copyIOVs ATLAS_NOT_THREAD_SAFE ( const std::string &  folder,
const std::string &  destfolder,
const cool::IFolderPtr &  sourcefl,
const CoraCoolFolderPtr sourceflc,
const cool::IFolderPtr &  destfl,
const CoraCoolFolderPtr destflc,
const std::string &  sourcetag,
const std::string &  desttag,
const cool::ValidityKey  since,
const cool::ValidityKey  until,
bool  timestamp,
bool  checkrefs,
bool  iscora,
const cool::PayloadMode::Mode  paymode,
bool  created 
)

Definition at line 882 of file AtlCoolCopy.cxx.

890  {
891 
892  std::cout << "Copying tag " << sourcetag << " of folder " << folder <<
893  " to destination tag " << desttag << std::endl;
894  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
895  cool::FolderVersioning::Mode dvermode=destfl->versioningMode();
896  // check for online mode
897  int updatemode=getUpdateMode(destfl->description(),desttag);
898  std::vector<std::string> dtaglist=destfl->listTags();
899  // if online mode, must have --getonline and either -truncate or -alliov
900  // only if destination DB is oracle
901  if (updatemode==1 && m_destdb.find("oracle")!=std::string::npos &&
902  (!m_getonline || (!m_truncate && !m_alliov))) {
903  if (m_ignoremode) {
904  std::cout << "Folder is online (UPD1) mode but IGNORING PROTECTION"
905  << std::endl;
906  } else {
907  if(find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
908  std::cout << "Folder is online mode (UPD1) and tag already exist - -getonline and -truncate or -alliov options MUST be used" << std::endl;
909  return 35;
910  }
911  else {
912  std::cout << "Folder is online mode (UPD1), new tag will be created" << std::endl;
913  }
914 
915  }
916  }
917  // if UPD4 mode, must have -getbulk and either -truncate or -alliov
918  if (updatemode==4 && m_destdb.find("oracle")!=std::string::npos &&
919  (!m_getbulk || (!m_truncate && !m_alliov))) {
920  if (m_ignoremode) {
921  std::cout << "Folder is bulkreco (UPD4) mode but IGNORING PROTECTION"
922  << std::endl;
923  } else {
924  if(find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
925  std::cout << "Folder is bulkreco mode (UPD4) - -getbulk and -truncate or -alliov options MUST be used" << std::endl;
926  return 35;
927  } else {
928  std::cout << "Folder is bulkreco mode (UPD4), new tag will be created" << std::endl;
929  }
930 
931  }
932  }
933 
934  // if destination tags are being checked, check if this tag already
935  // exists in destination folder, if so skip copy
936  // also skip if tag is HEAD, and folder was not newly created
937  if (m_checkdesttag) {
938  if (find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()
939  || (desttag=="HEAD" && !created)) {
940  std::cout << "Destination tag " << desttag <<
941  " already exists in folder " << folder << " - skip copy" << std::endl;
942  return 0;
943  }
944  }
945  bool relock=false; // tag must be relocked afterwards
946  bool prot=false; //update is protected, irrespective of if tag must be locked
947  if ((m_applock || (updatemode==1 && m_getonline) || updatemode==2 ||
948  (updatemode==4 && m_getbulk)) &&
949  dvermode==cool::FolderVersioning::MULTI_VERSION) {
950  // check if the destination tag exists and is locked
951  std::vector<std::string> dtaglist=destfl->listTags();
952  for (std::vector<std::string>::const_iterator itr=dtaglist.begin();
953  itr!=dtaglist.end();++itr) {
954  if (desttag==*itr) {
955  // tag exists - must protect it unless ignoremode
956  prot=!m_ignoremode;
957  // check lock status
958  if (destfl->tagLockStatus(desttag)==cool::HvsTagLock::LOCKED) {
959  std::cout << "Unlocking destination tag " << desttag <<
960  " for append or UPDx access" << std::endl;
961  if (m_applocksv) {
962  std::cout << "Appending according to SV folder rules" << std::endl;
963  if (updatemode!=3) {
964  std::cout <<
965  "ERROR: Only allowed for UPD3 mode tags" << std::endl;
966  return 36;
967  }
968  }
969  destfl->setTagLockStatus(desttag,cool::HvsTagLock::UNLOCKED);
970  relock=true;
971  }
972  }
973  }
974  }
975  int nobj=0;
976  int nbuf=0;
977  int nskip=0;
978  int nbad=0;
979  // set up tag to be used at point of insertion
980  std::string localtag="";
981  if (dvermode==cool::FolderVersioning::MULTI_VERSION &&
982  desttag!="HEAD" && m_usertags) localtag=desttag;
983  // now loop over IOVs - separate for CoraCool and COOL
984  try {
985  if (iscora) {
986  // branch for CoraCool
987  sourceflc->setPrefetchAll(false);
988  CoraCoolObjectIterPtr sourceitr;
989  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
990  sourceitr=sourceflc->browseObjects(since,until,m_chansel);
991  } else {
992  sourceitr=sourceflc->browseObjects(since,until,m_chansel,sourcetag);
993  }
994  // keep track of new FKs of inserted objects
995  std::map<int,int> insertkeymap;
996  unsigned int nref=0;
997  while (sourceitr->hasNext()) {
998  CoraCoolObjectPtr obj=sourceitr->next();
999  // check object should not be skipped as spreading outside select-IOV
1000  if (m_skipout &&
1001  (obj->since()<m_srunemin || obj->until()>m_srunemax)) {
1002  ++nskip;
1003  continue;
1004  }
1005  // check object should not be skipped as excluded channel
1006  if (!m_excludechans.empty()) {
1007  if (find(m_excludechans.begin(),m_excludechans.end(),
1008  obj->channelId())!=m_excludechans.end()) {
1009  ++nskip;
1010  continue;
1011  }
1012  }
1013  if (nbuf==0) {
1014  if (m_debug) std::cout <<
1015  "Setup new CoraCool storage buffer at object " << nobj << std::endl;
1016  destflc->setupStorageBuffer();
1017  }
1018  cool::ValidityKey newsince,newuntil;
1019  adjustIOVs(obj->since(),obj->until(),since,until,newsince,newuntil,
1020  timestamp);
1021  // skip negative/zero IOV lengths
1022  if (newsince>=newuntil) {
1023  std::cout << "WARNING: Skipping IOV with since " << newsince <<
1024  ">= until" << newuntil << std::endl;
1025  ++nbad;
1026  continue;
1027  }
1028  // find the old FK, check if we have already inserted this
1029  // in which case a reference can be added
1030  int oldfk=0;
1031  std::map<int,int>::const_iterator ikey=insertkeymap.end();
1032  bool foundkey=false;
1033  // ensure returned data is not of zero size - if so cannot extract
1034  // and remember the FK
1035  if (obj->size()>0) {
1036  oldfk=sourceflc->getAttrKey(
1037  (*obj->begin())[sourceflc->coralFKey()]);
1038  ikey=insertkeymap.find(oldfk);
1039  foundkey=true;
1040  }
1041  if (ikey==insertkeymap.end()) {
1042  int newfk=
1043  destflc->storeObject(newsince,newuntil,obj->begin(),obj->end(),
1044  obj->channelId(),localtag,
1045  (!m_userupdatehead && !localtag.empty()));
1046  if (foundkey) insertkeymap[oldfk]=newfk;
1047  } else {
1048  destflc->referenceObject(newsince,newuntil,ikey->second,
1049  obj->channelId(),localtag,
1050  (!m_userupdatehead && !localtag.empty()));
1051  ++nref;
1052  }
1053  ++nbuf;
1054  // flush buffer every m_bufsize objects
1055  if (nbuf==m_bufsize) {
1056  if (m_debug) std::cout << "Flush buffer after " << nobj << "," <<
1057  nbuf << " objects " << std::endl;
1058  // note CoraCool requires explicit re-setup of buffer after a flush
1059  destflc->flushStorageBuffer();
1060  destflc->setupStorageBuffer();
1061  nbuf=0;
1062  }
1063  ++nobj;
1064  }
1065  sourceitr->close();
1066  if (nref>0) std::cout << "Reference-to-existing used for " << nref
1067  << " payload objects" << std::endl;
1068  if (nbuf>0) {
1069  if (m_debug) std::cout << "Final buffer flush at " << nobj <<
1070  "," << nbuf << std::endl;
1071  destflc->flushStorageBuffer();
1072  }
1073  } else {
1074  // branch for pure COOL objects
1075  cool::IObjectIteratorPtr sourceitr;
1076  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1077  sourceitr=sourcefl->browseObjects(since,until,m_chansel);
1078  } else {
1079  sourceitr=sourcefl->browseObjects(since,until,m_chansel,sourcetag);
1080  }
1081  while (sourceitr->goToNext()) {
1082  const cool::IObject& obj=sourceitr->currentRef();
1083  // check object should not be skipped as spreading outside select-IOV
1084  if (m_skipout && (obj.since()<m_srunemin || obj.until()>m_srunemax)) {
1085  ++nskip;
1086  continue;
1087  }
1088  // check object should not be skipped as excluded channel
1089  if (!m_excludechans.empty()) {
1090  if (find(m_excludechans.begin(),m_excludechans.end(),
1091  obj.channelId())!=m_excludechans.end()) {
1092  ++nskip;
1093  continue;
1094  }
1095  }
1096  if (nbuf==0) {
1097  if (m_debug) std::cout << "Setup new storage buffer at object " <<
1098  nobj << std::endl;
1099  destfl->setupStorageBuffer();
1100  }
1101  cool::ValidityKey newsince;
1102  cool::ValidityKey newuntil;
1103  adjustIOVs(obj.since(),obj.until(),since,until,newsince,newuntil,
1104  timestamp);
1105  // skip negative/zero IOV lengths
1106  if (newsince>=newuntil) {
1107  std::cout << "WARNING: Skipping IOV with since " << newsince <<
1108  ">= until" << newuntil << std::endl;
1109  ++nbad;
1110  continue;
1111  }
1112  if (checkrefs) checkRef(obj.payload(),folder,sourcetag);
1113  if (prot) {
1114  // check existing data in destination DB to ensure no clash
1115  if (m_applocksv) {
1116  // appendlockedsv mode - all IOV ends must be infinite
1117  // and new since must be greater than old since
1118  if (newuntil!=cool::ValidityKeyMax) {
1119  std::cout << "New IOVs must have until=cool::ValidityKeyMax" << std::endl;
1120  throw cool::Exception("Illegal insert over locked IOV",
1121  "AtlCoolCopy");
1122  }
1123  cool::IObjectIteratorPtr checkitr=destfl->browseObjects(newsince,
1124  newuntil,obj.channelId(),desttag);
1125  while (checkitr->goToNext()) {
1126  const cool::IObject& checkobj=checkitr->currentRef();
1127  if (checkobj.since()>=newsince) {
1128  std::cout << "ERROR:: Attempt to insert SV overlapping IOV whilst appending to locked tag" << std::endl;
1129  throw cool::Exception("Illegal insert over locked IOV",
1130  "AtlCoolCopy");
1131  }
1132  if (checkobj.until()!=cool::ValidityKeyMax) {
1133  std::cout << "Existing IOVs must have until=cool::ValidityKeyMax" << std::endl;
1134  throw cool::Exception("Illegal insert over locked IOV",
1135  "AtlCoolCopy");
1136  }
1137  }
1138  checkitr->close();
1139  } else if (m_applock) {
1140  // appendlocked mode - just make sure no objects in this IOV
1141  const unsigned int nexist=
1142  destfl->countObjects(newsince,newuntil,obj.channelId(),desttag);
1143  if (nexist>0) {
1144  std::cout << "ERROR: Attempt to insert IOV over " << nexist <<
1145  " objects whilst appending to locked tag" << std::endl;
1146  throw cool::Exception("Illegal insert over locked IOV",
1147  "AtlCoolCopy");
1148  }
1149  } else if (updatemode==2) {
1150  // UPD2 tag - make sure run is on list of runs open for update
1151  // only for run/lumi based folders
1152  if (timestamp)
1153  throw cool::Exception(
1154  "Attempt to insert into locked UPD2 tag with timestamp format",
1155  "AtlCoolCopy");
1156  unsigned int run1=newsince >> 32;
1157  unsigned int run2=(newuntil-1) >> 32;
1158  for (unsigned int irun=run1;irun<=run2;++irun) {
1159  if (!std::binary_search(m_runlist.begin(),m_runlist.end(),irun))
1160  {
1161  std::cout << "Run " << irun << " from range [" << run1 << ","
1162  << run2 << "] not found in runfile list" << std::endl;
1163  throw cool::Exception("Illegal insert over locked IOV",
1164  "AtlCoolCopy");
1165  }
1166  }
1167  }
1168  // updatemode==1 will fall through, but no checks are done here
1169  }
1170  if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
1171  // cool vector payload object copy
1172  cool::IRecordIterator& pitr=obj.payloadIterator();
1173  const cool::IRecordVectorPtr vptr=pitr.fetchAllAsVector();
1174  destfl->storeObject(newsince,newuntil,
1175  *vptr,obj.channelId(),localtag,
1176  (!m_userupdatehead && !localtag.empty()));
1177  nbuf+=vptr->size();
1178  pitr.close();
1179  } else {
1180  // standard COOL object copy
1181  destfl->storeObject(newsince,newuntil,
1182  obj.payload(),obj.channelId(),localtag,
1183  (!m_userupdatehead && !localtag.empty()));
1184  ++nbuf;
1185  }
1186  // flush buffer every m_bufsize objects
1187  if (nbuf>=m_bufsize) {
1188  if (m_debug) std::cout << "Flush buffer after " << nobj << "," <<
1189  nbuf << " objects " << std::endl;
1190  destfl->flushStorageBuffer();
1191  nbuf=0;
1192  }
1193  ++nobj;
1194  }
1195  // finished with the iterator
1196  sourceitr->close();
1197  if (nbuf>0) {
1198  if (m_debug) std::cout << "Final buffer flush at " << nobj <<
1199  "," << nbuf << std::endl;
1200  destfl->flushStorageBuffer();
1201  } // end of COOL-specific part
1202  }
1203  std::cout << "Folder copied with " << nobj << " objects" << std::endl;
1204  if (nskip>0) std::cout << nskip <<
1205  " objects were skipped extending outside IOV selection or excluded channel"
1206  << std::endl;
1207  if (nbad>0) std::cout << nbad <<
1208  " objects were skipped having zero or negative IOV lengths" << std::endl;
1209  }
1210  // exceptions thrown from insert loop (both COOL or CORACOOL)
1211  catch (cool::Exception& e) {
1212  std::cout << "Exception thrown from copy loop: " << e.what() <<
1213  std::endl;
1214  if (relock) {
1215  std::cout << "Relocking destination tag " << desttag << std::endl;
1216  destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1217  }
1218  return 31;
1219  }
1220  // check if tag needs to be relocked
1221  if (relock) {
1222  std::cout << "Relocking destination tag " << desttag << std::endl;
1223  destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1224  }
1225 
1226  if (dvermode==cool::FolderVersioning::MULTI_VERSION && desttag!="HEAD") {
1227  if (!m_usertags) {
1228  // apply HEAD style tagging
1229  std::cout << "Tag folder with HEAD-style tagging for tag: " << desttag
1230  << std::endl;
1231  try {
1232  destfl->tagCurrentHead(desttag,sourcefl->tagDescription(sourcetag));
1233  }
1234  catch (cool::Exception& e) {
1235  std::cout << "Exception thrown in HEAD-style folder-tag: " <<
1236  e.what() << std::endl;
1237  return 32;
1238  }
1239  }
1240  // record information about this tag for later
1241  if (m_copytaginfo) m_cooltagmap.insert(
1242  CoolTagMap::value_type(desttag,CoolTagInfo(m_sourceDbPtr,folder,
1243  destfolder,sourcetag,desttag,m_taglabel)));
1244  // check if any hierarchical tags reference the just-copied tag
1245  if (!m_hitagmap.empty()) {
1246  for (HiTagMap::const_iterator imap=m_hitagmap.begin();
1247  imap!=m_hitagmap.end();++imap) {
1248  try {
1249  if (sourcefl->findTagRelation(imap->first)==sourcetag) {
1250  std::cout << "Create hierarchical tag between " << desttag <<
1251  " and " << imap->first << " in folder " << imap->second <<
1252  std::endl;
1253  // first check this relation has not already been created
1254  try {
1255  std::string etag=destfl->resolveTag(imap->first);
1256  if (etag==desttag) {
1257  std::cout << "This relation has already been created" << std::endl;
1258  } else {
1259  std::cout << "ERROR: Tag in parent already related to " <<
1260  desttag << std::endl;
1261  }
1262  }
1263  catch (cool::Exception& e ) {
1264  // only do creation if does not already exist - via exception
1265  try {
1266  destfl->createTagRelation(imap->first,desttag);
1267  // check if this implicates a new parent folder
1268  // to be checked for further tags up the tree
1269  if (find(m_hiparent.begin(),m_hiparent.end(),imap->second)==
1270  m_hiparent.end()) m_hiparent.push_back(imap->second);
1271  // store the original tag information to transfer
1272  // properties later
1273  if (m_copytaginfo &&
1274  m_cooltagmap.find(imap->first)==m_cooltagmap.end())
1275  m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
1276  CoolTagInfo(m_sourceDbPtr,imap->second,imap->second,
1277  imap->first,imap->first)));
1278  }
1279  catch (cool::Exception& e) {
1280  std::cout << "Cool exception " << e.what() <<
1281  "thrown in hierarchical tag creation" << std::endl;
1282  return 34;
1283  }
1284  }
1285  }
1286  }
1287  catch (cool::Exception& e) {
1288  }
1289  }
1290  }
1291  }
1292  return 0;
1293 }

◆ ATLAS_NOT_THREAD_SAFE() [3/4]

int AtlCoolCopy::copyFolder ATLAS_NOT_THREAD_SAFE ( const std::string &  folder,
const std::vector< std::string > &  taglist 
)

Definition at line 534 of file AtlCoolCopy.cxx.

535  {
536  // get source folder
537  cool::IFolderPtr sourcefl,destfl;
538  CoraCoolFolderPtr sourceflc,destflc;
539  try {
540  sourcefl=m_sourceDbPtr->getFolder(folder);
541  }
542  catch (cool::Exception& e) {
543  std::cout << "Could not get source folder: " << folder << std::endl;
544  return 20;
545  }
546  const std::string& sourcedesc=sourcefl->description();
547  // check if folder should be skipped from replication
548  if (m_skiprep && sourcedesc.find("<norep/>")!=std::string::npos) {
549  std::cout << "Folder " << folder <<
550  " skipped due to <norep/> metadata" << std::endl;
551  return 0;
552  }
553  const bool iscora=(m_coracool &&
554  sourcedesc.find("<coracool")!=std::string::npos);
555  if (iscora) {
556  // activate CoraCool and get source folder pointer
557  openCoraCool();
558  sourceflc=m_sourceCoraPtr->getFolder(folder);
559  }
560  // setup the channel-range
561  setChannelRange(sourcefl);
562 
563  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
564  cool::FolderVersioning::Mode dvermode=vermode;
565  if (m_forcesingle) {
566  dvermode=cool::FolderVersioning::SINGLE_VERSION;
567  std::cout << "Forcing destination folder to singleversion" << std::endl;
568  }
569  if (m_forcemulti) {
570  dvermode=cool::FolderVersioning::MULTI_VERSION;
571  std::cout << "Forcing destination folder to multiversion" << std::endl;
572  }
573  // setup type of folder (payload mode, inline, separate, or vector)
574  const cool::PayloadMode::Mode spaymode=
575  sourcefl->folderSpecification().payloadMode();
576  cool::PayloadMode::Mode dpaymode=spaymode;
577 
578  // set name for destination folder
579  std::string destfolder=folder;
580  if (!m_outfolder.empty()) {
581  destfolder=m_outfolder;
582  std::cout << "Destination folder will be renamed to " << destfolder <<
583  std::endl;
584  }
585 
586 // get destination folder, try to create if not there (and not verifying!)
587  bool created=false;
588  if (!m_nocopy && !m_root && !m_analyse) {
589  if (!m_destDbPtr->existsFolder(destfolder) && !m_verify) {
590  std::cout << "Creating folder " << destfolder << " payload-type " <<
591  dpaymode << " on destination" << std::endl;
592  created=true;
593  std::string metadata=sourcedesc;
594  if (m_forcerune || m_forcetime) {
595  std::string newmeta=m_forcerune ? "run-lumi" : "time";
596  std::string::size_type p1,p2;
597  p1=metadata.find("<timeStamp>");
598  p2=metadata.find("</timeStamp>");
599  if (p1!=std::string::npos && p2!=std::string::npos) {
600  metadata.replace(0,p2,"<timeStamp>"+newmeta);
601  std::cout << "Forced destination folder to " << newmeta << " : "
602  << metadata << std::endl;
603  } else {
604  std::cout <<
605  "ERROR: Could not parse metadata string to force timestamp type"
606  << std::endl;
607  }
608  }
609  // force separate or inline payload, but not if input is vector
610  if (m_forcepay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
611  dpaymode=cool::PayloadMode::SEPARATEPAYLOAD;
612  if (m_forcenopay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
613  dpaymode=cool::PayloadMode::INLINEPAYLOAD;
614 
615  try {
616  if (iscora) {
617  destflc=m_destCoraPtr->createFolder(destfolder,
618  sourceflc->coralTableName(),
619  sourceflc->fkSpecification(),
620  sourceflc->payloadSpecification(),
621  sourceflc->coralFKey(),
622  sourceflc->coralPKey(),
623  metadata,dvermode,true);
624  std::cout << "Created CoraCool folder" << std::endl;
625  destfl=m_destDbPtr->getFolder(destfolder);
626  } else {
627  destfl=m_destDbPtr->createFolder(destfolder,
628  cool::FolderSpecification(dvermode,
629  sourcefl->payloadSpecification(),dpaymode),
630  metadata,true);
631  }
632  }
633  catch (cool::Exception&e ) {
634  std::cout << "Create folder failed - aborting" << std::endl;
635  return 30;
636  }
637  }
638  // now get the pointer to destination folder if needed
639  // note both COOL and CoraCool pointers are set if needed
640  try {
641  if (iscora) destflc=m_destCoraPtr->getFolder(destfolder);
642  destfl=m_destDbPtr->getFolder(destfolder);
643  }
644  catch (cool::Exception& e) {
645  std::cout << "Could not get destination folder: " << destfolder
646  << std::endl;
647  return 21;
648  }
649  // check payload specifications of folders are the same
650  const cool::IRecordSpecification& sourcespec=
651  sourcefl->payloadSpecification();
652  const cool::IRecordSpecification& destspec=
653  destfl->payloadSpecification();
654  if (!(sourcespec==destspec)) {
655  bool badspec=false;
656  if (m_ignorespec) {
657  // specifications differ - check names are same
658  std::cout <<
659  "WARNING Source and destination folder specifications differ" <<
660  std::endl;
661  for (unsigned int i=0;i<sourcespec.size();++i) {
662  const std::string& sname=sourcespec[i].name();
663  if (!destspec.exists(sname)) {
664  std::cout << "ERROR: Field " << sname << " absent from destination"
665  << std::endl;
666  badspec=true;
667  }
668  }
669  } else {
670  badspec=true;
671  }
672  if (badspec) {
673  std::cout <<
674  "ERROR Source and destination folder specifications differ"
675  << std::endl;
676  return 22;
677  }
678  }
679  // check folder descriptions are the same - just print WARNING if not
680  const std::string& destdesc=destfl->description();
681  if (sourcedesc!=destdesc) {
682  std::cout << "WARNING: Source and destination folder descriptions (meta-data) differ" << std::endl;
683  std::cout << "Source folder: " << sourcedesc << std::endl;
684  std::cout << "Destn folder: " << destdesc << std::endl;
685  }
686  // check payload modes are same - print warning if not
687  const cool::PayloadMode::Mode dpaymode=
688  destfl->folderSpecification().payloadMode();
689  if (spaymode!=dpaymode) {
690  std::cout << "WARNING: Source (" << spaymode << ") and destination (" <<
691  dpaymode << " folder payload modes differ" << std::endl;
692  }
693 
694  // check/set channels table if requested
695  if (!m_nochannel) {
696  if (!checkChannels(folder,sourcefl,destfl,created)) return 23;
697  }
698  }
699  // if only copying structures, stop here
700  if (m_nodata) return 0;
701  // extract folder/range information
702  std::cout << "Start to process folder: " << folder;
703  // check for timestamp XML
704  bool timestamp=(sourcefl->description().find("<timeStamp>time")!=
705  std::string::npos);
706  cool::ValidityKey since,until;
707  if (timestamp) {
708  since=m_timemin;
709  until=m_timemax;
710  std::cout << " (timestamp)" << std::endl;
711  } else {
712  since=m_runemin;
713  until=m_runemax;
714  std::cout << " (run/lumi)" << std::endl;
715  }
716  if (m_alliov || m_truncate)
717  std::cout << "Output IOVs will be modified" << std::endl;
718  if (m_skipout)
719  std::cout << "IOVs extending outside selection range will be skipped"
720  << std::endl;
721 
722  bool checkrefs=false;
723  const std::string name0=sourcefl->payloadSpecification()[0].name();
724  if (m_checkrefs && (name0=="PoolRef" || name0=="fileGUID")) {
725  checkrefs=true;
726  std::cout << "Check POOL references in folder " << folder << std::endl;
727  }
728  // if nocopy, and checking POOL references, and folder is not a POOL ref one
729  // can skip the data access
730  if (m_checkrefs && m_nocopy && !checkrefs) return 0;
731 
732  // check for <fullrep/> metadata indicating copy all tags if -skiprep option
733  bool copyall=
734  (m_skiprep && (sourcedesc.find("<fullrep/>")!=std::string::npos));
735  if (copyall) std::cout <<
736  "All tags in folder will be copied due to <fullrep/> metadata" << std::endl;
737 
738  // set up true list of tags to copy
739  std::vector<std::string> tags;
740  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
741  std::cout << "Single version folder" << std::endl;
742  tags.emplace_back("HEAD");
743  } else {
744  std::cout << "Multi version folder: consider tags [ ";
745  // get list of tags in this node which are requested
746  const std::vector<std::string> foldertags=sourcefl->listTags();
747  for (std::vector<std::string>::const_iterator itag=foldertags.begin();
748  itag!=foldertags.end();++itag) {
749  bool copyit=copyall;
750  // if input taglist is empty, take all tags in folder
751  if (taglist.empty() ||
752  find(taglist.begin(),taglist.end(),*itag)!=taglist.end())
753  copyit=true;
754  for (std::vector<std::string>::const_iterator imtag=m_magic.begin();
755  imtag!=m_magic.end();++imtag) {
756  if (itag->find(*imtag)!=std::string::npos) copyit=true;
757  }
758  if (copyit) {
759  tags.push_back(*itag);
760  std::cout << *itag << " ";
761  }
762  }
763  // if no tags were found, or doing inclusive hierarchical tag copying
764  // try resolving input tags hierarchically
765  if (tags.empty() || m_hitag) {
766  for (std::vector<std::string>::const_iterator itag=taglist.begin();
767  itag!=taglist.end();++itag) {
768  try {
769  std::string htag=sourcefl->resolveTag(*itag);
770  if (find(tags.begin(),tags.end(),htag)==tags.end()) {
771  std::cout << *itag << "=" << htag << " ";
772  tags.push_back(std::move(htag));
773  }
774  }
775  // ignore exceptions indicating tag not defined here
776  // note std::exception rather than COOL exception to cover case
777  // when trying to resolve a leaf tag in the '/' folder, which
778  // throws a coral AttributeException
779  catch (std::exception& e) { }
780  }
781  }
782  std::cout << "]" << std::endl;
783  // if still no tags were found, or forced HEAD tag copying, or no
784  // tags in the folder at all (=MV folder being used as SV), add HEAD
785  if (((tags.empty() || m_includehead) && !m_excludehead) ||
786  foldertags.empty())
787  tags.emplace_back("HEAD");
788  }
789 
790  sourcefl->setPrefetchAll(false);
791  for (std::vector<std::string>::const_iterator itag=tags.begin();
792  itag!=tags.end();++itag) {
793  std::string outtag=*itag;
794  if (!m_outtag.empty()) outtag=m_outtag;
795  int retcode;
796  // verify or copy folder
797  if (m_verify) {
798  if (!destfl) {
799  retcode = 1;
800  }
801  else {
802  retcode=verifyIOVs(folder,sourcefl,sourceflc,destfl,destflc,
803  *itag,since,until,checkrefs,iscora,spaymode);
804  }
805  } else if (m_nocopy) {
806  retcode=nocopyIOVs(folder,sourcefl,*itag,since,until,checkrefs);
807  } else if (m_root) {
808  retcode=rootIOVs(folder,sourcefl,*itag,since,until,timestamp);
809  } else if (m_analyse) {
810  retcode=analyseIOVs(folder,sourcefl,*itag,since,until,timestamp);
811  } else {
812  retcode=copyIOVs(folder,destfolder,sourcefl,sourceflc,destfl,destflc,
813  *itag,outtag,since,until,timestamp,checkrefs,iscora,
814  spaymode,created);
815  }
816  if (retcode!=0) {
817  std::cout << "ERROR operation failed for folder " << folder << " tag " <<
818  *itag << std::endl;
819  return retcode;
820  }
821  }
822  return 0;
823 }

◆ ATLAS_NOT_THREAD_SAFE() [4/4]

int main ATLAS_NOT_THREAD_SAFE ( int  argc,
const char *  argv[] 
)

Definition at line 3483 of file AtlCoolCopy.cxx.

3483  {
3484  int retcode=0;
3485  if (argc<3) {
3486  printHelp();
3487  retcode=1;
3488  } else {
3489  AtlCoolCopy mycopy(argv[1],argv[2]);
3490  retcode=mycopy.setOpts(argc-3,&argv[3]);
3491  if (retcode==999) {
3492  printHelp();
3493  return 0;
3494  }
3495  if (retcode==0) retcode=mycopy.doCopy();
3496  }
3497  if (retcode>0) std::cout << "ERROR AtlCoolCopy.exe fails with exit code " <<
3498  retcode << std::endl;
3499  return retcode;
3500 }

◆ printHelp()

void printHelp ( )

Definition at line 3353 of file AtlCoolCopy.cxx.

3353  {
3354  std::cout << "usage: AtlCoolCopy.exe <sourceCoolDB> <destinationCoolDB> { <options> }" << std::endl;
3355  std::cout << "Options are (see doc/mainpage.h for more):" << std::endl <<
3356  "-a, -alliov : set IOVs on destination to [ValidityKeyMin, ValidityKeyMax]" << std::endl <<" or following settings of -nrls,-nrlu,-nts,-ntu" <<
3357  std::endl <<
3358  "-ag, -addguid <guid> : Add GUID to list accessed for POOL checks" <<
3359  std::endl <<
3360  "-alf, -addlfn <LFN> : Add LFN to list accessed for POOL checks" <<
3361  std::endl <<
3362  "-al, -appendlocked : Allow locked tags to be updated if no overlap" <<
3363  std::endl <<
3364  "-alsv, -appendlockedsv : Allow locked tag update for openended IOVs" <<
3365  std::endl <<
3366  "-ana, -analyse <delta_t>: produce analysis ROOT file (filename = dest DB argument)" << std::endl <<
3367 "-bs, -buffersize <size> : set size of bulkstorage output buf to <size> objs"
3368  << std::endl <<
3369  "-c, -create : create destination DB if not already existing"
3370  << std::endl <<
3371  "-cd, -checkdest : Check destination DB and skip already existing tags"
3372  << std::endl <<
3373  "-ch, -channel : restrict selection to given channel or range specified as c1:c2" << std::endl <<
3374  "-ch1, -channel1 : specify start of a range of channels" << std::endl <<
3375  "-ch2, -channel2 : specify end of a range of channels" << std::endl <<
3376  "-cf, -checkfiles : check POOL files can be opened and have correct GUID"
3377  << std::endl <<
3378  "-co, -checkoutput <file> : write POOL file check output on file"
3379  << std::endl << "-cti, -copytaginfo : Copy tag descriptions"
3380  << std::endl << "-ctl, -copytaglock : Copy tag locked status and descriptions"
3381  << std::endl << "-cr, -checkrefs : check POOL references"
3382  << std::endl << "-d, -debug : produce debug output" << std::endl;
3383  std::cout <<
3384  "-ds, -dataset : output register.sh for creating DQ2 datasets "
3385  << std::endl <<
3386  "-ec, -excludechannel : exclude given channel from copy" << std::endl <<
3387  "-eh, -excludehead : exclude HEAD tag even if no tags found in MV folders"
3388  << std::endl <<
3389  "-ih, -includehead : include HEAD as well as any tags in MV folders" <<
3390  std::endl <<
3391  "-e, -exclude <pattern> : exclude folders" << std::endl <<
3392  "-f, -folder <pattern> : include folders" << std::endl <<
3393  "-fs, -forcesingle : force destination folder to be singleversion"
3394  << std::endl <<
3395  "-fm, -forcemulti : force destination folder to be multiversion"
3396  << std::endl <<
3397  "-frl, -forcerunlumi : force destination folder to be run/LB indexed"
3398  << std::endl <<
3399  "-ftm, -forcetime : force destination folder to be timestamp indexed"
3400  << std::endl <<
3401  "-fp, -forcepayload : Force destn folders to be created with payload tbl" << std::endl <<
3402  "-fnp, -forcenopayload : Force destn folders to be created without payload tbl" << std::endl <<
3403  "-forcerecreate : delete and recreate destination database" << std::endl;
3404  std::cout << "-ht, -headtag : Use HEAD-style tagging"
3405  << std::endl <<
3406  "-go, -getonline : Set minimum run number (-rls setting) to next ONLINE run"
3407  << std::endl <<
3408  "-gb, -getbulk : Set minimum run number (-rls setting) to next bulk reco run"
3409  << std::endl <<
3410  "-gt, -gettime : Extract timestamp information for given run number range"
3411  << std::endl <<
3412  "-h, -help : print help and exit" << std::endl <<
3413  "-hi, -hitag : Copy hierrchical tags inclusively" << std::endl <<
3414  "-ignoremode <pwd> : Ignore UPDx mode protection (experts only)"
3415  << std::endl <<
3416  "-is, -ignorespec : Ignore differences in folder spec if names are all equal" << std::endl <<
3417  "-lo, -lockedonly : Only copy locked/partially-locked top-level tags"
3418  << std::endl <<
3419  "-nc, -nocopy : Do not actually copy, just read source DB" << std::endl <<
3420  "-nch, -nochannel : Do not check or copy channel information" << std::endl
3421 << "-ncr, -noclobroot: Do not copy CLOB data into ROOT files" << std::endl
3422 << "-noc, -nocoracool : Do not copy Coracool structure payloads " << std::endl <<
3423  "-nd, -nodata : Copy only folder structures, not data" << std::endl <<
3424  "-nh, -nohitag : Do not follow hierarchical tag relations" << std::endl <<
3425 "-mc, -mergecat <catfile> : specify POOL file catalogue for new dataset making"
3426  << std::endl;
3427  std::cout << "-of, -outfolder <folder> : rename folder on output"
3428  << std::endl <<
3429  "-onr, -onlinerun : Retrieve run number from online server (not replica)" <<std::endl <<
3430  "-ot, -outtag : " << "Rename tag on output" << std::endl;
3431  std::cout <<
3432  "-pa, -poolall : Output all POOL files (incl good) when checking files"
3433  << std::endl <<
3434 "-pc, -poolcat <catfile> : specify POOL file catalogue for ref checking"
3435  << std::endl <<
3436  "-pf, -parfile <file> : Read additional options/parameters from file" <<
3437  std::endl <<
3438 "-pt, -prunetags : Copy only hierarchical tags descending from specified toptags"
3439  << std::endl <<
3440 "-rdo, -readoracle : force data to be read from Oracle, using dbreplica.config"
3441  << std::endl <<
3442  "-sl, -seal <val>: Set SEAL (hence COOL/POOL) output level to <val>"
3443  << std::endl <<
3444  "-sr, -skiprep : Skip folders having <norep/> folder metadata tag" <<
3445  std::endl <<
3446  "-t, -tag <tag> : Copy multiversion data with tag <tag>" << std::endl <<
3447  "-mt, -magic <tag> : Include magic tags involving the given pattern " << std::endl <<
3448 "-tr, -truncate : Set destination IOVs outside query interval to query interval"
3449  << std::endl <<
3450 "-tl, -taglabel : Specify tag description to enter in destination DB"
3451  << std::endl <<
3452  "-uht, -userheadtag : Also copy user tag data to the HEAD" << std::endl <<
3453  "-v, -verify : Verify data present on destination rather than copying" <<
3454  std::endl <<
3455  "-ro, -root : Produce ROOT output file instead of copying to COOL"
3456  << std::endl <<
3457  "-zn, -zeronull : Zero NULLs in ROOT file instead of skipping them" <<
3458  std::endl <<
3459  "-rls, -runlumisince <run> <LB> : Set minimum IOV interval" << std::endl <<
3460  "-rlu, -runlumiuntil <run> <LB> : Set maximum IOV interval" << std::endl <<
3461  "-rs, -runsince <run> : Set minimum IOV interval" << std::endl <<
3462  "-ru, -rununtil <run> : Set maximum IOV interval" << std::endl <<
3463  "-r, -run <run> : Copy only run <run>" << std::endl <<
3464 "-srls, -skiprunlumisince <run> <LB> : Set minimum IOV for skipping IOV in copy"
3465  << std::endl <<
3466 "-srlu, -skiprunlumiuntil <run> <LB> : Set maximum IOV for skipping IOV in copy"
3467  << std::endl <<
3468  "-tdb, -timedb <dbconn> : Set database connection for time information" <<
3469  "-ts, -timesince <time> : Set minimum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3470  std::endl <<
3471  "-tu, -timeuntil <time> : Set maximum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3472  std::endl;
3473  std::cout <<
3474  "-nrls, -newrunlumisince <run> <LB> : Set minimum of output IOV interval (use with -alliov)" << std::endl <<
3475  "-nrlu, -newrunlumiuntil <run> <LB> : Set maximum of output IOV interval (use with --aliov)" << std::endl <<
3476  "-nts, -newtimesince <time> : Set minimum of outputIOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3477  std::endl <<
3478  "-ntu, -newtimeuntil <time> : Set maximum of output IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3479  std::endl;
3480  std::cout << "See http://twiki.cern.ch/twiki/bin/view/Atlas/AtlCoolCopy for more details" << std::endl;
3481 }

◆ WriteCallback()

size_t WriteCallback ( void *  contents,
size_t  size,
size_t  nmemb,
std::string *  s 
)

Definition at line 64 of file AtlCoolCopy.cxx.

64  {
65  size_t newLength = size * nmemb;
66 
67  try
68  {
69  s->append((char *)contents, newLength);
70  }
71  catch (std::bad_alloc &e)
72  {
73  // handle memory problem
74  return 0;
75  }
76  return newLength;
77 }
AllowedVariables::e
e
Definition: AsgElectronSelectorTool.cxx:37
AtlCoolCopy
Definition: AtlCoolCopy.cxx:80
CheckTagAssociation.localtag
localtag
Definition: CheckTagAssociation.py:19
find
std::string find(const std::string &s)
return a remapped string
Definition: hcg.cxx:135
TRTCalib_cfilter.p1
p1
Definition: TRTCalib_cfilter.py:130
CheckTagAssociation.taglist
taglist
Definition: CheckTagAssociation.py:103
dq_defect_copy_defect_database.since
def since
Definition: dq_defect_copy_defect_database.py:54
python.checkMetadata.metadata
metadata
Definition: checkMetadata.py:175
dq_defect_copy_defect_database.until
def until
Definition: dq_defect_copy_defect_database.py:55
python.TileCalibTools.copyFolder
def copyFolder(dbr, dbw, folder, tagr, tagw, chanNum, pointInTime1, pointInTime2)
Definition: TileCalibTools.py:446
python.setupRTTAlg.size
int size
Definition: setupRTTAlg.py:39
TRTCalib_cfilter.p2
p2
Definition: TRTCalib_cfilter.py:131
histSizes.code
code
Definition: histSizes.py:129
tags
std::vector< std::string > tags
Definition: hcg.cxx:102
lumiFormat.i
int i
Definition: lumiFormat.py:85
LArCellNtuple.argv
argv
Definition: LArCellNtuple.py:152
maskDeadModules.run1
run1
Definition: maskDeadModules.py:40
calibdata.exception
exception
Definition: calibdata.py:495
DQHistogramMergeRegExp.argc
argc
Definition: DQHistogramMergeRegExp.py:19
contents
void contents(std::vector< std::string > &keys, TDirectory *td, const std::string &directory, const std::string &pattern, const std::string &path)
Definition: computils.cxx:321
CoraCoolFolderPtr
boost::shared_ptr< CoraCoolFolder > CoraCoolFolderPtr
Definition: CoraCoolTypes.h:15
ReadCellNoiseFromCoolCompare.run2
run2
Definition: ReadCellNoiseFromCoolCompare.py:53
printHelp
void printHelp()
Definition: AtlCoolCopy.cxx:3353
EventContainers::Mode
Mode
Definition: IdentifiableContainerBase.h:13
mc.nskip
nskip
Definition: mc.PhPy8EG_Hto4l_NNLOPS_nnlo_30_ggH125_ZZ4l.py:41
CoolTagInfo
Definition: CoolTagInfo.h:16
TileSynchronizeBch.copyall
copyall
Definition: TileSynchronizeBch.py:47
CoraCoolObjectIterPtr
boost::shared_ptr< CoraCoolObjectIter > CoraCoolObjectIterPtr
Definition: CoraCoolTypes.h:21
CoraCoolObjectPtr
boost::shared_ptr< CoraCoolObject > CoraCoolObjectPtr
Definition: CoraCoolTypes.h:18
makeDTCalibBlob_pickPhase.outtag
string outtag
Definition: makeDTCalibBlob_pickPhase.py:286
python.SystemOfUnits.s
float s
Definition: SystemOfUnits.py:147
CaloCondBlobAlgs_fillNoiseFromASCII.folder
folder
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:55
python.PyAthena.obj
obj
Definition: PyAthena.py:132