AtlCoolCopy.cxx File Reference

#include <algorithm>
#include <cmath>
#include <ctime>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include <nlohmann/json.hpp>
#include <curl/curl.h>
#include "CoolKernel/DatabaseId.h"
#include "CoolKernel/Exception.h"
#include "CoolKernel/IDatabaseSvc.h"
#include "CoolKernel/IDatabase.h"
#include "CoolKernel/IFolder.h"
#include "CoolKernel/FolderSpecification.h"
#include "CoolKernel/IFolderSet.h"
#include "CoolKernel/IObject.h"
#include "CoolKernel/IObjectIterator.h"
#include "CoolKernel/IRecordIterator.h"
#include "CoolApplication/Application.h"
#include "CoralBase/AttributeListException.h"
#include "PersistencySvc/IFileCatalog.h"
#include "StorageSvc/SimpleUtilityBase.h"
#include "RelationalAccess/ConnectionService.h"
#include "RelationalAccess/IConnectionServiceConfiguration.h"
#include "RelationalAccess/ISessionProxy.h"
#include "RelationalAccess/ITransaction.h"
#include "RelationalAccess/ISchema.h"
#include "RelationalAccess/ITable.h"
#include "RelationalAccess/IQuery.h"
#include "RelationalAccess/ICursor.h"
#include "CoraCool/CoraCoolDatabase.h"
#include "CoraCool/CoraCoolFolder.h"
#include "CoraCool/CoraCoolObjectIter.h"
#include "CoraCool/CoraCoolObject.h"
#include "CxxUtils/checker_macros.h"
#include "PoolMapElement.h"
#include "ReplicaSorter.h"
#include "CoolTagInfo.h"
#include "TFile.h"
#include "TTree.h"
#include "TH1.h"
#include "TObjString.h"

Go to the source code of this file.

Classes
class	AtlCoolCopy

Functions
size_t	WriteCallback (void *contents, size_t size, size_t nmemb, std::string *s)
void	printHelp ()
int AtlCoolCopy::copyFolder	ATLAS_NOT_THREAD_SAFE (const std::string &folder, const std::vector< std::string > &taglist)
int AtlCoolCopy::copyIOVs	ATLAS_NOT_THREAD_SAFE (const std::string &folder, const std::string &destfolder, const cool::IFolderPtr &sourcefl, const CoraCoolFolderPtr &sourceflc, const cool::IFolderPtr &destfl, const CoraCoolFolderPtr &destflc, const std::string &sourcetag, const std::string &desttag, const cool::ValidityKey since, const cool::ValidityKey until, bool timestamp, bool checkrefs, bool iscora, const cool::PayloadMode::Mode paymode, bool created)
int AtlCoolCopy::doCopy	ATLAS_NOT_THREAD_SAFE ()
	Install fatal handler with default options.
int main	ATLAS_NOT_THREAD_SAFE (int argc, const char *argv[])

Function Documentation

ATLAS_NOT_THREAD_SAFE() [1/4]

int AtlCoolCopy::doCopy ATLAS_NOT_THREAD_SAFE ( void )

inline

Install fatal handler with default options.

This is meant to be easy to call from python via ctypes.

Install fatal handler with default options.

Definition at line 2094 of file AtlCoolCopy.cxx.

                                               {
  if (isOpen()) {
    int retcode=0;
    // execute copy for all defined folders
    for (std::vector<std::string>::const_iterator ifolder=
     m_folderlist.begin();ifolder!=m_folderlist.end();++ifolder) {
      int code=copyFolder(*ifolder,m_tags);
      if (code>retcode) retcode=code;
    }
    if (!m_hiparent.empty()) {
      // fill in any hierarchical tags for parent folders
      int code=tagParents();
      if (code>retcode) retcode=code;
    }
    if (m_copytaginfo) {
      // write information (tag description, lock status) to dest tags
      int code=writeTagInfo();
      if (code>retcode) retcode=code;
    }
    if (m_checkrefs) {
      int code=0;
      if (m_poolcat.empty()) {
    code=listPoolRefs();
    if (!m_checkoutputfile.empty()) filePoolRefs();
      } else {
    code=resolvePoolRefs();
      }
      if (code>retcode) retcode=code;
    }
    if (p_rootfile) {
      p_rootfile->Write();
      delete p_rootfile;
    }
    return retcode;
  } else {
    return 10;
  }
}

ATLAS_NOT_THREAD_SAFE() [2/4]

int AtlCoolCopy::copyIOVs ATLAS_NOT_THREAD_SAFE	(	const std::string &	folder,
		const std::string &	destfolder,
		const cool::IFolderPtr &	sourcefl,
		const CoraCoolFolderPtr &	sourceflc,
		const cool::IFolderPtr &	destfl,
		const CoraCoolFolderPtr &	destflc,
		const std::string &	sourcetag,
		const std::string &	desttag,
		const cool::ValidityKey	since,
		const cool::ValidityKey	until,
		bool	timestamp,
		bool	checkrefs,
		bool	iscora,
		const cool::PayloadMode::Mode	paymode,
		bool	created )

Definition at line 879 of file AtlCoolCopy.cxx.

                                                         {
 
  std::cout << "Copying tag " << sourcetag << " of folder " << folder << 
      " to destination tag " << desttag << std::endl;
  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
  cool::FolderVersioning::Mode dvermode=destfl->versioningMode();
  // check for online mode
  int updatemode=getUpdateMode(destfl->description(),desttag);
  std::vector<std::string> dtaglist=destfl->listTags();
  // if online mode, must have --getonline and either -truncate or -alliov
  // only if destination DB is oracle
  if (updatemode==1 && m_destdb.find("oracle")!=std::string::npos &&
     (!m_getonline || (!m_truncate && !m_alliov))) {
    if (m_ignoremode) {
      std::cout << "Folder is online (UPD1) mode but IGNORING PROTECTION"
        << std::endl;
    } else {
         if(find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
             std::cout << "Folder is online mode (UPD1) and tag already exist - -getonline and -truncate or -alliov options MUST be used" << std::endl;
             return 35;
         }
         else {
             std::cout << "Folder is online mode (UPD1), new tag will be created" << std::endl;
         }
 
    }
  }
  // if UPD4 mode, must have -getbulk and either -truncate or -alliov
  if (updatemode==4 && m_destdb.find("oracle")!=std::string::npos &&
     (!m_getbulk || (!m_truncate && !m_alliov))) {
    if (m_ignoremode) {
      std::cout << "Folder is bulkreco (UPD4) mode but IGNORING PROTECTION"
        << std::endl;
    } else {
        if(find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
            std::cout << "Folder is bulkreco mode (UPD4) - -getbulk and -truncate or -alliov options MUST be used" << std::endl;
            return 35;
        } else {
            std::cout << "Folder is bulkreco mode (UPD4), new tag will be created" << std::endl;
        }
 
    }
  }
 
  // if destination tags are being checked, check if this tag already 
  // exists in destination folder, if so skip copy
  // also skip if tag is HEAD, and folder was not newly created
  if (m_checkdesttag) {
    if (find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()
        || (desttag=="HEAD" && !created)) {
      std::cout << "Destination tag " << desttag << 
    " already exists in folder " << folder << " - skip copy" << std::endl;
      return 0;
    }
  }
  bool relock=false; // tag must be relocked afterwards
  bool prot=false; //update is protected, irrespective of if tag must be locked
  if ((m_applock || (updatemode==1 && m_getonline) || updatemode==2 ||
       (updatemode==4 && m_getbulk)) && 
      dvermode==cool::FolderVersioning::MULTI_VERSION) {
    // check if the destination tag exists and is locked
    std::vector<std::string> dtaglist=destfl->listTags();
    for (std::vector<std::string>::const_iterator itr=dtaglist.begin();
     itr!=dtaglist.end();++itr) {
      if (desttag==*itr) {
    // tag exists - must protect it unless ignoremode
    prot=!m_ignoremode;
    // check lock status
    if (destfl->tagLockStatus(desttag)==cool::HvsTagLock::LOCKED) {
      std::cout << "Unlocking destination tag " << desttag << 
        " for append or UPDx access" << std::endl;
      if (m_applocksv) {
        std::cout << "Appending according to SV folder rules" << std::endl;
        if (updatemode!=3) {
          std::cout << 
            "ERROR: Only allowed for UPD3 mode tags" << std::endl;
          return 36;
        }
      }
      destfl->setTagLockStatus(desttag,cool::HvsTagLock::UNLOCKED);
      relock=true;
    }
      }
    }
  }
  int nobj=0;
  int nbuf=0;
  int nskip=0;
  int nbad=0;
  // set up tag to be used at point of insertion
  std::string localtag="";
  if (dvermode==cool::FolderVersioning::MULTI_VERSION &&
      desttag!="HEAD" && m_usertags) localtag=desttag;
  // now loop over IOVs - separate for CoraCool and COOL
  try {
    if (iscora) {
      // branch for CoraCool
      sourceflc->setPrefetchAll(false);
      CoraCoolObjectIterPtr sourceitr;
      if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
    sourceitr=sourceflc->browseObjects(since,until,m_chansel);
      } else {
    sourceitr=sourceflc->browseObjects(since,until,m_chansel,sourcetag);
      }
      // keep track of new FKs of inserted objects
      std::map<int,int> insertkeymap;
      unsigned int nref=0;
      while (sourceitr->hasNext()) {
    CoraCoolObjectPtr obj=sourceitr->next();
    // check object should not be skipped as spreading outside select-IOV
    if (m_skipout && 
        (obj->since()<m_srunemin || obj->until()>m_srunemax)) {
      ++nskip;
      continue;
    }
    // check object should not be skipped as excluded channel
    if (!m_excludechans.empty()) {
      if (find(m_excludechans.begin(),m_excludechans.end(),
           obj->channelId())!=m_excludechans.end()) {
        ++nskip;
        continue;
      }
    }
        if (nbuf==0) {
          if (m_debug) std::cout << 
       "Setup new CoraCool storage buffer at object " << nobj << std::endl;
          destflc->setupStorageBuffer();
        }
    cool::ValidityKey newsince,newuntil;
    adjustIOVs(obj->since(),obj->until(),since,until,newsince,newuntil,
           timestamp);
    // skip negative/zero IOV lengths
    if (newsince>=newuntil) {
      std::cout << "WARNING: Skipping IOV with since " << newsince <<
        ">= until" << newuntil << std::endl;
      ++nbad;
      continue;
    }
    // find the old FK, check if we have already inserted this
    // in which case a reference can be added
    int oldfk=0;
    std::map<int,int>::const_iterator ikey=insertkeymap.end();
    bool foundkey=false;
    // ensure returned data is not of zero size - if so cannot extract
    // and remember the FK
    if (obj->size()>0) {
      oldfk=sourceflc->getAttrKey(
       (*obj->begin())[sourceflc->coralFKey()]);
      ikey=insertkeymap.find(oldfk);
      foundkey=true;
    }
    if (ikey==insertkeymap.end()) {
      int newfk=
        destflc->storeObject(newsince,newuntil,obj->begin(),obj->end(),
                   obj->channelId(),localtag,
                 (!m_userupdatehead && !localtag.empty()));
      if (foundkey) insertkeymap[oldfk]=newfk;
    } else {
      destflc->referenceObject(newsince,newuntil,ikey->second,
                   obj->channelId(),localtag,
                 (!m_userupdatehead && !localtag.empty()));
      ++nref;
    }
        ++nbuf;
        // flush buffer every m_bufsize objects
        if (nbuf==m_bufsize) {
          if (m_debug) std::cout << "Flush buffer after " << nobj << "," << 
            nbuf << " objects " << std::endl;
      // note CoraCool requires explicit re-setup of buffer after a flush
          destflc->flushStorageBuffer();
      destflc->setupStorageBuffer();
          nbuf=0;
        }
    ++nobj;
      }
      sourceitr->close();
      if (nref>0) std::cout << "Reference-to-existing used for " << nref
                << " payload objects" << std::endl;
      if (nbuf>0) {
        if (m_debug) std::cout << "Final buffer flush at " << nobj << 
          "," << nbuf << std::endl;
        destflc->flushStorageBuffer();
      }
    } else {
      // branch for pure COOL objects
      cool::IObjectIteratorPtr sourceitr;
      if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
        sourceitr=sourcefl->browseObjects(since,until,m_chansel);
      } else {
        sourceitr=sourcefl->browseObjects(since,until,m_chansel,sourcetag);
      }
      while (sourceitr->goToNext()) {
        const cool::IObject& obj=sourceitr->currentRef();
    // check object should not be skipped as spreading outside select-IOV
    if (m_skipout && (obj.since()<m_srunemin || obj.until()>m_srunemax)) {
      ++nskip;
      continue;
    }
    // check object should not be skipped as excluded channel
    if (!m_excludechans.empty()) {
      if (find(m_excludechans.begin(),m_excludechans.end(),
           obj.channelId())!=m_excludechans.end()) {
        ++nskip;
        continue;
      }
    }
        if (nbuf==0) {
          if (m_debug) std::cout << "Setup new storage buffer at object " <<
      nobj << std::endl;
          destfl->setupStorageBuffer();
        }
        cool::ValidityKey newsince;
        cool::ValidityKey newuntil;
    adjustIOVs(obj.since(),obj.until(),since,until,newsince,newuntil,
           timestamp);
    // skip negative/zero IOV lengths
    if (newsince>=newuntil) {
      std::cout << "WARNING: Skipping IOV with since " << newsince <<
        ">= until" << newuntil << std::endl;
      ++nbad;
      continue;
    }
        if (checkrefs) checkRef(obj.payload(),folder,sourcetag);
        if (prot) {
        // check existing data in destination DB to ensure no clash
      if (m_applocksv) {
        // appendlockedsv mode - all IOV ends must be infinite 
        // and new since must be greater than old since
        if (newuntil!=cool::ValidityKeyMax) {
        std::cout << "New IOVs must have until=cool::ValidityKeyMax" << std::endl;
        throw cool::Exception("Illegal insert over locked IOV",
                      "AtlCoolCopy");
        }
        cool::IObjectIteratorPtr checkitr=destfl->browseObjects(newsince,
               newuntil,obj.channelId(),desttag);
        while (checkitr->goToNext()) {
          const cool::IObject& checkobj=checkitr->currentRef();
          if (checkobj.since()>=newsince) {
        std::cout << "ERROR:: Attempt to insert SV overlapping IOV whilst appending to locked tag" << std::endl;
        throw cool::Exception("Illegal insert over locked IOV",
                      "AtlCoolCopy");
          }
          if (checkobj.until()!=cool::ValidityKeyMax) {
        std::cout << "Existing IOVs must have until=cool::ValidityKeyMax" << std::endl;
        throw cool::Exception("Illegal insert over locked IOV",
                      "AtlCoolCopy");
          }
        }
        checkitr->close();
      } else if (m_applock) {
        // appendlocked mode - just make sure no objects in this IOV
            const unsigned int nexist=
          destfl->countObjects(newsince,newuntil,obj.channelId(),desttag);
        if (nexist>0) {
          std::cout << "ERROR: Attempt to insert IOV over " << nexist << 
          " objects whilst appending to locked tag" << std::endl;
          throw cool::Exception("Illegal insert over locked IOV",
                "AtlCoolCopy");
        }
      } else if (updatemode==2) {
        // UPD2 tag - make sure run is on list of runs open for update
        // only for run/lumi based folders
        if (timestamp) 
          throw cool::Exception(
       "Attempt to insert into locked UPD2 tag with timestamp format",
                    "AtlCoolCopy");
        unsigned int run1=newsince >> 32;
        unsigned int run2=(newuntil-1) >> 32;
        for (unsigned int irun=run1;irun<=run2;++irun) {
          if (!std::binary_search(m_runlist.begin(),m_runlist.end(),irun))
          { 
        std::cout << "Run " << irun << " from range [" << run1 << ","
            << run2 << "] not found in runfile list" << std::endl;
        throw cool::Exception("Illegal insert over locked IOV",
                      "AtlCoolCopy");
          }
        }
      }
      // updatemode==1 will fall through, but no checks are done here
        }
    if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
      // cool vector payload object copy
      cool::IRecordIterator& pitr=obj.payloadIterator();
      const cool::IRecordVectorPtr vptr=pitr.fetchAllAsVector();
      destfl->storeObject(newsince,newuntil,
                 *vptr,obj.channelId(),localtag,
                (!m_userupdatehead && !localtag.empty()));
      nbuf+=vptr->size();
      pitr.close();
    } else {
      // standard COOL object copy
      destfl->storeObject(newsince,newuntil,
              obj.payload(),obj.channelId(),localtag,
                (!m_userupdatehead && !localtag.empty()));
      ++nbuf;
    }
        // flush buffer every m_bufsize objects
        if (nbuf>=m_bufsize) {
          if (m_debug) std::cout << "Flush buffer after " << nobj << "," << 
            nbuf << " objects " << std::endl;
          destfl->flushStorageBuffer();
          nbuf=0;
        }
        ++nobj;
      }
      // finished with the iterator
      sourceitr->close();
      if (nbuf>0) {
        if (m_debug) std::cout << "Final buffer flush at " << nobj << 
          "," << nbuf << std::endl;
        destfl->flushStorageBuffer();
      } // end of COOL-specific part
    }
    std::cout << "Folder copied with " << nobj << " objects" << std::endl;
    if (nskip>0) std::cout << nskip << 
   " objects were skipped extending outside IOV selection or excluded channel"
               << std::endl;
    if (nbad>0) std::cout << nbad << 
      " objects were skipped having zero or negative IOV lengths" << std::endl;
  }
  // exceptions thrown from insert loop (both COOL or CORACOOL)
  catch (cool::Exception& e) {
    std::cout << "Exception thrown from copy loop: " << e.what() <<
      std::endl;
    if (relock) {
      std::cout << "Relocking destination tag " << desttag << std::endl;
      destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
    }
    return 31;
  }
  // check if tag needs to be relocked
  if (relock) {
    std::cout << "Relocking destination tag " << desttag << std::endl;
    destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
  }
 
  if (dvermode==cool::FolderVersioning::MULTI_VERSION && desttag!="HEAD") {
    if (!m_usertags) {
      // apply HEAD style tagging
      std::cout << "Tag folder with HEAD-style tagging for tag: " << desttag
            << std::endl;
      try {
        destfl->tagCurrentHead(desttag,sourcefl->tagDescription(sourcetag));
      } 
      catch (cool::Exception& e) {
        std::cout << "Exception thrown in HEAD-style folder-tag: " <<
          e.what() << std::endl;
        return 32;
      }
    }
    // record information about this tag for later
    if (m_copytaginfo) m_cooltagmap.insert(
      CoolTagMap::value_type(desttag,CoolTagInfo(m_sourceDbPtr,folder,
                destfolder,sourcetag,desttag,m_taglabel)));
    // check if any hierarchical tags reference the just-copied tag
    if (!m_hitagmap.empty()) {
      for (HiTagMap::const_iterator imap=m_hitagmap.begin();
       imap!=m_hitagmap.end();++imap) {
    try {
      if (sourcefl->findTagRelation(imap->first)==sourcetag) {
        std::cout << "Create hierarchical tag between " << desttag <<
        " and " << imap->first << " in folder " << imap->second <<
        std::endl;
        // first check this relation has not already been created
        try {
          std::string etag=destfl->resolveTag(imap->first);
          if (etag==desttag) {
        std::cout << "This relation has already been created" << std::endl;
          } else {
        std::cout << "ERROR: Tag in parent already related to " << 
          desttag << std::endl;
          }
        }
        catch (cool::Exception& e ) {
          // only do creation if does not already exist - via exception
          try {
            destfl->createTagRelation(imap->first,desttag);
            // check if this implicates a new parent folder
            // to be checked for further tags up the tree
            if (find(m_hiparent.begin(),m_hiparent.end(),imap->second)==
          m_hiparent.end()) m_hiparent.push_back(imap->second);
        // store the original tag information to transfer
        // properties later
        if (m_copytaginfo && 
            m_cooltagmap.find(imap->first)==m_cooltagmap.end())
          m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
            CoolTagInfo(m_sourceDbPtr,imap->second,imap->second,
                     imap->first,imap->first)));
          }
          catch (cool::Exception& e) {
            std::cout << "Cool exception " << e.what() << 
        "thrown in hierarchical tag creation" << std::endl;
            return 34;
          }
        }
      }
    }
    catch (cool::Exception& e) {
    }
      }
    }
  }
  return 0;
}

ATLAS_NOT_THREAD_SAFE() [3/4]

int AtlCoolCopy::copyFolder ATLAS_NOT_THREAD_SAFE	(	const std::string &	folder,
		const std::vector< std::string > &	taglist )

Definition at line 531 of file AtlCoolCopy.cxx.

                                                                {
  // get source folder
  cool::IFolderPtr sourcefl,destfl;
  CoraCoolFolderPtr sourceflc,destflc;
  try {
    sourcefl=m_sourceDbPtr->getFolder(folder);
  }
  catch (cool::Exception& e) {
    std::cout << "Could not get source folder: " << folder << std::endl;
    return 20;
  }
  const std::string& sourcedesc=sourcefl->description();
  // check if folder should be skipped from replication
  if (m_skiprep && sourcedesc.find("<norep/>")!=std::string::npos) {
    std::cout << "Folder " << folder << 
      " skipped due to <norep/> metadata" << std::endl;
    return 0;
  }
  const bool iscora=(m_coracool && 
             sourcedesc.find("<coracool")!=std::string::npos);
  if (iscora) {
    // activate CoraCool and get source folder pointer
    openCoraCool();
    sourceflc=m_sourceCoraPtr->getFolder(folder);
  }
  // setup the channel-range
  setChannelRange(sourcefl);
 
  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
  cool::FolderVersioning::Mode dvermode=vermode;
  if (m_forcesingle) {
    dvermode=cool::FolderVersioning::SINGLE_VERSION;
    std::cout << "Forcing destination folder to singleversion" << std::endl;
  }
  if (m_forcemulti) {
    dvermode=cool::FolderVersioning::MULTI_VERSION;
    std::cout << "Forcing destination folder to multiversion" << std::endl;
  }
  // setup type of folder (payload mode, inline, separate, or vector)
  const cool::PayloadMode::Mode spaymode=
    sourcefl->folderSpecification().payloadMode();
  cool::PayloadMode::Mode dpaymode=spaymode;
 
  // set name for destination folder
  std::string destfolder=folder;
  if (!m_outfolder.empty()) {
    destfolder=m_outfolder;
    std::cout << "Destination folder will be renamed to " << destfolder <<
      std::endl;
  }
  
// get destination folder, try to create if not there (and not verifying!)
  bool created=false;
  if (!m_nocopy && !m_root && !m_analyse) {
    if (!m_destDbPtr->existsFolder(destfolder) && !m_verify) {
      std::cout << "Creating folder " << destfolder << " payload-type " << 
    dpaymode << " on destination" << std::endl;
      created=true;
      std::string metadata=sourcedesc;
      if (m_forcerune || m_forcetime) {
    std::string newmeta=m_forcerune ? "run-lumi" : "time";
    std::string::size_type p1,p2;
    p1=metadata.find("<timeStamp>");
    p2=metadata.find("</timeStamp>");
    if (p1!=std::string::npos && p2!=std::string::npos) {
      metadata.replace(0,p2,"<timeStamp>"+newmeta);
      std::cout << "Forced destination folder to " << newmeta << " : "
          << metadata << std::endl;
    } else {
      std::cout << 
        "ERROR: Could not parse metadata string to force timestamp type" 
            << std::endl;
    }
      }
      // force separate or inline payload, but not if input is vector
      if (m_forcepay && spaymode!=cool::PayloadMode::VECTORPAYLOAD) 
    dpaymode=cool::PayloadMode::SEPARATEPAYLOAD;
      if (m_forcenopay && spaymode!=cool::PayloadMode::VECTORPAYLOAD) 
    dpaymode=cool::PayloadMode::INLINEPAYLOAD;
 
      try {
    if (iscora) {
      destflc=m_destCoraPtr->createFolder(destfolder,
                      sourceflc->coralTableName(),
                      sourceflc->fkSpecification(),
                      sourceflc->payloadSpecification(),
                      sourceflc->coralFKey(),
                      sourceflc->coralPKey(),
                      metadata,dvermode,true);
      std::cout << "Created CoraCool folder" << std::endl;
      destfl=m_destDbPtr->getFolder(destfolder);
    } else {
      destfl=m_destDbPtr->createFolder(destfolder,
                  cool::FolderSpecification(dvermode,
                  sourcefl->payloadSpecification(),dpaymode),
                       metadata,true);
    }
      }
      catch (cool::Exception&e ) {
        std::cout << "Create folder failed - aborting" << std::endl;
        return 30;
      }
    }
    // now get the pointer to destination folder if needed
    // note both COOL and CoraCool pointers are set if needed
    try {
      if (iscora) destflc=m_destCoraPtr->getFolder(destfolder);
      destfl=m_destDbPtr->getFolder(destfolder);
    }
    catch (cool::Exception& e) {
      std::cout << "Could not get destination folder: " << destfolder 
        << std::endl;
      return 21;
    }
    // check payload specifications of folders are the same
    const cool::IRecordSpecification& sourcespec=
      sourcefl->payloadSpecification();
    const cool::IRecordSpecification& destspec=
      destfl->payloadSpecification();
    if (!(sourcespec==destspec)) {
      bool badspec=false;
      if (m_ignorespec) {
    // specifications differ - check names are same
    std::cout << 
      "WARNING Source and destination folder specifications differ" << 
      std::endl;
    for (unsigned int i=0;i<sourcespec.size();++i) {
      const std::string& sname=sourcespec[i].name();
      if (!destspec.exists(sname)) {
        std::cout << "ERROR: Field " << sname << " absent from destination"
        << std::endl;
        badspec=true;
      }
    }
      } else {
    badspec=true;
      }
      if (badspec) {
        std::cout << 
      "ERROR Source and destination folder specifications differ" 
          << std::endl;
        return 22;
      }
    }
    // check folder descriptions are the same - just print WARNING if not
    const std::string& destdesc=destfl->description();
    if (sourcedesc!=destdesc) {
      std::cout << "WARNING: Source and destination folder descriptions (meta-data) differ" << std::endl;
      std::cout << "Source folder: " << sourcedesc << std::endl;
      std::cout << "Destn  folder: " << destdesc << std::endl;
    }
    // check payload modes are same - print warning if not
    const cool::PayloadMode::Mode dpaymode=
      destfl->folderSpecification().payloadMode();
    if (spaymode!=dpaymode) {
      std::cout << "WARNING: Source (" << spaymode << ") and destination (" <<
    dpaymode << " folder payload modes differ" << std::endl;
    }
    
    // check/set channels table if requested
    if (!m_nochannel) {
      if (!checkChannels(folder,sourcefl,destfl,created)) return 23;
    }
  }
  // if only copying structures, stop here
  if (m_nodata) return 0;
  // extract folder/range information
  std::cout << "Start to process folder: " << folder;
  // check for timestamp XML 
  bool timestamp=(sourcefl->description().find("<timeStamp>time")!=
          std::string::npos);
  cool::ValidityKey since,until;
  if (timestamp) {
    since=m_timemin;
    until=m_timemax;
    std::cout << " (timestamp)" << std::endl;
  } else {
    since=m_runemin;
    until=m_runemax;
    std::cout << " (run/lumi)" << std::endl;
  }
  if (m_alliov || m_truncate) 
    std::cout << "Output IOVs will be modified" << std::endl;
  if (m_skipout) 
    std::cout << "IOVs extending outside selection range will be skipped"
          << std::endl;
 
  bool checkrefs=false;
  const std::string name0=sourcefl->payloadSpecification()[0].name();
  if (m_checkrefs && (name0=="PoolRef" || name0=="fileGUID")) {
    checkrefs=true;
    std::cout << "Check POOL references in folder " << folder << std::endl;
  }
  // if nocopy, and checking POOL references, and folder is not a POOL ref one
  // can skip the data access 
  if (m_checkrefs && m_nocopy && !checkrefs) return 0;
 
  // check for <fullrep/> metadata indicating copy all tags if -skiprep option
  bool copyall=
    (m_skiprep && (sourcedesc.find("<fullrep/>")!=std::string::npos));
  if (copyall) std::cout << 
   "All tags in folder will be copied due to <fullrep/> metadata" << std::endl;
 
  // set up true list of tags to copy
  std::vector<std::string> tags;
  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
    std::cout << "Single version folder" << std::endl;
    tags.emplace_back("HEAD");
  } else {
    std::cout << "Multi version folder: consider tags [ ";
    // get list of tags in this node which are requested
    const std::vector<std::string> foldertags=sourcefl->listTags();
    for (std::vector<std::string>::const_iterator itag=foldertags.begin();
     itag!=foldertags.end();++itag) {
      bool copyit=copyall;
      // if input taglist is empty, take all tags in folder
      if (taglist.empty() ||
      find(taglist.begin(),taglist.end(),*itag)!=taglist.end()) 
    copyit=true;
      for (std::vector<std::string>::const_iterator imtag=m_magic.begin();
       imtag!=m_magic.end();++imtag) {
    if (itag->find(*imtag)!=std::string::npos) copyit=true;
      }
      if (copyit) {
    tags.push_back(*itag);
        std::cout << *itag << " ";
      }
    }
    // if no tags were found, or doing inclusive hierarchical tag copying
    //  try resolving input tags hierarchically
    if (tags.empty() || m_hitag) {
      for (std::vector<std::string>::const_iterator itag=taglist.begin();
       itag!=taglist.end();++itag) {
    try {
      std::string htag=sourcefl->resolveTag(*itag);
      if (find(tags.begin(),tags.end(),htag)==tags.end()) {
        std::cout << *itag << "=" << htag << " ";
        tags.push_back(std::move(htag));
      }
    }
    // ignore exceptions indicating tag not defined here
    // note std::exception rather than COOL exception to cover case
    // when trying to resolve a leaf tag in the '/' folder, which
    // throws a coral AttributeException
    catch (std::exception& e) { }
      }
    }
    std::cout << "]" << std::endl;
    // if still no tags were found, or forced HEAD tag copying, or no
    // tags in the folder at all (=MV folder being used as SV), add HEAD
    if (((tags.empty() || m_includehead) && !m_excludehead) ||
    foldertags.empty())
      tags.emplace_back("HEAD");
  }
 
  sourcefl->setPrefetchAll(false);
  for (std::vector<std::string>::const_iterator itag=tags.begin();
       itag!=tags.end();++itag) {
    std::string outtag=*itag;
    if (!m_outtag.empty()) outtag=m_outtag;
    int retcode;
    // verify or copy folder
    if (m_verify) {
      if (!destfl) {
        retcode = 1;
      }
      else {
        retcode=verifyIOVs(folder,sourcefl,sourceflc,destfl,destflc,
                           *itag,since,until,checkrefs,iscora,spaymode);
      }
    } else if (m_nocopy) {
      retcode=nocopyIOVs(folder,sourcefl,*itag,since,until,checkrefs);
    } else if (m_root) {
      retcode=rootIOVs(folder,sourcefl,*itag,since,until,timestamp);
    } else if (m_analyse) {
      retcode=analyseIOVs(folder,sourcefl,*itag,since,until,timestamp);
    } else {
      retcode=copyIOVs(folder,destfolder,sourcefl,sourceflc,destfl,destflc,
               *itag,outtag,since,until,timestamp,checkrefs,iscora,
               spaymode,created);
    }
    if (retcode!=0) {
      std::cout << "ERROR operation failed for folder " << folder << " tag " <<
    *itag << std::endl;
      return retcode;
    }
  }
  return 0;
}

ATLAS_NOT_THREAD_SAFE() [4/4]

int main ATLAS_NOT_THREAD_SAFE	(	int	argc,
		const char *	argv[] )

Definition at line 3484 of file AtlCoolCopy.cxx.

                                                              {
  int retcode=0;
  if (argc<3) {
    printHelp();
    retcode=1;
  } else {
    AtlCoolCopy mycopy(argv[1],argv[2]);
    retcode=mycopy.setOpts(argc-3,&argv[3]);
    if (retcode==999) {
      printHelp();
      return 0;
    }
    if (retcode==0) retcode=mycopy.doCopy();
  }
  if (retcode>0) std::cout << "ERROR AtlCoolCopy.exe fails with exit code " << 
           retcode << std::endl;
  return retcode;
}

printHelp()

void printHelp

(

)

Definition at line 3354 of file AtlCoolCopy.cxx.

                 {
  std::cout << "usage: AtlCoolCopy.exe <sourceCoolDB> <destinationCoolDB> { <options> }" << std::endl;
  std::cout << "Options are (see doc/mainpage.h for more):" << std::endl <<
    "-a, -alliov : set IOVs on destination to [ValidityKeyMin, ValidityKeyMax]"   << std::endl <<" or following settings of -nrls,-nrlu,-nts,-ntu" << 
    std::endl <<
    "-ag, -addguid <guid> : Add GUID to list accessed for POOL checks" << 
    std::endl <<
    "-alf, -addlfn <LFN> : Add LFN to list accessed for POOL checks" << 
    std::endl <<
    "-al, -appendlocked : Allow locked tags to be updated if no overlap" << 
    std::endl << 
    "-alsv, -appendlockedsv : Allow locked tag update for openended IOVs" << 
    std::endl << 
    "-ana, -analyse <delta_t>: produce analysis ROOT file (filename = dest DB argument)" << std::endl << 
"-bs, -buffersize <size> : set size of bulkstorage output buf to <size> objs"
        << std::endl << 
    "-c, -create : create destination DB if not already existing" 
        << std::endl <<
    "-cd, -checkdest : Check destination DB and skip already existing tags"
        << std::endl <<
    "-ch, -channel : restrict selection to given channel or range specified as c1:c2" << std::endl <<
    "-ch1, -channel1 : specify start of a range of channels" << std::endl <<
    "-ch2, -channel2 : specify end of a range of channels" << std::endl <<
    "-cf, -checkfiles : check POOL files can be opened and have correct GUID" 
        << std::endl << 
    "-co, -checkoutput <file> : write POOL file check output on file"
        << std::endl << "-cti, -copytaginfo : Copy tag descriptions"
        << std::endl << "-ctl, -copytaglock : Copy tag locked status and descriptions"
        << std::endl << "-cr, -checkrefs : check POOL references"
        << std::endl << "-d, -debug : produce debug output" << std::endl;
  std::cout <<
    "-ds, -dataset : output register.sh for creating DQ2 datasets " 
        << std::endl <<
    "-ec, -excludechannel : exclude given channel from copy" << std::endl <<
    "-eh, -excludehead : exclude HEAD tag even if no tags found in MV folders" 
        <<  std::endl <<
    "-ih, -includehead : include HEAD as well as any tags in MV folders" << 
    std::endl <<
    "-e, -exclude <pattern> : exclude folders" << std::endl <<
    "-f, -folder <pattern> : include folders" << std::endl <<
    "-fs, -forcesingle : force destination folder to be singleversion" 
        << std::endl <<
    "-fm, -forcemulti : force destination folder to be multiversion" 
        << std::endl <<
    "-frl, -forcerunlumi : force destination folder to be run/LB indexed" 
        << std::endl <<
    "-ftm, -forcetime : force destination folder to be timestamp indexed" 
        << std::endl <<
    "-fp, -forcepayload : Force destn folders to be created with payload tbl" << std::endl <<
    "-fnp, -forcenopayload : Force destn folders to be created without payload tbl" << std::endl <<
    "-forcerecreate : delete and recreate destination database" << std::endl;
  std::cout << "-ht, -headtag : Use HEAD-style tagging"
        << std::endl <<
   "-go, -getonline : Set minimum run number (-rls setting) to next ONLINE run"
        << std::endl <<
 "-gb, -getbulk :  Set minimum run number (-rls setting) to next bulk reco run"
        << std::endl <<
    "-gt, -gettime : Extract timestamp information for given run number range" 
        << std::endl <<
    "-h, -help : print help and exit" << std::endl << 
    "-hi, -hitag : Copy hierrchical tags inclusively" << std::endl << 
    "-ignoremode <pwd> : Ignore UPDx mode protection (experts only)" 
        << std::endl <<
    "-is, -ignorespec : Ignore differences in folder spec if names are all equal" << std::endl <<
    "-lo, -lockedonly : Only copy locked/partially-locked top-level tags"
        << std::endl <<
    "-nc, -nocopy : Do not actually copy, just read source DB" << std::endl <<
    "-nch, -nochannel : Do not check or copy channel information" << std::endl 
<<  "-ncr, -noclobroot: Do not copy CLOB data into ROOT files" << std::endl
<< "-noc, -nocoracool : Do not copy Coracool structure payloads " << std::endl <<
    "-nd, -nodata : Copy only folder structures, not data" << std::endl <<
    "-nh, -nohitag : Do not follow hierarchical tag relations" << std::endl <<
"-mc, -mergecat <catfile> : specify POOL file catalogue for new dataset making"
        << std::endl;
  std::cout << "-of, -outfolder <folder> : rename folder on output" 
        << std::endl <<
    "-onr, -onlinerun : Retrieve run number from online server (not replica)" <<std::endl <<
    "-ot, -outtag : " << "Rename tag on output" << std::endl;
  std::cout <<
    "-pa, -poolall : Output all POOL files (incl good) when checking files"
        << std::endl <<
"-pc, -poolcat <catfile> : specify POOL file catalogue for ref checking" 
        << std::endl <<
    "-pf, -parfile <file> : Read additional options/parameters from file" << 
    std::endl << 
"-pt, -prunetags : Copy only hierarchical tags descending from specified toptags"
        << std::endl <<
"-rdo, -readoracle : force data to be read from Oracle, using dbreplica.config"
        << std::endl <<
    "-sl, -seal <val>: Set SEAL (hence COOL/POOL) output level to <val>"
        << std::endl << 
    "-sr, -skiprep : Skip folders having <norep/> folder metadata tag" <<
    std::endl << 
    "-t, -tag <tag> : Copy multiversion data with tag <tag>" << std::endl <<
    "-mt, -magic <tag> : Include magic tags involving the given pattern " << std::endl << 
"-tr, -truncate : Set destination IOVs outside query interval to query interval"
        << std::endl <<
"-tl, -taglabel : Specify tag description to enter in destination DB" 
        << std::endl << 
    "-uht, -userheadtag : Also copy user tag data to the HEAD" << std::endl <<
    "-v, -verify : Verify data present on destination rather than copying" <<
    std::endl << 
    "-ro, -root : Produce ROOT output file instead of copying to COOL"
        << std::endl <<
    "-zn, -zeronull : Zero NULLs in ROOT file instead of skipping them" << 
    std::endl << 
    "-rls, -runlumisince <run> <LB> : Set minimum IOV interval" << std::endl <<
    "-rlu, -runlumiuntil <run> <LB> : Set maximum IOV interval" << std::endl <<
    "-rs, -runsince <run> : Set minimum IOV interval" << std::endl << 
    "-ru, -rununtil <run> : Set maximum IOV interval" << std::endl <<
    "-r, -run <run> : Copy only run <run>" << std::endl <<
"-srls, -skiprunlumisince <run> <LB> : Set minimum IOV for skipping IOV in copy" 
        << std::endl <<
"-srlu, -skiprunlumiuntil <run> <LB> : Set maximum IOV for skipping IOV in copy" 
        << std::endl <<
    "-tdb, -timedb <dbconn> : Set database connection for time information" <<
    "-ts, -timesince <time> : Set minimum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" << 
    std::endl <<
    "-tu, -timeuntil <time> : Set maximum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" << 
    std::endl;
  std::cout << 
    "-nrls, -newrunlumisince <run> <LB> : Set minimum of output IOV interval (use with -alliov)" << std::endl <<
    "-nrlu, -newrunlumiuntil <run> <LB> : Set maximum of output IOV interval (use with --aliov)" << std::endl <<
    "-nts, -newtimesince <time> : Set minimum of outputIOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" << 
    std::endl <<
    "-ntu, -newtimeuntil <time> : Set maximum of output IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" << 
    std::endl;
  std::cout << "See http://twiki.cern.ch/twiki/bin/view/Atlas/AtlCoolCopy for more details" << std::endl;
}

WriteCallback()

size_t WriteCallback	(	void *	contents,
		size_t	size,
		size_t	nmemb,
		std::string *	s )

Definition at line 61 of file AtlCoolCopy.cxx.

                                                                              {
  size_t newLength = size * nmemb;
 
  try
  {
    s->append((char *)contents, newLength);
  }
  catch (std::bad_alloc &e)
  {
    // handle memory problem
    return 0;
  }
  return newLength;
}