AtlCoolCopy.cxx File Reference

#include <algorithm>
#include <cmath>
#include <ctime>
#include <fstream>
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include <nlohmann/json.hpp>
#include <curl/curl.h>
#include "CoolKernel/DatabaseId.h"
#include "CoolKernel/Exception.h"
#include "CoolKernel/IDatabaseSvc.h"
#include "CoolKernel/IDatabase.h"
#include "CoolKernel/IFolder.h"
#include "CoolKernel/FolderSpecification.h"
#include "CoolKernel/IFolderSet.h"
#include "CoolKernel/IObject.h"
#include "CoolKernel/IObjectIterator.h"
#include "CoolKernel/IRecordIterator.h"
#include "CoolApplication/Application.h"
#include "CoralBase/AttributeListException.h"
#include "FileCatalog/IFileCatalog.h"
#include "PersistencySvc/IDatabase.h"
#include "PersistencySvc/ISession.h"
#include "PersistencySvc/ITransaction.h"
#include "PersistencySvc/SimpleUtilityBase.h"
#include "RelationalAccess/ConnectionService.h"
#include "RelationalAccess/IConnectionServiceConfiguration.h"
#include "RelationalAccess/ISessionProxy.h"
#include "RelationalAccess/ITransaction.h"
#include "RelationalAccess/ISchema.h"
#include "RelationalAccess/ITable.h"
#include "RelationalAccess/IQuery.h"
#include "RelationalAccess/ICursor.h"
#include "CoraCool/CoraCoolDatabase.h"
#include "CoraCool/CoraCoolFolder.h"
#include "CoraCool/CoraCoolObjectIter.h"
#include "CoraCool/CoraCoolObject.h"
#include "CxxUtils/checker_macros.h"
#include "PoolMapElement.h"
#include "ReplicaSorter.h"
#include "CoolTagInfo.h"
#include "TFile.h"
#include "TTree.h"
#include "TH1.h"
#include "TObjString.h"

Go to the source code of this file.

Classes
class	AtlCoolCopy

Functions
size_t	WriteCallback (void *contents, size_t size, size_t nmemb, std::string *s)
void	printHelp ()
int AtlCoolCopy::copyFolder	ATLAS_NOT_THREAD_SAFE (const std::string &folder, const std::vector< std::string > &taglist)
int AtlCoolCopy::copyIOVs	ATLAS_NOT_THREAD_SAFE (const std::string &folder, const std::string &destfolder, const cool::IFolderPtr &sourcefl, const CoraCoolFolderPtr &sourceflc, const cool::IFolderPtr &destfl, const CoraCoolFolderPtr &destflc, const std::string &sourcetag, const std::string &desttag, const cool::ValidityKey since, const cool::ValidityKey until, bool timestamp, bool checkrefs, bool iscora, const cool::PayloadMode::Mode paymode, bool created)
int AtlCoolCopy::doCopy	ATLAS_NOT_THREAD_SAFE ()
	Install fatal handler with default options. More...
int main	ATLAS_NOT_THREAD_SAFE (int argc, const char *argv[])

Function Documentation

ATLAS_NOT_THREAD_SAFE() [1/4]

int AtlCoolCopy::doCopy ATLAS_NOT_THREAD_SAFE ( )

inline

Install fatal handler with default options.

This is meant to be easy to call from python via ctypes.

Definition at line 2092 of file AtlCoolCopy.cxx.

                                               {
  if (isOpen()) {
    int retcode=0;
    // execute copy for all defined folders
    for (std::vector<std::string>::const_iterator ifolder=
     m_folderlist.begin();ifolder!=m_folderlist.end();++ifolder) {
      int code=copyFolder(*ifolder,m_tags);
      if (code>retcode) retcode=code;
    }
    if (!m_hiparent.empty()) {
      // fill in any hierarchical tags for parent folders
      int code=tagParents();
      if (code>retcode) retcode=code;
    }
    if (m_copytaginfo) {
      // write information (tag description, lock status) to dest tags
      int code=writeTagInfo();
      if (code>retcode) retcode=code;
    }
    if (m_checkrefs) {
      int code=0;
      if (m_poolcat.empty()) {
    code=listPoolRefs();
    if (!m_checkoutputfile.empty()) filePoolRefs();
      } else {
    code=resolvePoolRefs();
      }
      if (code>retcode) retcode=code;
    }
    if (p_rootfile) {
      p_rootfile->Write();
      delete p_rootfile;
    }
    return retcode;
  } else {
    return 10;
  }
}

ATLAS_NOT_THREAD_SAFE() [2/4]

int AtlCoolCopy::copyIOVs ATLAS_NOT_THREAD_SAFE	(	const std::string &	folder,
		const std::string &	destfolder,
		const cool::IFolderPtr &	sourcefl,
		const CoraCoolFolderPtr &	sourceflc,
		const cool::IFolderPtr &	destfl,
		const CoraCoolFolderPtr &	destflc,
		const std::string &	sourcetag,
		const std::string &	desttag,
		const cool::ValidityKey	since,
		const cool::ValidityKey	until,
		bool	timestamp,
		bool	checkrefs,
		bool	iscora,
		const cool::PayloadMode::Mode	paymode,
		bool	created
	)

Definition at line 877 of file AtlCoolCopy.cxx.

                                                         {
 
  std::cout << "Copying tag " << sourcetag << " of folder " << folder << 
      " to destination tag " << desttag << std::endl;
  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
  cool::FolderVersioning::Mode dvermode=destfl->versioningMode();
  // check for online mode
  int updatemode=getUpdateMode(destfl->description(),desttag);
  std::vector<std::string> dtaglist=destfl->listTags();
  // if online mode, must have --getonline and either -truncate or -alliov
  // only if destination DB is oracle
  if (updatemode==1 && m_destdb.find("oracle")!=std::string::npos &&
     (!m_getonline || (!m_truncate && !m_alliov))) {
    if (m_ignoremode) {
      std::cout << "Folder is online (UPD1) mode but IGNORING PROTECTION"
        << std::endl;
    } else {
         if(find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
             std::cout << "Folder is online mode (UPD1) and tag already exist - -getonline and -truncate or -alliov options MUST be used" << std::endl;
             return 35;
         }
         else {
             std::cout << "Folder is online mode (UPD1), new tag will be created" << std::endl;
         }
 
    }
  }
  // if UPD4 mode, must have -getbulk and either -truncate or -alliov
  if (updatemode==4 && m_destdb.find("oracle")!=std::string::npos &&
     (!m_getbulk || (!m_truncate && !m_alliov))) {
    if (m_ignoremode) {
      std::cout << "Folder is bulkreco (UPD4) mode but IGNORING PROTECTION"
        << std::endl;
    } else {
        if(find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
            std::cout << "Folder is bulkreco mode (UPD4) - -getbulk and -truncate or -alliov options MUST be used" << std::endl;
            return 35;
        } else {
            std::cout << "Folder is bulkreco mode (UPD4), new tag will be created" << std::endl;
        }
 
    }
  }
 
  // if destination tags are being checked, check if this tag already 
  // exists in destination folder, if so skip copy
  // also skip if tag is HEAD, and folder was not newly created
  if (m_checkdesttag) {
    if (find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()
        || (desttag=="HEAD" && !created)) {
      std::cout << "Destination tag " << desttag << 
    " already exists in folder " << folder << " - skip copy" << std::endl;
      return 0;
    }
  }
  bool relock=false; // tag must be relocked afterwards
  bool prot=false; //update is protected, irrespective of if tag must be locked
  if ((m_applock || (updatemode==1 && m_getonline) || updatemode==2 ||
       (updatemode==4 && m_getbulk)) && 
      dvermode==cool::FolderVersioning::MULTI_VERSION) {
    // check if the destination tag exists and is locked
    std::vector<std::string> dtaglist=destfl->listTags();
    for (std::vector<std::string>::const_iterator itr=dtaglist.begin();
     itr!=dtaglist.end();++itr) {
      if (desttag==*itr) {
    // tag exists - must protect it unless ignoremode
    prot=!m_ignoremode;
    // check lock status
    if (destfl->tagLockStatus(desttag)==cool::HvsTagLock::LOCKED) {
      std::cout << "Unlocking destination tag " << desttag << 
        " for append or UPDx access" << std::endl;
      if (m_applocksv) {
        std::cout << "Appending according to SV folder rules" << std::endl;
        if (updatemode!=3) {
          std::cout << 
            "ERROR: Only allowed for UPD3 mode tags" << std::endl;
          return 36;
        }
      }
      destfl->setTagLockStatus(desttag,cool::HvsTagLock::UNLOCKED);
      relock=true;
    }
      }
    }
  }
  int nobj=0;
  int nbuf=0;
  int nskip=0;
  int nbad=0;
  // set up tag to be used at point of insertion
  std::string localtag="";
  if (dvermode==cool::FolderVersioning::MULTI_VERSION &&
      desttag!="HEAD" && m_usertags) localtag=desttag;
  // now loop over IOVs - separate for CoraCool and COOL
  try {
    if (iscora) {
      // branch for CoraCool
      sourceflc->setPrefetchAll(false);
      CoraCoolObjectIterPtr sourceitr;
      if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
    sourceitr=sourceflc->browseObjects(since,until,m_chansel);
      } else {
    sourceitr=sourceflc->browseObjects(since,until,m_chansel,sourcetag);
      }
      // keep track of new FKs of inserted objects
      std::map<int,int> insertkeymap;
      unsigned int nref=0;
      while (sourceitr->hasNext()) {
    CoraCoolObjectPtr obj=sourceitr->next();
    // check object should not be skipped as spreading outside select-IOV
    if (m_skipout && 
        (obj->since()<m_srunemin || obj->until()>m_srunemax)) {
      ++nskip;
      continue;
    }
    // check object should not be skipped as excluded channel
    if (!m_excludechans.empty()) {
      if (find(m_excludechans.begin(),m_excludechans.end(),
           obj->channelId())!=m_excludechans.end()) {
        ++nskip;
        continue;
      }
    }
        if (nbuf==0) {
          if (m_debug) std::cout << 
       "Setup new CoraCool storage buffer at object " << nobj << std::endl;
          destflc->setupStorageBuffer();
        }
    cool::ValidityKey newsince,newuntil;
    adjustIOVs(obj->since(),obj->until(),since,until,newsince,newuntil,
           timestamp);
    // skip negative/zero IOV lengths
    if (newsince>=newuntil) {
      std::cout << "WARNING: Skipping IOV with since " << newsince <<
        ">= until" << newuntil << std::endl;
      ++nbad;
      continue;
    }
    // find the old FK, check if we have already inserted this
    // in which case a reference can be added
    int oldfk=0;
    std::map<int,int>::const_iterator ikey=insertkeymap.end();
    bool foundkey=false;
    // ensure returned data is not of zero size - if so cannot extract
    // and remember the FK
    if (obj->size()>0) {
      oldfk=sourceflc->getAttrKey(
       (*obj->begin())[sourceflc->coralFKey()]);
      ikey=insertkeymap.find(oldfk);
      foundkey=true;
    }
    if (ikey==insertkeymap.end()) {
      int newfk=
        destflc->storeObject(newsince,newuntil,obj->begin(),obj->end(),
                   obj->channelId(),localtag,
                 (!m_userupdatehead && !localtag.empty()));
      if (foundkey) insertkeymap[oldfk]=newfk;
    } else {
      destflc->referenceObject(newsince,newuntil,ikey->second,
                   obj->channelId(),localtag,
                 (!m_userupdatehead && !localtag.empty()));
      ++nref;
    }
        ++nbuf;
        // flush buffer every m_bufsize objects
        if (nbuf==m_bufsize) {
          if (m_debug) std::cout << "Flush buffer after " << nobj << "," << 
            nbuf << " objects " << std::endl;
      // note CoraCool requires explicit re-setup of buffer after a flush
          destflc->flushStorageBuffer();
      destflc->setupStorageBuffer();
          nbuf=0;
        }
    ++nobj;
      }
      sourceitr->close();
      if (nref>0) std::cout << "Reference-to-existing used for " << nref
                << " payload objects" << std::endl;
      if (nbuf>0) {
        if (m_debug) std::cout << "Final buffer flush at " << nobj << 
          "," << nbuf << std::endl;
        destflc->flushStorageBuffer();
      }
    } else {
      // branch for pure COOL objects
      cool::IObjectIteratorPtr sourceitr;
      if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
        sourceitr=sourcefl->browseObjects(since,until,m_chansel);
      } else {
        sourceitr=sourcefl->browseObjects(since,until,m_chansel,sourcetag);
      }
      while (sourceitr->goToNext()) {
        const cool::IObject& obj=sourceitr->currentRef();
    // check object should not be skipped as spreading outside select-IOV
    if (m_skipout && (obj.since()<m_srunemin || obj.until()>m_srunemax)) {
      ++nskip;
      continue;
    }
    // check object should not be skipped as excluded channel
    if (!m_excludechans.empty()) {
      if (find(m_excludechans.begin(),m_excludechans.end(),
           obj.channelId())!=m_excludechans.end()) {
        ++nskip;
        continue;
      }
    }
        if (nbuf==0) {
          if (m_debug) std::cout << "Setup new storage buffer at object " <<
      nobj << std::endl;
          destfl->setupStorageBuffer();
        }
        cool::ValidityKey newsince;
        cool::ValidityKey newuntil;
    adjustIOVs(obj.since(),obj.until(),since,until,newsince,newuntil,
           timestamp);
    // skip negative/zero IOV lengths
    if (newsince>=newuntil) {
      std::cout << "WARNING: Skipping IOV with since " << newsince <<
        ">= until" << newuntil << std::endl;
      ++nbad;
      continue;
    }
        if (checkrefs) checkRef(obj.payload(),folder,sourcetag);
        if (prot) {
        // check existing data in destination DB to ensure no clash
      if (m_applocksv) {
        // appendlockedsv mode - all IOV ends must be infinite 
        // and new since must be greater than old since
        if (newuntil!=cool::ValidityKeyMax) {
        std::cout << "New IOVs must have until=cool::ValidityKeyMax" << std::endl;
        throw cool::Exception("Illegal insert over locked IOV",
                      "AtlCoolCopy");
        }
        cool::IObjectIteratorPtr checkitr=destfl->browseObjects(newsince,
               newuntil,obj.channelId(),desttag);
        while (checkitr->goToNext()) {
          const cool::IObject& checkobj=checkitr->currentRef();
          if (checkobj.since()>=newsince) {
        std::cout << "ERROR:: Attempt to insert SV overlapping IOV whilst appending to locked tag" << std::endl;
        throw cool::Exception("Illegal insert over locked IOV",
                      "AtlCoolCopy");
          }
          if (checkobj.until()!=cool::ValidityKeyMax) {
        std::cout << "Existing IOVs must have until=cool::ValidityKeyMax" << std::endl;
        throw cool::Exception("Illegal insert over locked IOV",
                      "AtlCoolCopy");
          }
        }
        checkitr->close();
      } else if (m_applock) {
        // appendlocked mode - just make sure no objects in this IOV
            const unsigned int nexist=
          destfl->countObjects(newsince,newuntil,obj.channelId(),desttag);
        if (nexist>0) {
          std::cout << "ERROR: Attempt to insert IOV over " << nexist << 
          " objects whilst appending to locked tag" << std::endl;
          throw cool::Exception("Illegal insert over locked IOV",
                "AtlCoolCopy");
        }
      } else if (updatemode==2) {
        // UPD2 tag - make sure run is on list of runs open for update
        // only for run/lumi based folders
        if (timestamp) 
          throw cool::Exception(
       "Attempt to insert into locked UPD2 tag with timestamp format",
                    "AtlCoolCopy");
        unsigned int run1=newsince >> 32;
        unsigned int run2=(newuntil-1) >> 32;
        for (unsigned int irun=run1;irun<=run2;++irun) {
          if (!std::binary_search(m_runlist.begin(),m_runlist.end(),irun))
          { 
        std::cout << "Run " << irun << " from range [" << run1 << ","
            << run2 << "] not found in runfile list" << std::endl;
        throw cool::Exception("Illegal insert over locked IOV",
                      "AtlCoolCopy");
          }
        }
      }
      // updatemode==1 will fall through, but no checks are done here
        }
    if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
      // cool vector payload object copy
      cool::IRecordIterator& pitr=obj.payloadIterator();
      const cool::IRecordVectorPtr vptr=pitr.fetchAllAsVector();
      destfl->storeObject(newsince,newuntil,
                 *vptr,obj.channelId(),localtag,
                (!m_userupdatehead && !localtag.empty()));
      nbuf+=vptr->size();
      pitr.close();
    } else {
      // standard COOL object copy
      destfl->storeObject(newsince,newuntil,
              obj.payload(),obj.channelId(),localtag,
                (!m_userupdatehead && !localtag.empty()));
      ++nbuf;
    }
        // flush buffer every m_bufsize objects
        if (nbuf>=m_bufsize) {
          if (m_debug) std::cout << "Flush buffer after " << nobj << "," << 
            nbuf << " objects " << std::endl;
          destfl->flushStorageBuffer();
          nbuf=0;
        }
        ++nobj;
      }
      // finished with the iterator
      sourceitr->close();
      if (nbuf>0) {
        if (m_debug) std::cout << "Final buffer flush at " << nobj << 
          "," << nbuf << std::endl;
        destfl->flushStorageBuffer();
      } // end of COOL-specific part
    }
    std::cout << "Folder copied with " << nobj << " objects" << std::endl;
    if (nskip>0) std::cout << nskip << 
   " objects were skipped extending outside IOV selection or excluded channel"
               << std::endl;
    if (nbad>0) std::cout << nbad << 
      " objects were skipped having zero or negative IOV lengths" << std::endl;
  }
  // exceptions thrown from insert loop (both COOL or CORACOOL)
  catch (cool::Exception& e) {
    std::cout << "Exception thrown from copy loop: " << e.what() <<
      std::endl;
    if (relock) {
      std::cout << "Relocking destination tag " << desttag << std::endl;
      destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
    }
    return 31;
  }
  // check if tag needs to be relocked
  if (relock) {
    std::cout << "Relocking destination tag " << desttag << std::endl;
    destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
  }
 
  if (dvermode==cool::FolderVersioning::MULTI_VERSION && desttag!="HEAD") {
    if (!m_usertags) {
      // apply HEAD style tagging
      std::cout << "Tag folder with HEAD-style tagging for tag: " << desttag
            << std::endl;
      try {
        destfl->tagCurrentHead(desttag,sourcefl->tagDescription(sourcetag));
      } 
      catch (cool::Exception& e) {
        std::cout << "Exception thrown in HEAD-style folder-tag: " <<
          e.what() << std::endl;
        return 32;
      }
    }
    // record information about this tag for later
    if (m_copytaginfo) m_cooltagmap.insert(
      CoolTagMap::value_type(desttag,CoolTagInfo(m_sourceDbPtr,folder,
                destfolder,sourcetag,desttag,m_taglabel)));
    // check if any hierarchical tags reference the just-copied tag
    if (!m_hitagmap.empty()) {
      for (HiTagMap::const_iterator imap=m_hitagmap.begin();
       imap!=m_hitagmap.end();++imap) {
    try {
      if (sourcefl->findTagRelation(imap->first)==sourcetag) {
        std::cout << "Create hierarchical tag between " << desttag <<
        " and " << imap->first << " in folder " << imap->second <<
        std::endl;
        // first check this relation has not already been created
        try {
          std::string etag=destfl->resolveTag(imap->first);
          if (etag==desttag) {
        std::cout << "This relation has already been created" << std::endl;
          } else {
        std::cout << "ERROR: Tag in parent already related to " << 
          desttag << std::endl;
          }
        }
        catch (cool::Exception& e ) {
          // only do creation if does not already exist - via exception
          try {
            destfl->createTagRelation(imap->first,desttag);
            // check if this implicates a new parent folder
            // to be checked for further tags up the tree
            if (find(m_hiparent.begin(),m_hiparent.end(),imap->second)==
          m_hiparent.end()) m_hiparent.push_back(imap->second);
        // store the original tag information to transfer
        // properties later
        if (m_copytaginfo && 
            m_cooltagmap.find(imap->first)==m_cooltagmap.end())
          m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
            CoolTagInfo(m_sourceDbPtr,imap->second,imap->second,
                     imap->first,imap->first)));
          }
          catch (cool::Exception& e) {
            std::cout << "Cool exception " << e.what() << 
        "thrown in hierarchical tag creation" << std::endl;
            return 34;
          }
        }
      }
    }
    catch (cool::Exception& e) {
    }
      }
    }
  }
  return 0;
}

ATLAS_NOT_THREAD_SAFE() [3/4]

int AtlCoolCopy::copyFolder ATLAS_NOT_THREAD_SAFE	(	const std::string &	folder,
		const std::vector< std::string > &	taglist
	)

Definition at line 534 of file AtlCoolCopy.cxx.

                                                                {
  // get source folder
  cool::IFolderPtr sourcefl,destfl;
  CoraCoolFolderPtr sourceflc,destflc;
  try {
    sourcefl=m_sourceDbPtr->getFolder(folder);
  }
  catch (cool::Exception& e) {
    std::cout << "Could not get source folder: " << folder << std::endl;
    return 20;
  }
  const std::string& sourcedesc=sourcefl->description();
  // check if folder should be skipped from replication
  if (m_skiprep && sourcedesc.find("<norep/>")!=std::string::npos) {
    std::cout << "Folder " << folder << 
      " skipped due to <norep/> metadata" << std::endl;
    return 0;
  }
  const bool iscora=(m_coracool && 
             sourcedesc.find("<coracool")!=std::string::npos);
  if (iscora) {
    // activate CoraCool and get source folder pointer
    openCoraCool();
    sourceflc=m_sourceCoraPtr->getFolder(folder);
  }
  // setup the channel-range
  setChannelRange(sourcefl);
 
  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
  cool::FolderVersioning::Mode dvermode=vermode;
  if (m_forcesingle) {
    dvermode=cool::FolderVersioning::SINGLE_VERSION;
    std::cout << "Forcing destination folder to singleversion" << std::endl;
  }
  if (m_forcemulti) {
    dvermode=cool::FolderVersioning::MULTI_VERSION;
    std::cout << "Forcing destination folder to multiversion" << std::endl;
  }
  // setup type of folder (payload mode, inline, separate, or vector)
  const cool::PayloadMode::Mode spaymode=
    sourcefl->folderSpecification().payloadMode();
  cool::PayloadMode::Mode dpaymode=spaymode;
 
  // set name for destination folder
  std::string destfolder=folder;
  if (!m_outfolder.empty()) {
    destfolder=m_outfolder;
    std::cout << "Destination folder will be renamed to " << destfolder <<
      std::endl;
  }
  
// get destination folder, try to create if not there (and not verifying!)
  bool created=false;
  if (!m_nocopy && !m_root && !m_analyse) {
    if (!m_destDbPtr->existsFolder(destfolder) && !m_verify) {
      std::cout << "Creating folder " << destfolder << " payload-type " << 
    dpaymode << " on destination" << std::endl;
      created=true;
      std::string metadata=sourcedesc;
      if (m_forcerune || m_forcetime) {
    std::string newmeta=m_forcerune ? "run-lumi" : "time";
    std::string::size_type p1,p2;
    p1=metadata.find("<timeStamp>");
    p2=metadata.find("</timeStamp>");
    if (p1!=std::string::npos && p2!=std::string::npos) {
      metadata.replace(0,p2,"<timeStamp>"+newmeta);
      std::cout << "Forced destination folder to " << newmeta << " : "
          << metadata << std::endl;
    } else {
      std::cout << 
        "ERROR: Could not parse metadata string to force timestamp type" 
            << std::endl;
    }
      }
      // force separate or inline payload, but not if input is vector
      if (m_forcepay && spaymode!=cool::PayloadMode::VECTORPAYLOAD) 
    dpaymode=cool::PayloadMode::SEPARATEPAYLOAD;
      if (m_forcenopay && spaymode!=cool::PayloadMode::VECTORPAYLOAD) 
    dpaymode=cool::PayloadMode::INLINEPAYLOAD;
 
      try {
    if (iscora) {
      destflc=m_destCoraPtr->createFolder(destfolder,
                      sourceflc->coralTableName(),
                      sourceflc->fkSpecification(),
                      sourceflc->payloadSpecification(),
                      sourceflc->coralFKey(),
                      sourceflc->coralPKey(),
                      metadata,dvermode,true);
      std::cout << "Created CoraCool folder" << std::endl;
      destfl=m_destDbPtr->getFolder(destfolder);
    } else {
      destfl=m_destDbPtr->createFolder(destfolder,
                  cool::FolderSpecification(dvermode,
                  sourcefl->payloadSpecification(),dpaymode),
                       metadata,true);
    }
      }
      catch (cool::Exception&e ) {
        std::cout << "Create folder failed - aborting" << std::endl;
        return 30;
      }
    }
    // now get the pointer to destination folder if needed
    // note both COOL and CoraCool pointers are set if needed
    try {
      if (iscora) destflc=m_destCoraPtr->getFolder(destfolder);
      destfl=m_destDbPtr->getFolder(destfolder);
    }
    catch (cool::Exception& e) {
      std::cout << "Could not get destination folder: " << destfolder 
        << std::endl;
      return 21;
    }
    // check payload specifications of folders are the same
    const cool::IRecordSpecification& sourcespec=
      sourcefl->payloadSpecification();
    const cool::IRecordSpecification& destspec=
      destfl->payloadSpecification();
    if (!(sourcespec==destspec)) {
      bool badspec=false;
      if (m_ignorespec) {
    // specifications differ - check names are same
    std::cout << 
      "WARNING Source and destination folder specifications differ" << 
      std::endl;
    for (unsigned int i=0;i<sourcespec.size();++i) {
      const std::string& sname=sourcespec[i].name();
      if (!destspec.exists(sname)) {
        std::cout << "ERROR: Field " << sname << " absent from destination"
        << std::endl;
        badspec=true;
      }
    }
      } else {
    badspec=true;
      }
      if (badspec) {
        std::cout << 
      "ERROR Source and destination folder specifications differ" 
          << std::endl;
        return 22;
      }
    }
    // check folder descriptions are the same - just print WARNING if not
    const std::string& destdesc=destfl->description();
    if (sourcedesc!=destdesc) {
      std::cout << "WARNING: Source and destination folder descriptions (meta-data) differ" << std::endl;
      std::cout << "Source folder: " << sourcedesc << std::endl;
      std::cout << "Destn  folder: " << destdesc << std::endl;
    }
    // check payload modes are same - print warning if not
    const cool::PayloadMode::Mode dpaymode=
      destfl->folderSpecification().payloadMode();
    if (spaymode!=dpaymode) {
      std::cout << "WARNING: Source (" << spaymode << ") and destination (" <<
    dpaymode << " folder payload modes differ" << std::endl;
    }
    
    // check/set channels table if requested
    if (!m_nochannel) {
      if (!checkChannels(folder,sourcefl,destfl,created)) return 23;
    }
  }
  // if only copying structures, stop here
  if (m_nodata) return 0;
  // extract folder/range information
  std::cout << "Start to process folder: " << folder;
  // check for timestamp XML 
  bool timestamp=(sourcefl->description().find("<timeStamp>time")!=
          std::string::npos);
  cool::ValidityKey since,until;
  if (timestamp) {
    since=m_timemin;
    until=m_timemax;
    std::cout << " (timestamp)" << std::endl;
  } else {
    since=m_runemin;
    until=m_runemax;
    std::cout << " (run/lumi)" << std::endl;
  }
  if (m_alliov || m_truncate) 
    std::cout << "Output IOVs will be modified" << std::endl;
  if (m_skipout) 
    std::cout << "IOVs extending outside selection range will be skipped"
          << std::endl;
 
  bool checkrefs=false;
  const std::string name0=sourcefl->payloadSpecification()[0].name();
  if (m_checkrefs && (name0=="PoolRef" || name0=="fileGUID")) {
    checkrefs=true;
    std::cout << "Check POOL references in folder " << folder << std::endl;
  }
  // if nocopy, and checking POOL references, and folder is not a POOL ref one
  // can skip the data access 
  if (m_checkrefs && m_nocopy && !checkrefs) return 0;
 
  // check for <fullrep/> metadata indicating copy all tags if -skiprep option
  bool copyall=
    (m_skiprep && (sourcedesc.find("<fullrep/>")!=std::string::npos));
  if (copyall) std::cout << 
   "All tags in folder will be copied due to <fullrep/> metadata" << std::endl;
 
  // set up true list of tags to copy
  std::vector<std::string> tags;
  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
    std::cout << "Single version folder" << std::endl;
    tags.emplace_back("HEAD");
  } else {
    std::cout << "Multi version folder: consider tags [ ";
    // get list of tags in this node which are requested
    const std::vector<std::string> foldertags=sourcefl->listTags();
    for (std::vector<std::string>::const_iterator itag=foldertags.begin();
     itag!=foldertags.end();++itag) {
      bool copyit=copyall;
      // if input taglist is empty, take all tags in folder
      if (taglist.empty() ||
      find(taglist.begin(),taglist.end(),*itag)!=taglist.end()) 
    copyit=true;
      for (std::vector<std::string>::const_iterator imtag=m_magic.begin();
       imtag!=m_magic.end();++imtag) {
    if (itag->find(*imtag)!=std::string::npos) copyit=true;
      }
      if (copyit) {
    tags.push_back(*itag);
        std::cout << *itag << " ";
      }
    }
    // if no tags were found, or doing inclusive hierarchical tag copying
    //  try resolving input tags hierarchically
    if (tags.empty() || m_hitag) {
      for (std::vector<std::string>::const_iterator itag=taglist.begin();
       itag!=taglist.end();++itag) {
    try {
      std::string htag=sourcefl->resolveTag(*itag);
      if (find(tags.begin(),tags.end(),htag)==tags.end()) {
        std::cout << *itag << "=" << htag << " ";
        tags.push_back(htag);
      }
    }
    // ignore exceptions indicating tag not defined here
    // note std::exception rather than COOL exception to cover case
    // when trying to resolve a leaf tag in the '/' folder, which
    // throws a coral AttributeException
    catch (std::exception& e) { }
      }
    }
    std::cout << "]" << std::endl;
    // if still no tags were found, or forced HEAD tag copying, or no
    // tags in the folder at all (=MV folder being used as SV), add HEAD
    if (((tags.empty() || m_includehead) && !m_excludehead) ||
    foldertags.empty())
      tags.emplace_back("HEAD");
  }
 
  sourcefl->setPrefetchAll(false);
  for (std::vector<std::string>::const_iterator itag=tags.begin();
       itag!=tags.end();++itag) {
    std::string outtag=*itag;
    if (!m_outtag.empty()) outtag=m_outtag;
    int retcode;
    // verify or copy folder
    if (m_verify) {
      retcode=verifyIOVs(folder,sourcefl,sourceflc,destfl,destflc,
             *itag,since,until,checkrefs,iscora,spaymode);
    } else if (m_nocopy) {
      retcode=nocopyIOVs(folder,sourcefl,*itag,since,until,checkrefs);
    } else if (m_root) {
      retcode=rootIOVs(folder,sourcefl,*itag,since,until,timestamp);
    } else if (m_analyse) {
      retcode=analyseIOVs(folder,sourcefl,*itag,since,until,timestamp);
    } else {
      retcode=copyIOVs(folder,destfolder,sourcefl,sourceflc,destfl,destflc,
               *itag,outtag,since,until,timestamp,checkrefs,iscora,
               spaymode,created);
    }
    if (retcode!=0) {
      std::cout << "ERROR operation failed for folder " << folder << " tag " <<
    *itag << std::endl;
      return retcode;
    }
  }
  return 0;
}

ATLAS_NOT_THREAD_SAFE() [4/4]

int main ATLAS_NOT_THREAD_SAFE	(	int	argc,
		const char *	argv[]
	)

Definition at line 3475 of file AtlCoolCopy.cxx.

                                                              {
  int retcode=0;
  if (argc<3) {
    printHelp();
    retcode=1;
  } else {
    AtlCoolCopy mycopy(argv[1],argv[2]);
    retcode=mycopy.setOpts(argc-3,&argv[3]);
    if (retcode==999) {
      printHelp();
      return 0;
    }
    if (retcode==0) retcode=mycopy.doCopy();
  }
  if (retcode>0) std::cout << "ERROR AtlCoolCopy.exe fails with exit code " << 
           retcode << std::endl;
  return retcode;
}

printHelp()

void printHelp

(

)

Definition at line 3345 of file AtlCoolCopy.cxx.

                 {
  std::cout << "usage: AtlCoolCopy.exe <sourceCoolDB> <destinationCoolDB> { <options> }" << std::endl;
  std::cout << "Options are (see doc/mainpage.h for more):" << std::endl <<
    "-a, -alliov : set IOVs on destination to [ValidityKeyMin, ValidityKeyMax]"   << std::endl <<" or following settings of -nrls,-nrlu,-nts,-ntu" << 
    std::endl <<
    "-ag, -addguid <guid> : Add GUID to list accessed for POOL checks" << 
    std::endl <<
    "-alf, -addlfn <LFN> : Add LFN to list accessed for POOL checks" << 
    std::endl <<
    "-al, -appendlocked : Allow locked tags to be updated if no overlap" << 
    std::endl << 
    "-alsv, -appendlockedsv : Allow locked tag update for openended IOVs" << 
    std::endl << 
    "-ana, -analyse <delta_t>: produce analysis ROOT file (filename = dest DB argument)" << std::endl << 
"-bs, -buffersize <size> : set size of bulkstorage output buf to <size> objs"
        << std::endl << 
    "-c, -create : create destination DB if not already existing" 
        << std::endl <<
    "-cd, -checkdest : Check destination DB and skip already existing tags"
        << std::endl <<
    "-ch, -channel : restrict selection to given channel or range specified as c1:c2" << std::endl <<
    "-ch1, -channel1 : specify start of a range of channels" << std::endl <<
    "-ch2, -channel2 : specify end of a range of channels" << std::endl <<
    "-cf, -checkfiles : check POOL files can be opened and have correct GUID" 
        << std::endl << 
    "-co, -checkoutput <file> : write POOL file check output on file"
        << std::endl << "-cti, -copytaginfo : Copy tag descriptions"
        << std::endl << "-ctl, -copytaglock : Copy tag locked status and descriptions"
        << std::endl << "-cr, -checkrefs : check POOL references"
        << std::endl << "-d, -debug : produce debug output" << std::endl;
  std::cout <<
    "-ds, -dataset : output register.sh for creating DQ2 datasets " 
        << std::endl <<
    "-ec, -excludechannel : exclude given channel from copy" << std::endl <<
    "-eh, -excludehead : exclude HEAD tag even if no tags found in MV folders" 
        <<  std::endl <<
    "-ih, -includehead : include HEAD as well as any tags in MV folders" << 
    std::endl <<
    "-e, -exclude <pattern> : exclude folders" << std::endl <<
    "-f, -folder <pattern> : include folders" << std::endl <<
    "-fs, -forcesingle : force destination folder to be singleversion" 
        << std::endl <<
    "-fm, -forcemulti : force destination folder to be multiversion" 
        << std::endl <<
    "-frl, -forcerunlumi : force destination folder to be run/LB indexed" 
        << std::endl <<
    "-ftm, -forcetime : force destination folder to be timestamp indexed" 
        << std::endl <<
    "-fp, -forcepayload : Force destn folders to be created with payload tbl" << std::endl <<
    "-fnp, -forcenopayload : Force destn folders to be created without payload tbl" << std::endl <<
    "-forcerecreate : delete and recreate destination database" << std::endl;
  std::cout << "-ht, -headtag : Use HEAD-style tagging"
        << std::endl <<
   "-go, -getonline : Set minimum run number (-rls setting) to next ONLINE run"
        << std::endl <<
 "-gb, -getbulk :  Set minimum run number (-rls setting) to next bulk reco run"
        << std::endl <<
    "-gt, -gettime : Extract timestamp information for given run number range" 
        << std::endl <<
    "-h, -help : print help and exit" << std::endl << 
    "-hi, -hitag : Copy hierrchical tags inclusively" << std::endl << 
    "-ignoremode <pwd> : Ignore UPDx mode protection (experts only)" 
        << std::endl <<
    "-is, -ignorespec : Ignore differences in folder spec if names are all equal" << std::endl <<
    "-lo, -lockedonly : Only copy locked/partially-locked top-level tags"
        << std::endl <<
    "-nc, -nocopy : Do not actually copy, just read source DB" << std::endl <<
    "-nch, -nochannel : Do not check or copy channel information" << std::endl 
<<  "-ncr, -noclobroot: Do not copy CLOB data into ROOT files" << std::endl
<< "-noc, -nocoracool : Do not copy Coracool structure payloads " << std::endl <<
    "-nd, -nodata : Copy only folder structures, not data" << std::endl <<
    "-nh, -nohitag : Do not follow hierarchical tag relations" << std::endl <<
"-mc, -mergecat <catfile> : specify POOL file catalogue for new dataset making"
        << std::endl;
  std::cout << "-of, -outfolder <folder> : rename folder on output" 
        << std::endl <<
    "-onr, -onlinerun : Retrieve run number from online server (not replica)" <<std::endl <<
    "-ot, -outtag : " << "Rename tag on output" << std::endl;
  std::cout <<
    "-pa, -poolall : Output all POOL files (incl good) when checking files"
        << std::endl <<
"-pc, -poolcat <catfile> : specify POOL file catalogue for ref checking" 
        << std::endl <<
    "-pf, -parfile <file> : Read additional options/parameters from file" << 
    std::endl << 
"-pt, -prunetags : Copy only hierarchical tags descending from specified toptags"
        << std::endl <<
"-rdo, -readoracle : force data to be read from Oracle, using dbreplica.config"
        << std::endl <<
    "-sl, -seal <val>: Set SEAL (hence COOL/POOL) output level to <val>"
        << std::endl << 
    "-sr, -skiprep : Skip folders having <norep/> folder metadata tag" <<
    std::endl << 
    "-t, -tag <tag> : Copy multiversion data with tag <tag>" << std::endl <<
    "-mt, -magic <tag> : Include magic tags involving the given pattern " << std::endl << 
"-tr, -truncate : Set destination IOVs outside query interval to query interval"
        << std::endl <<
"-tl, -taglabel : Specify tag description to enter in destination DB" 
        << std::endl << 
    "-uht, -userheadtag : Also copy user tag data to the HEAD" << std::endl <<
    "-v, -verify : Verify data present on destination rather than copying" <<
    std::endl << 
    "-ro, -root : Produce ROOT output file instead of copying to COOL"
        << std::endl <<
    "-zn, -zeronull : Zero NULLs in ROOT file instead of skipping them" << 
    std::endl << 
    "-rls, -runlumisince <run> <LB> : Set minimum IOV interval" << std::endl <<
    "-rlu, -runlumiuntil <run> <LB> : Set maximum IOV interval" << std::endl <<
    "-rs, -runsince <run> : Set minimum IOV interval" << std::endl << 
    "-ru, -rununtil <run> : Set maximum IOV interval" << std::endl <<
    "-r, -run <run> : Copy only run <run>" << std::endl <<
"-srls, -skiprunlumisince <run> <LB> : Set minimum IOV for skipping IOV in copy" 
        << std::endl <<
"-srlu, -skiprunlumiuntil <run> <LB> : Set maximum IOV for skipping IOV in copy" 
        << std::endl <<
    "-tdb, -timedb <dbconn> : Set database connection for time information" <<
    "-ts, -timesince <time> : Set minimum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" << 
    std::endl <<
    "-tu, -timeuntil <time> : Set maximum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" << 
    std::endl;
  std::cout << 
    "-nrls, -newrunlumisince <run> <LB> : Set minimum of output IOV interval (use with -alliov)" << std::endl <<
    "-nrlu, -newrunlumiuntil <run> <LB> : Set maximum of output IOV interval (use with --aliov)" << std::endl <<
    "-nts, -newtimesince <time> : Set minimum of outputIOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" << 
    std::endl <<
    "-ntu, -newtimeuntil <time> : Set maximum of output IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" << 
    std::endl;
  std::cout << "See http://twiki.cern.ch/twiki/bin/view/Atlas/AtlCoolCopy for more details" << std::endl;
}

WriteCallback()

size_t WriteCallback	(	void *	contents,
		size_t	size,
		size_t	nmemb,
		std::string *	s
	)

Definition at line 64 of file AtlCoolCopy.cxx.

                                                                              {
  size_t newLength = size * nmemb;
 
  try
  {
    s->append((char *)contents, newLength);
  }
  catch (std::bad_alloc &e)
  {
    // handle memory problem
    return 0;
  }
  return newLength;
}