ATLAS Offline Software
AtlCoolCopy.cxx
Go to the documentation of this file.
1 /*
2  Copyright (C) 2002-2025 CERN for the benefit of the ATLAS collaboration
3 */
4 
5 // AtlCoolCopy.cxx
6 // COOL copying utility, C++ based on PyCoolCopy from Sven Schmidt
7 // Richard Hawkings, started 30/5/06
8 // compiles in offline cmt framework to binary executable, needs offline env
9 
10 #include <algorithm>
11 #include <cmath>
12 #include <ctime>
13 #include <fstream>
14 #include <iostream>
15 #include <sstream>
16 #include <string>
17 #include <vector>
18 #include <nlohmann/json.hpp>
19 #include <curl/curl.h>
20 
21 #include "CoolKernel/DatabaseId.h"
22 #include "CoolKernel/Exception.h"
23 #include "CoolKernel/IDatabaseSvc.h"
24 #include "CoolKernel/IDatabase.h"
25 #include "CoolKernel/IFolder.h"
26 #include "CoolKernel/FolderSpecification.h"
27 #include "CoolKernel/IFolderSet.h"
28 #include "CoolKernel/IObject.h"
29 #include "CoolKernel/IObjectIterator.h"
30 #include "CoolKernel/IRecordIterator.h"
31 #include "CoolApplication/Application.h"
32 #include "CoralBase/AttributeListException.h"
33 
39 #include "RelationalAccess/ConnectionService.h"
40 #include "RelationalAccess/IConnectionServiceConfiguration.h"
41 #include "RelationalAccess/ISessionProxy.h"
42 #include "RelationalAccess/ITransaction.h"
43 #include "RelationalAccess/ISchema.h"
44 #include "RelationalAccess/ITable.h"
45 #include "RelationalAccess/IQuery.h"
46 #include "RelationalAccess/ICursor.h"
47 
52 
54 
55 #include "PoolMapElement.h"
56 #include "ReplicaSorter.h"
57 #include "CoolTagInfo.h"
58 
59 #include "TFile.h"
60 #include "TTree.h"
61 #include "TH1.h"
62 #include "TObjString.h"
63 
64 size_t WriteCallback(void *contents, size_t size, size_t nmemb, std::string *s) {
65  size_t newLength = size * nmemb;
66 
67  try
68  {
69  s->append((char *)contents, newLength);
70  }
71  catch (std::bad_alloc &e)
72  {
73  // handle memory problem
74  return 0;
75  }
76  return newLength;
77 }
78 
79 
80 class AtlCoolCopy {
81  public:
82  AtlCoolCopy(const std::string& sourcedb, const std::string& destdb,
83  bool allowcreate=false);
84  bool isOpen() const;
85  bool addFolder(const std::string& folder,const bool onlyTags);
86  bool addExclude(const std::string& folder);
88  (const std::string& folder,const std::vector<std::string>& taglist);
89  int doCopy ATLAS_NOT_THREAD_SAFE ();
90  int setOpts(int argc, const char* argv[]);
91 
92  private:
93  // methods
94  bool openConnections(const std::string& sourcedb, const std::string& destdb,
95  bool allowcreate);
96  bool openCoraCool();
97  bool getLockedTags();
98  bool procOptVector(const int argc, const char* argv[],
99  std::vector<std::string>& folders);
100  static std::string transConn(const std::string& inconn);
101  void setChannelRange(const cool::IFolderPtr& sourcefl);
102  static cool::ChannelId channelID(const cool::IFolderPtr& folder,
103  const std::string& chanstring);
105  (const std::string& folder,const std::string& destfolder,
106  const cool::IFolderPtr& sourcefl,const CoraCoolFolderPtr& sourceflc,
107  const cool::IFolderPtr& destfl,const CoraCoolFolderPtr& destflc,
108  const std::string& sourcetag,const std::string& desttag,
109  const cool::ValidityKey since,const cool::ValidityKey until,
110  bool timestamp,bool checkrefs,bool iscora,
111  const cool::PayloadMode::Mode paymode, bool created);
112  void adjustIOVs(const cool::ValidityKey& since,
113  const cool::ValidityKey& until,
114  const cool::ValidityKey& qsince,
115  const cool::ValidityKey& quntil,
116  cool::ValidityKey& newsince,
117  cool::ValidityKey& newuntil,
118  const bool timestamp) const;
119  int nocopyIOVs(const std::string& folder,
120  const cool::IFolderPtr& sourcefl,const std::string& sourcetag,
121  const cool::ValidityKey since,const cool::ValidityKey until,
122  bool checkrefs);
123  int verifyIOVs(const std::string& folder,
124  const cool::IFolderPtr& sourcefl,const CoraCoolFolderPtr& sourceflc,
125  const cool::IFolderPtr& destfl,const CoraCoolFolderPtr& destflc,
126  const std::string& sourcetag,
127  const cool::ValidityKey since,const cool::ValidityKey until,
128  const bool checkrefs,const bool iscora,
129  const cool::PayloadMode::Mode paymode);
130  int rootIOVs(const std::string& folder,
131  const cool::IFolderPtr& sourcefl,const std::string& sourcetag,
132  const cool::ValidityKey since,const cool::ValidityKey until,
133  const bool timestamp);
134  std::string rootDirs(const std::string& folder, const std::string& toproot);
135  bool rootAllocate(const cool::IFieldSpecification& spec,
136  void*& sptr,char& rootID) const;
137  void rootWrite(void* sptr, const cool::IField& field) const;
138  static bool isNumeric(const char* input);
139  static bool equalRecord(const cool::IRecord& lhs,
140  const cool::IRecord& rhs);
141  int analyseIOVs(const std::string& folder,
142  const cool::IFolderPtr& sourcefl,const std::string& sourcetag,
143  const cool::ValidityKey since,const cool::ValidityKey until,
144  const bool timestamp);
145  static TH1F* bookOrFindTH1F(const std::string& hID, const std::string& htitle,
146  const int chan, const float xlow, const float xhigh);
147  static cool::ValidityKey timeVal(const char* input);
148  static std::string timeString(const cool::ValidityKey iovtime);
149  static cool::ValidityKey runLBVal(const char* input1, const char* input2);
150  bool getTimeFromRun();
151  bool getOnlineRun();
152  bool getBulkRun();
153  bool getRunList();
154 
155  static int getUpdateMode(std::string_view desc, std::string_view tag);
156 
157  bool checkChannels(const std::string& folder,
158  const cool::IFolderPtr& sourcefl,const cool::IFolderPtr& destfl,
159  bool newfolder);
160  void checkRef(const cool::IRecord& payload,
161  const std::string& folder,const std::string& tag);
162  int tagParents();
163  int writeTagInfo();
164  int listPoolRefs();
165  int resolvePoolRefs ATLAS_NOT_THREAD_SAFE ();
166  static std::string getCoolHistGUID(const std::string& file);
167  void filePoolRefs();
168  static pool::IFileCatalog* setupCatalog(const std::vector<std::string>& catvec);
169 
170  // input parameters
171  std::string m_sourcedb;
172  std::string m_destdb;
179  bool m_debug;
180  bool m_alliov;
181  bool m_verify;
182  bool m_root;
184  bool m_analyse;
186  bool m_listpfn;
188  bool m_poolall;
189  bool m_nocopy;
190  bool m_nodata;
192  bool m_chdesc;
193  bool m_hitag;
194  bool m_nohitag;
203  bool m_skipout;
204  bool m_skiprep;
205  bool m_applock;
208  bool m_gettime;
211  bool m_getbulk;
221  cool::ValidityKey m_runemin;
222  cool::ValidityKey m_runemax;
223  cool::ValidityKey m_timemin;
224  cool::ValidityKey m_timemax;
225  cool::ValidityKey m_newrunemin;
226  cool::ValidityKey m_newrunemax;
227  cool::ValidityKey m_newtimemin;
228  cool::ValidityKey m_newtimemax;
229  cool::ValidityKey m_srunemin;
230  cool::ValidityKey m_srunemax;
231  std::vector<std::string> m_channelRange;
232  std::string m_channel1;
233  std::string m_channel2;
236  long long m_anadelt;
237  std::string m_outfolder;
238  std::string m_outtag;
239  std::string m_newdataset;
240  std::string m_checkoutputfile;
241  std::string m_timedb;
242  std::string m_taglabel;
243  std::string m_runinfohost;
244  std::vector<std::string> m_addguid; // additional guids to be processsed
245  std::vector<std::string> m_addlfn; // additional LFNs to be processsed
246  std::vector<std::string> m_parfile; // list of additional files with params
247  std::vector<cool::ChannelId> m_excludechans; // list of channels to exclude
248  std::vector<std::string> m_runfile; // list of filenames with run numbers
249  std::vector<unsigned int> m_runlist; // list of runs open for UPD2 tags
250 
251  // internal variables
252  coral::ConnectionService m_coralsvc;
253  cool::Application m_coolapp;
254  cool::IDatabaseSvc* m_dbSvc;
256  cool::IDatabasePtr m_sourceDbPtr;
257  cool::IDatabasePtr m_destDbPtr;
260  std::vector<std::string> m_folderlist; // list of leaf folders to process
261  std::vector<std::string> m_folderexcl; // list of leaf folders to exclude
262  std::vector<std::string> m_tags; // list of tags
263  std::vector<std::string> m_magic; // list magic tags fragments to match
264  std::vector<std::string> m_poolcat; // list of POOL catalogues for input
265  std::vector<std::string> m_mergecat; // list of POOL catalogues for mergechk
266  bool m_open;
267  cool::ChannelSelection m_chansel;
268  typedef std::map<std::string,PoolMapElement> PoolMap;
269  PoolMap::iterator m_poollast; // pointer to last POOL ref updated
270  PoolMap m_poolrefs; // POOL refs and usage counts
271  // hierarchical tags - tag name and folder
272  using HiTagMap = std::map<std::string, std::string>;
274  // tags indirectly used in the hierarchy and have to be treated at end
275  std::vector<std::string> m_hiparent;
276  // map of CoolTagInfo objects - tags which have have their descriptions
277  // and lock status set in the destination DB at the end
278  using CoolTagMap = std::map<std::string, CoolTagInfo>;
280  // output root file for ROOT file export and IOV analysis
281  TFile* p_rootfile;
282 
283  // ROOT ntuple variables.
284  ULong64_t m_nt_since = 0;
285  ULong64_t m_nt_until = 0;
286  UInt_t m_nt_runsince = 0;
287  UInt_t m_nt_rununtil = 0;
288  UInt_t m_nt_lbsince = 0;
289  UInt_t m_nt_lbuntil = 0;
290  UInt_t m_nt_channel = 0;
291  char m_nt_tagid[256] = {0};
292  std::string m_nt_treename;
293  std::vector<void*> m_nt_bufferptr;
294 };
295 
296 inline bool AtlCoolCopy::isOpen() const { return m_open; }
297 
298 void printHelp();
299 
300 AtlCoolCopy::AtlCoolCopy(const std::string& sourcedb, const std::string& destdb,
301  bool allowcreate) :
302  m_sourcedb(sourcedb),m_destdb(destdb),m_allowcreate(allowcreate),
303  m_recreate(false),
304  m_includehead(false),m_excludehead(false),
305  m_usertags(true),m_userupdatehead(false),m_debug(false),m_alliov(false),
306  m_verify(false),m_root(false),m_zeronull(false),m_analyse(false),
307  m_checkrefs(false),m_listpfn(false),m_poolopen(false),m_poolall(false),
308  m_nocopy(false),m_nodata(false),m_nochannel(false),m_chdesc(false),
309  m_hitag(false),m_nohitag(false),m_forcesingle(false),m_forcemulti(false),
310  m_forcerune(false),m_forcetime(false),m_forcepay(false),m_forcenopay(false),
311  m_sourceread(true),m_truncate(false),m_skipout(false),m_skiprep(false),
312  m_applock(false),m_applocksv(false),
313  m_readoracle(false),m_gettime(false),m_getonline(false),m_onlinerun(false),
314  m_getbulk(false),
315  m_prunetags(false),m_lockedonly(false),m_copytaginfo(false),
316  m_copytaglock(false),m_coracool(true),m_ignoremode(false),
317  m_ignorespec(false),m_checkdesttag(false),m_noclobroot(false),
318  m_runemin(cool::ValidityKeyMin),m_runemax(cool::ValidityKeyMax),
319  m_timemin(cool::ValidityKeyMin),m_timemax(cool::ValidityKeyMax),
320  m_newrunemin(cool::ValidityKeyMin),m_newrunemax(cool::ValidityKeyMax),
321  m_newtimemin(cool::ValidityKeyMin),m_newtimemax(cool::ValidityKeyMax),
322  m_srunemin(cool::ValidityKeyMin),m_srunemax(cool::ValidityKeyMax),
323  m_channel1(""),m_channel2(""),m_bufsize(1000),m_sealmsg(5),
324  m_anadelt(-1),m_outfolder(""),m_outtag(""),m_newdataset(""),
325  m_checkoutputfile(""),m_timedb(""),m_taglabel(""),
326  m_runinfohost("http://atlas-run-info-api.web.cern.ch/api"),
327  m_coolapp(&m_coralsvc),
328  m_dbSvc(&(m_coolapp.databaseService())),m_repsort(nullptr),
329  m_open(false),m_chansel(cool::ChannelSelection::all()),p_rootfile(nullptr)
330 {
331  m_poolrefs.clear();
332  m_poollast=m_poolrefs.end();
333  // configure CORAL components
334  coral::IConnectionServiceConfiguration& csconfig=m_coralsvc.configuration();
335  csconfig.disablePoolAutomaticCleanUp();
336  csconfig.setConnectionTimeOut(0);
337 }
338 
339 bool AtlCoolCopy::openConnections(const std::string& sourcedb,
340  const std::string& destdb,bool allowcreate) {
341  // check connections not already open
342  if (m_open) return true;
343  // initialise replica sorter if it was requested
344  if (m_readoracle) {
345  m_repsort=new ReplicaSorter();
346  coral::IConnectionServiceConfiguration& csconfig=m_coralsvc.configuration();
347  csconfig.setReplicaSortingAlgorithm(*m_repsort);
348  }
349  // cool::IDatabaseSvc& dbSvc=cool::DatabaseSvcFactory::databaseService();
350  // open source database
351  std::cout << "Open source database: " << sourcedb << std::endl;
352  if (!m_sourceread) std::cout << "... in UPDATE mode" << std::endl;
353  try {
354  m_sourceDbPtr=m_dbSvc->openDatabase(transConn(sourcedb),m_sourceread);
355  }
356  catch (std::exception& e) {
357  std::cout << "Cool exception caught: " << e.what() << std::endl;
358  return false;
359  }
360  // open destination database
361  if (m_nocopy) {
362  m_open=true;
363  return true;
364  } else if (m_root || m_analyse) {
365  std::cout << "Open destination ROOT file: " << destdb << std::endl;
366  p_rootfile=new TFile(destdb.c_str(),"RECREATE");
367  if (p_rootfile==nullptr) std::cout << "ERROR: Could not open ROOT file" <<
368  std::endl;
369  m_open=(p_rootfile!=nullptr);
370  return m_open;
371  }
372  std::string tdestdb=transConn(destdb);
373  std::cout << "Open destination database: " << tdestdb << std::endl;
374  try {
375  m_destDbPtr=m_dbSvc->openDatabase(tdestdb,m_verify);
376  // if the open succeeds and we are using recreate mode, drop/delete
377  // the existing database first
378  if (m_recreate) {
379  std::cout <<
380  "Forcing recreation of destination database - deleting existing data!"
381  << std::endl;
382  m_destDbPtr.reset();
383  m_dbSvc->dropDatabase(tdestdb);
384  // go into catch to recrete database
385  throw cool::DatabaseDoesNotExist("old database deleted");
386  }
387  }
388  catch (std::exception& e) {
389  std::cout << "COOL exception caught: " << e.what() << std::endl;
390  // try to recover by creating new DB if possible and requested
391  if (allowcreate || m_recreate) {
392  std::cout << "Try to create new conditions DB" << std::endl;
393  try {
394  m_destDbPtr=m_dbSvc->createDatabase(tdestdb);
395  std::cout << "Creation succeeded" << std::endl;
396  }
397  catch (cool::Exception& e) {
398  std::cout << "Creation failed" << std::endl;
399  return false;
400  }
401  } else {
402  return false;
403  }
404  }
405  m_open=true;
406  return true;
407 }
408 
410  if (m_sourceCoraPtr.get()==nullptr) {
411  std::cout << "Attempt to open source CoraCool DB " << m_sourcedb <<
412  std::endl;
415  std::cout << "Opened CoraCool source DB" << std::endl;
416  }
417  // skip destination DB if not copying
418  if (m_nocopy) return true;
419  if (m_destCoraPtr.get()==nullptr) {
420  std::cout << "Attempt to open destination CoraCool DB " << m_destdb <<
421  std::endl;
424  std::cout << "Opened CoraCool dest DB" << std::endl;
425  }
426  return true;
427 }
428 
430  // set m_tags to list of top level tags which are locked
431  m_tags.clear();
432  cool::IFolderSetPtr topfolder=m_sourceDbPtr->getFolderSet("/");
433  const std::vector<std::string>& toptaglist=topfolder->listTags();
434  for (std::vector<std::string>::const_iterator toptag=toptaglist.begin();
435  toptag!=toptaglist.end();++toptag) {
436  cool::HvsTagLock::Status tstat=topfolder->tagLockStatus(*toptag);
437  if (tstat==cool::HvsTagLock::LOCKED ||
438  tstat==cool::HvsTagLock::PARTIALLYLOCKED) {
439  std::cout << "Top-level tag " << *toptag << " will be copied" <<
440  std::endl;
441  m_tags.push_back(*toptag);
442  }
443  }
444  std::cout << "Total of " << m_tags.size() << " top-level tags to be copied"
445  << std::endl;
446  return (!m_tags.empty());
447 }
448 
449 std::string AtlCoolCopy::transConn(const std::string& inconn) {
450  // translate simple connection string (no slash) to mycool.db with given
451  // instance name, all others are left alone
452  if (inconn.find('/')==std::string::npos) {
453  return "sqlite://X;schema=mycool.db;dbname="+inconn;
454  } else {
455  return inconn;
456  }
457 }
458 
459 
460 bool AtlCoolCopy::addFolder(const std::string& folder,const bool onlyTags) {
461  std::cout << "Add folders in path:" << folder << " [ ";
462  const std::vector<std::string> nodelist=m_sourceDbPtr->listAllNodes();
463  // find all matching leaf folders
464  for (std::vector<std::string>::const_iterator nodeitr=nodelist.begin();
465  nodeitr!=nodelist.end();++nodeitr) {
466  // match exact folder name or leading part of path
467  // for leading part matches, next char in folder name must be '/'
468  // so /CALO/SetA matches /CALO/SetA/X but not /CALO/SetAB
469  if (*nodeitr==folder || folder=="/" ||
470  (nodeitr->compare(0,folder.size(),folder)==0 &&
471  nodeitr->size()>folder.size() && nodeitr->compare(folder.size(),1,"/")==0)) {
472  // check if folder on exclude list
473  bool exclude=false;
474  for (std::vector<std::string>::const_iterator iexcl=m_folderexcl.begin();
475  iexcl!=m_folderexcl.end();++iexcl) {
476  if (iexcl->compare(0,1,"/")==0) {
477  // exclude pattern starting / matches folder path (/SCT or /SCT/DCS)
478  exclude=(exclude || nodeitr->compare(0,iexcl->size(),*iexcl)==0);
479  } else {
480  // exclude pattern without leading / matches anywhere in folder
481  exclude=(exclude || (nodeitr->find(*iexcl)!=std::string::npos));
482  }
483  }
484  // check folder exists (is a leaf folder), and not excluded
485  if (m_sourceDbPtr->existsFolder(*nodeitr) && !exclude && !onlyTags) {
486  // only add if folder not already on list
487  if (find(m_folderlist.begin(),m_folderlist.end(),*nodeitr)==
488  m_folderlist.end()) {
489  std::cout << *nodeitr << " ";
490  m_folderlist.push_back(*nodeitr);
491  }
492  }
493  // if its a folderset, check for any hierarchical tags
494  if (!m_nohitag && m_sourceDbPtr->existsFolderSet(*nodeitr) && !exclude) {
495  if (!m_prunetags) {
496  const std::vector<std::string> foldersettags=
497  m_sourceDbPtr->getFolderSet(*nodeitr)->listTags();
498  if (!foldersettags.empty()) {
499  for (std::vector<std::string>::const_iterator
500  itr=foldersettags.begin();itr!=foldersettags.end();++itr) {
501  m_hitagmap[*itr]=*nodeitr;
502  }
503  }
504  } else {
505  // only take tags in this folder which are referenced from one of
506  // the input tags (assumed to be top-level tags)
507  cool::IFolderSetPtr sfolder=m_sourceDbPtr->getFolderSet(*nodeitr);
508  for (std::vector<std::string>::const_iterator toptag=m_tags.begin();
509  toptag!=m_tags.end();++toptag) {
510  try {
511  std::string rtag=sfolder->resolveTag(*toptag);
512  m_hitagmap[rtag]=*nodeitr;
513  }
514  // catch exceptions indicating tag defines nothing here
515  // note std::exception rather than COOL exception to cover case
516  // when trying to resolve a leaf tag in the '/' folder, which
517  // throws a coral AttributeException
518  catch (std::exception& e) {}
519  }
520  }
521  }
522  }
523  }
524  std::cout << "]" << std::endl;
525  return true;
526 }
527 
528 bool AtlCoolCopy::addExclude(const std::string& folder) {
529  std::cout << "Adding folder to exclude list: " << folder << std::endl;
530  m_folderexcl.push_back(folder);
531  return true;
532 }
533 
535  (const std::string& folder,const std::vector<std::string>& taglist) {
536  // get source folder
537  cool::IFolderPtr sourcefl,destfl;
538  CoraCoolFolderPtr sourceflc,destflc;
539  try {
540  sourcefl=m_sourceDbPtr->getFolder(folder);
541  }
542  catch (cool::Exception& e) {
543  std::cout << "Could not get source folder: " << folder << std::endl;
544  return 20;
545  }
546  const std::string& sourcedesc=sourcefl->description();
547  // check if folder should be skipped from replication
548  if (m_skiprep && sourcedesc.find("<norep/>")!=std::string::npos) {
549  std::cout << "Folder " << folder <<
550  " skipped due to <norep/> metadata" << std::endl;
551  return 0;
552  }
553  const bool iscora=(m_coracool &&
554  sourcedesc.find("<coracool")!=std::string::npos);
555  if (iscora) {
556  // activate CoraCool and get source folder pointer
557  openCoraCool();
558  sourceflc=m_sourceCoraPtr->getFolder(folder);
559  }
560  // setup the channel-range
561  setChannelRange(sourcefl);
562 
563  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
564  cool::FolderVersioning::Mode dvermode=vermode;
565  if (m_forcesingle) {
566  dvermode=cool::FolderVersioning::SINGLE_VERSION;
567  std::cout << "Forcing destination folder to singleversion" << std::endl;
568  }
569  if (m_forcemulti) {
570  dvermode=cool::FolderVersioning::MULTI_VERSION;
571  std::cout << "Forcing destination folder to multiversion" << std::endl;
572  }
573  // setup type of folder (payload mode, inline, separate, or vector)
574  const cool::PayloadMode::Mode spaymode=
575  sourcefl->folderSpecification().payloadMode();
576  cool::PayloadMode::Mode dpaymode=spaymode;
577 
578  // set name for destination folder
579  std::string destfolder=folder;
580  if (!m_outfolder.empty()) {
581  destfolder=m_outfolder;
582  std::cout << "Destination folder will be renamed to " << destfolder <<
583  std::endl;
584  }
585 
586 // get destination folder, try to create if not there (and not verifying!)
587  bool created=false;
588  if (!m_nocopy && !m_root && !m_analyse) {
589  if (!m_destDbPtr->existsFolder(destfolder) && !m_verify) {
590  std::cout << "Creating folder " << destfolder << " payload-type " <<
591  dpaymode << " on destination" << std::endl;
592  created=true;
593  std::string metadata=sourcedesc;
594  if (m_forcerune || m_forcetime) {
595  std::string newmeta=m_forcerune ? "run-lumi" : "time";
596  std::string::size_type p1,p2;
597  p1=metadata.find("<timeStamp>");
598  p2=metadata.find("</timeStamp>");
599  if (p1!=std::string::npos && p2!=std::string::npos) {
600  metadata.replace(0,p2,"<timeStamp>"+newmeta);
601  std::cout << "Forced destination folder to " << newmeta << " : "
602  << metadata << std::endl;
603  } else {
604  std::cout <<
605  "ERROR: Could not parse metadata string to force timestamp type"
606  << std::endl;
607  }
608  }
609  // force separate or inline payload, but not if input is vector
610  if (m_forcepay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
611  dpaymode=cool::PayloadMode::SEPARATEPAYLOAD;
612  if (m_forcenopay && spaymode!=cool::PayloadMode::VECTORPAYLOAD)
613  dpaymode=cool::PayloadMode::INLINEPAYLOAD;
614 
615  try {
616  if (iscora) {
617  destflc=m_destCoraPtr->createFolder(destfolder,
618  sourceflc->coralTableName(),
619  sourceflc->fkSpecification(),
620  sourceflc->payloadSpecification(),
621  sourceflc->coralFKey(),
622  sourceflc->coralPKey(),
623  metadata,dvermode,true);
624  std::cout << "Created CoraCool folder" << std::endl;
625  destfl=m_destDbPtr->getFolder(destfolder);
626  } else {
627  destfl=m_destDbPtr->createFolder(destfolder,
628  cool::FolderSpecification(dvermode,
629  sourcefl->payloadSpecification(),dpaymode),
630  metadata,true);
631  }
632  }
633  catch (cool::Exception&e ) {
634  std::cout << "Create folder failed - aborting" << std::endl;
635  return 30;
636  }
637  }
638  // now get the pointer to destination folder if needed
639  // note both COOL and CoraCool pointers are set if needed
640  try {
641  if (iscora) destflc=m_destCoraPtr->getFolder(destfolder);
642  destfl=m_destDbPtr->getFolder(destfolder);
643  }
644  catch (cool::Exception& e) {
645  std::cout << "Could not get destination folder: " << destfolder
646  << std::endl;
647  return 21;
648  }
649  // check payload specifications of folders are the same
650  const cool::IRecordSpecification& sourcespec=
651  sourcefl->payloadSpecification();
652  const cool::IRecordSpecification& destspec=
653  destfl->payloadSpecification();
654  if (!(sourcespec==destspec)) {
655  bool badspec=false;
656  if (m_ignorespec) {
657  // specifications differ - check names are same
658  std::cout <<
659  "WARNING Source and destination folder specifications differ" <<
660  std::endl;
661  for (unsigned int i=0;i<sourcespec.size();++i) {
662  const std::string& sname=sourcespec[i].name();
663  if (!destspec.exists(sname)) {
664  std::cout << "ERROR: Field " << sname << " absent from destination"
665  << std::endl;
666  badspec=true;
667  }
668  }
669  } else {
670  badspec=true;
671  }
672  if (badspec) {
673  std::cout <<
674  "ERROR Source and destination folder specifications differ"
675  << std::endl;
676  return 22;
677  }
678  }
679  // check folder descriptions are the same - just print WARNING if not
680  const std::string& destdesc=destfl->description();
681  if (sourcedesc!=destdesc) {
682  std::cout << "WARNING: Source and destination folder descriptions (meta-data) differ" << std::endl;
683  std::cout << "Source folder: " << sourcedesc << std::endl;
684  std::cout << "Destn folder: " << destdesc << std::endl;
685  }
686  // check payload modes are same - print warning if not
687  const cool::PayloadMode::Mode dpaymode=
688  destfl->folderSpecification().payloadMode();
689  if (spaymode!=dpaymode) {
690  std::cout << "WARNING: Source (" << spaymode << ") and destination (" <<
691  dpaymode << " folder payload modes differ" << std::endl;
692  }
693 
694  // check/set channels table if requested
695  if (!m_nochannel) {
696  if (!checkChannels(folder,sourcefl,destfl,created)) return 23;
697  }
698  }
699  // if only copying structures, stop here
700  if (m_nodata) return 0;
701  // extract folder/range information
702  std::cout << "Start to process folder: " << folder;
703  // check for timestamp XML
704  bool timestamp=(sourcefl->description().find("<timeStamp>time")!=
705  std::string::npos);
706  cool::ValidityKey since,until;
707  if (timestamp) {
708  since=m_timemin;
709  until=m_timemax;
710  std::cout << " (timestamp)" << std::endl;
711  } else {
712  since=m_runemin;
713  until=m_runemax;
714  std::cout << " (run/lumi)" << std::endl;
715  }
716  if (m_alliov || m_truncate)
717  std::cout << "Output IOVs will be modified" << std::endl;
718  if (m_skipout)
719  std::cout << "IOVs extending outside selection range will be skipped"
720  << std::endl;
721 
722  bool checkrefs=false;
723  const std::string name0=sourcefl->payloadSpecification()[0].name();
724  if (m_checkrefs && (name0=="PoolRef" || name0=="fileGUID")) {
725  checkrefs=true;
726  std::cout << "Check POOL references in folder " << folder << std::endl;
727  }
728  // if nocopy, and checking POOL references, and folder is not a POOL ref one
729  // can skip the data access
730  if (m_checkrefs && m_nocopy && !checkrefs) return 0;
731 
732  // check for <fullrep/> metadata indicating copy all tags if -skiprep option
733  bool copyall=
734  (m_skiprep && (sourcedesc.find("<fullrep/>")!=std::string::npos));
735  if (copyall) std::cout <<
736  "All tags in folder will be copied due to <fullrep/> metadata" << std::endl;
737 
738  // set up true list of tags to copy
739  std::vector<std::string> tags;
740  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
741  std::cout << "Single version folder" << std::endl;
742  tags.emplace_back("HEAD");
743  } else {
744  std::cout << "Multi version folder: consider tags [ ";
745  // get list of tags in this node which are requested
746  const std::vector<std::string> foldertags=sourcefl->listTags();
747  for (std::vector<std::string>::const_iterator itag=foldertags.begin();
748  itag!=foldertags.end();++itag) {
749  bool copyit=copyall;
750  // if input taglist is empty, take all tags in folder
751  if (taglist.empty() ||
752  find(taglist.begin(),taglist.end(),*itag)!=taglist.end())
753  copyit=true;
754  for (std::vector<std::string>::const_iterator imtag=m_magic.begin();
755  imtag!=m_magic.end();++imtag) {
756  if (itag->find(*imtag)!=std::string::npos) copyit=true;
757  }
758  if (copyit) {
759  tags.push_back(*itag);
760  std::cout << *itag << " ";
761  }
762  }
763  // if no tags were found, or doing inclusive hierarchical tag copying
764  // try resolving input tags hierarchically
765  if (tags.empty() || m_hitag) {
766  for (std::vector<std::string>::const_iterator itag=taglist.begin();
767  itag!=taglist.end();++itag) {
768  try {
769  std::string htag=sourcefl->resolveTag(*itag);
770  if (find(tags.begin(),tags.end(),htag)==tags.end()) {
771  std::cout << *itag << "=" << htag << " ";
772  tags.push_back(htag);
773  }
774  }
775  // ignore exceptions indicating tag not defined here
776  // note std::exception rather than COOL exception to cover case
777  // when trying to resolve a leaf tag in the '/' folder, which
778  // throws a coral AttributeException
779  catch (std::exception& e) { }
780  }
781  }
782  std::cout << "]" << std::endl;
783  // if still no tags were found, or forced HEAD tag copying, or no
784  // tags in the folder at all (=MV folder being used as SV), add HEAD
785  if (((tags.empty() || m_includehead) && !m_excludehead) ||
786  foldertags.empty())
787  tags.emplace_back("HEAD");
788  }
789 
790  sourcefl->setPrefetchAll(false);
791  for (std::vector<std::string>::const_iterator itag=tags.begin();
792  itag!=tags.end();++itag) {
793  std::string outtag=*itag;
794  if (!m_outtag.empty()) outtag=m_outtag;
795  int retcode;
796  // verify or copy folder
797  if (m_verify) {
798  retcode=verifyIOVs(folder,sourcefl,sourceflc,destfl,destflc,
799  *itag,since,until,checkrefs,iscora,spaymode);
800  } else if (m_nocopy) {
801  retcode=nocopyIOVs(folder,sourcefl,*itag,since,until,checkrefs);
802  } else if (m_root) {
803  retcode=rootIOVs(folder,sourcefl,*itag,since,until,timestamp);
804  } else if (m_analyse) {
805  retcode=analyseIOVs(folder,sourcefl,*itag,since,until,timestamp);
806  } else {
807  retcode=copyIOVs(folder,destfolder,sourcefl,sourceflc,destfl,destflc,
808  *itag,outtag,since,until,timestamp,checkrefs,iscora,
809  spaymode,created);
810  }
811  if (retcode!=0) {
812  std::cout << "ERROR operation failed for folder " << folder << " tag " <<
813  *itag << std::endl;
814  return retcode;
815  }
816  }
817  return 0;
818 }
819 
820 void AtlCoolCopy::setChannelRange(const cool::IFolderPtr& sourcefl) {
821  // add range specified via -ch1 -ch2 if given
822  if (!m_channel1.empty() && !m_channel2.empty()) {
823  m_channelRange.clear();
824  m_channelRange.push_back(m_channel1+":"+m_channel2);
825  }
827  const size_t nChanRange=m_channelRange.size();
828  for (size_t i=0;i<nChanRange;i++) {
829  size_t cpos=m_channelRange[i].find(':');
830  if (cpos==std::string::npos || cpos > m_channelRange[i].size()-1) {
831  // single channel
832  std::cout << "Adding channel " << m_channelRange[i] <<
833  " to channel selection" << std::endl;
834  if (m_chansel.allChannels())
835  m_chansel=cool::ChannelSelection(channelID(sourcefl,
836  m_channelRange[i]));
837  else
838  m_chansel.addChannel(channelID(sourcefl,m_channelRange[i]));
839  }
840  else {
841  // Channel Range
842  std::string c1=m_channelRange[i].substr(0,cpos);
843  std::string c2=m_channelRange[i].substr(1+cpos);
844  std::cout << "Adding channel range " << c1 << " to " << c2 <<
845  " to channel selection" << std::endl;
846  if (m_chansel.allChannels())
847  m_chansel=cool::ChannelSelection(channelID(sourcefl,c1),
848  channelID(sourcefl,c2));
849  else
850  m_chansel.addRange(channelID(sourcefl,c1),channelID(sourcefl,c2));
851  }
852  }//end loop over channel ranges
853 }
854 
855 cool::ChannelId AtlCoolCopy::channelID(const cool::IFolderPtr& folder,
856  const std::string& chanstring) {
857  const char* cstr=chanstring.c_str();
858  if (isNumeric(cstr)) {
859  // channel is a number
860  return cool::ChannelId(strtoul(cstr,nullptr,10));
861  } else {
862  cool::ChannelId chan=0;
863  try {
864  chan=folder->channelId(chanstring);
865  std::cout << "Channel name " << chanstring << " maps to channelID "
866  << chan << std::endl;
867  }
868  catch (cool::Exception& e) {
869  std::cout << "ERROR: Channel name " << chanstring <<
870  " not defined in folder " << std::endl;
871  }
872  return chan;
873  }
874 }
875 
876 // Calls non-thread-safe functions of CoraCoolFolder.
877 int AtlCoolCopy::copyIOVs ATLAS_NOT_THREAD_SAFE
878  (const std::string& folder,
879  const std::string& destfolder,
880  const cool::IFolderPtr& sourcefl,const CoraCoolFolderPtr& sourceflc,
881  const cool::IFolderPtr& destfl,const CoraCoolFolderPtr& destflc,
882  const std::string& sourcetag,const std::string& desttag,
883  const cool::ValidityKey since,const cool::ValidityKey until,
884  bool timestamp,bool checkrefs,bool iscora,
885  const cool::PayloadMode::Mode paymode ,bool created) {
886 
887  std::cout << "Copying tag " << sourcetag << " of folder " << folder <<
888  " to destination tag " << desttag << std::endl;
889  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
890  cool::FolderVersioning::Mode dvermode=destfl->versioningMode();
891  // check for online mode
892  int updatemode=getUpdateMode(destfl->description(),desttag);
893  std::vector<std::string> dtaglist=destfl->listTags();
894  // if online mode, must have --getonline and either -truncate or -alliov
895  // only if destination DB is oracle
896  if (updatemode==1 && m_destdb.find("oracle")!=std::string::npos &&
897  (!m_getonline || (!m_truncate && !m_alliov))) {
898  if (m_ignoremode) {
899  std::cout << "Folder is online (UPD1) mode but IGNORING PROTECTION"
900  << std::endl;
901  } else {
902  if(find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
903  std::cout << "Folder is online mode (UPD1) and tag already exist - -getonline and -truncate or -alliov options MUST be used" << std::endl;
904  return 35;
905  }
906  else {
907  std::cout << "Folder is online mode (UPD1), new tag will be created" << std::endl;
908  }
909 
910  }
911  }
912  // if UPD4 mode, must have -getbulk and either -truncate or -alliov
913  if (updatemode==4 && m_destdb.find("oracle")!=std::string::npos &&
914  (!m_getbulk || (!m_truncate && !m_alliov))) {
915  if (m_ignoremode) {
916  std::cout << "Folder is bulkreco (UPD4) mode but IGNORING PROTECTION"
917  << std::endl;
918  } else {
919  if(find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()) {
920  std::cout << "Folder is bulkreco mode (UPD4) - -getbulk and -truncate or -alliov options MUST be used" << std::endl;
921  return 35;
922  } else {
923  std::cout << "Folder is bulkreco mode (UPD4), new tag will be created" << std::endl;
924  }
925 
926  }
927  }
928 
929  // if destination tags are being checked, check if this tag already
930  // exists in destination folder, if so skip copy
931  // also skip if tag is HEAD, and folder was not newly created
932  if (m_checkdesttag) {
933  if (find(dtaglist.begin(),dtaglist.end(),desttag)!=dtaglist.end()
934  || (desttag=="HEAD" && !created)) {
935  std::cout << "Destination tag " << desttag <<
936  " already exists in folder " << folder << " - skip copy" << std::endl;
937  return 0;
938  }
939  }
940  bool relock=false; // tag must be relocked afterwards
941  bool prot=false; //update is protected, irrespective of if tag must be locked
942  if ((m_applock || (updatemode==1 && m_getonline) || updatemode==2 ||
943  (updatemode==4 && m_getbulk)) &&
944  dvermode==cool::FolderVersioning::MULTI_VERSION) {
945  // check if the destination tag exists and is locked
946  std::vector<std::string> dtaglist=destfl->listTags();
947  for (std::vector<std::string>::const_iterator itr=dtaglist.begin();
948  itr!=dtaglist.end();++itr) {
949  if (desttag==*itr) {
950  // tag exists - must protect it unless ignoremode
951  prot=!m_ignoremode;
952  // check lock status
953  if (destfl->tagLockStatus(desttag)==cool::HvsTagLock::LOCKED) {
954  std::cout << "Unlocking destination tag " << desttag <<
955  " for append or UPDx access" << std::endl;
956  if (m_applocksv) {
957  std::cout << "Appending according to SV folder rules" << std::endl;
958  if (updatemode!=3) {
959  std::cout <<
960  "ERROR: Only allowed for UPD3 mode tags" << std::endl;
961  return 36;
962  }
963  }
964  destfl->setTagLockStatus(desttag,cool::HvsTagLock::UNLOCKED);
965  relock=true;
966  }
967  }
968  }
969  }
970  int nobj=0;
971  int nbuf=0;
972  int nskip=0;
973  int nbad=0;
974  // set up tag to be used at point of insertion
975  std::string localtag="";
976  if (dvermode==cool::FolderVersioning::MULTI_VERSION &&
977  desttag!="HEAD" && m_usertags) localtag=desttag;
978  // now loop over IOVs - separate for CoraCool and COOL
979  try {
980  if (iscora) {
981  // branch for CoraCool
982  sourceflc->setPrefetchAll(false);
983  CoraCoolObjectIterPtr sourceitr;
984  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
985  sourceitr=sourceflc->browseObjects(since,until,m_chansel);
986  } else {
987  sourceitr=sourceflc->browseObjects(since,until,m_chansel,sourcetag);
988  }
989  // keep track of new FKs of inserted objects
990  std::map<int,int> insertkeymap;
991  unsigned int nref=0;
992  while (sourceitr->hasNext()) {
993  CoraCoolObjectPtr obj=sourceitr->next();
994  // check object should not be skipped as spreading outside select-IOV
995  if (m_skipout &&
996  (obj->since()<m_srunemin || obj->until()>m_srunemax)) {
997  ++nskip;
998  continue;
999  }
1000  // check object should not be skipped as excluded channel
1001  if (!m_excludechans.empty()) {
1002  if (find(m_excludechans.begin(),m_excludechans.end(),
1003  obj->channelId())!=m_excludechans.end()) {
1004  ++nskip;
1005  continue;
1006  }
1007  }
1008  if (nbuf==0) {
1009  if (m_debug) std::cout <<
1010  "Setup new CoraCool storage buffer at object " << nobj << std::endl;
1011  destflc->setupStorageBuffer();
1012  }
1013  cool::ValidityKey newsince,newuntil;
1014  adjustIOVs(obj->since(),obj->until(),since,until,newsince,newuntil,
1015  timestamp);
1016  // skip negative/zero IOV lengths
1017  if (newsince>=newuntil) {
1018  std::cout << "WARNING: Skipping IOV with since " << newsince <<
1019  ">= until" << newuntil << std::endl;
1020  ++nbad;
1021  continue;
1022  }
1023  // find the old FK, check if we have already inserted this
1024  // in which case a reference can be added
1025  int oldfk=0;
1026  std::map<int,int>::const_iterator ikey=insertkeymap.end();
1027  bool foundkey=false;
1028  // ensure returned data is not of zero size - if so cannot extract
1029  // and remember the FK
1030  if (obj->size()>0) {
1031  oldfk=sourceflc->getAttrKey(
1032  (*obj->begin())[sourceflc->coralFKey()]);
1033  ikey=insertkeymap.find(oldfk);
1034  foundkey=true;
1035  }
1036  if (ikey==insertkeymap.end()) {
1037  int newfk=
1038  destflc->storeObject(newsince,newuntil,obj->begin(),obj->end(),
1039  obj->channelId(),localtag,
1040  (!m_userupdatehead && !localtag.empty()));
1041  if (foundkey) insertkeymap[oldfk]=newfk;
1042  } else {
1043  destflc->referenceObject(newsince,newuntil,ikey->second,
1044  obj->channelId(),localtag,
1045  (!m_userupdatehead && !localtag.empty()));
1046  ++nref;
1047  }
1048  ++nbuf;
1049  // flush buffer every m_bufsize objects
1050  if (nbuf==m_bufsize) {
1051  if (m_debug) std::cout << "Flush buffer after " << nobj << "," <<
1052  nbuf << " objects " << std::endl;
1053  // note CoraCool requires explicit re-setup of buffer after a flush
1054  destflc->flushStorageBuffer();
1055  destflc->setupStorageBuffer();
1056  nbuf=0;
1057  }
1058  ++nobj;
1059  }
1060  sourceitr->close();
1061  if (nref>0) std::cout << "Reference-to-existing used for " << nref
1062  << " payload objects" << std::endl;
1063  if (nbuf>0) {
1064  if (m_debug) std::cout << "Final buffer flush at " << nobj <<
1065  "," << nbuf << std::endl;
1066  destflc->flushStorageBuffer();
1067  }
1068  } else {
1069  // branch for pure COOL objects
1070  cool::IObjectIteratorPtr sourceitr;
1071  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1072  sourceitr=sourcefl->browseObjects(since,until,m_chansel);
1073  } else {
1074  sourceitr=sourcefl->browseObjects(since,until,m_chansel,sourcetag);
1075  }
1076  while (sourceitr->goToNext()) {
1077  const cool::IObject& obj=sourceitr->currentRef();
1078  // check object should not be skipped as spreading outside select-IOV
1079  if (m_skipout && (obj.since()<m_srunemin || obj.until()>m_srunemax)) {
1080  ++nskip;
1081  continue;
1082  }
1083  // check object should not be skipped as excluded channel
1084  if (!m_excludechans.empty()) {
1085  if (find(m_excludechans.begin(),m_excludechans.end(),
1086  obj.channelId())!=m_excludechans.end()) {
1087  ++nskip;
1088  continue;
1089  }
1090  }
1091  if (nbuf==0) {
1092  if (m_debug) std::cout << "Setup new storage buffer at object " <<
1093  nobj << std::endl;
1094  destfl->setupStorageBuffer();
1095  }
1096  cool::ValidityKey newsince;
1097  cool::ValidityKey newuntil;
1098  adjustIOVs(obj.since(),obj.until(),since,until,newsince,newuntil,
1099  timestamp);
1100  // skip negative/zero IOV lengths
1101  if (newsince>=newuntil) {
1102  std::cout << "WARNING: Skipping IOV with since " << newsince <<
1103  ">= until" << newuntil << std::endl;
1104  ++nbad;
1105  continue;
1106  }
1107  if (checkrefs) checkRef(obj.payload(),folder,sourcetag);
1108  if (prot) {
1109  // check existing data in destination DB to ensure no clash
1110  if (m_applocksv) {
1111  // appendlockedsv mode - all IOV ends must be infinite
1112  // and new since must be greater than old since
1113  if (newuntil!=cool::ValidityKeyMax) {
1114  std::cout << "New IOVs must have until=cool::ValidityKeyMax" << std::endl;
1115  throw cool::Exception("Illegal insert over locked IOV",
1116  "AtlCoolCopy");
1117  }
1118  cool::IObjectIteratorPtr checkitr=destfl->browseObjects(newsince,
1119  newuntil,obj.channelId(),desttag);
1120  while (checkitr->goToNext()) {
1121  const cool::IObject& checkobj=checkitr->currentRef();
1122  if (checkobj.since()>=newsince) {
1123  std::cout << "ERROR:: Attempt to insert SV overlapping IOV whilst appending to locked tag" << std::endl;
1124  throw cool::Exception("Illegal insert over locked IOV",
1125  "AtlCoolCopy");
1126  }
1127  if (checkobj.until()!=cool::ValidityKeyMax) {
1128  std::cout << "Existing IOVs must have until=cool::ValidityKeyMax" << std::endl;
1129  throw cool::Exception("Illegal insert over locked IOV",
1130  "AtlCoolCopy");
1131  }
1132  }
1133  checkitr->close();
1134  } else if (m_applock) {
1135  // appendlocked mode - just make sure no objects in this IOV
1136  const unsigned int nexist=
1137  destfl->countObjects(newsince,newuntil,obj.channelId(),desttag);
1138  if (nexist>0) {
1139  std::cout << "ERROR: Attempt to insert IOV over " << nexist <<
1140  " objects whilst appending to locked tag" << std::endl;
1141  throw cool::Exception("Illegal insert over locked IOV",
1142  "AtlCoolCopy");
1143  }
1144  } else if (updatemode==2) {
1145  // UPD2 tag - make sure run is on list of runs open for update
1146  // only for run/lumi based folders
1147  if (timestamp)
1148  throw cool::Exception(
1149  "Attempt to insert into locked UPD2 tag with timestamp format",
1150  "AtlCoolCopy");
1151  unsigned int run1=newsince >> 32;
1152  unsigned int run2=(newuntil-1) >> 32;
1153  for (unsigned int irun=run1;irun<=run2;++irun) {
1154  if (!std::binary_search(m_runlist.begin(),m_runlist.end(),irun))
1155  {
1156  std::cout << "Run " << irun << " from range [" << run1 << ","
1157  << run2 << "] not found in runfile list" << std::endl;
1158  throw cool::Exception("Illegal insert over locked IOV",
1159  "AtlCoolCopy");
1160  }
1161  }
1162  }
1163  // updatemode==1 will fall through, but no checks are done here
1164  }
1165  if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
1166  // cool vector payload object copy
1167  cool::IRecordIterator& pitr=obj.payloadIterator();
1168  const cool::IRecordVectorPtr vptr=pitr.fetchAllAsVector();
1169  destfl->storeObject(newsince,newuntil,
1170  *vptr,obj.channelId(),localtag,
1171  (!m_userupdatehead && !localtag.empty()));
1172  nbuf+=vptr->size();
1173  pitr.close();
1174  } else {
1175  // standard COOL object copy
1176  destfl->storeObject(newsince,newuntil,
1177  obj.payload(),obj.channelId(),localtag,
1178  (!m_userupdatehead && !localtag.empty()));
1179  ++nbuf;
1180  }
1181  // flush buffer every m_bufsize objects
1182  if (nbuf>=m_bufsize) {
1183  if (m_debug) std::cout << "Flush buffer after " << nobj << "," <<
1184  nbuf << " objects " << std::endl;
1185  destfl->flushStorageBuffer();
1186  nbuf=0;
1187  }
1188  ++nobj;
1189  }
1190  // finished with the iterator
1191  sourceitr->close();
1192  if (nbuf>0) {
1193  if (m_debug) std::cout << "Final buffer flush at " << nobj <<
1194  "," << nbuf << std::endl;
1195  destfl->flushStorageBuffer();
1196  } // end of COOL-specific part
1197  }
1198  std::cout << "Folder copied with " << nobj << " objects" << std::endl;
1199  if (nskip>0) std::cout << nskip <<
1200  " objects were skipped extending outside IOV selection or excluded channel"
1201  << std::endl;
1202  if (nbad>0) std::cout << nbad <<
1203  " objects were skipped having zero or negative IOV lengths" << std::endl;
1204  }
1205  // exceptions thrown from insert loop (both COOL or CORACOOL)
1206  catch (cool::Exception& e) {
1207  std::cout << "Exception thrown from copy loop: " << e.what() <<
1208  std::endl;
1209  if (relock) {
1210  std::cout << "Relocking destination tag " << desttag << std::endl;
1211  destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1212  }
1213  return 31;
1214  }
1215  // check if tag needs to be relocked
1216  if (relock) {
1217  std::cout << "Relocking destination tag " << desttag << std::endl;
1218  destfl->setTagLockStatus(desttag,cool::HvsTagLock::LOCKED);
1219  }
1220 
1221  if (dvermode==cool::FolderVersioning::MULTI_VERSION && desttag!="HEAD") {
1222  if (!m_usertags) {
1223  // apply HEAD style tagging
1224  std::cout << "Tag folder with HEAD-style tagging for tag: " << desttag
1225  << std::endl;
1226  try {
1227  destfl->tagCurrentHead(desttag,sourcefl->tagDescription(sourcetag));
1228  }
1229  catch (cool::Exception& e) {
1230  std::cout << "Exception thrown in HEAD-style folder-tag: " <<
1231  e.what() << std::endl;
1232  return 32;
1233  }
1234  }
1235  // record information about this tag for later
1236  if (m_copytaginfo) m_cooltagmap.insert(
1237  CoolTagMap::value_type(desttag,CoolTagInfo(m_sourceDbPtr,folder,
1238  destfolder,sourcetag,desttag,m_taglabel)));
1239  // check if any hierarchical tags reference the just-copied tag
1240  if (!m_hitagmap.empty()) {
1241  for (HiTagMap::const_iterator imap=m_hitagmap.begin();
1242  imap!=m_hitagmap.end();++imap) {
1243  try {
1244  if (sourcefl->findTagRelation(imap->first)==sourcetag) {
1245  std::cout << "Create hierarchical tag between " << desttag <<
1246  " and " << imap->first << " in folder " << imap->second <<
1247  std::endl;
1248  // first check this relation has not already been created
1249  try {
1250  std::string etag=destfl->resolveTag(imap->first);
1251  if (etag==desttag) {
1252  std::cout << "This relation has already been created" << std::endl;
1253  } else {
1254  std::cout << "ERROR: Tag in parent already related to " <<
1255  desttag << std::endl;
1256  }
1257  }
1258  catch (cool::Exception& e ) {
1259  // only do creation if does not already exist - via exception
1260  try {
1261  destfl->createTagRelation(imap->first,desttag);
1262  // check if this implicates a new parent folder
1263  // to be checked for further tags up the tree
1264  if (find(m_hiparent.begin(),m_hiparent.end(),imap->second)==
1265  m_hiparent.end()) m_hiparent.push_back(imap->second);
1266  // store the original tag information to transfer
1267  // properties later
1268  if (m_copytaginfo &&
1269  m_cooltagmap.find(imap->first)==m_cooltagmap.end())
1270  m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
1271  CoolTagInfo(m_sourceDbPtr,imap->second,imap->second,
1272  imap->first,imap->first)));
1273  }
1274  catch (cool::Exception& e) {
1275  std::cout << "Cool exception " << e.what() <<
1276  "thrown in hierarchical tag creation" << std::endl;
1277  return 34;
1278  }
1279  }
1280  }
1281  }
1282  catch (cool::Exception& e) {
1283  }
1284  }
1285  }
1286  }
1287  return 0;
1288 }
1289 
1290 void AtlCoolCopy::adjustIOVs(const cool::ValidityKey& since,
1291  const cool::ValidityKey& until,
1292  const cool::ValidityKey& qsince,
1293  const cool::ValidityKey& quntil,
1294  cool::ValidityKey& newsince,
1295  cool::ValidityKey& newuntil,
1296  bool timestamp) const {
1297  // set newsince/until to since/until
1298  // but doing any required truncation/adjustment
1299  newsince=since;
1300  newuntil=until;
1301  if (m_alliov) {
1302  // make IOV cover specified range
1303  if (timestamp) {
1304  newsince=m_newtimemin;
1305  newuntil=m_newtimemax;
1306  } else {
1307  newsince=m_newrunemin;
1308  newuntil=m_newrunemax;
1309  }
1310  }
1311  // check for IOV truncation to range of query
1312  if (m_truncate) {
1313  if (newsince<qsince) newsince=qsince;
1314  if (newuntil>quntil) newuntil=quntil;
1315  }
1316  // check not requesting an IOV with negative length
1317  if (newuntil<newsince) throw cool::Exception(
1318  "Attempt to insert IOV with -ve length","AtlCoolCopy::adjustIOVs");
1319 }
1320 
1321 int AtlCoolCopy::nocopyIOVs(const std::string& folder,
1322  const cool::IFolderPtr& sourcefl,const std::string& sourcetag,
1323  const cool::ValidityKey since,const cool::ValidityKey until,
1324  bool checkrefs) {
1325 
1326  std::cout << "Reading tag " << sourcetag << " of folder " << folder <<
1327  " (no copy)" << std::endl;
1328  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
1329  int nobj=0;
1330  cool::IObjectIteratorPtr sourceitr;
1331  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1332  sourceitr=sourcefl->browseObjects(since,until,m_chansel);
1333  } else {
1334  sourceitr=sourcefl->browseObjects(since,until,m_chansel,sourcetag);
1335  }
1336  try {
1337  while (sourceitr->goToNext()) {
1338  const cool::IObject& obj=sourceitr->currentRef();
1339  if (checkrefs) checkRef(obj.payload(),folder,sourcetag);
1340  ++nobj;
1341  }
1342  std::cout << "Folder scanned with " << nobj << " objects" << std::endl;
1343  }
1344  // exceptions thrown from read loop
1345  catch (cool::Exception& e) {
1346  std::cout << "Exception thrown from read loop: " << e.what() <<
1347  std::endl;
1348  return 33;
1349  }
1350  // finished with the iterator
1351  sourceitr->close();
1352  return 0;
1353 }
1354 
1355 int AtlCoolCopy::verifyIOVs(const std::string& folder,
1356  const cool::IFolderPtr& sourcefl,const CoraCoolFolderPtr& sourceflc,
1357  const cool::IFolderPtr& destfl,const CoraCoolFolderPtr& destflc,
1358  const std::string& sourcetag,
1359  const cool::ValidityKey since,const cool::ValidityKey until,
1360  const bool checkrefs,const bool iscora,
1361  const cool::PayloadMode::Mode paymode) {
1362 
1363  std::cout << "Verifying tag " << sourcetag << " of folder " << folder <<
1364  std::endl;
1365  destfl->setPrefetchAll(false);
1366  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
1367  int nobj=0;
1368  cool::IObjectIteratorPtr sourceitr,destitr;
1369  CoraCoolObjectIterPtr csourceitr,cdestitr;
1370  std::string tag=sourcetag;
1371  if (vermode==cool::FolderVersioning::SINGLE_VERSION || sourcetag=="HEAD")
1372  tag="";
1373  try {
1374  if (iscora) {
1375  sourceflc->setPrefetchAll(false);
1376  csourceitr=sourceflc->browseObjects(since,until,m_chansel,tag);
1377  cdestitr=destflc->browseObjects(since,until,m_chansel,tag);
1378  } else {
1379  sourceitr=sourcefl->browseObjects(since,until,m_chansel,tag);
1380  destitr=destfl->browseObjects(since,until,m_chansel,tag);
1381  }
1382  }
1383  catch (std::exception& e) {
1384  std::cout << "Exception thrown from verify iterator setup: " << e.what() <<
1385  std::endl;
1386  return 105;
1387  }
1388  try {
1389  // the check algorithm assumes the two databases will give back the
1390  // results of the query in the same order, which is currently the
1391  // case in COOL 1.3. If this changes, the verification will be more complex
1392  if (iscora) {
1393  const std::string& cfkey=sourceflc->coralFKey();
1394  const std::string& cpkey=sourceflc->coralPKey();
1395  while (csourceitr->hasNext()) {
1396  CoraCoolObjectPtr sobj=csourceitr->next();
1397  // check object should not be skipped as excluded channel
1398  if (!m_excludechans.empty()) {
1399  if (find(m_excludechans.begin(),m_excludechans.end(),
1400  sobj->channelId())!=m_excludechans.end()) {
1401  continue;
1402  }
1403  }
1404  if (cdestitr->hasNext()) {
1405  CoraCoolObjectPtr dobj=cdestitr->next();
1406  // check objects are the same
1407  int iret=0;
1408  // check IOV equality
1409  if (sobj->since()!=dobj->since() || sobj->until()!=dobj->until())
1410  iret=100;
1411  // check channel equality
1412  if (sobj->channelId()!=dobj->channelId())
1413  iret=101;
1414  // check size of payloads
1415  if (sobj->size()!=dobj->size()) {
1416  std::cout << "ERROR CoraCool object " << nobj <<
1417  " sizes do not match: " << sobj->size() << " " << dobj->size()
1418  << std::endl;
1419  iret=102;
1420  }
1421  // check payload equality (does not check attribute names)
1422  CoraCoolObject::const_iterator ditr=dobj->begin();
1423  for (CoraCoolObject::const_iterator sitr=sobj->begin();
1424  sitr!=sobj->end();++sitr,++ditr) {
1425  // loop over all the attributes, in order to skip PK and FK
1426  for (coral::AttributeList::const_iterator aitr=sitr->begin();
1427  aitr!=sitr->end();++aitr) {
1428  const std::string& aname=aitr->specification().name();
1429  if (aname!=cfkey && aname!=cpkey) {
1430  try {
1431  if (*aitr!=(*ditr)[aname]) {
1432  std::cout << "ERROR Values of attriute " << aname <<
1433  " differ" << std::endl;
1434  iret=102;
1435  }
1436  }
1437  catch (coral::AttributeListException& e) {
1438  std::cout << "ERROR: CoraCool attribute " << aname <<
1439  " not found in destination!" << std::endl;
1440  iret=102;
1441  }
1442  }
1443  }
1444  }
1445  if (iret!=0) {
1446  std::cout << "ERROR database entries do not match: since: " <<
1447  sobj->since() << "," << dobj->since() << " until: " <<
1448  sobj->until() << "," << dobj->until() << " channel: " <<
1449  sobj->channelId() << "," << dobj->channelId() << std::endl;
1450  return iret;
1451  }
1452  } else {
1453  std::cout <<
1454  "ERROR destination folder no matching CoraCool iterator for object "
1455  << nobj << std::endl;
1456  return 103;
1457  }
1458  ++ nobj;
1459  }
1460  } else {
1461  while (sourceitr->goToNext()) {
1462  const cool::IObject& sobj=sourceitr->currentRef();
1463  // check object should not be skipped as excluded channel
1464  if (!m_excludechans.empty()) {
1465  if (find(m_excludechans.begin(),m_excludechans.end(),
1466  sobj.channelId())!=m_excludechans.end()) {
1467  continue;
1468  }
1469  }
1470  if (checkrefs) checkRef(sobj.payload(),folder,sourcetag);
1471  if (destitr->goToNext()) {
1472  const cool::IObject& dobj=destitr->currentRef();
1473  // check objects are the same
1474  int iret=0;
1475  // check IOV equality
1476  if (sobj.since()!=dobj.since() || sobj.until()!=dobj.until())
1477  iret=100;
1478  // check channel equality
1479  if (sobj.channelId()!=dobj.channelId())
1480  iret=101;
1481  // check payload equality (does not check attribute names)
1482  // do not use cool::IRecord equality operator as does not correctly
1483  // handle null values
1484  if (paymode==cool::PayloadMode::VECTORPAYLOAD) {
1485  cool::IRecordIterator& spitr=sobj.payloadIterator();
1486  const cool::IRecordVectorPtr svptr=spitr.fetchAllAsVector();
1487  cool::IRecordIterator& dpitr=dobj.payloadIterator();
1488  const cool::IRecordVectorPtr dvptr=dpitr.fetchAllAsVector();
1489  if (svptr->size()!=dvptr->size()) {
1490  iret=102;
1491  } else {
1492  // loop through the payloads, checking equality - assumes
1493  // order is significant and same in source and destination DBs
1494  cool::IRecordVector::const_iterator svitr=svptr->begin();
1495  cool::IRecordVector::const_iterator svend=svptr->end();
1496  cool::IRecordVector::const_iterator dvitr=dvptr->begin();
1497  for (;svitr!=svend;++svitr,++dvitr) {
1498  if (!equalRecord(**svitr,**dvitr)) iret=102;
1499  }
1500  }
1501  if (iret!=0) {
1502  std::cout << "ERROR vector payloads do not match (size " << svptr->size() << "," << dvptr->size() << ")" << std::endl;
1503  }
1504 
1505  } else {
1506  // standard COOL folder - simple check of payloads
1507  if (!equalRecord(sobj.payload(),dobj.payload())) iret=102;
1508  }
1509 
1510  if (iret!=0) {
1511  std::cout << "ERROR database entries do not match: since: " <<
1512  sobj.since() << "," << dobj.since() << " until: " <<
1513  sobj.until() << "," << dobj.until() << " channel: " <<
1514  sobj.channelId() << "," << dobj.channelId() << std::endl;
1515  std::cout << "Source payload:" << std::endl;
1516  sobj.payload().attributeList().toOutputStream(std::cout);
1517  std::cout << std::endl << "Destination payload:" << std::endl;
1518  dobj.payload().attributeList().toOutputStream(std::cout);
1519  std::cout << std::endl;
1520  return iret;
1521  }
1522  } else {
1523  std::cout <<
1524  "ERROR destination folder no matching iterator for object "
1525  << nobj << std::endl;
1526  return 103;
1527  }
1528  ++nobj;
1529  }
1530  }
1531  }
1532  catch (cool::Exception& e) {
1533  std::cout << "Exception thrown from verify loop: " << e.what() <<
1534  std::endl;
1535  return 104;
1536  }
1537  // finished with the iterators
1538  if (iscora) {
1539  } else {
1540  sourceitr->close();
1541  destitr->close();
1542  }
1543  std::cout << "Verification of folder " << folder << " tag " <<
1544  sourcetag << " OK (" << nobj << " objects)" << std::endl;
1545  return 0;
1546 }
1547 
1548 
1549 
1550 int AtlCoolCopy::rootIOVs(const std::string& folder,
1551  const cool::IFolderPtr& sourcefl,const std::string& sourcetag,
1552  const cool::ValidityKey since,const cool::ValidityKey until,
1553  const bool timestamp) {
1554  // copy this selection to ROOT
1555  std::cout << "Write tag " << sourcetag << " of folder " << folder <<
1556  " to ROOT file" << std::endl;
1557  // create the directory structure - top directory COOL
1558  std::string treename=rootDirs(folder,"COOL");
1559  bool timestamp2=timestamp;
1560  if (m_forcetime) timestamp2=true;
1561  if (m_forcerune) timestamp2=false;
1562  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
1563  // create TTree with appropriate structure - only if it does not exist
1564  // could have been created from a previous tag in the same folder
1565  TTree* tree=static_cast<TTree*>(gDirectory->FindObject(treename.c_str()));
1566  if (tree==nullptr) {
1567  std::cout << "Book TTree " << treename << std::endl;
1568  tree=new TTree(treename.c_str(),"COOL datadump");
1569  if (timestamp2) {
1570  tree->Branch("IOVSince",&m_nt_since,"IOVSince/l");
1571  tree->Branch("IOVUntil",&m_nt_until,"IOVUntil/l");
1572  } else {
1573  tree->Branch("RunSince",&m_nt_runsince,"RunSince/i");
1574  tree->Branch("RunUntil",&m_nt_rununtil,"RunUntil/i");
1575  tree->Branch("LBSince",&m_nt_lbsince,"LBSince/i");
1576  tree->Branch("LBUntil",&m_nt_lbuntil,"LBUntil/i");
1577  }
1578  tree->Branch("ChannelID",&m_nt_channel,"ChannelID/i");
1579  if (vermode==cool::FolderVersioning::MULTI_VERSION)
1580  tree->Branch("TagID",m_nt_tagid,"TagID/C");
1581  // now loop through specification, creating payload buffer and tree
1582  const cool::IRecordSpecification& spec=
1583  (sourcefl->folderSpecification()).payloadSpecification();
1584  unsigned int ncolumns=spec.size();
1585  // clear the buffer of pointers - note this leaks the memory of the
1586  // previous buffers, but to do this properly would have to remember
1587  // the type of each object and delete appropriately
1588  m_nt_treename=treename;
1589  m_nt_bufferptr.clear();
1590  for (unsigned int icol=0;icol<ncolumns;++icol) {
1591  const cool::IFieldSpecification& fieldspec=spec[icol];
1592  void* ptr=nullptr;
1593  char rootID;
1594  if (rootAllocate(fieldspec,ptr,rootID)) {
1595  tree->Branch(fieldspec.name().c_str(),ptr,
1596  (fieldspec.name()+"/"+rootID).c_str());
1597  if (m_debug) std::cout << "Defining column for " << spec[icol].name()
1598  << " of type " << spec[icol].storageType().name() << std::endl;
1599  } else {
1600  std::cout << "Attribute " << spec[icol].name() << " of type " <<
1601  spec[icol].storageType().name() << " will be skipped" << std::endl;
1602  }
1603  m_nt_bufferptr.push_back(ptr);
1604  }
1605  } else {
1606  std::cout << "TTree " << treename << " already exists" << std::endl;
1607  // check we have seen and defined this tree
1608  unsigned int size=
1609  (sourcefl->folderSpecification()).payloadSpecification().size();
1610  if (treename!=m_nt_treename || size!=m_nt_bufferptr.size()) {
1611  std::cout << "ERROR in tree buffer definition: expect " << treename <<
1612  " size " << size << " but buffer has " << m_nt_treename << " size "
1613  << m_nt_bufferptr.size() << std::endl;
1614  return 123;
1615  }
1616  }
1617 
1618  int nobj=0;
1619  int nex=0;
1620  cool::IObjectIteratorPtr sourceitr;
1621  try {
1622  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1623  sourceitr=sourcefl->browseObjects(since,until,m_chansel);
1624  } else {
1625  sourceitr=sourcefl->browseObjects(since,until,m_chansel,sourcetag);
1626  }
1627  }
1628  catch (cool::Exception& e) {
1629  std::cout << "Exception thrown from ROOT copy iterator setup: " <<
1630  e.what() << std::endl;
1631  return 125;
1632  }
1633  // now loop over all IOVs to copy them
1634  try {
1635  while (sourceitr->goToNext()) {
1636  const cool::IObject& sobj=sourceitr->currentRef();
1637  if (timestamp2) {
1638  m_nt_since=sobj.since();
1639  m_nt_until=sobj.until();
1640  } else {
1641  m_nt_runsince=(sobj.since() >> 32);
1642  m_nt_rununtil=(sobj.until() >> 32);
1643  m_nt_lbsince=(sobj.since() & 0xFFFFFFFF);
1644  m_nt_lbuntil=(sobj.until() & 0xFFFFFFFF);
1645  }
1646  m_nt_channel=sobj.channelId();
1647  if (vermode==cool::FolderVersioning::MULTI_VERSION) {
1648  // truncate the string first to avoid coverity complaining about
1649  // potential buffer overruns
1650  std::string sourcetag2=sourcetag.substr(0,255);
1651  strncpy(m_nt_tagid,sourcetag2.c_str(),sizeof(m_nt_tagid)-1);
1652  }
1653  // loop over the payload elements and fill the ones for which buffers
1654  // are defined
1655  try {
1656  const cool::IRecord& record=sobj.payload();
1657  for (unsigned int icol=0;icol<record.size();++icol) {
1658  if (m_nt_bufferptr[icol]!=nullptr) rootWrite(m_nt_bufferptr[icol],record[icol]);
1659  }
1660  tree->Fill();
1661  ++nobj;
1662  }
1663  catch (cool::Exception& e) {
1664  // can get exceptions due to NULL values
1665  // catch them and continue...
1666  ++nex;
1667  }
1668  }
1669  sourceitr->close();
1670  std::cout << "Written " << nobj << " objects to ROOT TTree with " << nex
1671  << " nulls/exceptions" << std::endl;
1672  }
1673  catch (cool::Exception& e) {
1674  std::cout << "Exception thrown from ROOT file writing loop: " <<
1675  e.what() << std::endl;
1676  return 124;
1677  }
1678  return 0;
1679 }
1680 
1681 std::string AtlCoolCopy::rootDirs(const std::string& folder,
1682  const std::string& toproot) {
1683  // create a ROOT directory structure start with toproot (=COOL or COOLANA)
1684  // and cd to the directory where a tree should be created
1685  // return the tree name (=leaf of the COOL foldername)
1686  p_rootfile->cd();
1687  if (p_rootfile->FindObject(toproot.c_str())==nullptr) {
1688  p_rootfile->mkdir(toproot.c_str());
1689  std::cout << "Made top directory " << toproot << std::endl;
1690  }
1691  gDirectory->cd(toproot.c_str());
1692  // now parse the COOL folder name to create intermediate directories
1693  // assume folder names begin with '/', to be skipped
1694  std::string::size_type iofs1=1;
1695  std::string::size_type iofs2=1;
1696  std::string treename;
1697  while (iofs2!=std::string::npos) {
1698  iofs2=folder.find('/',iofs1);
1699  if (iofs2==std::string::npos) {
1700  // no more /, so current part of string is leaf tree name
1701  treename=folder.substr(iofs1);
1702  } else {
1703  std::string dirname=folder.substr(iofs1,iofs2-iofs1);
1704  iofs1=iofs2+1;
1705  if (gDirectory->FindObject(dirname.c_str())==nullptr) {
1706  std::cout << "Make directory " << dirname << std::endl;
1707  gDirectory->mkdir(dirname.c_str());
1708  }
1709  gDirectory->cd(dirname.c_str());
1710  }
1711  }
1712  return treename;
1713 }
1714 
1715 bool AtlCoolCopy::rootAllocate(const cool::IFieldSpecification& spec,
1716  void*& sptr,char& rootID) const {
1717  const cool::StorageType& stype=spec.storageType();
1718  rootID=' ';
1719  sptr=nullptr;
1720  if (stype==cool::StorageType::Bool) {
1721  // map bool to uint32 for now
1722  sptr=static_cast<void*>(new unsigned int(0));
1723  rootID='i';
1724  } else if (stype==cool::StorageType::UChar) {
1725  sptr=static_cast<void*>(new unsigned char(' '));
1726  rootID='C';
1727  } else if (stype==cool::StorageType::Int16) {
1728  sptr=static_cast<void*>(new short(0));
1729  rootID='S';
1730  } else if (stype==cool::StorageType::UInt16) {
1731  sptr=static_cast<void*>(new unsigned short(0));
1732  rootID='s';
1733  } else if (stype==cool::StorageType::Int32) {
1734  sptr=static_cast<void*>(new int(0));
1735  rootID='I';
1736  } else if (stype==cool::StorageType::UInt32) {
1737  sptr=static_cast<void*>(new unsigned int(0));
1738  rootID='i';
1739  } else if (stype==cool::StorageType::Int64) {
1740  sptr=static_cast<void*>(new long long int(0));
1741  rootID='L';
1742  } else if (stype==cool::StorageType::UInt63) {
1743  sptr=static_cast<void*>(new unsigned long long int(0));
1744  rootID='l';
1745  } else if (stype==cool::StorageType::Float) {
1746  sptr=static_cast<void*>(new float(0.));
1747  rootID='F';
1748  } else if (stype==cool::StorageType::Double) {
1749  sptr=static_cast<void*>(new double(0.));
1750  rootID='D';
1751  } else if (stype==cool::StorageType::String4k ||
1752  stype==cool::StorageType::String255) {
1753  sptr=static_cast<void*>(new char[4100]);
1754  rootID='C';
1755  } else if (stype==cool::StorageType::String64k && !m_noclobroot) {
1756  sptr=static_cast<void*>(new char[65536]);
1757  rootID='C';
1758  } else if (stype==cool::StorageType::String16M && !m_noclobroot) {
1759  sptr=static_cast<void*>(new char[67108864]);
1760  rootID='C';
1761  } else if (stype==cool::StorageType::Blob64k && !m_noclobroot) {
1762  sptr=static_cast<void*>(new char[65536]);
1763  rootID='C';
1764  } else {
1765  std::cout << "rootAllocate: Unsupported storage type for attribute: " <<
1766  spec.name() << std::endl;
1767  }
1768  return (sptr!=nullptr);
1769 }
1770 
1771 void AtlCoolCopy::rootWrite(void* sptr,const cool::IField& field) const {
1772  // check for NULL value
1773  bool recnull=(field.isNull() && m_zeronull);
1774  const cool::StorageType& stype=field.storageType();
1775  if (stype==cool::StorageType::Bool) {
1776  if (recnull) {
1777  *(static_cast<int*>(sptr))=0;
1778  } else {
1779  *(static_cast<int*>(sptr))=(field.data<bool>() ? 1 : 0);
1780  }
1781  } else if (stype==cool::StorageType::UChar) {
1782  if (recnull) {
1783  *(static_cast<unsigned char*>(sptr))=0;
1784  } else {
1785  *(static_cast<unsigned char*>(sptr))=field.data<unsigned char>();
1786  }
1787  } else if (stype==cool::StorageType::Int16) {
1788  if (recnull) {
1789  *(static_cast<short*>(sptr))=0;
1790  } else {
1791  *(static_cast<short*>(sptr))=field.data<short>();
1792  }
1793  } else if (stype==cool::StorageType::UInt16) {
1794  if (recnull) {
1795  *(static_cast<unsigned short*>(sptr))=0;
1796  } else {
1797  *(static_cast<unsigned short*>(sptr))=field.data<unsigned short>();
1798  }
1799  } else if (stype==cool::StorageType::Int32) {
1800  if (recnull) {
1801  *(static_cast<int*>(sptr))=0;
1802  } else {
1803  *(static_cast<int*>(sptr))=field.data<int>();
1804  }
1805  } else if (stype==cool::StorageType::UInt32) {
1806  if (recnull) {
1807  *(static_cast<unsigned int*>(sptr))=0;
1808  } else {
1809  *(static_cast<unsigned int*>(sptr))=field.data<unsigned int>();
1810  }
1811  } else if (stype==cool::StorageType::Int64) {
1812  if (recnull) {
1813  *(static_cast<long long*>(sptr))=0;
1814  } else {
1815  *(static_cast<long long*>(sptr))=field.data<long long>();
1816  }
1817  } else if (stype==cool::StorageType::UInt63) {
1818  if (recnull) {
1819  *(static_cast<unsigned long long*>(sptr))=0;
1820  } else {
1821  *(static_cast<unsigned long long*>(sptr))=field.data<unsigned long long>();
1822  }
1823  } else if (stype==cool::StorageType::Float) {
1824  if (recnull) {
1825  *(static_cast<float*>(sptr))=0.;
1826  } else {
1827  *(static_cast<float*>(sptr))=field.data<float>();
1828  }
1829  } else if (stype==cool::StorageType::Double) {
1830  if (recnull) {
1831  *(static_cast<double*>(sptr))=0.;
1832  } else {
1833  *(static_cast<double*>(sptr))=field.data<double>();
1834  }
1835  } else if (stype==cool::StorageType::String255 ||
1836  stype==cool::StorageType::String4k ||
1837  stype==cool::StorageType::String64k ||
1838  stype==cool::StorageType::String16M) {
1839  if (recnull) {
1840  strcpy(static_cast<char*>(sptr),"NULL");
1841  } else {
1842  strcpy(static_cast<char*>(sptr),field.data<std::string>().c_str());
1843  }
1844  } else if (stype==cool::StorageType::Blob64k && !m_noclobroot) {
1845  if (recnull) {
1846  strcpy(static_cast<char*>(sptr),"NULL");
1847  } else {
1848  auto blob = field.data<coral::Blob>();
1849  std::string blobStr((char*)blob.startingAddress(), blob.size());
1850  strcpy(static_cast<char*>(sptr), blobStr.c_str());
1851  }
1852  } else {
1853  std::cout << "ERROR: Unknown storage type in rootWrite!" << std::endl;
1854  }
1855 }
1856 
1857 int AtlCoolCopy::analyseIOVs(const std::string& folder,
1858  const cool::IFolderPtr& sourcefl,const std::string& sourcetag,
1859  const cool::ValidityKey since,const cool::ValidityKey until,
1860  const bool timestamp) {
1861  // analyse the IOV and channel structure of this folder/tag, make ROOT histos
1862  std::cout << "Analyse tag " << sourcetag << " of folder " << folder <<
1863  std::endl;
1864  bool anatime=(m_anadelt>=0);
1865  if (anatime) {
1866  std::cout << "Analyse time structures with tolerance of " <<
1867  m_anadelt/1.E9 << " seconds " << std::endl;
1868  }
1869  // create COOLANA/folder structure and create leaf directory for histograms
1870  std::string dirname=rootDirs(folder,"COOLANA");
1871  if (gDirectory->FindObject(dirname.c_str())==nullptr) {
1872  std::cout << "Make directory " << dirname << std::endl;
1873  gDirectory->mkdir(dirname.c_str());
1874  }
1875  gDirectory->cd(dirname.c_str());
1876  // get channels
1877  std::vector<cool::ChannelId> channels=sourcefl->listChannels();
1878  unsigned int nchan=channels.size();
1879  // end of last IOV seen on this channel
1880  long long* lastiov=new long long[nchan];
1881  long long* iovtotlen=new long long[nchan];
1882  long long* iovtotgap=new long long[nchan];
1883  int* iovn=new int[nchan];
1884  for (unsigned int i=0;i<nchan;++i) {
1885  lastiov[i]=-1; // no IOV seen on this channel yet
1886  iovtotlen[i]=0;
1887  iovtotgap[i]=0;
1888  iovn[i]=0;
1889  }
1890  cool::ValidityKey globsince=cool::ValidityKeyMax;
1891  cool::ValidityKey globuntil=cool::ValidityKeyMin;
1892 
1893  // book histograms - only if the objects do not already exist (previous tag)
1894  TH1F* h_iovchan=bookOrFindTH1F("IOVSperChannel","IOVs per channel",
1895  nchan,-0.5,nchan-0.5);
1896  TH1F* h_iovname=bookOrFindTH1F("IOVSperChannelName",
1897  "IOVs per channel-name (copy of IOVSperChannel)",
1898  nchan,-0.5,nchan-0.5);
1899  TH1F* h_iovlength=bookOrFindTH1F("IOVLogLength","IOV log10 length",
1900  100,0.,20.);
1901  TH1F* h_iovgap=bookOrFindTH1F("IOVLogGapLength","IOV log10 gap length",
1902  100,0.,20.);
1903  // histograms for time analysis if needed
1904  TH1F* h_anadelt=nullptr;
1905  TH1F* h_chandelt=nullptr;
1906  if (anatime) {
1907  h_anadelt=bookOrFindTH1F("IOVAlignLogDelT","IOV alignment log deltaT",
1908  100,0.,20.);
1909  // add factor 2 in number of channels, to catch those where a channel
1910  // changes many times within the m_anadelt window (i.e. noise/jitter)
1911  h_chandelt=bookOrFindTH1F("AlignedChanPerIOV","Aligned channels per IOV",
1912  nchan*2,0.5,nchan*2-0.5);
1913  }
1914  std::cout << "Booked histograms for folder with " << nchan << " channels"
1915  << std::endl;
1916  // setup storage for IOV time analysis
1917  using IOVTimeMap = std::map<cool::ValidityKey, int>;
1918  IOVTimeMap iov_time_map;
1919 
1920  // setup iterator to loop over objects
1921  int nobj=0;
1922  cool::FolderVersioning::Mode vermode=sourcefl->versioningMode();
1923  cool::IObjectIteratorPtr sourceitr;
1924  try {
1925  if (vermode==cool::FolderVersioning::SINGLE_VERSION) {
1926  sourceitr=sourcefl->browseObjects(since,until,m_chansel);
1927  } else {
1928  sourceitr=sourcefl->browseObjects(since,until,m_chansel,sourcetag);
1929  }
1930  }
1931  catch (cool::Exception& e) {
1932  std::cout << "Exception thrown from analysis copy iterator setup: " <<
1933  e.what() << std::endl;
1934  return 135;
1935  }
1936  // look up names of all the channels in one go
1937  std::map<cool::ChannelId,std::string>
1938  chanmap=sourcefl->listChannelsWithNames();
1939  // now loop over all IOVs and write them
1940  try {
1941  while (sourceitr->goToNext()) {
1942  const cool::IObject& sobj=sourceitr->currentRef();
1943  // get channel and find its index
1944  cool::ChannelId ichanid=sobj.channelId();
1945  const std::string& cname=chanmap[ichanid];
1948  std::equal_range(channels.begin(),channels.end(),ichanid);
1949  if (chanitr.first!=chanitr.second) {
1950  cool::ChannelId ichan=std::distance(channels.begin(),chanitr.first);
1951  Int_t bin=h_iovchan->Fill(ichan);
1952  if (ichan && !h_iovname->GetBinContent(bin)) {
1953  h_iovname->GetXaxis()->SetBinLabel(bin,cname.c_str());
1954  }
1955  h_iovname->Fill(ichan);
1956  // analyse IOV start/stop
1957  cool::ValidityKey since2=sobj.since();
1958  cool::ValidityKey until2=sobj.until();
1959  if (since2<globsince) globsince=since2;
1960  // dont count IOV is open-ended, so length statistics will make sense
1961  if (until2!=cool::ValidityKeyMax) {
1962  if (until2>globuntil) globuntil=until2;
1963  long long len=until2-since2;
1964  h_iovlength->Fill(log10(len));
1965  iovn[ichan]+=1;
1966  iovtotlen[ichan]+=len;
1967  }
1968  if (lastiov[ichan]<static_cast<long long>(since2) &&
1969  lastiov[ichan]>=0) {
1970  // have a gap in the IOV structure for this channel
1971  long long gap=since2-lastiov[ichan];
1972  iovtotgap[ichan]+=gap;
1973  h_iovgap->Fill(log10(gap));
1974  std::cout << "Gap of " << gap << std::endl;
1975  }
1976  if (until2!=cool::ValidityKeyMax) {
1977  lastiov[ichan]=until2;
1978  } else {
1979  lastiov[ichan]=since2;
1980  }
1981  // analyse IOV alignment
1982  if (anatime) {
1983  long long del1=-1;
1984  long long del2=-1;
1985  // find the nearest time to this one
1986  // this gives the first value exceeding since2
1987  IOVTimeMap::iterator hiitr=iov_time_map.lower_bound(since2);
1988  IOVTimeMap::iterator lowitr=hiitr;
1989  if (hiitr!=iov_time_map.end()) {
1990  // del1 is +ve time interval to next one
1991  del1=hiitr->first-since2;
1992  }
1993  if (lowitr!=iov_time_map.begin()) {
1994  // del2 is +ve time interval to previous one
1995  --lowitr;
1996  del2=since2-lowitr->first;
1997  }
1998  long long del=-1;
1999  IOVTimeMap::iterator moditr;
2000  bool domod=false;
2001  if (del1<=m_anadelt && del1>-1) {
2002  moditr=hiitr;
2003  del=del1;
2004  domod=true;
2005  }
2006  if (del2<=m_anadelt && del2>-1 && (del2<del || del==-1)) {
2007  moditr=lowitr;
2008  del=del2;
2009  domod=true;
2010  }
2011  if (domod) {
2012  ++(moditr->second);
2013  if (del>0) {
2014  h_anadelt->Fill(log10(del));
2015  } else {
2016  h_anadelt->Fill(0.);
2017  }
2018  } else {
2019  iov_time_map[since2]=0;
2020  }
2021  }
2022  } else {
2023  // channel ID not found
2024  std::cout << "ERROR :Channel " << ichanid <<
2025  " not found in channel list - ignored" << std::endl;
2026  }
2027  ++nobj;
2028  }
2029  std::cout << "Finished analysis with " << nobj << " objects" << std::endl;
2030  if (timestamp) {
2031  std::cout << "IOV timestamp range: " << globsince << " " <<
2032  timeString(globsince) << "to " << globuntil << " " <<
2033  timeString(globuntil) << std::endl;
2034  } else {
2035  std::cout << "IOV run/LB range [" << (globsince >> 32) << "," <<
2036  (globsince & 0xFFFFFFFF) << "] to [" << (globuntil >> 32) << "," <<
2037  (globuntil & 0xFFFFFFFF) << "]" << std::endl;
2038  }
2039  // calculate statistics for each channel
2040  TH1F* h_iovoccchan=bookOrFindTH1F("ChannelOcc","Channel Occupancy",
2041  nchan,-0.5,nchan-0.5);
2042  TH1F* h_iovlenchan=bookOrFindTH1F("IOVLenChan",
2043  "Mean IOV length per channel",nchan,-0.5,nchan-0.5);
2044  for (unsigned int i=0;i<nchan;++i) {
2045  float occ=0.;
2046  float avlen=0.;
2047  if (iovn[i]>0) {
2048  occ=1.-float(iovtotgap[i])/float(iovtotlen[i]+iovtotgap[i]);
2049  avlen=float(iovtotlen[i])/iovn[i];
2050  }
2051  h_iovoccchan->Fill(i,occ);
2052  h_iovlenchan->Fill(i,avlen);
2053  }
2054  if (anatime) {
2055  std::cout << "Alignment analysis: " << iov_time_map.size()
2056  << " seperate IOV starts (tolerance " << m_anadelt/1.E9 <<
2057  " seconds)" << std::endl;
2058  // fill histogram of number of channels incremented per IOV
2059  for (IOVTimeMap::const_iterator itr=iov_time_map.begin();
2060  itr!=iov_time_map.end();++itr) {
2061  h_chandelt->Fill(itr->second);
2062  }
2063  }
2064  }
2065  catch (cool::Exception& e) {
2066  std::cout << "Exception thrown from folder analysis reading loop: " <<
2067  e.what() << std::endl;
2068  return 134;
2069  }
2070  sourceitr->close();
2071  delete [] lastiov;
2072  delete [] iovtotlen;
2073  delete [] iovtotgap;
2074  delete[] iovn; iovn=nullptr;
2075  return 0;
2076 }
2077 
2078 TH1F* AtlCoolCopy::bookOrFindTH1F(const std::string& hID,
2079  const std::string& htitle,
2080  const int nchan, const float xlow, const float xhigh) {
2081  // check histogram does not exist in current directory
2082  // if yes, return pointer to it, else book and return pointer
2083  TObject* obj=gDirectory->FindObject(hID.c_str());
2084  if (obj==nullptr) {
2085  return new TH1F(hID.c_str(),htitle.c_str(),nchan,xlow,xhigh);
2086  } else {
2087  return static_cast<TH1F*>(obj);
2088  }
2089 }
2090 
2091 
2092 int AtlCoolCopy::doCopy ATLAS_NOT_THREAD_SAFE () {
2093  if (isOpen()) {
2094  int retcode=0;
2095  // execute copy for all defined folders
2096  for (std::vector<std::string>::const_iterator ifolder=
2097  m_folderlist.begin();ifolder!=m_folderlist.end();++ifolder) {
2098  int code=copyFolder(*ifolder,m_tags);
2099  if (code>retcode) retcode=code;
2100  }
2101  if (!m_hiparent.empty()) {
2102  // fill in any hierarchical tags for parent folders
2103  int code=tagParents();
2104  if (code>retcode) retcode=code;
2105  }
2106  if (m_copytaginfo) {
2107  // write information (tag description, lock status) to dest tags
2108  int code=writeTagInfo();
2109  if (code>retcode) retcode=code;
2110  }
2111  if (m_checkrefs) {
2112  int code=0;
2113  if (m_poolcat.empty()) {
2114  code=listPoolRefs();
2115  if (!m_checkoutputfile.empty()) filePoolRefs();
2116  } else {
2117  code=resolvePoolRefs();
2118  }
2119  if (code>retcode) retcode=code;
2120  }
2121  if (p_rootfile) {
2122  p_rootfile->Write();
2123  delete p_rootfile;
2124  }
2125  return retcode;
2126  } else {
2127  return 10;
2128  }
2129 }
2130 
2131 bool AtlCoolCopy::procOptVector(const int argc, const char* argv[],
2132  std::vector<std::string>& folders) {
2133  // process an array of options, return True if OK, False if not
2134  int ic=0;
2135  bool error=false;
2136  while (ic<argc) {
2137  int ir=argc-ic;
2138  std::string_view par0=argv[ic];
2139  // strip double "--" to achieve compatability with python-style options
2140  if (par0.compare(0,2,"--")==0) par0=par0.substr(1);
2141  if ((par0=="-f" || par0=="-folder") && ir>1) {
2142  folders.emplace_back(argv[ic+1]);
2143  ++ic;
2144  } else if ((par0=="-e" || par0=="-exclude") && ir>1) {
2145  addExclude(argv[ic+1]);
2146  ++ic;
2147  } else if ((par0=="-t" || par0=="-tag") && ir>1) {
2148  m_tags.emplace_back(argv[ic+1]);
2149  ++ic;
2150  } else if ((par0=="-mt" || par0=="-magic") && ir>1) {
2151  m_magic.push_back("_"+std::string(argv[ic+1]));
2152  ++ic;
2153  } else if ((par0=="-of" || par0=="-outfolder") && ir>1) {
2154  m_outfolder=argv[ic+1];
2155  ++ic;
2156  } else if ((par0=="-ot" || par0=="-outtag") && ir>1) {
2157  m_outtag=argv[ic+1];
2158  ++ic;
2159  } else if ((par0=="-bs" || par0=="-buffersize") && ir>1) {
2160  m_bufsize=atoi(argv[ic+1]);
2161  ++ic;
2162  } else if ((par0=="-sl" || par0=="-seal") && ir>1) {
2163  m_sealmsg=atoi(argv[ic+1]);
2164  ++ic;
2165  } else if ((par0=="-rls" || par0=="-runlumisince") && ir>2) {
2166  m_runemin=runLBVal(argv[ic+1],argv[ic+2]);
2167  ic+=2;
2168  } else if ((par0=="-rlu" || par0=="-runlumiuntil") && ir>2) {
2169  m_runemax=runLBVal(argv[ic+1],argv[ic+2]);
2170  ic+=2;
2171  } else if ((par0=="-nrls" || par0=="-newrunlumisince") && ir>2) {
2173  ic+=2;
2174  } else if ((par0=="-nrlu" || par0=="-newrunlumiuntil") && ir>2) {
2176  ic+=2;
2177  } else if ((par0=="-srls" || par0=="-skiprunlumisince") && ir>2) {
2178  m_srunemin=runLBVal(argv[ic+1],argv[ic+2]);
2179  m_skipout=true;
2180  ic+=2;
2181  } else if ((par0=="-srlu" || par0=="-skiprunlumiuntil") && ir>2) {
2182  m_srunemax=runLBVal(argv[ic+1],argv[ic+2]);
2183  m_skipout=true;
2184  ic+=2;
2185  } else if (par0=="-ro" || par0=="-root") {
2186  m_root=true;
2187  } else if (par0=="-zn" || par0=="-zeronull") {
2188  m_zeronull=true;
2189  } else if (par0=="-ana" || par0=="-analyse") {
2190  m_analyse=true;
2191  float ttime=atof(argv[ic+1])*1.E9;
2192  m_anadelt=static_cast<long long>(ttime);
2193  ++ic;
2194  } else if ((par0=="-rs" || par0=="-runsince") && ir>1) {
2195  m_runemin=(static_cast<long long>(atol(argv[ic+1])) << 32);
2196  ++ic;
2197  } else if ((par0=="-ru" || par0=="-rununtil") && ir>1) {
2198  m_runemax=(static_cast<long long>(1+atol(argv[ic+1])) << 32)-1;
2199  ++ic;
2200  } else if ((par0=="-r" || par0=="-run") && ir>1) {
2201  m_runemin=(static_cast<long long>(atol(argv[ic+1])) << 32);
2202  m_runemax=(static_cast<long long>(1+atol(argv[ic+1])) << 32)-1;
2203  ++ic;
2204  } else if ((par0=="-ts" || par0=="-timesince") && ir>1) {
2205  m_timemin=timeVal(argv[ic+1]);
2206  ++ic;
2207  } else if ((par0=="-tu" || par0=="-timeuntil") && ir>1) {
2208  m_timemax=timeVal(argv[ic+1]);
2209  ++ic;
2210  } else if ((par0=="-nts" || par0=="-newtimesince") && ir>1) {
2212  ++ic;
2213  } else if ((par0=="-ntu" || par0=="-newtimeuntil") && ir>1) {
2215  ++ic;
2216  } else if (par0=="-c" || par0=="-create") {
2217  m_allowcreate=true;
2218  } else if ((par0=="-ch" || par0=="-channel") && ir>1) {
2219  m_channelRange.emplace_back(argv[ic+1]);
2220  ++ic;
2221  } else if ((par0=="-ch1" || par0=="-channel1") && ir>1) {
2222  m_channel1=argv[ic+1];
2223  ++ic;
2224  } else if ((par0=="-ch2" || par0=="-channel2") && ir>1) {
2225  m_channel2=argv[ic+1];
2226  ++ic;
2227  } else if (par0=="-chd" || par0=="-channeldesc") {
2228  m_chdesc=true;
2229  // no abbreviaton for this one - dangerous option
2230  } else if (par0=="-forcerecreate") {
2231  m_recreate=true;
2232  } else if (par0=="-d" || par0=="-debug") {
2233  m_debug=true;
2234  } else if (par0=="-a" || par0=="-alliov") {
2235  m_alliov=true;
2236  } else if (par0=="-ih" || par0=="-includehead") {
2237  m_includehead=true;
2238  } else if (par0=="-eh" || par0=="-excludehead") {
2239  m_excludehead=true;
2240  } else if (par0=="-ht" || par0=="-headtag") {
2241  m_usertags=false;
2242  } else if (par0=="-uht" || par0=="-userheadtag") {
2243  m_userupdatehead=true;
2244  } else if (par0=="-v" || par0=="-verify") {
2245  m_verify=true;
2246  } else if (par0=="-nc" || par0=="-nocopy") {
2247  m_nocopy=true;
2248  } else if (par0=="-noc" || par0=="-nocoracool") {
2249  m_coracool=false;
2250  } else if (par0=="-nch" || par0=="-nochannel") {
2251  m_nochannel=true;
2252  } else if (par0=="-ncr" || par0=="-noclobroot") {
2253  m_noclobroot=true;
2254  } else if (par0=="-nd" || par0=="-nodata") {
2255  m_nodata=true;
2256  } else if (par0=="-nh" || par0=="-nohitag") {
2257  m_nohitag=true;
2258  } else if (par0=="-hi" || par0=="-hitag") {
2259  m_hitag=true;
2260  } else if ((par0=="-ec" || par0=="-excludechannel") && ir>1) {
2261  m_excludechans.push_back(strtoul(argv[ic+1],nullptr,10));
2262  ++ic;
2263  } else if (par0=="-fs" || par0=="-forcesingle") {
2264  m_forcesingle=true;
2265  } else if (par0=="-fm" || par0=="-forcemulti") {
2266  m_forcemulti=true;
2267  } else if (par0=="-frl" || par0=="-forcerunlumi") {
2268  m_forcerune=true;
2269  } else if (par0=="-ftm" || par0=="-forcetime") {
2270  m_forcetime=true;
2271  } else if (par0=="-fp" || par0=="-forcepayload") {
2272  m_forcepay=true;
2273  } else if (par0=="-fnp" || par0=="-forcenopayload") {
2274  m_forcenopay=true;
2275  } else if (par0=="-cr" || par0=="-checkrefs") {
2276  m_checkrefs=true;
2277  } else if (par0=="-lp" || par0=="-listpfn") {
2278  m_listpfn=true;
2279  m_checkrefs=true;
2280  } else if (par0=="-cf" || par0=="-checkfiles") {
2281  m_poolopen=true;
2282  m_checkrefs=true;
2283  } else if (par0=="-pa" || par0=="-poolall") {
2284  m_poolall=true;
2285  } else if ((par0=="-co" || par0=="-checkoutput") && ir>1) {
2287  ++ic;
2288  } else if ((par0=="-pc" || par0=="-poolcat") && ir>1) {
2289  m_poolcat.emplace_back(argv[ic+1]);
2290  ++ic;
2291  } else if ((par0=="-mc" || par0=="-mergecat") && ir>1) {
2292  m_mergecat.emplace_back(argv[ic+1]);
2293  ++ic;
2294  } else if ((par0=="-ds" || par0=="-dataset") && ir>1) {
2295  m_newdataset=argv[ic+1];
2296  ++ic;
2297  } else if (par0=="-us" || par0=="-updatesource") {
2298  m_sourceread=false;
2299  } else if (par0=="-cd" || par0=="-checkdest") {
2300  m_checkdesttag=true;
2301  } else if (par0=="-tr" || par0=="-truncate") {
2302  m_truncate=true;
2303  } else if (par0=="-al" || par0=="-appendlocked") {
2304  m_applock=true;
2305  } else if (par0=="-alsv" || par0=="-appendlockedsv") {
2306  m_applock=true;
2307  m_applocksv=true;
2308  } else if (par0=="-rdo" || par0=="-readoracle") {
2309  m_readoracle=true;
2310  } else if (par0=="-skiprep" || par0=="-sr") {
2311  m_skiprep=true;
2312  } else if (par0=="-go" || par0=="-getonline") {
2313  m_getonline=true;
2314  } else if (par0=="-onr" || par0=="-onlinerun") {
2315  m_onlinerun=true;
2316  } else if (par0=="-gb" || par0=="-getbulk") {
2317  m_getbulk=true;
2318  } else if (par0=="-gt" || par0=="-gettime") {
2319  m_gettime=true;
2320  } else if ((par0=="-tdb" || par0=="-timedb") && ir>1) {
2321  m_timedb=argv[ic+1];
2322  ++ic;
2323  } else if (par0=="-ignoremode" && ir>1) {
2324  std::cout << "Ignoremode password is " << argv[ic+1] << ":end" << std::endl;
2325  if (strcmp(argv[ic+1],"BackDoor")==0) {
2326  m_ignoremode=true;
2327  } else {
2328  std::cout << "ERROR: Incorrect password for -ignoremode" << std::endl;
2329  error=true;
2330  }
2331  ++ic;
2332  } else if (par0=="-is" || par0=="-ignorespec") {
2333  m_ignorespec=true;
2334  } else if (par0=="-pt" || par0=="-prunetags") {
2335  m_prunetags=true;
2336  m_excludehead=true;
2337  } else if (par0=="-lo" || par0=="-lockedonly") {
2338  m_lockedonly=true;
2339  m_prunetags=true;
2340  m_excludehead=true;
2341  } else if (par0=="-cti" || par0=="-copytaginfo") {
2342  m_copytaginfo=true;
2343  } else if (par0=="-ctl" || par0=="-copytaglock") {
2344  m_copytaginfo=true;
2345  m_copytaglock=true;
2346  } else if ((par0=="-tl" || par0=="-taglabel") && ir>1) {
2347  m_taglabel=argv[ic+1];
2348  ++ic;
2349  m_copytaginfo=true;
2350  } else if ((par0=="-ag" || par0=="-addguid") && ir>1) {
2351  m_addguid.emplace_back(argv[ic+1]);
2352  ++ic;
2353  } else if ((par0=="-alf" || par0=="-addlfn") && ir>1) {
2354  m_addlfn.emplace_back(argv[ic+1]);
2355  ++ic;
2356  } else if ((par0=="-pf" || par0=="-parfile") && ir>1) {
2357  m_parfile.emplace_back(argv[ic+1]);
2358  ++ic;
2359  } else if ((par0=="-rf" || par0=="-runfile") && ir>1) {
2360  m_runfile.emplace_back(argv[ic+1]);
2361  ++ic;
2362  } else if ((par0=="-ws" || par0=="-runinfohost") && ir>1) {
2363  m_runinfohost=argv[ic+1];
2364  ++ic;
2365  } else if (par0=="-h" || par0=="-help") {
2366  // help printout triggered by -999 return code
2367  return 999;
2368  } else {
2369  std::cout << "Parameter error for argument: " << par0 << std::endl;
2370  error=true;
2371  }
2372  ++ic;
2373  }
2374  return !error;
2375 }
2376 
2377 
2378 int AtlCoolCopy::setOpts(int argc, const char* argv[]) {
2379  // accumulate raw folder list
2380  std::vector<std::string> folders;
2381  // process options given on command-line
2382  if (!procOptVector(argc,argv,folders)) return 2;
2383 
2384  // parse any parameters in files via parfile option
2385  for (std::vector<std::string>::const_iterator ifile=m_parfile.begin();
2386  ifile!=m_parfile.end();++ifile) {
2387  std::cout << "Reading parameters from file " << *ifile << std::endl;
2388  FILE* p_inp=fopen(ifile->c_str(),"r");
2389  if (p_inp==nullptr) {
2390  std::cout << "File not found" << std::endl;
2391  return 3;
2392  }
2393  char* p_buf=new char[999];
2394  while (!feof(p_inp)) {
2395  char* p_line=fgets(p_buf,999,p_inp);
2396  if (p_line!=nullptr) {
2397  int fargc=0;
2398  const char* fargv[99];
2399  // split this line into tokens
2400  char* p_start=nullptr;
2401  while (*p_line!='\0') {
2402  if (*p_line==' ' || *p_line=='\n') {
2403  // pointing at a space/newline, marking the end of a parameter
2404  if (p_start!=nullptr) {
2405  // if we have a parameter, mark the end and store it
2406  *p_line='\0';
2407  fargv[fargc]=p_start;
2408  fargc++;
2409  p_start=nullptr;
2410  }
2411  } else {
2412  // mark the start of a parameter
2413  if (p_start==nullptr) p_start=p_line;
2414  }
2415  ++p_line;
2416  }
2417  if (fargc>0) {
2418  if (!procOptVector(fargc,&fargv[0],folders)) {
2419  fclose(p_inp);
2420  return 3;
2421  }
2422  }
2423  }
2424  }
2425  std::cout << "Close file" << std::endl;
2426  fclose(p_inp);
2427  delete[] p_buf;
2428  }
2429 
2430  // now open the database so folder lookup will work
2432  std::cout << "Problem opening connections" << std::endl;
2433  return 10;
2434  } else {
2435  // having assembled the raw list of folders and exclude list, construct
2436  // real list of folders (allows exclude to be specified after folders on
2437  // command line), if none given, add '/'
2438  if (m_lockedonly) {
2439  if (getLockedTags()==0) return 11;
2440  }
2441  if (folders.empty()) {
2442  // no folders specified, take all with tags
2443  addFolder("/",false);
2444  } else {
2445  // explicit list of folders
2446  for (std::vector<std::string>::const_iterator ifold=folders.begin();
2447  ifold!=folders.end();++ifold) addFolder(*ifold,false);
2448  // need to process only tags in root folder, if inclusive hiearchicals
2449  if (m_hitag) addFolder("/",true);
2450  }
2451  }
2452  if (m_getonline) {
2453  if (!getOnlineRun()) return 6;
2454  }
2455  if (m_getbulk) {
2456  if (!getBulkRun()) return 6;
2457  }
2458  if (m_gettime) {
2459  if (!getTimeFromRun()) return 5;
2460  }
2461  if (!m_runfile.empty()) {
2462  if (!getRunList()) return 7;
2463  }
2464  // list out parameter changes
2465  if (m_runemin!=cool::ValidityKeyMin || m_runemax!=cool::ValidityKeyMax)
2466  std::cout << "Source run/LB range [" << (m_runemin >> 32) << "," <<
2467  (m_runemin & 0xFFFFFFFF) << "] to [" << (m_runemax >> 32) << "," <<
2468  (m_runemax & 0xFFFFFFFF) << "]" << std::endl;
2469  if (m_timemin!=cool::ValidityKeyMin || m_timemax!=cool::ValidityKeyMax)
2470  std::cout << "Source timestamp range " << m_timemin << " " <<
2471  timeString(m_timemin) << "to " << m_timemax << " " <<
2472  timeString(m_timemax) << std::endl;
2473  if (m_alliov)
2474  std::cout << "Change o/p run/LB range [" << (m_newrunemin >> 32) << "," <<
2475  (m_newrunemin & 0xFFFFFFFF) << "] to [" << (m_newrunemax >> 32) << "," <<
2476  (m_newrunemax & 0xFFFFFFFF) << "]" << std::endl;
2477  if (m_alliov)
2478  std::cout << "Change o/p timestamp range " << m_newtimemin << " " <<
2479  timeString(m_newtimemin) << "to " << m_newtimemax << " " <<
2480  timeString(m_newtimemax) << std::endl;
2481  if (m_skipout) {
2482  std::cout << "Skip IOVs extending outside run/LB range [" <<
2483  (m_srunemin >> 32) << "," << (m_srunemin & 0xFFFFFFFF) << "] to [" <<
2484  (m_srunemax >> 32) << "," << (m_srunemax & 0xFFFFFFFF) << "]" <<
2485  std::endl;
2486  }
2487  if (!m_excludechans.empty()) {
2488  for (std::vector<cool::ChannelId>::const_iterator itr=
2489  m_excludechans.begin();itr!=m_excludechans.end();++itr)
2490  std::cout << "Channel " << *itr << " will be excluded" << std::endl;
2491  }
2492  if (m_hitag) std::cout << "All hierarchical tags connecting to referenced tags will be copied" << std::endl;
2493  if (m_nohitag) std::cout << "Hierarchical tag relations will not be copied"
2494  << std::endl;
2495  return 0;
2496 }
2497 
2498 bool AtlCoolCopy::isNumeric(const char* input) {
2499  // determine if input string is numeric or string
2500  bool isnum=true;
2501  const char* cptr=input;
2502  while (*cptr!='\0') {
2503  if (!isdigit(*cptr)) { isnum=false; break;}
2504  ++cptr;
2505  }
2506  return isnum;
2507 }
2508 
2509 bool AtlCoolCopy::equalRecord(const cool::IRecord& lhs,
2510  const cool::IRecord& rhs) {
2511  // equality test for COOL IRecords, handling NULL string attributes correctly
2512  // tests values and types of Attributes, not names
2513  if (lhs.size()!=rhs.size()) return false;
2514  for (size_t i=0;i<lhs.size();++i) {
2515  // types must match - have to test explicitly first
2516  if (lhs[i].specification().storageType()!=
2517  rhs[i].specification().storageType())
2518  return false;
2519  // now use IField equality to for final test (spec again and data)
2520  if (lhs[i]!=rhs[i]) {
2521  // if not equal, check for special case of strings and compare directly
2522  // types are known to be equal so only test LHS
2523  const cool::StorageType& stype=lhs[i].specification().storageType();
2524  if (stype==cool::StorageType::String255 ||
2525  stype==cool::StorageType::String4k ||
2526  stype==cool::StorageType::String64k ||
2527  stype==cool::StorageType::String16M) {
2528  // check if string data payloads are really equal or not
2529  if (lhs[i].data<std::string>()!=rhs[i].data<std::string>())
2530  return false;
2531  } else {
2532  // if not string, trust the result of IField !=operator
2533  return false;
2534  }
2535  }
2536  }
2537  return true;
2538 }
2539 
2540 
2541 cool::ValidityKey AtlCoolCopy::timeVal(const char* input) {
2542  // convert input char* string to 64bit COOL validityKey
2543  // input either represents a string in seconds, or a date in the form
2544  // yyyy/mm/dd:hh:mm:ss
2545  // first determine if input is a number
2546  if (isNumeric(input)) {
2547  return static_cast<long long>(atol(input))*
2548  static_cast<long long>(1.E9);
2549  } else {
2550  struct tm mytm{},mytm2{};
2551  char* result=strptime(input,"%Y-%m-%d:%T",&mytm);
2552  if (result!=nullptr) {
2553  // make the DST field zero, so the time is interpreted without daylight
2554  // savings time
2555  mytm.tm_isdst=0;
2556  // now have to correct for the local time zone - do this by also
2557  // calculating the time since epoch for 2/1/1970 midnight (no DST)
2558  cool::ValidityKey itime=static_cast<cool::ValidityKey>(mktime(&mytm));
2559  strptime("1970-01-02:00:00:00","%Y-%m-%d:%T",&mytm2);
2560  cool::ValidityKey caltime=static_cast<cool::ValidityKey>(mktime(&mytm2));
2561  itime+=24*60*60-caltime;
2562  return itime*static_cast<cool::ValidityKey>(1.E9);
2563  } else {
2564  std::cout <<
2565  "ERROR in format of time value, use e.g. 2007-05-25:14:01:00" <<
2566  std::endl;
2567  return 0;
2568  }
2569  }
2570 }
2571 
2572 std::string AtlCoolCopy::timeString(const cool::ValidityKey iovtime) {
2573  if (iovtime==cool::ValidityKeyMin) {
2574  return "ValidityKeyMin ";
2575  } else if (iovtime==cool::ValidityKeyMax) {
2576  return "ValidityKeyMax ";
2577  } else {
2578  time_t time=static_cast<time_t>(iovtime/1E9);
2579  struct tm result;
2580  char buf[32];
2581  return "UTC "+std::string(asctime_r(gmtime_r(&time, &result), buf));
2582  }
2583 }
2584 
2585 cool::ValidityKey AtlCoolCopy::runLBVal(const char* input1,const char* input2) {
2586  // parse the inputs as run/LB numbers to generate a validitykey
2587  // used in processing -rls-type options
2588  cool::ValidityKey val;
2589  if (input1[0]=='M' || input1[0]=='m') {
2590  val=((1LL << 31)-1) << 32;
2591  } else {
2592  val=static_cast<long long>(atol(input1)) << 32;
2593  }
2594  if (input2[0]=='M' || input2[0]=='m') {
2595  val+=(1LL << 32);
2596  } else {
2597  val+=atoll(input2);
2598  }
2599  if (val>cool::ValidityKeyMax) val=cool::ValidityKeyMax;
2600  return val;
2601 }
2602 
2603 
2605  // extract time limits for a run range using the expected information
2606  // in the /TDAQ/RunCtrl/SOR_Params and EOR_Params on the given DB connection
2607 
2608  // choose database instance based on run number switchover if none is given
2609  if (m_timedb.empty()) {
2610  if ((m_runemin >> 32)>=236107) {
2611  m_timedb="COOLONL_TDAQ/CONDBR2";
2612  } else {
2613  m_timedb="COOLONL_TDAQ/COMP200";
2614  }
2615  }
2616  // select foldername based on database instance
2617  std::string sorfolder="/TDAQ/RunCtrl/SOR_Params";
2618  std::string eorfolder="/TDAQ/RunCtrl/EOR_Params";
2619  if (m_timedb.find ("CONDBR2")!=std::string::npos) {
2620  sorfolder="/TDAQ/RunCtrl/SOR";
2621  eorfolder="/TDAQ/RunCtrl/EOR";
2622  }
2623  std::cout << "Extracting times for run range [" << (m_runemin >> 32) <<
2624  "," << (m_runemax >> 32) << "] from database " << m_timedb <<
2625  " folder " << sorfolder << std::endl;
2626  if (m_runemin==cool::ValidityKeyMin || m_runemax==cool::ValidityKeyMax) {
2627  std::cout << "ERROR: Run range not correctly specified" << std::endl;
2628  return false;
2629  }
2630  // open database connection
2631  cool::IDatabasePtr tdaqdb;
2632  try {
2633  tdaqdb=m_dbSvc->openDatabase(transConn(m_timedb),m_sourceread);
2634  std::cout << "Opened database connection" << std::endl;
2635  }
2636  catch (std::exception& e) {
2637  std::cout << "COOL exception caught: " << e.what() << std::endl;
2638  return false;
2639  }
2640  // start of run information
2641  try {
2642  cool::IFolderPtr folder=tdaqdb->getFolder(sorfolder);
2643  // channel number is arbitrary - have to loop
2644  cool::IObjectIteratorPtr itr=folder->browseObjects(m_runemin,m_runemin,
2646  int nobj=0;
2647  while (itr->goToNext()) {
2648  const cool::IRecord& payload=itr->currentRef().payload();
2649  m_timemin=payload["SORTime"].data<unsigned long long>();
2650  ++nobj;
2651  }
2652  itr->close();
2653  if (nobj!=1) {
2654  std::cout << "ERROR: Found " << nobj << " SOR records" << std::endl;
2655  return false;
2656  }
2657  }
2658  catch (std::exception& e) {
2659  std::cout << "Exception accessing SOR information: " << e.what() <<
2660  std::endl;
2661  return false;
2662  }
2663  // end of run information
2664  try {
2665  cool::IFolderPtr folder=tdaqdb->getFolder(eorfolder);
2666  // channel number is arbitrary - have to loop
2667  cool::IObjectIteratorPtr itr=folder->browseObjects(((m_runemax >> 32) << 32),((m_runemax >> 32) << 32),
2669  int nobj=0;
2670  while (itr->goToNext()) {
2671  const cool::IRecord& payload=itr->currentRef().payload();
2672  m_timemax=payload["EORTime"].data<unsigned long long>();
2673  ++nobj;
2674  }
2675  itr->close();
2676  if (nobj!=1) {
2677  std::cout << "ERROR: Found " << nobj << " EOR records" << std::endl;
2678  return false;
2679  }
2680  }
2681  catch (std::exception& e) {
2682  std::cout << "Exception accessing SOR information: " << e.what() <<
2683  std::endl;
2684  return false;
2685  }
2686  tdaqdb->closeDatabase();
2687  std::cout << "Timestamp range set to " << m_timemin << " " <<
2688  timeString(m_timemin) << "to " << m_timemax << " " <<
2689  timeString(m_timemax) << std::endl;
2690  return true;
2691 }
2692 
2694  // open using the ATLAS_COOLONL_GLOBAL schema, since we have authentication
2695  // information for this one by default
2696  // get minimum run-number/timestamp for bulk reco update
2697  std::cout << "Extracting current run-number from ATLAS_RUN_NUMBER @ ATONR_ADG ... " <<
2698  // Initialize libcurl
2699  curl_global_init(CURL_GLOBAL_ALL);
2700  using uniqueCurl_t = std::unique_ptr<CURL,decltype(&curl_easy_cleanup)>;
2701  uniqueCurl_t curl(curl_easy_init(), curl_easy_cleanup);
2702  CURLcode res = CURLE_OK;
2703  if (curl) {
2704  std::string url = m_runinfohost + "/runs?sort=runnumber:DESC&size=1";
2705  // Set the URL
2706  res = curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());
2707  if (res != CURLE_OK) {
2708  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2709  return false;
2710  }
2711  // Follow HTTP redirections
2712  res = curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1L);
2713  if (res != CURLE_OK) {
2714  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2715  return false;
2716  }
2717  // Response data buffer
2718  std::string response;
2719 
2720  // Set the callback function to receive response data
2721  res = curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, WriteCallback);
2722  if (res != CURLE_OK) {
2723  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2724  return false;
2725  }
2726  res = curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &response);
2727  if (res != CURLE_OK) {
2728  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2729  return false;
2730  }
2731 
2732  // Perform the request
2733  res = curl_easy_perform(curl.get());
2734  if (res != CURLE_OK) {
2735  std::cerr << "Failed to perform request: " << curl_easy_strerror(res) << ":" << url.c_str() << std::endl;
2736  return false;
2737  } else {
2738  // Print the received response
2739  std::cout << "Response: " << std::endl;
2740  std::cout << response << std::endl;
2741  // Parse the JSON string
2742  try {
2744 
2745  // Extract the runnumber field
2746  int runNumber = jsonData["resources"][0]["runnumber"];
2747  int nextrun=runNumber+1;
2748  std::cout << "Next run started will be " << nextrun << std::endl;
2749  const long long rtime=time(nullptr);
2750  std::cout << "Epoch time extracted " << rtime << std::endl;
2751  if (m_alliov) {
2752  // if overwriting IOVs, set the new IOV lower limit
2753  m_newrunemin=(static_cast<long long>(nextrun)) << 32;
2754  m_newtimemin=rtime*static_cast<long long>(1E9);
2755  } else {
2756  // set the query lower limit - for use with truncate option
2757  m_runemin=(static_cast<long long>(nextrun)) << 32;
2758  m_timemin=rtime*static_cast<long long>(1E9);
2759  m_truncate=true;
2760  }
2761  } catch (nlohmann::json::parse_error& e) {
2762  std::cerr << "Failed to parse JSON response: " << e.what() << std::endl;
2763  return false;
2764  } catch (nlohmann::json::type_error& e) {
2765  std::cerr << "Failed to extract data from JSON response: " << e.what() << std::endl;
2766  return false;
2767  } catch (std::exception& e) {
2768  std::cerr << "Failed to extract run and timestamp from JSON response: " << e.what() << std::endl;
2769  return false;
2770  }
2771  }
2772  // Clean up done by unique_ptr d'tor
2773  } else {
2774  std::cerr << "Failed to initialize libcurl." << std::endl;
2775  return false;
2776  }
2777  // Cleanup libcurl
2778  curl_global_cleanup();
2779  return true;
2780 }
2782  // get minimum run-number/timestamp for bulk reco update
2783  if (m_getonline) {
2784  std::cout << "ERROR: -getonline and -getbulk cannot be used simultaneously"
2785  << std::endl;
2786  return false;
2787  }
2788  std::cout << "Call getbulk using URL" << std::endl;
2789  // Initialize libcurl
2790  curl_global_init(CURL_GLOBAL_ALL);
2791  using uniqueCurl_t = std::unique_ptr<CURL,decltype(&curl_easy_cleanup)>;
2792  uniqueCurl_t curl(curl_easy_init(), curl_easy_cleanup);
2793  //CURL *curl = curl_easy_init();
2794  CURLcode res = CURLE_OK;
2795 
2796  if (curl) {
2797  std::string url = m_runinfohost + "/runs/nemop/sync";
2798  // Set the URL
2799  res = curl_easy_setopt(curl.get(), CURLOPT_URL, url.c_str());
2800  if (res != CURLE_OK) {
2801  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2802  return false;
2803  }
2804  // Follow HTTP redirections
2805  res = curl_easy_setopt(curl.get(), CURLOPT_FOLLOWLOCATION, 1L);
2806  if (res != CURLE_OK) {
2807  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2808  return false;
2809  }
2810  // Response data buffer
2811  std::string response;
2812 
2813  // Set the callback function to receive response data
2814  res = curl_easy_setopt(curl.get(), CURLOPT_WRITEFUNCTION, WriteCallback);
2815  if (res != CURLE_OK) {
2816  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2817  return false;
2818  }
2819  res = curl_easy_setopt(curl.get(), CURLOPT_WRITEDATA, &response);
2820  if (res != CURLE_OK) {
2821  std::cerr << "Failed to perform request: curl_easy_setopt, line "<<__LINE__<<std::endl;
2822  return false;
2823  }
2824  // Perform the request
2825  res = curl_easy_perform(curl.get());
2826  if (res != CURLE_OK) {
2827  std::cerr << "Failed to perform request: " << curl_easy_strerror(res) << ":" << url.c_str() << std::endl;
2828  return false;
2829  } else {
2830  // Print the received response
2831  std::cout << "Response: " << std::endl;
2832  std::cout << response << std::endl;
2833  try {
2834  // Split the response into two fields
2836  // Extract the run and timestamp fields
2837  int nextrun = jsonData["run"];
2838  long long rtime = jsonData["timestamp"];
2839  std::cout << "Next run started will be " << nextrun << std::endl;
2840  std::cout << "Epoch time extracted " << rtime << std::endl;
2841  if (m_alliov) {
2842  // if overwriting IOVs, set the new IOV lower limit
2843  m_newrunemin=(static_cast<long long>(nextrun)) << 32;
2844  m_newtimemin=rtime*static_cast<long long>(1E9);
2845  } else {
2846  // set the query lower limit - for use with truncate option
2847  m_runemin=(static_cast<long long>(nextrun)) << 32;
2848  m_timemin=rtime*static_cast<long long>(1E9);
2849  m_truncate=true;
2850  }
2851  } catch (nlohmann::json::parse_error& e) {
2852  std::cerr << "Failed to parse JSON response: " << e.what() << std::endl;
2853  return false;
2854  } catch (nlohmann::json::type_error& e) {
2855  std::cerr << "Failed to extract data from JSON response: " << e.what() << std::endl;
2856  return false;
2857  } catch (std::exception& e) {
2858  std::cerr << "Failed to extract run and timestamp from JSON response: " << e.what() << std::endl;
2859  return false;
2860  }
2861  }
2862  // Clean up done by unique_ptr
2863  } else {
2864  std::cerr << "Failed to initialize libcurl." << std::endl;
2865  return false;
2866  }
2867  // Cleanup libcurl
2868  curl_global_cleanup();
2869 
2870  return true;
2871 }
2872 
2874  // loop over all given filenames
2875  for (std::vector<std::string>::const_iterator itr=m_runfile.begin();
2876  itr!=m_runfile.end();++itr) {
2877  std::cout << "Reading allowed run numbers from file " << *itr << std::endl;
2878  FILE* p_inp=fopen(itr->c_str(),"r");
2879  if (p_inp==nullptr) {
2880  std::cout << "File not found" << std::endl;
2881  return false;
2882  }
2883  char* p_buf=new char[999];
2884  while (!feof(p_inp)) {
2885  char* p_line=fgets(p_buf,999,p_inp);
2886  if (p_line!=nullptr) {
2887  unsigned int run=atoi(p_line);
2888  m_runlist.push_back(run);
2889  }
2890  }
2891  fclose(p_inp);
2892  delete[] p_buf;
2893  }
2894  std::sort(m_runlist.begin(),m_runlist.end());
2895  std::cout << "Read list of " << m_runlist.size() << " runs from " <<
2896  m_runfile.size() << " input runlist files" << std::endl;
2897  for (std::vector<unsigned int>::const_iterator itr=m_runlist.begin();
2898  itr!=m_runlist.end();++itr) std::cout <<
2899  "Update allowed for run " << *itr << std::endl;
2900  return true;
2901 }
2902 
2903 int AtlCoolCopy::getUpdateMode(std::string_view desc,
2904  std::string_view tag) {
2905  // analyse the folder description and tag name for updateMode flags
2906  // return 1 for online mode, 0 otherwise
2907  std::string_view modestr="";
2908  int mode=0;
2909  std::string::size_type iofs1=desc.find("<updateMode>");
2910  std::string::size_type iofs2=desc.find("</updateMode>");
2911  if (iofs1!=std::string::npos && iofs2!=std::string::npos && iofs2>iofs1)
2912  modestr=desc.substr(iofs1+12,iofs2-iofs1-12);
2913  if (modestr=="UPD1" || tag.find("UPD1")!=std::string::npos) mode=1;
2914  if (modestr=="UPD2" || tag.find("UPD2")!=std::string::npos) mode=2;
2915  if (modestr=="UPD3" || tag.find("UPD3")!=std::string::npos) mode=3;
2916  if (modestr=="UPD4" || tag.find("UPD4")!=std::string::npos) mode=4;
2917  return mode;
2918 }
2919 
2920 
2921 bool AtlCoolCopy::checkChannels(const std::string& folder,
2922  const cool::IFolderPtr& sourcefl,const cool::IFolderPtr& destfl,
2923  bool newfolder) {
2924  // Channel manipulation is slow due to lack of bulk API for creating them
2925  // this routine therefore does the minimum work, not checking destination
2926  // if folder is newly created, and not copying description unless requested
2927  // get map of all channels
2928  const std::map<cool::ChannelId,std::string> chanmap=
2929  sourcefl->listChannelsWithNames();
2930  if (m_debug) std::cout << "Checking channels table for folder " << folder
2931  << " with " << chanmap.size() << " channels" << std::endl;
2932  int ncreate=0;
2933  int nmodify=0;
2934  // iterate over all channels
2935  for (std::map<cool::ChannelId,std::string>::const_iterator
2936  itr=chanmap.begin();itr!=chanmap.end();++itr) {
2937  // assume now ALL channels are in channels table (COOL 2.4 and above)
2938  const cool::ChannelId chan=itr->first;
2939  // check if channel should be skipped
2940  if (!m_excludechans.empty()) {
2941  if (find(m_excludechans.begin(),m_excludechans.end(),
2942  chan)!=m_excludechans.end()) continue;
2943  }
2944  if (!m_chansel.inSelection(chan)) continue;
2945 
2946  const std::string& sourcename=itr->second;
2947  // only look up channel description if we are copying it
2948  std::string sourcedesc="";
2949  if (m_chdesc) sourcedesc=sourcefl->channelDescription(chan);
2950  // only check destination if folder alreay existed
2951  if (!newfolder && destfl->existsChannel(chan)) {
2952  const std::string & destname=destfl->channelName(chan);
2953  std::string destdesc="";
2954  if (m_chdesc) destdesc=destfl->channelDescription(chan);
2955  // if sourcename is not set, and destination is, dont worry
2956  if (sourcename!=destname && !sourcename.empty()) {
2957  // channel names differ - error if verifying, set if copying
2958  if (m_verify) {
2959  std::cout << "ERROR: Channel " << chan << " names differ: " <<
2960  sourcename << " " << destname << std::endl;
2961  return false;
2962  }
2963  destfl->setChannelName(chan,sourcename);
2964  std::cout << "Modify channel " << chan << " name from "
2965  << destname << " to " << sourcename << std::endl;
2966  ++nmodify;
2967  }
2968  if (sourcedesc!=destdesc && !sourcedesc.empty()) {
2969  // channel descriptions differ - error if verifying, set if copying
2970  if (m_verify) {
2971  std::cout << "ERROR: Channel " << chan << " descriptions differ: "
2972  << sourcedesc << " " << destdesc << std::endl;
2973  return false;
2974  }
2975  std::cout << "Modify channel " << chan << " description from "
2976  << sourcedesc << " to " << destdesc << std::endl;
2977  destfl->setChannelDescription(chan,sourcedesc);
2978  ++nmodify;
2979  }
2980  } else {
2981  // channel does not exist on destination - if verifying, this is error
2982  if (m_verify) return false;
2983  // otherwise create it
2984  if (m_debug) std::cout << "Create new channel " << chan << " name "
2985  << sourcename << " description" << sourcedesc << std::endl;
2986  destfl->createChannel(chan,sourcename,sourcedesc);
2987  ++ncreate;
2988  }
2989  }
2990  if (ncreate>0) std::cout << "Created " << ncreate << " new channels for "
2991  << folder << std::endl;
2992  if (nmodify>0) std::cout << "Modified " << nmodify << " channel info for "
2993  << folder << std::endl;
2994  return true;
2995 }
2996 
2997 
2998 void AtlCoolCopy::checkRef(const cool::IRecord& payload,
2999  const std::string& folder, const std::string& tag) {
3000  const std::string name=payload[0].name();
3001  if (name=="PoolRef" || name=="fileGUID") {
3002  std::string poolref;
3003  std::string addr=payload[0].data<std::string>();
3004  if (name=="PoolRef") {
3005  std::string::size_type iofs1=addr.find("[DB=");
3006  std::string::size_type iofs2=addr.find(']',iofs1);
3007  if (iofs1!=std::string::npos && iofs2!=std::string::npos && iofs2>iofs1) {
3008  poolref=addr.substr(iofs1+4,iofs2-iofs1-4);
3009  } else {
3010  std::cout << "ERROR: Badly formed POOL object reference " <<
3011  addr << std::endl;
3012  }
3013  } else {
3014  // for fileGUID, POOL GUID is just the string
3015  poolref=addr;
3016  }
3017  std::string foldertag=folder+":"+tag;
3018  // insert into list, first check if same as before
3019  if (m_poollast!=m_poolrefs.end() && m_poollast->first==poolref) {
3020  m_poollast->second.inccount();
3021  m_poollast->second.addfoldertag(foldertag);
3022  } else {
3023  m_poollast=m_poolrefs.find(poolref);
3024  if (m_poollast!=m_poolrefs.end()) {
3025  m_poollast->second.inccount();
3026  m_poollast->second.addfoldertag(foldertag);
3027  } else {
3028  m_poolrefs[poolref]=PoolMapElement(1,foldertag);
3029  m_poollast=m_poolrefs.find(poolref);
3030  }
3031  }
3032  }
3033 }
3034 
3036  std::cout << "Copying additional hierarchical tags of parent folders"
3037  << std::endl;
3038  while (!m_hiparent.empty()) {
3039  // take first element and erase it
3040  std::string folderset=*m_hiparent.begin();
3041  m_hiparent.erase(m_hiparent.begin());
3042  // process any tags defined for this folderset
3043  if (m_sourceDbPtr->existsFolderSet(folderset)) {
3044  cool::IFolderSetPtr sourcefl=m_sourceDbPtr->getFolderSet(folderset);
3045  cool::IFolderSetPtr destfl=m_destDbPtr->getFolderSet(folderset);
3046  std::cout << "Processing folderset " << folderset << std::endl;
3047  for (HiTagMap::const_iterator imap=m_hitagmap.begin();
3048  imap!=m_hitagmap.end();++imap) {
3049  try {
3050  // will throw exception if this tag is not pointing to the folder
3051  std::string sourcetag=sourcefl->findTagRelation(imap->first);
3052  std::cout << "Create hierarchical tag between " << sourcetag <<
3053  " and " << imap->first << " in folder " << imap->second <<
3054  std::endl;
3055  try {
3056  // first check this relation has not already been created
3057  std::string etag=destfl->resolveTag(imap->first);
3058  if (etag==sourcetag) {
3059  std::cout << "This relation has already been created" << std::endl;
3060  } else {
3061  std::cout << "ERROR: Tag in parent already related to " <<
3062  etag << std::endl;
3063  return 34;
3064  }
3065  }
3066  catch (cool::Exception& e ) {
3067  // only do creation if does not already exist - via exception
3068  try {
3069  destfl->createTagRelation(imap->first,sourcetag);
3070  // check if this implicates yet another parent folder to be checked
3071  if (find(m_hiparent.begin(),m_hiparent.end(),imap->second)==
3072  m_hiparent.end()) m_hiparent.push_back(imap->second);
3073  if (m_copytaginfo &&
3074  m_cooltagmap.find(imap->first)==m_cooltagmap.end())
3075  m_cooltagmap.insert(CoolTagMap::value_type(imap->first,
3076  CoolTagInfo(m_sourceDbPtr,imap->second,imap->second,
3077  imap->first,imap->first)));
3078  }
3079  catch (cool::Exception& e) {
3080  std::cout << "Cool exception " << e.what() <<
3081  "thrown in hierarchical tag creation" << std::endl;
3082  return 34;
3083  }
3084  }
3085  }
3086  // exceptions from findTag - do nothing
3087  catch (cool::Exception& e) { }
3088  }
3089  } else {
3090  std::cout << "ERROR Folderset " << folderset << " not found" <<
3091  std::endl;
3092  }
3093  }
3094  return 0;
3095 }
3096 
3098  std::cout << "Write tag description ";
3099  if (m_copytaglock) std::cout << "and lock info ";
3100  std::cout << "for " << m_cooltagmap.size() << " tags" << std::endl;
3101  for (CoolTagMap::const_iterator itr=m_cooltagmap.begin();
3102  itr!=m_cooltagmap.end();++itr) {
3103  itr->second.write(m_destDbPtr,m_copytaglock);
3104  }
3105  return 0;
3106 }
3107 
3109  std::cout << "Total of " << m_poolrefs.size() << " POOL files referenced"
3110  << std::endl;
3111  for (PoolMap::const_iterator ipool=m_poolrefs.begin();
3112  ipool!=m_poolrefs.end();++ipool) {
3113  std::cout << "Ref " << ipool->first << " (" << ipool->second.count()
3114  << ")" << std::endl;
3115  }
3116  return 0;
3117 }
3118 
3119 int AtlCoolCopy::resolvePoolRefs ATLAS_NOT_THREAD_SAFE () {
3120  std::cout << "Total of " << m_poolrefs.size() << " POOL Files referenced"
3121  << std::endl;
3122  pool::IFileCatalog* catalog=setupCatalog(m_poolcat);
3123  if (catalog==nullptr) return 110;
3124  pool::SimpleUtilityBase pool_utility;
3125  if (m_poolopen) {
3126  // prepare POOL session
3127  pool_utility.startSession();
3128  }
3129  // inject additional GUIDs/LFNs if needed
3130  if (!m_addguid.empty()) {
3131  for (std::vector<std::string>::const_iterator itr=m_addguid.begin();
3132  itr!=m_addguid.end();++itr) {
3133  m_poolrefs[*itr]=PoolMapElement(1,"ADDGUID");
3134  std::cout << "Added POOL file GUID: " << *itr << std::endl;
3135  }
3136  }
3137  if (!m_addlfn.empty()) {
3138  for (std::vector<std::string>::const_iterator itr=m_addlfn.begin();
3139  itr!=m_addlfn.end();++itr) {
3140  std::string guid;
3141  catalog->lookupFileByLFN(*itr,guid);
3142  std::cout << "Add POOL file GUID: " << guid << " from LFN " << *itr
3143  << std::endl;
3144  m_poolrefs[guid]=PoolMapElement(1,"ADDLFN");
3145  }
3146  }
3147 
3148  // set up for making new dataset if needed
3149  using LFNGVec = std::vector<std::pair<std::string, std::string> >;
3150  LFNGVec dsfound;
3151  bool dscopy=!m_newdataset.empty();
3152 
3153  // loop through GUIDs and attempt to resolve
3154  int nbad=0;
3155  for (PoolMap::iterator ipool=m_poolrefs.begin();
3156  ipool!=m_poolrefs.end();++ipool) {
3157  pool::FileCatalog::FileID guid=ipool->first;
3159  catalog->getLFNs( guid, lfns );
3160  if( !lfns.empty() ) {
3161  // file found in cataloge - print LFN and usage count
3162  const std::string lfn = lfns[0].first;
3163  ipool->second.setlfn(lfn);
3164  std::cout << "LFN: " << lfn << " (" << ipool->second.count()
3165  << ")" << std::endl;
3166  if (dscopy)
3167  dsfound.push_back(std::pair<std::string,std::string>(lfn,guid));
3168  } else {
3169  // error bit 0 - no LFN
3170  if (!m_listpfn) ipool->second.setErrorBit(0);
3171  }
3172  if (m_listpfn || m_poolopen) {
3173  std::string pfn, tech;
3174  catalog->getFirstPFN( guid, pfn, tech );
3175  if( !pfn.empty() ) {
3176  ipool->second.setpfn(pfn);
3177  std::cout << "PFN: " << pfn << " (" << ipool->second.count()
3178  << ")" << std::endl;
3179  if (m_poolopen) {
3180  // first try the file as a CoolHist file
3181  std::string hguid=getCoolHistGUID(pfn);
3182  if (!hguid.empty()) {
3183  // successful get of CoolHist GUID
3184  if (hguid!=ipool->first) {
3185  std::cout << "ERROR File CoolHist GUID " << hguid <<
3186  " inconsistent with catalogue " << ipool->first << std::endl;
3187  ipool->second.setErrorBit(3);
3188  }
3189  } else {
3190  // try the file as a genuine POOL file
3191  try {
3192  const std::string fid = pool_utility.readFileGUID( pfn );
3193  if( fid != ipool->first ) {
3194  std::cout << "ERROR File GUID " << fid <<
3195  " inconsistent with catalogue " << ipool->first << std::endl;
3196  // file GUID inconsistent with catalogue - error bit 3
3197  ipool->second.setErrorBit(3);
3198  }
3199  }
3200  catch( std::runtime_error& e ) {
3201  std::cout << "Cannot open file for reading!" << std::endl;
3202  std::cout << e.what() << std::endl;
3203  // File cannot be opened: set error bit 2
3204  ipool->second.setErrorBit(2);
3205  }
3206  }
3207  } // end of actions opening POOL file
3208  } else {
3209  // PFN not found - set bit 1
3210  ipool->second.setErrorBit(1);
3211  }
3212  }
3213  // check file error code
3214  if (ipool->second.errcode()>0) ++nbad;
3215  }
3216  catalog->commit();
3217  catalog->disconnect();
3218  delete catalog;
3219 
3220  // produce definition of new dataset if needed
3221  if (dscopy) {
3222  // start up a new catalogue instance with the merge catalogues
3223  // which indicate which files are already available and don't need to
3224  // put in the output dataset definition
3225  catalog=setupCatalog(m_mergecat);
3226  if (catalog==nullptr) return 110;
3227  const std::string dssname="register.sh";
3228  std::cout << "Write DQ2 registerFileInDataset commands to " << dssname
3229  << " for registration in dataset " << m_newdataset << std::endl;
3230  std::ofstream dsstream(dssname.c_str());
3231  for (LFNGVec::const_iterator itr=dsfound.begin();
3232  itr!=dsfound.end();++itr) {
3233  const std::string& lfn=itr->first;
3234  const std::string& guid=itr->second;
3236  catalog->getLFNs( guid, lfns );
3237  if( !lfns.empty() ) {
3238  // file is already registered - check logical names are consistent
3239  const std::string lfn2 = lfns[0].first;
3240  if (lfn2!=lfn) std::cout << "WARNING: LFNs for GUID " << guid <<
3241  " differ in input/merge datasets: " << lfn << " vs "
3242  << lfn2 << std::endl;
3243  } else {
3244  // file is not registered - add to output
3245  dsstream << "dq2-register-files " << m_newdataset << " "
3246  << lfn << " " << guid << std::endl;
3247  }
3248  }
3249  catalog->commit();
3250  catalog->disconnect();
3251  delete catalog;
3252  }
3253 
3254  int retcode=0;
3255  if (nbad==0) {
3256  std::cout << "All POOL references were resolved by catalogues"
3257  << std::endl;
3258  } else {
3259  std::cout << "ERROR Could not resolve " << nbad << " files" << std::endl;
3260  retcode=111;
3261  }
3262  if (nbad>0 || m_poolall) filePoolRefs();
3263  return retcode;
3264 }
3265 
3266 std::string AtlCoolCopy::getCoolHistGUID(const std::string& file) {
3267  // attempt to extract COOL Hist GUID from file
3268  std::string hguid="";
3269  TFile* myfile=TFile::Open(file.c_str(),"READ");
3270  if (myfile!=nullptr) {
3271  TObjString* oguid;
3272  myfile->GetObject("fileGUID",oguid);
3273  if (oguid!=nullptr) {
3274  hguid=oguid->GetString();
3275  std::cout << "CoolHist GUID found to be " << hguid << std::endl;
3276  }
3277  myfile->Close();
3278  }
3279  return hguid;
3280 }
3281 
3283  // list POOL refs and error codes, optionally to file
3284  if (!m_checkoutputfile.empty()) {
3285  std::cout << "Write POOL file checkoutput on " << m_checkoutputfile <<
3286  std::endl;
3287  } else {
3288  m_checkoutputfile="/dev/null";
3289  }
3290  std::ofstream chkostream(m_checkoutputfile.c_str());
3291  std::cout << "ErrCode GUID Count LFN PFN Folders ..." << std::endl;
3292  for (PoolMap::const_iterator ipool=m_poolrefs.begin();
3293  ipool!=m_poolrefs.end();++ipool) {
3294  if (m_poolall || ipool->second.errcode()>0) {
3295  std::ostringstream line;
3296  // write error code, GUID, usage count
3297  line << ipool->second.errcode() << " " << ipool->first << " " <<
3298  ipool->second.count() << " ";
3299  if (!ipool->second.lfn().empty()) {
3300  line << ipool->second.lfn() << " ";
3301  } else {
3302  line << "noLFN ";
3303  }
3304  if (!ipool->second.pfn().empty()) {
3305  line << ipool->second.pfn() << " ";
3306  } else {
3307  line << "noPFN ";
3308  }
3309  for (std::vector<std::string>::const_iterator
3310  itr=ipool->second.foldertag().begin();
3311  itr!=ipool->second.foldertag().end();++itr)
3312  line << *itr << " ";
3313  std::cout << line.str() << std::endl;
3314  chkostream << line.str() << std::endl;
3315  }
3316  }
3317 }
3318 
3320  const std::vector<std::string>& catvec) {
3322  try {
3323  catalog->setWriteCatalog("file:PoolFileCatalog.xml");
3324  for (std::vector<std::string>::const_iterator icat=catvec.begin();
3325  icat!=catvec.end();++icat) {
3326  std::cout << "Add catalogue: " << *icat << std::endl;
3327  // if catalogue contains no ":" specifier, assume plain file
3328  if (icat->find(':')==std::string::npos) {
3329  catalog->addReadCatalog("file:"+(*icat));
3330  } else {
3331  catalog->addReadCatalog(*icat);
3332  }
3333  }
3334  catalog->connect();
3335  catalog->start();
3336  return catalog;
3337  }
3338  catch (std::exception& e) {
3339  std::cout << "Could not setup POOL catalogues, exception:" << e.what()
3340  << std::endl;
3341  return nullptr;
3342  }
3343 }
3344 
3345 void printHelp() {
3346  std::cout << "usage: AtlCoolCopy.exe <sourceCoolDB> <destinationCoolDB> { <options> }" << std::endl;
3347  std::cout << "Options are (see doc/mainpage.h for more):" << std::endl <<
3348  "-a, -alliov : set IOVs on destination to [ValidityKeyMin, ValidityKeyMax]" << std::endl <<" or following settings of -nrls,-nrlu,-nts,-ntu" <<
3349  std::endl <<
3350  "-ag, -addguid <guid> : Add GUID to list accessed for POOL checks" <<
3351  std::endl <<
3352  "-alf, -addlfn <LFN> : Add LFN to list accessed for POOL checks" <<
3353  std::endl <<
3354  "-al, -appendlocked : Allow locked tags to be updated if no overlap" <<
3355  std::endl <<
3356  "-alsv, -appendlockedsv : Allow locked tag update for openended IOVs" <<
3357  std::endl <<
3358  "-ana, -analyse <delta_t>: produce analysis ROOT file (filename = dest DB argument)" << std::endl <<
3359 "-bs, -buffersize <size> : set size of bulkstorage output buf to <size> objs"
3360  << std::endl <<
3361  "-c, -create : create destination DB if not already existing"
3362  << std::endl <<
3363  "-cd, -checkdest : Check destination DB and skip already existing tags"
3364  << std::endl <<
3365  "-ch, -channel : restrict selection to given channel or range specified as c1:c2" << std::endl <<
3366  "-ch1, -channel1 : specify start of a range of channels" << std::endl <<
3367  "-ch2, -channel2 : specify end of a range of channels" << std::endl <<
3368  "-cf, -checkfiles : check POOL files can be opened and have correct GUID"
3369  << std::endl <<
3370  "-co, -checkoutput <file> : write POOL file check output on file"
3371  << std::endl << "-cti, -copytaginfo : Copy tag descriptions"
3372  << std::endl << "-ctl, -copytaglock : Copy tag locked status and descriptions"
3373  << std::endl << "-cr, -checkrefs : check POOL references"
3374  << std::endl << "-d, -debug : produce debug output" << std::endl;
3375  std::cout <<
3376  "-ds, -dataset : output register.sh for creating DQ2 datasets "
3377  << std::endl <<
3378  "-ec, -excludechannel : exclude given channel from copy" << std::endl <<
3379  "-eh, -excludehead : exclude HEAD tag even if no tags found in MV folders"
3380  << std::endl <<
3381  "-ih, -includehead : include HEAD as well as any tags in MV folders" <<
3382  std::endl <<
3383  "-e, -exclude <pattern> : exclude folders" << std::endl <<
3384  "-f, -folder <pattern> : include folders" << std::endl <<
3385  "-fs, -forcesingle : force destination folder to be singleversion"
3386  << std::endl <<
3387  "-fm, -forcemulti : force destination folder to be multiversion"
3388  << std::endl <<
3389  "-frl, -forcerunlumi : force destination folder to be run/LB indexed"
3390  << std::endl <<
3391  "-ftm, -forcetime : force destination folder to be timestamp indexed"
3392  << std::endl <<
3393  "-fp, -forcepayload : Force destn folders to be created with payload tbl" << std::endl <<
3394  "-fnp, -forcenopayload : Force destn folders to be created without payload tbl" << std::endl <<
3395  "-forcerecreate : delete and recreate destination database" << std::endl;
3396  std::cout << "-ht, -headtag : Use HEAD-style tagging"
3397  << std::endl <<
3398  "-go, -getonline : Set minimum run number (-rls setting) to next ONLINE run"
3399  << std::endl <<
3400  "-gb, -getbulk : Set minimum run number (-rls setting) to next bulk reco run"
3401  << std::endl <<
3402  "-gt, -gettime : Extract timestamp information for given run number range"
3403  << std::endl <<
3404  "-h, -help : print help and exit" << std::endl <<
3405  "-hi, -hitag : Copy hierrchical tags inclusively" << std::endl <<
3406  "-ignoremode <pwd> : Ignore UPDx mode protection (experts only)"
3407  << std::endl <<
3408  "-is, -ignorespec : Ignore differences in folder spec if names are all equal" << std::endl <<
3409  "-lo, -lockedonly : Only copy locked/partially-locked top-level tags"
3410  << std::endl <<
3411  "-nc, -nocopy : Do not actually copy, just read source DB" << std::endl <<
3412  "-nch, -nochannel : Do not check or copy channel information" << std::endl
3413 << "-ncr, -noclobroot: Do not copy CLOB data into ROOT files" << std::endl
3414 << "-noc, -nocoracool : Do not copy Coracool structure payloads " << std::endl <<
3415  "-nd, -nodata : Copy only folder structures, not data" << std::endl <<
3416  "-nh, -nohitag : Do not follow hierarchical tag relations" << std::endl <<
3417 "-mc, -mergecat <catfile> : specify POOL file catalogue for new dataset making"
3418  << std::endl;
3419  std::cout << "-of, -outfolder <folder> : rename folder on output"
3420  << std::endl <<
3421  "-onr, -onlinerun : Retrieve run number from online server (not replica)" <<std::endl <<
3422  "-ot, -outtag : " << "Rename tag on output" << std::endl;
3423  std::cout <<
3424  "-pa, -poolall : Output all POOL files (incl good) when checking files"
3425  << std::endl <<
3426 "-pc, -poolcat <catfile> : specify POOL file catalogue for ref checking"
3427  << std::endl <<
3428  "-pf, -parfile <file> : Read additional options/parameters from file" <<
3429  std::endl <<
3430 "-pt, -prunetags : Copy only hierarchical tags descending from specified toptags"
3431  << std::endl <<
3432 "-rdo, -readoracle : force data to be read from Oracle, using dbreplica.config"
3433  << std::endl <<
3434  "-sl, -seal <val>: Set SEAL (hence COOL/POOL) output level to <val>"
3435  << std::endl <<
3436  "-sr, -skiprep : Skip folders having <norep/> folder metadata tag" <<
3437  std::endl <<
3438  "-t, -tag <tag> : Copy multiversion data with tag <tag>" << std::endl <<
3439  "-mt, -magic <tag> : Include magic tags involving the given pattern " << std::endl <<
3440 "-tr, -truncate : Set destination IOVs outside query interval to query interval"
3441  << std::endl <<
3442 "-tl, -taglabel : Specify tag description to enter in destination DB"
3443  << std::endl <<
3444  "-uht, -userheadtag : Also copy user tag data to the HEAD" << std::endl <<
3445  "-v, -verify : Verify data present on destination rather than copying" <<
3446  std::endl <<
3447  "-ro, -root : Produce ROOT output file instead of copying to COOL"
3448  << std::endl <<
3449  "-zn, -zeronull : Zero NULLs in ROOT file instead of skipping them" <<
3450  std::endl <<
3451  "-rls, -runlumisince <run> <LB> : Set minimum IOV interval" << std::endl <<
3452  "-rlu, -runlumiuntil <run> <LB> : Set maximum IOV interval" << std::endl <<
3453  "-rs, -runsince <run> : Set minimum IOV interval" << std::endl <<
3454  "-ru, -rununtil <run> : Set maximum IOV interval" << std::endl <<
3455  "-r, -run <run> : Copy only run <run>" << std::endl <<
3456 "-srls, -skiprunlumisince <run> <LB> : Set minimum IOV for skipping IOV in copy"
3457  << std::endl <<
3458 "-srlu, -skiprunlumiuntil <run> <LB> : Set maximum IOV for skipping IOV in copy"
3459  << std::endl <<
3460  "-tdb, -timedb <dbconn> : Set database connection for time information" <<
3461  "-ts, -timesince <time> : Set minimum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3462  std::endl <<
3463  "-tu, -timeuntil <time> : Set maximum IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss)" <<
3464  std::endl;
3465  std::cout <<
3466  "-nrls, -newrunlumisince <run> <LB> : Set minimum of output IOV interval (use with -alliov)" << std::endl <<
3467  "-nrlu, -newrunlumiuntil <run> <LB> : Set maximum of output IOV interval (use with --aliov)" << std::endl <<
3468  "-nts, -newtimesince <time> : Set minimum of outputIOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3469  std::endl <<
3470  "-ntu, -newtimeuntil <time> : Set maximum of output IOV interval (UTC SECONDs or UTC YYYY-MM-DD:hh:mm:ss) (use with --alliov)" <<
3471  std::endl;
3472  std::cout << "See http://twiki.cern.ch/twiki/bin/view/Atlas/AtlCoolCopy for more details" << std::endl;
3473 }
3474 
3475 int main ATLAS_NOT_THREAD_SAFE (int argc, const char* argv[]) {
3476  int retcode=0;
3477  if (argc<3) {
3478  printHelp();
3479  retcode=1;
3480  } else {
3481  AtlCoolCopy mycopy(argv[1],argv[2]);
3482  retcode=mycopy.setOpts(argc-3,&argv[3]);
3483  if (retcode==999) {
3484  printHelp();
3485  return 0;
3486  }
3487  if (retcode==0) retcode=mycopy.doCopy();
3488  }
3489  if (retcode>0) std::cout << "ERROR AtlCoolCopy.exe fails with exit code " <<
3490  retcode << std::endl;
3491  return retcode;
3492 }
SimpleUtilityBase.h
AtlCoolCopy::analyseIOVs
int analyseIOVs(const std::string &folder, const cool::IFolderPtr &sourcefl, const std::string &sourcetag, const cool::ValidityKey since, const cool::ValidityKey until, const bool timestamp)
Definition: AtlCoolCopy.cxx:1857
xAOD::iterator
JetConstituentVector::iterator iterator
Definition: JetConstituentVector.cxx:68
AtlCoolCopy::runLBVal
static cool::ValidityKey runLBVal(const char *input1, const char *input2)
Definition: AtlCoolCopy.cxx:2585
AtlCoolCopy::channelID
static cool::ChannelId channelID(const cool::IFolderPtr &folder, const std::string &chanstring)
Definition: AtlCoolCopy.cxx:855
AtlCoolCopy::m_skipout
bool m_skipout
Definition: AtlCoolCopy.cxx:203
AllowedVariables::e
e
Definition: AsgElectronSelectorTool.cxx:37
AtlCoolCopy::rootWrite
void rootWrite(void *sptr, const cool::IField &field) const
Definition: AtlCoolCopy.cxx:1771
AtlCoolCopy::m_magic
std::vector< std::string > m_magic
Definition: AtlCoolCopy.cxx:263
AtlCoolCopy::m_getbulk
bool m_getbulk
Definition: AtlCoolCopy.cxx:211
pool::SimpleUtilityBase::readFileGUID
virtual std::string readFileGUID(const std::string &pfn)
AtlCoolCopy::verifyIOVs
int verifyIOVs(const std::string &folder, const cool::IFolderPtr &sourcefl, const CoraCoolFolderPtr &sourceflc, const cool::IFolderPtr &destfl, const CoraCoolFolderPtr &destflc, const std::string &sourcetag, const cool::ValidityKey since, const cool::ValidityKey until, const bool checkrefs, const bool iscora, const cool::PayloadMode::Mode paymode)
Definition: AtlCoolCopy.cxx:1355
AtlCoolCopy::m_nt_rununtil
UInt_t m_nt_rununtil
Definition: AtlCoolCopy.cxx:287
AtlCoolCopy
Definition: AtlCoolCopy.cxx:80
AtlCoolCopy::m_outtag
std::string m_outtag
Definition: AtlCoolCopy.cxx:238
get_generator_info.result
result
Definition: get_generator_info.py:21
pool::IFileCatalog::Files
Gaudi::IFileCatalog::Files Files
Definition: IFileCatalog.h:26
AtlCoolCopy::m_channel1
std::string m_channel1
Definition: AtlCoolCopy.cxx:232
AtlCoolCopy::ATLAS_NOT_THREAD_SAFE
int doCopy ATLAS_NOT_THREAD_SAFE()
python.rtime.rtime
def rtime(fn, *args, **kw)
Definition: rtime.py:15
CheckTagAssociation.localtag
localtag
Definition: CheckTagAssociation.py:19
AtlCoolCopy::m_verify
bool m_verify
Definition: AtlCoolCopy.cxx:181
AtlCoolCopy::m_nt_lbuntil
UInt_t m_nt_lbuntil
Definition: AtlCoolCopy.cxx:289
AtlCoolCopy::m_srunemax
cool::ValidityKey m_srunemax
Definition: AtlCoolCopy.cxx:230
find
std::string find(const std::string &s)
return a remapped string
Definition: hcg.cxx:135
CoraCoolDatabase
Definition: CoraCoolDatabase.h:24
AtlCoolCopy::m_coralsvc
coral::ConnectionService m_coralsvc
Definition: AtlCoolCopy.cxx:252
json
nlohmann::json json
Definition: HistogramDef.cxx:9
AtlCoolCopy::m_copytaglock
bool m_copytaglock
Definition: AtlCoolCopy.cxx:215
AtlCoolCopy::getTimeFromRun
bool getTimeFromRun()
Definition: AtlCoolCopy.cxx:2604
response
MDT_Response response
Definition: MDT_ResponseTest.cxx:28
CaloCondBlobAlgs_fillNoiseFromASCII.spec
spec
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:46
AtlCoolCopy::m_readoracle
bool m_readoracle
Definition: AtlCoolCopy.cxx:207
xAOD::short
short
Definition: Vertex_v1.cxx:165
AtlCoolCopy::m_destdb
std::string m_destdb
Definition: AtlCoolCopy.cxx:172
AtlCoolCopy::m_repsort
ReplicaSorter * m_repsort
Definition: AtlCoolCopy.cxx:255
AtlCoolCopy::m_getonline
bool m_getonline
Definition: AtlCoolCopy.cxx:209
AtlCoolCopy::getUpdateMode
static int getUpdateMode(std::string_view desc, std::string_view tag)
Definition: AtlCoolCopy.cxx:2903
mergePhysValFiles.newfolder
newfolder
Definition: DataQuality/DataQualityUtils/scripts/mergePhysValFiles.py:82
AtlCoolCopy::m_checkoutputfile
std::string m_checkoutputfile
Definition: AtlCoolCopy.cxx:240
pool::FileCatalog::FileID
std::string FileID
Definition: IFileCatalog.h:20
defineDB.ichan
ichan
Definition: JetTagCalibration/share/defineDB.py:28
AtlCoolCopy::setChannelRange
void setChannelRange(const cool::IFolderPtr &sourcefl)
Definition: AtlCoolCopy.cxx:820
xAOD::char
char
Definition: TrigDecision_v1.cxx:38
parse
std::map< std::string, std::string > parse(const std::string &list)
Definition: egammaLayerRecalibTool.cxx:1113
extractSporadic.c1
c1
Definition: extractSporadic.py:133
tree
TChain * tree
Definition: tile_monitor.h:30
IDatabase.h
AtlCoolCopy::m_excludehead
bool m_excludehead
Definition: AtlCoolCopy.cxx:176
TRTCalib_cfilter.p1
p1
Definition: TRTCalib_cfilter.py:130
IFileCatalog.h
dirname
std::string dirname(std::string name)
Definition: utils.cxx:200
AtlCoolCopy::rootDirs
std::string rootDirs(const std::string &folder, const std::string &toproot)
Definition: AtlCoolCopy.cxx:1681
AtlCoolCopy::m_truncate
bool m_truncate
Definition: AtlCoolCopy.cxx:202
AtlCoolCopy::ATLAS_NOT_THREAD_SAFE
int copyFolder ATLAS_NOT_THREAD_SAFE(const std::string &folder, const std::vector< std::string > &taglist)
AtlCoolCopy::m_timemax
cool::ValidityKey m_timemax
Definition: AtlCoolCopy.cxx:224
bin
Definition: BinsDiffFromStripMedian.h:43
AtlCoolCopy::m_hitag
bool m_hitag
Definition: AtlCoolCopy.cxx:193
AtlCoolCopy::m_chansel
cool::ChannelSelection m_chansel
Definition: AtlCoolCopy.cxx:267
PlotCalibFromCool.nchan
nchan
Definition: PlotCalibFromCool.py:564
AtlCoolCopy::openConnections
bool openConnections(const std::string &sourcedb, const std::string &destdb, bool allowcreate)
Definition: AtlCoolCopy.cxx:339
AtlCoolCopy::m_userupdatehead
bool m_userupdatehead
Definition: AtlCoolCopy.cxx:178
python.subdetectors.tile.Blob
Blob
Definition: tile.py:17
CheckTagAssociation.taglist
taglist
Definition: CheckTagAssociation.py:103
AtlCoolCopy::m_parfile
std::vector< std::string > m_parfile
Definition: AtlCoolCopy.cxx:246
pool::IFileCatalog::disconnect
void disconnect()
Definition: IFileCatalog.h:40
AtlCoolCopy::m_hitagmap
HiTagMap m_hitagmap
Definition: AtlCoolCopy.cxx:273
pool::IFileCatalog::commit
void commit()
Save catalog to file.
Definition: IFileCatalog.h:49
AtlCoolCopy::m_dbSvc
cool::IDatabaseSvc * m_dbSvc
Definition: AtlCoolCopy.cxx:254
exclude
std::set< std::string > exclude
list of directories to be excluded
Definition: hcg.cxx:95
ReadOfcFromCool.field
field
Definition: ReadOfcFromCool.py:48
AtlCoolCopy::m_newrunemin
cool::ValidityKey m_newrunemin
Definition: AtlCoolCopy.cxx:225
dbg::ptr
void * ptr(T *p)
Definition: SGImplSvc.cxx:74
AtlCoolCopy::m_usertags
bool m_usertags
Definition: AtlCoolCopy.cxx:177
AtlCoolCopy::m_srunemin
cool::ValidityKey m_srunemin
Definition: AtlCoolCopy.cxx:229
AtlCoolCopy::m_sourcedb
std::string m_sourcedb
Definition: AtlCoolCopy.cxx:171
CaloSwCorrections.gap
def gap(flags, cells_name, *args, **kw)
Definition: CaloSwCorrections.py:212
AtlCoolCopy::m_forcemulti
bool m_forcemulti
Definition: AtlCoolCopy.cxx:196
ITransaction.h
physics_parameters.url
string url
Definition: physics_parameters.py:27
ATLAS_NOT_THREAD_SAFE
int AtlCoolCopy::copyFolder ATLAS_NOT_THREAD_SAFE(const std::string &folder, const std::vector< std::string > &taglist)
Definition: AtlCoolCopy.cxx:535
AtlCoolCopy::equalRecord
static bool equalRecord(const cool::IRecord &lhs, const cool::IRecord &rhs)
Definition: AtlCoolCopy.cxx:2509
dq_defect_bulk_create_defects.line
line
Definition: dq_defect_bulk_create_defects.py:27
AtlCoolCopy::m_ignoremode
bool m_ignoremode
Definition: AtlCoolCopy.cxx:217
atlasStyleMacro.icol
int icol
Definition: atlasStyleMacro.py:13
AtlCoolCopy::m_newtimemax
cool::ValidityKey m_newtimemax
Definition: AtlCoolCopy.cxx:228
CoraCoolObject.h
cool
Definition: CoolTagInfo.h:12
dq_defect_copy_defect_database.channels
def channels
Definition: dq_defect_copy_defect_database.py:56
AtlCoolCopy::timeVal
static cool::ValidityKey timeVal(const char *input)
Definition: AtlCoolCopy.cxx:2541
AtlCoolCopy::m_coracool
bool m_coracool
Definition: AtlCoolCopy.cxx:216
AtlCoolCopy::m_applocksv
bool m_applocksv
Definition: AtlCoolCopy.cxx:206
pool::IFileCatalog::lookupFileByLFN
void lookupFileByLFN(const std::string &lfn, std::string &fid) const
Return the status of a LFName.
Definition: IFileCatalog.h:82
python.Dumpers.aname
string aname
Definition: Dumpers.py:5543
AtlCoolCopy::getCoolHistGUID
static std::string getCoolHistGUID(const std::string &file)
Definition: AtlCoolCopy.cxx:3266
dq_defect_copy_defect_database.since
def since
Definition: dq_defect_copy_defect_database.py:54
AtlCoolCopy::m_forcesingle
bool m_forcesingle
Definition: AtlCoolCopy.cxx:195
AtlCoolCopy::tagParents
int tagParents()
Definition: AtlCoolCopy.cxx:3035
python.checkMetadata.metadata
metadata
Definition: checkMetadata.py:175
AtlCoolCopy::m_gettime
bool m_gettime
Definition: AtlCoolCopy.cxx:208
AtlCoolCopy::checkRef
void checkRef(const cool::IRecord &payload, const std::string &folder, const std::string &tag)
Definition: AtlCoolCopy.cxx:2998
dq_defect_copy_defect_database.until
def until
Definition: dq_defect_copy_defect_database.py:55
AtlCoolCopy::m_checkrefs
bool m_checkrefs
Definition: AtlCoolCopy.cxx:185
ReplicaSorter.h
python.TileCalibTools.copyFolder
def copyFolder(dbr, dbw, folder, tagr, tagw, chanNum, pointInTime1, pointInTime2)
Definition: TileCalibTools.py:446
AtlCoolCopy::setupCatalog
static pool::IFileCatalog * setupCatalog(const std::vector< std::string > &catvec)
Definition: AtlCoolCopy.cxx:3319
python.setupRTTAlg.size
int size
Definition: setupRTTAlg.py:39
AtlCoolCopy::m_poolrefs
PoolMap m_poolrefs
Definition: AtlCoolCopy.cxx:270
skel.input1
tuple input1
Definition: skel.GENtoEVGEN.py:775
main
int main(int, char **)
Main class for all the CppUnit test classes
Definition: CppUnit_SGtestdriver.cxx:141
AtlCoolCopy::m_nt_lbsince
UInt_t m_nt_lbsince
Definition: AtlCoolCopy.cxx:288
TRTCalib_cfilter.p2
p2
Definition: TRTCalib_cfilter.py:131
AtlCoolCopy::m_newrunemax
cool::ValidityKey m_newrunemax
Definition: AtlCoolCopy.cxx:226
CoraCoolObject::const_iterator
AttrListVec::const_iterator const_iterator
Definition: CoraCoolObject.h:23
AtlCoolCopy::m_nt_since
ULong64_t m_nt_since
Definition: AtlCoolCopy.cxx:284
histSizes.code
code
Definition: histSizes.py:129
tags
std::vector< std::string > tags
Definition: hcg.cxx:102
AtlCoolCopy::m_nohitag
bool m_nohitag
Definition: AtlCoolCopy.cxx:194
pool::IFileCatalog
Definition: IFileCatalog.h:23
rerun_display.lfn
lfn
Definition: rerun_display.py:60
AtlCoolCopy::m_open
bool m_open
Definition: AtlCoolCopy.cxx:266
CaloCondBlobAlgs_fillNoiseFromASCII.desc
desc
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:53
AtlCoolCopy::m_poolall
bool m_poolall
Definition: AtlCoolCopy.cxx:188
AtlCoolCopy::m_nt_runsince
UInt_t m_nt_runsince
Definition: AtlCoolCopy.cxx:286
ParseInputs.gDirectory
gDirectory
Definition: Final2012/ParseInputs.py:133
AtlCoolCopy::m_skiprep
bool m_skiprep
Definition: AtlCoolCopy.cxx:204
lumiFormat.i
int i
Definition: lumiFormat.py:85
AtlCoolCopy::CoolTagMap
std::map< std::string, CoolTagInfo > CoolTagMap
Definition: AtlCoolCopy.cxx:278
ReadCellNoiseFromCool.chan
chan
Definition: ReadCellNoiseFromCool.py:52
LArCellNtuple.argv
argv
Definition: LArCellNtuple.py:152
python.LArBadChannelDBAlg.xFFFFFFFF
xFFFFFFFF
Definition: LArBadChannelDBAlg.py:73
AtlCoolCopy::m_alliov
bool m_alliov
Definition: AtlCoolCopy.cxx:180
AtlCoolCopy::m_nt_until
ULong64_t m_nt_until
Definition: AtlCoolCopy.cxx:285
maskDeadModules.run1
run1
Definition: maskDeadModules.py:40
AtlCoolCopy::m_channelRange
std::vector< std::string > m_channelRange
Definition: AtlCoolCopy.cxx:231
AtlCoolCopy::m_channel2
std::string m_channel2
Definition: AtlCoolCopy.cxx:233
AtlCoolCopy::m_ignorespec
bool m_ignorespec
Definition: AtlCoolCopy.cxx:218
AtlCoolCopy::m_checkdesttag
bool m_checkdesttag
Definition: AtlCoolCopy.cxx:219
AtlCoolCopy::m_poolcat
std::vector< std::string > m_poolcat
Definition: AtlCoolCopy.cxx:264
pool::IFileCatalog::start
void start()
redirect to init() for Gaudi FC
Definition: IFileCatalog.h:45
PlotPulseshapeFromCool.input
input
Definition: PlotPulseshapeFromCool.py:106
calibdata.exception
exception
Definition: calibdata.py:495
res
std::pair< std::vector< unsigned int >, bool > res
Definition: JetGroupProductTest.cxx:11
AtlCoolCopy::m_sourceread
bool m_sourceread
Definition: AtlCoolCopy.cxx:201
AtlCoolCopy::m_debug
bool m_debug
Definition: AtlCoolCopy.cxx:179
file
TFile * file
Definition: tile_monitor.h:29
AtlCoolCopy::m_excludechans
std::vector< cool::ChannelId > m_excludechans
Definition: AtlCoolCopy.cxx:247
ISession.h
AtlCoolCopy::adjustIOVs
void adjustIOVs(const cool::ValidityKey &since, const cool::ValidityKey &until, const cool::ValidityKey &qsince, const cool::ValidityKey &quntil, cool::ValidityKey &newsince, cool::ValidityKey &newuntil, const bool timestamp) const
Definition: AtlCoolCopy.cxx:1290
AtlCoolCopy::m_outfolder
std::string m_outfolder
Definition: AtlCoolCopy.cxx:237
Preparation.mode
mode
Definition: Preparation.py:107
AtlCoolCopy::ATLAS_NOT_THREAD_SAFE
int copyIOVs ATLAS_NOT_THREAD_SAFE(const std::string &folder, const std::string &destfolder, const cool::IFolderPtr &sourcefl, const CoraCoolFolderPtr &sourceflc, const cool::IFolderPtr &destfl, const CoraCoolFolderPtr &destflc, const std::string &sourcetag, const std::string &desttag, const cool::ValidityKey since, const cool::ValidityKey until, bool timestamp, bool checkrefs, bool iscora, const cool::PayloadMode::Mode paymode, bool created)
pool::IFileCatalog::setWriteCatalog
void setWriteCatalog(const std::string &connect)
Access to the (first) writable file catalog.
run
Definition: run.py:1
AtlCoolCopy::m_lockedonly
bool m_lockedonly
Definition: AtlCoolCopy.cxx:213
DQHistogramMergeRegExp.argc
argc
Definition: DQHistogramMergeRegExp.py:19
xAOD::double
double
Definition: CompositeParticle_v1.cxx:159
PoolMapElement.h
AtlCoolCopy::m_coolapp
cool::Application m_coolapp
Definition: AtlCoolCopy.cxx:253
AtlCoolCopy::m_listpfn
bool m_listpfn
Definition: AtlCoolCopy.cxx:186
AtlCoolCopy::procOptVector
bool procOptVector(const int argc, const char *argv[], std::vector< std::string > &folders)
Definition: AtlCoolCopy.cxx:2131
AtlCoolCopy::m_forcerune
bool m_forcerune
Definition: AtlCoolCopy.cxx:197
AtlCoolCopy::m_sourceDbPtr
cool::IDatabasePtr m_sourceDbPtr
Definition: AtlCoolCopy.cxx:256
contents
void contents(std::vector< std::string > &keys, TDirectory *td, const std::string &directory, const std::string &pattern, const std::string &path)
Definition: computils.cxx:321
pool::IFileCatalog::connect
void connect()
Definition: IFileCatalog.h:39
CoraCoolFolderPtr
boost::shared_ptr< CoraCoolFolder > CoraCoolFolderPtr
Definition: CoraCoolTypes.h:15
AtlCoolCopy::m_runemax
cool::ValidityKey m_runemax
Definition: AtlCoolCopy.cxx:222
AtlCoolCopy::m_folderlist
std::vector< std::string > m_folderlist
Definition: AtlCoolCopy.cxx:260
AtlCoolCopy::m_poolopen
bool m_poolopen
Definition: AtlCoolCopy.cxx:187
AtlCoolCopy::m_runlist
std::vector< unsigned int > m_runlist
Definition: AtlCoolCopy.cxx:249
CxxUtils::atof
double atof(std::string_view str)
Converts a string into a double / float.
Definition: Control/CxxUtils/Root/StringUtils.cxx:91
pool_uuid.guid
guid
Definition: pool_uuid.py:112
grepfile.ic
int ic
Definition: grepfile.py:33
ReadCellNoiseFromCoolCompare.run2
run2
Definition: ReadCellNoiseFromCoolCompare.py:53
AtlCoolCopy::m_onlinerun
bool m_onlinerun
Definition: AtlCoolCopy.cxx:210
CoraCoolDatabasePtr
boost::shared_ptr< CoraCoolDatabase > CoraCoolDatabasePtr
Definition: CoraCoolTypes.h:12
CoraCoolFolder.h
AtlCoolCopy::m_sealmsg
int m_sealmsg
Definition: AtlCoolCopy.cxx:235
AtlCoolCopy::m_sourceCoraPtr
CoraCoolDatabasePtr m_sourceCoraPtr
Definition: AtlCoolCopy.cxx:258
AtlCoolCopy::m_applock
bool m_applock
Definition: AtlCoolCopy.cxx:205
AtlCoolCopy::ATLAS_NOT_THREAD_SAFE
int resolvePoolRefs ATLAS_NOT_THREAD_SAFE()
AtlCoolCopy::listPoolRefs
int listPoolRefs()
Definition: AtlCoolCopy.cxx:3108
AtlCoolCopy::PoolMap
std::map< std::string, PoolMapElement > PoolMap
Definition: AtlCoolCopy.cxx:268
AtlCoolCopy::checkChannels
bool checkChannels(const std::string &folder, const cool::IFolderPtr &sourcefl, const cool::IFolderPtr &destfl, bool newfolder)
Definition: AtlCoolCopy.cxx:2921
AtlCoolCopy::m_taglabel
std::string m_taglabel
Definition: AtlCoolCopy.cxx:242
AtlCoolCopy::m_timemin
cool::ValidityKey m_timemin
Definition: AtlCoolCopy.cxx:223
name
std::string name
Definition: Control/AthContainers/Root/debug.cxx:240
printHelp
void printHelp()
Definition: AtlCoolCopy.cxx:3345
AtlCoolCopy::m_destDbPtr
cool::IDatabasePtr m_destDbPtr
Definition: AtlCoolCopy.cxx:257
AtlCoolCopy::nocopyIOVs
int nocopyIOVs(const std::string &folder, const cool::IFolderPtr &sourcefl, const std::string &sourcetag, const cool::ValidityKey since, const cool::ValidityKey until, bool checkrefs)
Definition: AtlCoolCopy.cxx:1321
AtlCoolCopy::filePoolRefs
void filePoolRefs()
Definition: AtlCoolCopy.cxx:3282
AtlCoolCopy::m_nt_bufferptr
std::vector< void * > m_nt_bufferptr
Definition: AtlCoolCopy.cxx:293
AtlCoolCopy::getBulkRun
bool getBulkRun()
Definition: AtlCoolCopy.cxx:2781
EventContainers::Mode
Mode
Definition: IdentifiableContainerBase.h:13
pool::SimpleUtilityBase
Common base class for POOL utilities.
Definition: SimpleUtilityBase.h:23
AtlCoolCopy::HiTagMap
std::map< std::string, std::string > HiTagMap
Definition: AtlCoolCopy.cxx:272
AtlCoolCopy::m_runemin
cool::ValidityKey m_runemin
Definition: AtlCoolCopy.cxx:221
AtlCoolCopy::writeTagInfo
int writeTagInfo()
Definition: AtlCoolCopy.cxx:3097
AtlCoolCopy::m_prunetags
bool m_prunetags
Definition: AtlCoolCopy.cxx:212
AtlCoolCopy::p_rootfile
TFile * p_rootfile
Definition: AtlCoolCopy.cxx:281
mc.nskip
nskip
Definition: mc.PhPy8EG_Hto4l_NNLOPS_nnlo_30_ggH125_ZZ4l.py:41
AtlCoolCopy::m_bufsize
int m_bufsize
Definition: AtlCoolCopy.cxx:234
AtlCoolCopy::m_timedb
std::string m_timedb
Definition: AtlCoolCopy.cxx:241
AtlCoolCopy::m_analyse
bool m_analyse
Definition: AtlCoolCopy.cxx:184
AtlCoolCopy::m_poollast
PoolMap::iterator m_poollast
Definition: AtlCoolCopy.cxx:269
PixelModuleFeMask_create_db.payload
string payload
Definition: PixelModuleFeMask_create_db.py:69
AtlCoolCopy::m_tags
std::vector< std::string > m_tags
Definition: AtlCoolCopy.cxx:262
AtlCoolCopy::m_nochannel
bool m_nochannel
Definition: AtlCoolCopy.cxx:191
AtlCoolCopy::m_mergecat
std::vector< std::string > m_mergecat
Definition: AtlCoolCopy.cxx:265
pool::IFileCatalog::getFirstPFN
void getFirstPFN(const std::string &fid, std::string &pfn, std::string &tech) const
Get the first PFN + filetype for the given FID.
ir
int ir
counter of the current depth
Definition: fastadd.cxx:49
AtlCoolCopy::m_runinfohost
std::string m_runinfohost
Definition: AtlCoolCopy.cxx:243
CoolTagInfo
Definition: CoolTagInfo.h:16
DeMoAtlasDataLoss.runNumber
string runNumber
Definition: DeMoAtlasDataLoss.py:64
AtlCoolCopy::m_nt_tagid
char m_nt_tagid[256]
Definition: AtlCoolCopy.cxx:291
AtlCoolCopy::m_forcepay
bool m_forcepay
Definition: AtlCoolCopy.cxx:199
python.DataFormatRates.c2
c2
Definition: DataFormatRates.py:123
AtlCoolCopy::m_nodata
bool m_nodata
Definition: AtlCoolCopy.cxx:190
TileSynchronizeBch.copyall
copyall
Definition: TileSynchronizeBch.py:47
AtlCoolCopy::bookOrFindTH1F
static TH1F * bookOrFindTH1F(const std::string &hID, const std::string &htitle, const int chan, const float xlow, const float xhigh)
Definition: AtlCoolCopy.cxx:2078
AtlCoolCopy::m_runfile
std::vector< std::string > m_runfile
Definition: AtlCoolCopy.cxx:248
Athena::Status
Status
Athena specific StatusCode values.
Definition: AthStatusCode.h:22
python.CaloAddPedShiftConfig.int
int
Definition: CaloAddPedShiftConfig.py:45
CoraCoolObjectIterPtr
boost::shared_ptr< CoraCoolObjectIter > CoraCoolObjectIterPtr
Definition: CoraCoolTypes.h:21
CaloSwCorrections.time
def time(flags, cells_name, *args, **kw)
Definition: CaloSwCorrections.py:242
CaloCellTimeCorrFiller.foldertag
string foldertag
Definition: CaloCellTimeCorrFiller.py:20
makeDTCalibBlob_pickPhase.folders
folders
Definition: makeDTCalibBlob_pickPhase.py:346
AtlCoolCopy::addFolder
bool addFolder(const std::string &folder, const bool onlyTags)
Definition: AtlCoolCopy.cxx:460
Pythia8_RapidityOrderMPI.val
val
Definition: Pythia8_RapidityOrderMPI.py:14
AtlCoolCopy::m_cooltagmap
CoolTagMap m_cooltagmap
Definition: AtlCoolCopy.cxx:279
AtlCoolCopy::m_copytaginfo
bool m_copytaginfo
Definition: AtlCoolCopy.cxx:214
AtlCoolCopy::transConn
static std::string transConn(const std::string &inconn)
Definition: AtlCoolCopy.cxx:449
AtlCoolCopy::m_includehead
bool m_includehead
Definition: AtlCoolCopy.cxx:175
AtlCoolCopy::getRunList
bool getRunList()
Definition: AtlCoolCopy.cxx:2873
CoraCoolObjectPtr
boost::shared_ptr< CoraCoolObject > CoraCoolObjectPtr
Definition: CoraCoolTypes.h:18
AtlCoolCopy::m_newdataset
std::string m_newdataset
Definition: AtlCoolCopy.cxx:239
makeDTCalibBlob_pickPhase.outtag
string outtag
Definition: makeDTCalibBlob_pickPhase.py:286
AtlCoolCopy::m_nt_treename
std::string m_nt_treename
Definition: AtlCoolCopy.cxx:292
AtlCoolCopy::m_root
bool m_root
Definition: AtlCoolCopy.cxx:182
python.SystemOfUnits.s
float s
Definition: SystemOfUnits.py:147
CoraCoolDatabase.h
CaloCondBlobAlgs_fillNoiseFromASCII.folder
folder
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:55
AtlCoolCopy::isOpen
bool isOpen() const
Definition: AtlCoolCopy.cxx:296
AtlCoolCopy::m_recreate
bool m_recreate
Definition: AtlCoolCopy.cxx:174
AtlCoolCopy::rootAllocate
bool rootAllocate(const cool::IFieldSpecification &spec, void *&sptr, char &rootID) const
Definition: AtlCoolCopy.cxx:1715
AtlCoolCopy::m_destCoraPtr
CoraCoolDatabasePtr m_destCoraPtr
Definition: AtlCoolCopy.cxx:259
CxxUtils::atoi
int atoi(std::string_view str)
Helper functions to unpack numbers decoded in string into integers and doubles The strings are requir...
Definition: Control/CxxUtils/Root/StringUtils.cxx:85
CoolTagInfo.h
CoraCoolObjectIter.h
WriteCallback
size_t WriteCallback(void *contents, size_t size, size_t nmemb, std::string *s)
Definition: AtlCoolCopy.cxx:64
AtlCoolCopy::m_nocopy
bool m_nocopy
Definition: AtlCoolCopy.cxx:189
python.TrigEgammaMonitorHelper.TH1F
def TH1F(name, title, nxbins, bins_par2, bins_par3=None, path='', **kwargs)
Definition: TrigEgammaMonitorHelper.py:24
LArCellNtuple.ifile
string ifile
Definition: LArCellNtuple.py:133
AtlCoolCopy::getLockedTags
bool getLockedTags()
Definition: AtlCoolCopy.cxx:429
AtlCoolCopy::m_nt_channel
UInt_t m_nt_channel
Definition: AtlCoolCopy.cxx:290
AtlCoolCopy::m_zeronull
bool m_zeronull
Definition: AtlCoolCopy.cxx:183
CaloCondBlobAlgs_fillNoiseFromASCII.tag
string tag
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:23
AtlCoolCopy::AtlCoolCopy
AtlCoolCopy(const std::string &sourcedb, const std::string &destdb, bool allowcreate=false)
Definition: AtlCoolCopy.cxx:300
AtlCoolCopy::m_noclobroot
bool m_noclobroot
Definition: AtlCoolCopy.cxx:220
AtlCoolCopy::setOpts
int setOpts(int argc, const char *argv[])
Definition: AtlCoolCopy.cxx:2378
AtlCoolCopy::m_addlfn
std::vector< std::string > m_addlfn
Definition: AtlCoolCopy.cxx:245
get_generator_info.error
error
Definition: get_generator_info.py:40
AtlCoolCopy::addExclude
bool addExclude(const std::string &folder)
Definition: AtlCoolCopy.cxx:528
AtlCoolCopy::m_anadelt
long long m_anadelt
Definition: AtlCoolCopy.cxx:236
checker_macros.h
Define macros for attributes used to control the static checker.
python.PyAthena.obj
obj
Definition: PyAthena.py:132
Cut::all
@ all
Definition: SUSYToolsAlg.cxx:67
error
Definition: IImpactPoint3dEstimator.h:70
AtlCoolCopy::m_hiparent
std::vector< std::string > m_hiparent
Definition: AtlCoolCopy.cxx:275
Amg::distance
float distance(const Amg::Vector3D &p1, const Amg::Vector3D &p2)
calculates the distance between two point in 3D space
Definition: GeoPrimitivesHelpers.h:54
AtlCoolCopy::m_chdesc
bool m_chdesc
Definition: AtlCoolCopy.cxx:192
AtlCoolCopy::timeString
static std::string timeString(const cool::ValidityKey iovtime)
Definition: AtlCoolCopy.cxx:2572
AtlCoolCopy::rootIOVs
int rootIOVs(const std::string &folder, const cool::IFolderPtr &sourcefl, const std::string &sourcetag, const cool::ValidityKey since, const cool::ValidityKey until, const bool timestamp)
Definition: AtlCoolCopy.cxx:1550
pool::IFileCatalog::addReadCatalog
void addReadCatalog(const std::string &connect)
Add new catalog identified by name to the existing ones.
Definition: IFileCatalog.h:116
ReplicaSorter
Definition: Database/CoolConvUtilities/src/ReplicaSorter.h:12
AtlCoolCopy::m_folderexcl
std::vector< std::string > m_folderexcl
Definition: AtlCoolCopy.cxx:261
AtlCoolCopy::m_allowcreate
bool m_allowcreate
Definition: AtlCoolCopy.cxx:173
AtlCoolCopy::m_forcetime
bool m_forcetime
Definition: AtlCoolCopy.cxx:198
CaloCondBlobAlgs_fillNoiseFromASCII.blob
blob
Definition: CaloCondBlobAlgs_fillNoiseFromASCII.py:95
AtlCoolCopy::getOnlineRun
bool getOnlineRun()
Definition: AtlCoolCopy.cxx:2693
python.SystemOfUnits.L
float L
Definition: SystemOfUnits.py:92
python.LArMinBiasAlgConfig.float
float
Definition: LArMinBiasAlgConfig.py:65
AtlCoolCopy::m_forcenopay
bool m_forcenopay
Definition: AtlCoolCopy.cxx:200
AtlCoolCopy::isNumeric
static bool isNumeric(const char *input)
Definition: AtlCoolCopy.cxx:2498
AtlCoolCopy::m_addguid
std::vector< std::string > m_addguid
Definition: AtlCoolCopy.cxx:244
AtlCoolCopy::openCoraCool
bool openCoraCool()
Definition: AtlCoolCopy.cxx:409
AtlCoolCopy::m_newtimemin
cool::ValidityKey m_newtimemin
Definition: AtlCoolCopy.cxx:227
PoolMapElement
Definition: PoolMapElement.h:14
pool::IFileCatalog::getLFNs
void getLFNs(const std::string &fid, Files &files) const
Get all logical names for a given FID. Return pairs <LFN,FID>
Definition: IFileCatalog.h:61